Expand Zarr output class to use v2/v3

.gitignore

@@ -142,3 +142,4 @@ docs/doctrees/
 sphinx/_build/
 resources/
 !resources/params.yml
+/src/ome-zarr

OmeSliCC/Zarr.py

@@ -1,6 +1,7 @@
 import numpy as np
 import pathlib
 import zarr
+from zarr.store import make_store_path
 
 from OmeSliCC.image_util import *
 from OmeSliCC.ome_zarr_util import *
@@ -13,15 +14,19 @@ def __init__(self, filename):
         self.ome = ('ome' == self.filename.split('.')[1].lower())
         self.data = []
 
-    def create(self, source, tile_size=None,
-               npyramid_add=0, pyramid_downsample=2, compression=[]):
+    def create(self, source, tile_size=None, npyramid_add=0, pyramid_downsample=2, compression=[],
+               v3=False):
         # create empty dataset
         dimension_order = source.get_dimension_order()
         self.dimension_order = dimension_order
         self.npyramid_add = npyramid_add
         self.pyramid_downsample = pyramid_downsample
-        file_url = pathlib.Path(self.filename).as_uri()
-        self.zarr_root = zarr.open_group(file_url, mode='w', storage_options={'dimension_separator': '/'})
+        if v3:
+            store_path = make_store_path(self.filename, mode='w')
+            self.zarr_root = zarr.Group.create(store=store_path, zarr_format=3)
+        else:
+            file_url = pathlib.Path(self.filename, mode='w').as_uri()
+            self.zarr_root = zarr.open_group(store=file_url, mode='w', storage_options={'dimension_separator': '/'})
         xyzct = source.get_size_xyzct()
         shape0 = [xyzct['xyzct'.index(dimension)] for dimension in dimension_order]
         dtype = source.pixel_types[0]
@@ -37,8 +42,15 @@ def create(self, source, tile_size=None,
             tile_size = [1, 1, 1] + list(np.flip(tile_size))
         for pathi in range(1 + npyramid_add):
             shape = calc_shape_scale(shape0, dimension_order, scale)
-            self.data.append(self.zarr_root.create_dataset(str(pathi), shape=shape, chunks=tile_size, dtype=dtype,
-                                                           compressor=compressor, filters=compression_filters))
+            if v3:
+                shape = np.array(shape).tolist()    # convert to basic int
+                tile_size = np.array(tile_size).tolist()  # convert to basic int
+                dataset = self.zarr_root.create_array(str(pathi), shape=shape, chunks=tile_size, dtype=dtype,
+                                                      compressor=compressor, codecs=compression_filters)
+            else:
+                dataset = self.zarr_root.create_dataset(str(pathi), shape=shape, chunks=tile_size, dtype=dtype,
+                                                        compressor=compressor, filters=compression_filters)
+            self.data.append(dataset)
             # used for ome metadata:
             datasets.append({
                 'path': str(pathi),

OmeSliCC/conversion.py

@@ -18,9 +18,9 @@
 from OmeSliCC.util import *
 
 
-def create_source(source_ref: str, params: dict, omero = None) -> OmeSource:
-    source_pixel_size = split_value_unit_list(params['input'].get('pixel_size'))
-    target_pixel_size = split_value_unit_list(params['output'].get('pixel_size'))
+def create_source(source_ref: str, params: dict, omero=None) -> OmeSource:
+    source_pixel_size = split_value_unit_list(params.get('input', {}).get('pixel_size'))
+    target_pixel_size = split_value_unit_list(params.get('output', {}).get('pixel_size'))
     ext = os.path.splitext(source_ref)[1].lower()
     if omero is not None:
         from OmeSliCC.OmeroSource import OmeroSource
@@ -228,7 +228,8 @@ def save_image_as_ome_zarr(source: OmeSource, data: np.ndarray, output_filename:
                npyramid_add=npyramid_add, pyramid_downsample=pyramid_downsample)
 
 
-def save_image_as_zarr(source: OmeSource, data: np.ndarray, output_filename: str, output_params: dict):
+def save_image_as_zarr(source: OmeSource, data: np.ndarray, output_filename: str, output_params: dict,
+                       v3: bool = False):
     # ome-zarr: https://ngff.openmicroscopy.org/latest/
     tile_size = output_params.get('tile_size')
     compression = output_params.get('compression')
@@ -237,7 +238,7 @@ def save_image_as_zarr(source: OmeSource, data: np.ndarray, output_filename: str
 
     zarr = Zarr(output_filename)
     zarr.create(source, tile_size=tile_size, npyramid_add=npyramid_add, pyramid_downsample=pyramid_downsample,
-                compression=compression)
+                compression=compression, v3=v3)
     zarr.set(data)
 
 

tests/ome_zarr_test.py

@@ -1,13 +1,13 @@
 import numpy as np
 from tqdm import tqdm
 
+from OmeSliCC.conversion import create_source, save_image_as_zarr
 from OmeSliCC.image_util import *
-from OmeSliCC.TiffSource import TiffSource
 from OmeSliCC.Zarr import Zarr
 from OmeSliCC.OmeZarrSource import OmeZarrSource
 
 
-def simple_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample):
+def create_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample):
     w, h = source.get_size()
     zarr = Zarr(output_filename)
     zarr.create(source, tile_size=[1, 1, 1, tile_size, tile_size],
@@ -24,21 +24,37 @@ def simple_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsa
             zarr.set(x0, y0, x1, y1, image)
 
 
-def open_zarr_source(filename):
+def open_omezarr_source(filename):
     source = OmeZarrSource(filename)
     image = source._asarray_level(0, x0=15000, x1=16000, y0=15000, y1=16000)
     show_image(image)
     image = source._asarray_level(4)
     show_image(image)
 
 
+def save_zarr_v3(source, data, filename, output_params):
+    save_image_as_zarr(source, data, filename, output_params, v3=True)
+
+
 if __name__ == '__main__':
-    source = TiffSource('E:/Personal/Crick/slides/TCGA_KIRC/0f450938-5604-4af6-8783-c385ea647569/TCGA-A3-3358-01Z-00-DX1.1bd1c720-f6db-4837-8f83-e7476dd2b0a3.svs')
+    #filename = 'E:/Personal/Crick/slides/TCGA_KIRC/0f450938-5604-4af6-8783-c385ea647569/TCGA-A3-3358-01Z-00-DX1.1bd1c720-f6db-4837-8f83-e7476dd2b0a3.svs'
+    filename = 'E:/Personal/Crick/slides/test_images/zarr test.zarr'
+    source = create_source(filename, {})
     w, h = source.get_size()
     output_filename = 'D:/slides/test/test.ome.zarr'
     npyramid_add = 4
     pyramid_downsample = 2
-    tile_size = 2048
-
-    simple_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample)
-    open_zarr_source(output_filename)
+    tile_size = 16
+    compression = None
+    output_params = {
+        'tile_size': [tile_size, tile_size],
+        'npyramid_add': npyramid_add,
+        'pyramid_downsample': pyramid_downsample,
+        'compression': compression
+    }
+
+    #create_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample)
+    #open_omezarr_source(output_filename)
+
+    data = source.get_source_dask()[0]
+    save_zarr_v3(source, data, output_filename, output_params)