From 623b4ab7b54ba4194180b6663ab473a78eef90c1 Mon Sep 17 00:00:00 2001 From: Joost de Folter Date: Thu, 27 Jun 2024 12:58:15 +0200 Subject: [PATCH] Expand Zarr output class to use v2/v3 --- .gitignore | 1 + OmeSliCC/Zarr.py | 24 ++++++++++++++++++------ OmeSliCC/conversion.py | 11 ++++++----- tests/ome_zarr_test.py | 32 ++++++++++++++++++++++++-------- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 4a79a0d..f80a6bf 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,4 @@ docs/doctrees/ sphinx/_build/ resources/ !resources/params.yml +/src/ome-zarr diff --git a/OmeSliCC/Zarr.py b/OmeSliCC/Zarr.py index 2c487d7..09ec168 100644 --- a/OmeSliCC/Zarr.py +++ b/OmeSliCC/Zarr.py @@ -1,6 +1,7 @@ import numpy as np import pathlib import zarr +from zarr.store import make_store_path from OmeSliCC.image_util import * from OmeSliCC.ome_zarr_util import * @@ -13,15 +14,19 @@ def __init__(self, filename): self.ome = ('ome' == self.filename.split('.')[1].lower()) self.data = [] - def create(self, source, tile_size=None, - npyramid_add=0, pyramid_downsample=2, compression=[]): + def create(self, source, tile_size=None, npyramid_add=0, pyramid_downsample=2, compression=[], + v3=False): # create empty dataset dimension_order = source.get_dimension_order() self.dimension_order = dimension_order self.npyramid_add = npyramid_add self.pyramid_downsample = pyramid_downsample - file_url = pathlib.Path(self.filename).as_uri() - self.zarr_root = zarr.open_group(file_url, mode='w', storage_options={'dimension_separator': '/'}) + if v3: + store_path = make_store_path(self.filename, mode='w') + self.zarr_root = zarr.Group.create(store=store_path, zarr_format=3) + else: + file_url = pathlib.Path(self.filename, mode='w').as_uri() + self.zarr_root = zarr.open_group(store=file_url, mode='w', storage_options={'dimension_separator': '/'}) xyzct = source.get_size_xyzct() shape0 = [xyzct['xyzct'.index(dimension)] for dimension in dimension_order] dtype = source.pixel_types[0] @@ -37,8 +42,15 @@ def create(self, source, tile_size=None, tile_size = [1, 1, 1] + list(np.flip(tile_size)) for pathi in range(1 + npyramid_add): shape = calc_shape_scale(shape0, dimension_order, scale) - self.data.append(self.zarr_root.create_dataset(str(pathi), shape=shape, chunks=tile_size, dtype=dtype, - compressor=compressor, filters=compression_filters)) + if v3: + shape = np.array(shape).tolist() # convert to basic int + tile_size = np.array(tile_size).tolist() # convert to basic int + dataset = self.zarr_root.create_array(str(pathi), shape=shape, chunks=tile_size, dtype=dtype, + compressor=compressor, codecs=compression_filters) + else: + dataset = self.zarr_root.create_dataset(str(pathi), shape=shape, chunks=tile_size, dtype=dtype, + compressor=compressor, filters=compression_filters) + self.data.append(dataset) # used for ome metadata: datasets.append({ 'path': str(pathi), diff --git a/OmeSliCC/conversion.py b/OmeSliCC/conversion.py index d101341..4b1951c 100644 --- a/OmeSliCC/conversion.py +++ b/OmeSliCC/conversion.py @@ -18,9 +18,9 @@ from OmeSliCC.util import * -def create_source(source_ref: str, params: dict, omero = None) -> OmeSource: - source_pixel_size = split_value_unit_list(params['input'].get('pixel_size')) - target_pixel_size = split_value_unit_list(params['output'].get('pixel_size')) +def create_source(source_ref: str, params: dict, omero=None) -> OmeSource: + source_pixel_size = split_value_unit_list(params.get('input', {}).get('pixel_size')) + target_pixel_size = split_value_unit_list(params.get('output', {}).get('pixel_size')) ext = os.path.splitext(source_ref)[1].lower() if omero is not None: from OmeSliCC.OmeroSource import OmeroSource @@ -228,7 +228,8 @@ def save_image_as_ome_zarr(source: OmeSource, data: np.ndarray, output_filename: npyramid_add=npyramid_add, pyramid_downsample=pyramid_downsample) -def save_image_as_zarr(source: OmeSource, data: np.ndarray, output_filename: str, output_params: dict): +def save_image_as_zarr(source: OmeSource, data: np.ndarray, output_filename: str, output_params: dict, + v3: bool = False): # ome-zarr: https://ngff.openmicroscopy.org/latest/ tile_size = output_params.get('tile_size') compression = output_params.get('compression') @@ -237,7 +238,7 @@ def save_image_as_zarr(source: OmeSource, data: np.ndarray, output_filename: str zarr = Zarr(output_filename) zarr.create(source, tile_size=tile_size, npyramid_add=npyramid_add, pyramid_downsample=pyramid_downsample, - compression=compression) + compression=compression, v3=v3) zarr.set(data) diff --git a/tests/ome_zarr_test.py b/tests/ome_zarr_test.py index edc6761..be24ced 100644 --- a/tests/ome_zarr_test.py +++ b/tests/ome_zarr_test.py @@ -1,13 +1,13 @@ import numpy as np from tqdm import tqdm +from OmeSliCC.conversion import create_source, save_image_as_zarr from OmeSliCC.image_util import * -from OmeSliCC.TiffSource import TiffSource from OmeSliCC.Zarr import Zarr from OmeSliCC.OmeZarrSource import OmeZarrSource -def simple_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample): +def create_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample): w, h = source.get_size() zarr = Zarr(output_filename) zarr.create(source, tile_size=[1, 1, 1, tile_size, tile_size], @@ -24,7 +24,7 @@ def simple_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsa zarr.set(x0, y0, x1, y1, image) -def open_zarr_source(filename): +def open_omezarr_source(filename): source = OmeZarrSource(filename) image = source._asarray_level(0, x0=15000, x1=16000, y0=15000, y1=16000) show_image(image) @@ -32,13 +32,29 @@ def open_zarr_source(filename): show_image(image) +def save_zarr_v3(source, data, filename, output_params): + save_image_as_zarr(source, data, filename, output_params, v3=True) + + if __name__ == '__main__': - source = TiffSource('E:/Personal/Crick/slides/TCGA_KIRC/0f450938-5604-4af6-8783-c385ea647569/TCGA-A3-3358-01Z-00-DX1.1bd1c720-f6db-4837-8f83-e7476dd2b0a3.svs') + #filename = 'E:/Personal/Crick/slides/TCGA_KIRC/0f450938-5604-4af6-8783-c385ea647569/TCGA-A3-3358-01Z-00-DX1.1bd1c720-f6db-4837-8f83-e7476dd2b0a3.svs' + filename = 'E:/Personal/Crick/slides/test_images/zarr test.zarr' + source = create_source(filename, {}) w, h = source.get_size() output_filename = 'D:/slides/test/test.ome.zarr' npyramid_add = 4 pyramid_downsample = 2 - tile_size = 2048 - - simple_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample) - open_zarr_source(output_filename) + tile_size = 16 + compression = None + output_params = { + 'tile_size': [tile_size, tile_size], + 'npyramid_add': npyramid_add, + 'pyramid_downsample': pyramid_downsample, + 'compression': compression + } + + #create_zarr(source, output_filename, tile_size, npyramid_add, pyramid_downsample) + #open_omezarr_source(output_filename) + + data = source.get_source_dask()[0] + save_zarr_v3(source, data, output_filename, output_params)