From e882a620d575bffdca21144a41bb990ab2039d8e Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 11 Jul 2023 04:18:28 +0100 Subject: [PATCH 01/10] Add name_by option to allow name.ome.zarr --- src/omero_zarr/cli.py | 9 +++++++++ src/omero_zarr/raw_pixels.py | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index ae279b2..5f84d81 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -278,6 +278,15 @@ def _configure(self, parser: Parser) -> None: default=None, help="Maximum number of workers (only for use with bioformats2raw)", ) + export.add_argument( + "--name_by", + default="id", + choices=["id", "name"], + help=( + "How to name the Image or Plate zarr. Default 'id' is [ID].zarr. " + "'name' is [NAME].ome.zarr" + ), + ) export.add_argument( "object", type=ProxyStringType("Image"), diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index c49d987..91e3792 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -45,8 +45,12 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) - target_dir = args.output tile_width = args.tile_width tile_height = args.tile_height + name_by = args.name_by - name = os.path.join(target_dir, "%s.zarr" % image.id) + if name_by == "name": + name = os.path.join(target_dir, "%s.ome.zarr" % image.name) + else: + name = os.path.join(target_dir, "%s.zarr" % image.id) print(f"Exporting to {name} ({VERSION})") store = open_store(name) root = open_group(store) @@ -245,7 +249,13 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1) target_dir = args.output - name = os.path.join(target_dir, "%s.zarr" % plate.id) + name_by = args.name_by + + if name_by == "name": + name = os.path.join(target_dir, "%s.ome.zarr" % plate.name) + else: + name = os.path.join(target_dir, "%s.zarr" % plate.id) + store = open_store(name) print(f"Exporting to {name} ({VERSION})") root = open_group(store) From d80d276d12b3bf29d39fccf53806148358df150d Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 11 Jul 2023 07:31:23 +0100 Subject: [PATCH 02/10] Remove .pattern from image name --- src/omero_zarr/raw_pixels.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 91e3792..dd29090 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -48,7 +48,9 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) - name_by = args.name_by if name_by == "name": - name = os.path.join(target_dir, "%s.ome.zarr" % image.name) + # remove .pattern from name, add .ome.zarr + img_name = image.name.replace(".pattern", "") + name = os.path.join(target_dir, "%s.ome.zarr" % img_name) else: name = os.path.join(target_dir, "%s.zarr" % image.id) print(f"Exporting to {name} ({VERSION})") From 1d726264f44e2b6cb833bcc23603e2b7e56121b5 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 7 Aug 2023 15:59:41 +0100 Subject: [PATCH 03/10] Handle first plate Wells having no fields --- src/omero_zarr/raw_pixels.py | 2 ++ src/omero_zarr/util.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index dd29090..597ff45 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -308,6 +308,8 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) print_status(int(t0), int(time.time()), count, total) # Update plate_metadata after each Well + if len(well_paths) == 0: + continue write_plate_metadata( root, row_names, diff --git a/src/omero_zarr/util.py b/src/omero_zarr/util.py index 0a4e382..647e2bc 100644 --- a/src/omero_zarr/util.py +++ b/src/omero_zarr/util.py @@ -32,7 +32,7 @@ def print_status(t0: int, t: int, count: int, total: int) -> None: """ percent_done = float(count) * 100 / total dt = t - t0 - if dt > 0: + if dt > 0 and count > 0: rate = float(count) / (t - t0) eta_f = float(total - count) / rate eta = time.strftime("%H:%M:%S", time.gmtime(eta_f)) From 6290b4b2590055832aacefba291d4853dd3e4d09 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 8 Aug 2023 07:51:16 +0100 Subject: [PATCH 04/10] Don't try to remove .pattern from image names --- src/omero_zarr/raw_pixels.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 597ff45..acdeb41 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -48,9 +48,7 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) - name_by = args.name_by if name_by == "name": - # remove .pattern from name, add .ome.zarr - img_name = image.name.replace(".pattern", "") - name = os.path.join(target_dir, "%s.ome.zarr" % img_name) + name = os.path.join(target_dir, "%s.ome.zarr" % image.name) else: name = os.path.join(target_dir, "%s.zarr" % image.id) print(f"Exporting to {name} ({VERSION})") From c73d40046536f8b5cc62908ebdaa86d097a30d0b Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 15 Aug 2023 13:11:53 +0100 Subject: [PATCH 05/10] Replace [] with () in zarr names --- src/omero_zarr/raw_pixels.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index acdeb41..bd44bf2 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -41,6 +41,12 @@ from .util import marshal_axes, marshal_transformations, open_store, print_status +def sanitize_name(zarr_name: str) -> str: + # Avoids re.compile errors when writing Zarr data with the named root + # https://github.com/ome/omero-cli-zarr/pull/147#issuecomment-1669075660 + return zarr_name.replace("[", "(").replace("]", ")") + + def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -> None: target_dir = args.output tile_width = args.tile_width @@ -48,7 +54,8 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) - name_by = args.name_by if name_by == "name": - name = os.path.join(target_dir, "%s.ome.zarr" % image.name) + img_name = sanitize_name(image.name) + name = os.path.join(target_dir, "%s.ome.zarr" % img_name) else: name = os.path.join(target_dir, "%s.zarr" % image.id) print(f"Exporting to {name} ({VERSION})") @@ -252,7 +259,8 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) name_by = args.name_by if name_by == "name": - name = os.path.join(target_dir, "%s.ome.zarr" % plate.name) + plate_name = sanitize_name(plate.name) + name = os.path.join(target_dir, "%s.ome.zarr" % plate_name) else: name = os.path.join(target_dir, "%s.zarr" % plate.id) From 57da8bbc5f595e73763b5053aa52b3a16ad01c9c Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 23 Aug 2023 11:53:00 +0100 Subject: [PATCH 06/10] Add get_zarr_name(). Always use .ome.zarr --- src/omero_zarr/raw_pixels.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index bd44bf2..af7073b 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -47,17 +47,23 @@ def sanitize_name(zarr_name: str) -> str: return zarr_name.replace("[", "(").replace("]", ")") -def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -> None: +def get_zarr_name( + obj: omero.gateway.BlitzObjectWrapper, args: argparse.Namespace +) -> str: target_dir = args.output - tile_width = args.tile_width - tile_height = args.tile_height name_by = args.name_by - if name_by == "name": - img_name = sanitize_name(image.name) - name = os.path.join(target_dir, "%s.ome.zarr" % img_name) + obj_name = sanitize_name(obj.name) + name = os.path.join(target_dir, "%s.ome.zarr" % obj_name) else: - name = os.path.join(target_dir, "%s.zarr" % image.id) + name = os.path.join(target_dir, "%s.ome.zarr" % obj.id) + return name + + +def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -> None: + tile_width = args.tile_width + tile_height = args.tile_height + name = get_zarr_name(image, args) print(f"Exporting to {name} ({VERSION})") store = open_store(name) root = open_group(store) @@ -254,15 +260,7 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) n_cols = gs["columns"] n_fields = plate.getNumberOfFields() total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1) - - target_dir = args.output - name_by = args.name_by - - if name_by == "name": - plate_name = sanitize_name(plate.name) - name = os.path.join(target_dir, "%s.ome.zarr" % plate_name) - else: - name = os.path.join(target_dir, "%s.zarr" % plate.id) + name = get_zarr_name(plate, args) store = open_store(name) print(f"Exporting to {name} ({VERSION})") From 36e66c43332d96682b99f8244ae148008e982e43 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 23 Aug 2023 11:53:31 +0100 Subject: [PATCH 07/10] Update README with --name_by option --- README.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 5d43e47..8972c62 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,7 @@ from OMERO as zarr files, according to the spec at https://github.com/ome/omero-ms-zarr/blob/master/spec.md as well as Masks associated with Images. -Images are 5D arrays of shape `(t, c, z, y, x)`. +Images are nD arrays of shape, up to `(t, c, z, y, x)`. Plates are a hierarchy of `plate/row/column/field(image)`. Masks are 2D bitmasks which can exist on muliplte planes of an Image. In `ome-zarr` sets of Masks are collected together into "labels". @@ -20,8 +20,7 @@ In `ome-zarr` sets of Masks are collected together into "labels". It supports export using 2 alternative methods: - By default the OMERO API is used to load planes as numpy arrays - and the zarr file is created from this data. NB: currently, large - tiled images are not supported by this method. + and the zarr file is created from this data. - Alternatively, if you can read directly from the OMERO binary repository and have installed https://github.com/glencoesoftware/bioformats2raw @@ -37,16 +36,19 @@ Images and Plates To export Images or Plates via the OMERO API:: - # Image will be saved in current directory as 1.zarr + # Image will be saved in current directory as 1.ome.zarr $ omero zarr export Image:1 - # Plate will be saved in current directory as 2.zarr + # Plate will be saved in current directory as 2.ome.zarr $ omero zarr export Plate:2 + # Use the Image or Plate 'name' to save e.g. my_image.ome.zarr + $ omero zarr --name_by name export Image:1 + # Specify an output directory $ omero zarr --output /home/user/zarr_files export Image:1 - # By default, a tile size of 1024 is used. Specify values with + # By default, a tile (chunk) size of 1024 is used. Specify values with $ omero zarr export Image:1 --tile_width 256 --tile_height 256 From 3ca2c58fca5c6a4fb90a26d5a0e51df041890033 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 23 Aug 2023 12:41:57 +0100 Subject: [PATCH 08/10] Update README and masks.py to use ID.ome.zarr --- README.rst | 13 +++++++------ src/omero_zarr/masks.py | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 8972c62..50c66b8 100644 --- a/README.rst +++ b/README.rst @@ -68,24 +68,25 @@ Masks and Polygons To export Masks or Polygons for an Image or Plate, use the `masks` or `polygons` command:: - # Saved under 1.zarr/labels/0 - 1.zarr/ must already exist + # Saved under 1.ome.zarr/labels/0 + # 1.ome.zarr/ should already exist or specify path with --source-image $ omero zarr masks Image:1 - # Labels saved under each image. e.g 2.zarr/A/1/0/labels/0 - # Plate should already be exported + # Labels saved under each image. e.g 2.ome.zarr/A/1/0/labels/0 + # 2.ome.zarr should already be exported or specify path with --source-image $ omero zarr masks Plate:2 - # Saved under zarr_files/1.zarr/labels/0 + # Saved under zarr_files/1.ome.zarr/labels/0 $ omero zarr --output /home/user/zarr_files masks Image:1 # Specify the label-name. (default is '0') - # e.g. Export to 1.zarr/labels/A + # e.g. Export to 1.ome.zarr/labels/A $ omero zarr masks Image:1 --label-name=A # Allow overlapping masks or polygons (overlap will be maximum value of the dtype) $ omero zarr polygons Image:1 --overlaps=dtype_max -The default behaviour is to export all masks or polygons on the Image to a single 5D +The default behaviour is to export all masks or polygons on the Image to a single nD "labeled" zarr array, with a different value for each Shape. An exception will be thrown if any of the masks overlap, unless the `--overlaps` option is used as above. diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index d54a07d..bf8e668 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -289,9 +289,9 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: ignored_dimensions.add(d) if self.plate: - filename = f"{self.plate.id}.zarr" + filename = f"{self.plate.id}.ome.zarr" else: - filename = f"{self.image.id}.zarr" + filename = f"{self.image.id}.ome.zarr" # Verify that we are linking this mask to a real ome-zarr source_image = self.source_image From 470ad2b197c989dda533e7bd3095d0c6c4cafe68 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 14 Dec 2023 09:19:38 +0000 Subject: [PATCH 09/10] Use .ome.zarr in args help --- src/omero_zarr/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 5f84d81..c855d8a 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -283,7 +283,7 @@ def _configure(self, parser: Parser) -> None: default="id", choices=["id", "name"], help=( - "How to name the Image or Plate zarr. Default 'id' is [ID].zarr. " + "How to name the Image or Plate zarr. Default 'id' is [ID].ome.zarr. " "'name' is [NAME].ome.zarr" ), ) From a14f62a940f4974a305b3f5858049ee493b898ef Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 14 Dec 2023 11:19:15 +0000 Subject: [PATCH 10/10] Improve source-image example in README --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 50c66b8..bc3f708 100644 --- a/README.rst +++ b/README.rst @@ -69,9 +69,12 @@ Masks and Polygons To export Masks or Polygons for an Image or Plate, use the `masks` or `polygons` command:: # Saved under 1.ome.zarr/labels/0 - # 1.ome.zarr/ should already exist or specify path with --source-image + # 1.ome.zarr/ should already exist... $ omero zarr masks Image:1 + # ...or specify path with --source-image + $ omero zarr masks Image:1 --source-image my_image.ome.zarr + # Labels saved under each image. e.g 2.ome.zarr/A/1/0/labels/0 # 2.ome.zarr should already be exported or specify path with --source-image $ omero zarr masks Plate:2