From f860abc9c94b132ca3aad37a45a4f53386140d88 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 31 May 2022 13:24:40 +0100 Subject: [PATCH 1/7] First attempt at using bioformats2raw 0.5.0 to generate HCS NGFF --- src/omero_zarr/cli.py | 64 +++++++++++++++++++++--------------- src/omero_zarr/raw_pixels.py | 31 +++++++++-------- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 5d94f72..f049657 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -307,7 +307,11 @@ def export(self, args: argparse.Namespace) -> None: image_to_zarr(image, args) elif isinstance(args.object, PlateI): plate = self._lookup(self.gateway, "Plate", args.object.id) - plate_to_zarr(plate, args) + if args.bf or args.bfpath: + self._bf_export(plate, args) + plate_to_zarr(plate, args, write_binary=False) + else: + plate_to_zarr(plate, args, write_binary=True) def _lookup( self, gateway: BlitzGateway, otype: str, oid: int @@ -319,25 +323,25 @@ def _lookup( self.ctx.die(110, f"No such {otype}: {oid}") return obj - def _bf_export(self, image: BlitzObjectWrapper, args: argparse.Namespace) -> None: + def _bf_export(self, obj: BlitzObjectWrapper, args: argparse.Namespace) -> None: if args.bfpath: abs_path = Path(args.bfpath) - elif image.getInplaceImport(): - p = image.getImportedImageFilePaths()["client_paths"][0] + elif obj.getInplaceImport(): + p = obj.getImportedImageFilePaths()["client_paths"][0] abs_path = Path("/") / Path(p) else: if self.client is None: raise Exception("This cannot happen") # mypy is confused prx, desc = self.client.getManagedRepository(description=True) - p = image.getImportedImageFilePaths()["server_paths"][0] + p = obj.getImportedImageFilePaths()["server_paths"][0] abs_path = Path(desc._path._val) / Path(desc._name._val) / Path(p) - temp_target = (Path(args.output) or Path.cwd()) / f"{image.id}.tmp" - image_target = (Path(args.output) or Path.cwd()) / f"{image.id}.zarr" + temp_target = (Path(args.output) or Path.cwd()) / f"{obj.id}.tmp" + zarr_target = (Path(args.output) or Path.cwd()) / f"{obj.id}.zarr" if temp_target.exists(): self.ctx.die(111, f"{temp_target.resolve()} already exists") - if image_target.exists(): - self.ctx.die(111, f"{image_target.resolve()} already exists") + if zarr_target.exists(): + self.ctx.die(111, f"{zarr_target.resolve()} already exists") cmd: List[str] = [ "bioformats2raw", @@ -353,9 +357,10 @@ def _bf_export(self, image: BlitzObjectWrapper, args: argparse.Namespace) -> Non cmd.append(f"--resolutions={args.resolutions}") if args.max_workers: cmd.append(f"--max_workers={args.max_workers}") - cmd.append(f"--series={image.series}") - cmd.append("--no-root-group") - cmd.append("--no-ome-meta-export") + if obj.OMERO_CLASS == "Image": + cmd.append(f"--series={obj.series}") + cmd.append("--no-root-group") + cmd.append("--no-ome-meta-export") self.ctx.dbg(" ".join(cmd)) process = subprocess.Popen( @@ -367,21 +372,26 @@ def _bf_export(self, image: BlitzObjectWrapper, args: argparse.Namespace) -> Non if stderr: self.ctx.err(stderr.decode("utf-8")) if process.returncode == 0: - image_source = temp_target / "0" - image_source.rename(image_target) - temp_target.rmdir() - self.ctx.out(f"Image exported to {image_target.resolve()}") - - # Add OMERO metadata - store = FSStore( - str(image_target.resolve()), - auto_mkdir=False, - normalize_keys=False, - mode="w", - ) - root = open_group(store) - add_omero_metadata(root, image) - add_toplevel_metadata(root) + if obj.OMERO_CLASS == "Image": + image_source = temp_target / "0" + image_source.rename(zarr_target) + temp_target.rmdir() + else: + temp_target.rename(zarr_target) + self.ctx.out(f"{obj.OMERO_CLASS} exported to {zarr_target.resolve()}") + + if obj.OMERO_CLASS == "Image": + # Add OMERO metadata + store = FSStore( + str(image_target.resolve()), + auto_mkdir=False, + normalize_keys=False, + mode="w", + ) + + root = open_group(store) + add_omero_metadata(root, obj) + add_toplevel_metadata(root) try: diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index b1bcc7b..3b4897f 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -210,7 +210,7 @@ def marshal_acquisition(acquisition: omero.gateway._PlateAcquisitionWrapper) -> return acq -def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) -> None: +def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace, write_binary=True) -> None: """ Exports a plate to a zarr file using the hierarchy discussed here ('Option 3'): https://github.com/ome/omero-ms-zarr/issues/73#issuecomment-706770955 @@ -262,23 +262,26 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) row_group = root.require_group(row) col_group = row_group.require_group(col) field_group = col_group.require_group(field_name) - add_image(img, field_group, cache_dir=cache_dir) + if write_binary: + add_image(img, field_group, cache_dir=cache_dir) add_omero_metadata(field_group, img) - # Update Well metadata after each image - write_well_metadata(col_group, fields) + if write_binary: + # Update Well metadata after each image + write_well_metadata(col_group, fields) max_fields = max(max_fields, field + 1) print_status(int(t0), int(time.time()), count, total) - # Update plate_metadata after each Well - write_plate_metadata( - root, - row_names, - col_names, - wells=list(well_paths), - field_count=max_fields, - acquisitions=plate_acq, - name=plate.name, - ) + if write_binary: + # Update plate_metadata after each Well + write_plate_metadata( + root, + row_names, + col_names, + wells=list(well_paths), + field_count=max_fields, + acquisitions=plate_acq, + name=plate.name, + ) add_toplevel_metadata(root) print("Finished.") From 2c9acb611c4ec305553486788998afc195b1602e Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 31 May 2022 16:40:36 +0100 Subject: [PATCH 2/7] Handle imported file paths for plates --- src/omero_zarr/cli.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index f049657..16f47f0 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -323,17 +323,24 @@ def _lookup( self.ctx.die(110, f"No such {otype}: {oid}") return obj + def _get_first_imported_filepath(self, obj: BlitzObjectWrapper,path_type: str): + if obj.OMERO_CLASS == "Plate": + for well in obj.listChildren(): + for ws in well.listChildren(): + return _get_first_imported_path(ws.getImage(), path_type) + return obj.getImportedImageFilePaths()[path_type][0] + def _bf_export(self, obj: BlitzObjectWrapper, args: argparse.Namespace) -> None: if args.bfpath: abs_path = Path(args.bfpath) elif obj.getInplaceImport(): - p = obj.getImportedImageFilePaths()["client_paths"][0] + p = _get_first_imported_filepath(obj, "client_paths") abs_path = Path("/") / Path(p) else: if self.client is None: raise Exception("This cannot happen") # mypy is confused prx, desc = self.client.getManagedRepository(description=True) - p = obj.getImportedImageFilePaths()["server_paths"][0] + p = _get_first_imported_filepath(obj, "server_paths") abs_path = Path(desc._path._val) / Path(desc._name._val) / Path(p) temp_target = (Path(args.output) or Path.cwd()) / f"{obj.id}.tmp" zarr_target = (Path(args.output) or Path.cwd()) / f"{obj.id}.zarr" From 852da70f989567d8fb0732686294bfa62bd11652 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 31 May 2022 16:45:21 +0100 Subject: [PATCH 3/7] Return first image for check if in-place imported --- src/omero_zarr/cli.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 16f47f0..f1515a9 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -323,25 +323,29 @@ def _lookup( self.ctx.die(110, f"No such {otype}: {oid}") return obj - def _get_first_imported_filepath(self, obj: BlitzObjectWrapper,path_type: str): - if obj.OMERO_CLASS == "Plate": - for well in obj.listChildren(): - for ws in well.listChildren(): - return _get_first_imported_path(ws.getImage(), path_type) - return obj.getImportedImageFilePaths()[path_type][0] - def _bf_export(self, obj: BlitzObjectWrapper, args: argparse.Namespace) -> None: + + def _get_first_image(obj): + if obj.OMERO_CLASS == "Plate": + for well in obj.listChildren(): + for ws in well.listChildren(): + return ws.getImage() + else: + return obj + if args.bfpath: abs_path = Path(args.bfpath) - elif obj.getInplaceImport(): - p = _get_first_imported_filepath(obj, "client_paths") - abs_path = Path("/") / Path(p) else: - if self.client is None: - raise Exception("This cannot happen") # mypy is confused - prx, desc = self.client.getManagedRepository(description=True) - p = _get_first_imported_filepath(obj, "server_paths") - abs_path = Path(desc._path._val) / Path(desc._name._val) / Path(p) + first_image = _get_first_image(obj) + if first_image.getInplaceImport(): + p = first_image.getImportedImageFilePaths()["client_paths"][0] + abs_path = Path("/") / Path(p) + else: + if self.client is None: + raise Exception("This cannot happen") # mypy is confused + prx, desc = self.client.getManagedRepository(description=True) + p = first_image.getImportedImageFilePaths()["server_paths"][0] + abs_path = Path(desc._path._val) / Path(desc._name._val) / Path(p) temp_target = (Path(args.output) or Path.cwd()) / f"{obj.id}.tmp" zarr_target = (Path(args.output) or Path.cwd()) / f"{obj.id}.zarr" From 36d7303d6490d8d40f9cb08fc9e61085cf14c8b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 May 2022 20:40:09 +0000 Subject: [PATCH 4/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/omero_zarr/cli.py | 1 - src/omero_zarr/raw_pixels.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index f1515a9..d9c3091 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -324,7 +324,6 @@ def _lookup( return obj def _bf_export(self, obj: BlitzObjectWrapper, args: argparse.Namespace) -> None: - def _get_first_image(obj): if obj.OMERO_CLASS == "Plate": for well in obj.listChildren(): diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 3b4897f..c29ea9d 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -210,7 +210,9 @@ def marshal_acquisition(acquisition: omero.gateway._PlateAcquisitionWrapper) -> return acq -def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace, write_binary=True) -> None: +def plate_to_zarr( + plate: omero.gateway._PlateWrapper, args: argparse.Namespace, write_binary=True +) -> None: """ Exports a plate to a zarr file using the hierarchy discussed here ('Option 3'): https://github.com/ome/omero-ms-zarr/issues/73#issuecomment-706770955 From 63e7f2c31bd246ddaeab19771362572ff4cc8ef3 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 31 May 2022 22:31:35 +0100 Subject: [PATCH 5/7] Fix pre-commit errors --- src/omero_zarr/cli.py | 6 +++--- src/omero_zarr/raw_pixels.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index d9c3091..739002f 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -6,7 +6,7 @@ from typing import Any, Callable, List from omero.cli import CLI, BaseControl, Parser, ProxyStringType -from omero.gateway import BlitzGateway, BlitzObjectWrapper +from omero.gateway import BlitzGateway, BlitzObjectWrapper, ImageWrapper from omero.model import ImageI, PlateI from zarr.hierarchy import open_group from zarr.storage import FSStore @@ -324,7 +324,7 @@ def _lookup( return obj def _bf_export(self, obj: BlitzObjectWrapper, args: argparse.Namespace) -> None: - def _get_first_image(obj): + def _get_first_image(obj: BlitzObjectWrapper) -> ImageWrapper: if obj.OMERO_CLASS == "Plate": for well in obj.listChildren(): for ws in well.listChildren(): @@ -393,7 +393,7 @@ def _get_first_image(obj): if obj.OMERO_CLASS == "Image": # Add OMERO metadata store = FSStore( - str(image_target.resolve()), + str(zarr_target.resolve()), auto_mkdir=False, normalize_keys=False, mode="w", diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index c29ea9d..f450e5a 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -211,7 +211,9 @@ def marshal_acquisition(acquisition: omero.gateway._PlateAcquisitionWrapper) -> def plate_to_zarr( - plate: omero.gateway._PlateWrapper, args: argparse.Namespace, write_binary=True + plate: omero.gateway._PlateWrapper, + args: argparse.Namespace, + write_binary: Optional[bool] = True, ) -> None: """ Exports a plate to a zarr file using the hierarchy discussed here ('Option 3'): From ea5f0a208d92b679ae5981ee992b56c55c107de2 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 31 May 2022 22:32:32 +0100 Subject: [PATCH 6/7] Store image ID --- src/omero_zarr/raw_pixels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index f450e5a..362a0f7 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -294,7 +294,7 @@ def plate_to_zarr( def add_omero_metadata(zarr_root: Group, image: omero.gateway.ImageWrapper) -> None: image_data = { - "id": 1, + "id": image.getId(), "channels": [channelMarshal(c) for c in image.getChannels()], "rdefs": { "model": (image.isGreyscaleRenderingModel() and "greyscale" or "color"), From faf9262474f4979ea158b9acb05086552d9c24f0 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 31 May 2022 22:35:18 +0100 Subject: [PATCH 7/7] Fix setup.py dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4393a32..0818202 100755 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def get_long_description() -> str: author="The Open Microscopy Team", author_email="", python_requires=">=3", - install_requires=["omero-py>=5.6.0", "ome-zarr>=0.3.1.dev0"], + install_requires=["omero-py>=5.6.0", "ome-zarr>=0.3.0"], long_description=long_description, keywords=["OMERO.CLI", "plugin"], url="https://github.com/ome/omero-cli-zarr/",