From 8f55263a4b88bf209bf59fc310fea65feb155840 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Wed, 16 Dec 2020 11:46:17 +0000 Subject: [PATCH 1/2] Single renderdef file for multiple named projects/datasets --- src/omero_cli_render.py | 137 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 130 insertions(+), 7 deletions(-) diff --git a/src/omero_cli_render.py b/src/omero_cli_render.py index 3fb77f1..5f3b8b3 100755 --- a/src/omero_cli_render.py +++ b/src/omero_cli_render.py @@ -134,6 +134,51 @@ # be updated individually. """ +BATCHSET_HELP = """Set rendering settings from a hierachy of rendering + definitions corresponding to an OMERO container hierarchy. + + 'projects' and 'datasets' are supported as top level containers, 'name' + must be provided. + If multiple containers have the same name they will all be processed. + Top level keys beginning with '_' are ignored, so they could for example + be used as YAML anchors. + + Definitions can be applied to projects, datasets, or datasets in projects. + + 'renderdef' is a rendering definition as passed to 'set', except that + 'version' is omitted. + + Example: + + projects: + # Apply renderdef to all Images in Project A + - name: Project A + renderdef: + channels: + ... + + # Apply renderdef to all Images in Dataset 1 in Project B + - name: Project B + datasets: + - name: Dataset 1 + renderdef: + channels: + ... + + datasets: + # Apply the channel_group1 renderdef anchor to all Images in Dataset 2 + - name: Dataset 2 + renderdef: + channels: + *channel_group1 + + _anchors: &channel_group1 + 1: + active: true + label: Plk1 + +""" + TEST_HELP = """Test that underlying pixel data is available The syntax for specifying objects is: : @@ -372,6 +417,7 @@ def _configure(self, parser): info = parser.add(sub, self.info, INFO_HELP) copy = parser.add(sub, self.copy, COPY_HELP) set_cmd = parser.add(sub, self.set, SET_HELP) + batchset = parser.add(sub, self.batchset, BATCHSET_HELP) edit = parser.add(sub, self.edit, EDIT_HELP) test = parser.add(sub, self.test, TEST_HELP) @@ -386,7 +432,10 @@ def _configure(self, parser): x.add_argument("object", type=render_type, help=tgt_help, nargs="+") - for x in (copy, set_cmd, edit): + batchset.add_argument("path", help=( + "Directory containing subdirectories with renderdefs.")) + + for x in (copy, set_cmd, edit, batchset): x.add_argument( "--skipthumbs", help="Do not regenerate thumbnails " "immediately", action="store_true") @@ -400,13 +449,14 @@ def _configure(self, parser): copy.add_argument("target", type=render_type, help=tgt_help, nargs="+") - for x in (set_cmd, edit): + for x in (set_cmd, edit, batchset): x.add_argument( "--disable", help="Disable non specified channels ", action="store_true") x.add_argument( "--ignore-errors", help="Do not error on mismatching" " rendering settings", action="store_true") + for x in (set_cmd, edit): x.add_argument( "channels", help="Local file or OriginalFile:ID which specifies the " @@ -430,6 +480,11 @@ def _lookup(self, gateway, type, oid): self.ctx.die(110, "No such %s: %s" % (type, oid)) return obj + def _lookup_name(self, gateway, type, name): + gateway.SERVICE_OPTS.setOmeroGroup('-1') + objs = gateway.getObjects(type, attributes={'name': name}) + return list(objs) + def render_images(self, gateway, object, batch=100): """ Get the images. @@ -667,6 +722,74 @@ def set(self, args): """ Implements the 'set' command """ data = self._load_rendering_settings( args.channels, session=self.client.getSession()) + self._set(data, args.object, + args.ignore_errors, args.disable, args.skipthumbs) + + def _get_batchset_targets(self, otype, descriptor): + """ Gets targets and renderdefs for the 'batchset' command """ + def require_one(required, keys): + if sum((r in keys) for r in required) != 1: + raise ValueError( + 'Exactly one of {} required'.format(required)) + + descriptor_keys = list(descriptor.keys()) + + require_one(['name'], descriptor_keys) + if otype == 'Project': + require_one(['datasets', 'renderdef'], descriptor_keys) + else: + require_one(['renderdef'], descriptor_keys) + + parents = self._lookup_name(self.gateway, otype, descriptor['name']) + if not parents: + raise Exception(f"No match for Project: {descriptor['name']}") + for parent in parents: + if 'renderdef' in descriptor_keys: + yield parent, descriptor['renderdef'] + else: + dataset_renderdefs = dict( + (d['name'], d) for d in descriptor['datasets']) + expected_names = set(dataset_renderdefs.keys()) + for dataset in parent.listChildren(): + if dataset.name in dataset_renderdefs: + for target in self._get_batchset_targets( + 'Dataset', dataset_renderdefs[dataset.name]): + yield target + try: + expected_names.remove(dataset.name) + except KeyError: + # Multiple containers have the same name + pass + if expected_names: + raise Exception( + 'No match for datasets: {}'.format(expected_names)) + + @gateway_required + def batchset(self, args): + """ Implements the 'batchset' command """ + try: + batch_data = pydict_text_io.load( + args.path, session=self.client.getSession()) + except Exception as e: + self.ctx.dbg(e) + self.ctx.die(103, "Could not read %s" % args.path) + for key, containers in batch_data.items(): + if key.startswith('_'): + continue + if key not in {'projects', 'datasets'}: + raise NotImplementedError( + 'Invalid batchset container: {}'.format(key)) + for container in containers: + for c, renderdef in self._get_batchset_targets( + f'{key[0].upper()}{key[1:-1]}', container): + self.ctx.out('Applying settings to ' + f'{c.OMERO_CLASS}:{c.id} {c.name}') + self._set( + renderdef, c._obj, + args.ignore_errors, args.disable, args.skipthumbs) + + def _set(self, data, target, ignore_errors, disable, skipthumbs): + """ Utility method to apply rendering settings """ (namedict, cindices, rangelist, colourlist, minmaxlist) = \ self._read_channels(data) greyscale = data.get('greyscale', None) @@ -674,14 +797,14 @@ def set(self, args): self.ctx.dbg('greyscale=%s' % greyscale) iids = [] - for img in self.render_images(self.gateway, args.object, batch=1): + for img in self.render_images(self.gateway, target, batch=1): iids.append(img.id) (def_z, def_t) = self._read_default_planes( - img, data, ignore_errors=args.ignore_errors) + img, data, ignore_errors=ignore_errors) active_channels = [] - if not args.disable: + if not disable: # Calling set_active_channels will disable channels which # are not specified. # Need to reset ALL active channels after set_active_channels() @@ -730,7 +853,7 @@ def set(self, args): img.saveDefaults() self.ctx.dbg( "Updated rendering settings for Image:%s" % img.id) - if not args.skipthumbs: + if not skipthumbs: self._generate_thumbs([img]) except Exception as e: self.ctx.err('ERROR: %s' % e) @@ -739,7 +862,7 @@ def set(self, args): if not iids: self.ctx.die(113, "ERROR: No images found for %s %d" % - (args.object.__class__.__name__, args.object.id._val)) + (target.__class__.__name__, target.id._val)) if namedict: self._update_channel_names(self.gateway, iids, namedict) From d3ba8c98631566e548efb76154c1a334a28cc807 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 28 Jan 2021 17:24:51 +0000 Subject: [PATCH 2/2] Add jsonschema for validating schemas --- MANIFEST.in | 1 + setup.py | 4 +- src/omero_cli_render.py | 6 + src/omero_render/__init__.py | 2 + src/omero_render/renderdef-batch-schema.yaml | 115 +++++++++++++++++++ src/omero_render/renderdef-schema.yaml | 61 ++++++++++ src/omero_render/schema.py | 31 +++++ 7 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 src/omero_render/__init__.py create mode 100644 src/omero_render/renderdef-batch-schema.yaml create mode 100644 src/omero_render/renderdef-schema.yaml create mode 100644 src/omero_render/schema.py diff --git a/MANIFEST.in b/MANIFEST.in index 131793c..e57a5da 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include *.txt include *.rst prune dist prune build +include src/omero_render/*.yaml diff --git a/setup.py b/setup.py index c3fb4e3..5080cad 100644 --- a/setup.py +++ b/setup.py @@ -96,8 +96,9 @@ def read(fname): setup( version=version, - packages=['', 'omero.plugins'], + packages=['', 'omero.plugins', 'omero_render'], package_dir={"": "src"}, + include_package_data=True, name='omero-cli-render', description="Plugin for use in the OMERO CLI.", long_description=read('README.rst'), @@ -120,6 +121,7 @@ def read(fname): install_requires=[ 'PyYAML', 'omero-py>=5.6.0', + 'jsonschema', 'future' ], python_requires='>=3', diff --git a/src/omero_cli_render.py b/src/omero_cli_render.py index 5f3b8b3..3ea8ff6 100755 --- a/src/omero_cli_render.py +++ b/src/omero_cli_render.py @@ -45,6 +45,8 @@ from omero import UnloadedEntityException +from omero_render import validate_renderdef, validate_renderdef_batch + HELP = "Tools for working with rendering settings" INFO_HELP = """Show details of a rendering setting @@ -667,6 +669,8 @@ def _load_rendering_settings(self, source, session=None): self.ctx.dbg(e) self.ctx.die(103, "Could not read %s" % source) + validate_renderdef(data) + if 'channels' not in data: self.ctx.die(104, "ERROR: No channels found in %s" % source) @@ -773,6 +777,8 @@ def batchset(self, args): except Exception as e: self.ctx.dbg(e) self.ctx.die(103, "Could not read %s" % args.path) + validate_renderdef_batch(batch_data) + for key, containers in batch_data.items(): if key.startswith('_'): continue diff --git a/src/omero_render/__init__.py b/src/omero_render/__init__.py new file mode 100644 index 0000000..47d9b0b --- /dev/null +++ b/src/omero_render/__init__.py @@ -0,0 +1,2 @@ +from .schema import validate_renderdef, validate_renderdef_batch +__all__ = ['validate_renderdef', 'validate_renderdef_batch'] diff --git a/src/omero_render/renderdef-batch-schema.yaml b/src/omero_render/renderdef-batch-schema.yaml new file mode 100644 index 0000000..648208f --- /dev/null +++ b/src/omero_render/renderdef-batch-schema.yaml @@ -0,0 +1,115 @@ +--- + +$schema: https://json-schema.org/schema# +$id: https://github.com/ome/omero-cli-render#renderdef-batch +title: jsonschema for batch rendering configuration +type: object +additionalProperties: false +# Top level objects beginning with _ will be ignored +patternProperties: + "^_": + type: + object + +# Python jsonschema doesn't support $ref so use YAML anchors instead +definitions: + channel: &channel + type: object + additionalProperties: false + properties: + active: + title: Active channel + type: boolean + color: + title: Channel color as HTML RGB triplet + type: string + label: + title: Channel name + type: string + start: + title: Start of rendering window, optional (needs end) + type: number + end: + title: End of rendering window, optional (needs start) + type: number + min: + title: Minimum pixel intensity, optional (needs max) + type: number + max: + title: Maximum pixel intensity, optional (needs min) + type: number + + renderdef: &renderdef + type: object + additionalProperties: false + required: + - channels + properties: + channels: + title: Dictionary of channels + type: object + additionalProperties: false + patternProperties: + "^[0-9]+$": + title: Channel index, 1-based + type: object + properties: *channel + # $ref: "#/definitions/channel" + greyscale: + title: Greyscale rendering, optional + type: boolean + z: + title: Default Z plane index, 1-based, optional + type: integer + t: + title: Default T plane index, 1-based, optional + type: integer + version: + title: Version of the renderdef specification + type: integer + + dataset: &dataset + type: object + additionalProperties: false + required: + - name + - renderdef + properties: + name: + title: Name of Dataset + type: string + renderdef: *renderdef + # $ref: "#/definitions/renderdef" + + project: &project + type: object + additionalProperties: false + required: + - name + properties: + name: + title: Name of Project + type: string + datasets: + title: List of Datasets + type: array + minItems: 1 + items: *dataset + # $ref: "#/definitions/dataset" + renderdef: *renderdef + # $ref: "#/definitions/renderdef" + +properties: + projects: + title: List of Projects + type: array + minItems: 1 + items: *project + # $ref: "#/definitions/project" + + datasets: + title: List of Datasets + type: array + minItems: 1 + items: *dataset + # $ref: "#/definitions/dataset" diff --git a/src/omero_render/renderdef-schema.yaml b/src/omero_render/renderdef-schema.yaml new file mode 100644 index 0000000..7984578 --- /dev/null +++ b/src/omero_render/renderdef-schema.yaml @@ -0,0 +1,61 @@ +--- + +$schema: https://json-schema.org/schema# +$id: https://github.com/ome/omero-cli-render#renderdef +title: jsonschema for render configuration file +type: object +additionalProperties: false +required: + - channels + +# Python jsonschema doesn't support $ref so use YAML anchors instead +definitions: + channel: &channel + type: object + additionalProperties: false + properties: + active: + title: Active channel + type: boolean + color: + title: Channel color as HTML RGB triplet + type: string + label: + title: Channel name + type: string + start: + title: Start of rendering window, optional (needs end) + type: number + end: + title: End of rendering window, optional (needs start) + type: number + min: + title: Minimum pixel intensity, optional (needs max) + type: number + max: + title: Maximum pixel intensity, optional (needs min) + type: number + +properties: + channels: + title: Dictionary of channels + type: object + additionalProperties: false + patternProperties: + "^[0-9]+$": + title: Channel index, 1-based + type: object + properties: + channel: *channel + greyscale: + title: Greyscale rendering, optional + type: boolean + z: + title: Default Z plane index, 1-based, optional + type: integer + t: + title: Default T plane index, 1-based, optional + type: integer + version: + title: Version of the renderdef specification + type: integer diff --git a/src/omero_render/schema.py b/src/omero_render/schema.py new file mode 100644 index 0000000..bc1268e --- /dev/null +++ b/src/omero_render/schema.py @@ -0,0 +1,31 @@ +import json +from jsonschema import Draft7Validator +from pkgutil import get_data +import yaml + + +def _validate(schema_name, renderdef): + schemastr = get_data('omero_render', schema_name) + yml = yaml.safe_load(schemastr) + # Hack to expand anchors + # https://stackoverflow.com/a/64993515 + schema = json.loads(json.dumps(yml)) + v = Draft7Validator(schema) + # print(yaml.dump(schema)) + # print(renderdef) + # Hack to ensure YAML integer keys are JSON strings + renderdef_json = json.loads(json.dumps(renderdef)) + + if not v.is_valid(renderdef_json): + errs = '\n\n** '.join( + ['Invalid definition'] + + ['\n\n'.join(str(e) for e in v.iter_errors(renderdef_json))]) + raise ValueError(errs) + + +def validate_renderdef(renderdef): + _validate('renderdef-schema.yaml', renderdef) + + +def validate_renderdef_batch(renderdef): + _validate('renderdef-batch-schema.yaml', renderdef)