Add batchset for rendering a hierarchy of containers (single file)

MANIFEST.in

@@ -2,3 +2,4 @@ include *.txt
 include *.rst
 prune dist
 prune build
+include src/omero_render/*.yaml

setup.py

@@ -96,8 +96,9 @@ def read(fname):
 
 setup(
     version=version,
-    packages=['', 'omero.plugins'],
+    packages=['', 'omero.plugins', 'omero_render'],
     package_dir={"": "src"},
+    include_package_data=True,
     name='omero-cli-render',
     description="Plugin for use in the OMERO CLI.",
     long_description=read('README.rst'),
@@ -120,6 +121,7 @@ def read(fname):
     install_requires=[
         'PyYAML',
         'omero-py>=5.6.0',
+        'jsonschema',
         'future'
         ],
     python_requires='>=3',

src/omero_cli_render.py

@@ -45,6 +45,8 @@
 
 from omero import UnloadedEntityException
 
+from omero_render import validate_renderdef, validate_renderdef_batch
+
 HELP = "Tools for working with rendering settings"
 
 INFO_HELP = """Show details of a rendering setting
@@ -134,6 +136,51 @@
     # be updated individually.
 """
 
+BATCHSET_HELP = """Set rendering settings from a hierachy of rendering
+    definitions corresponding to an OMERO container hierarchy.
+
+    'projects' and 'datasets' are supported as top level containers, 'name'
+    must be provided.
+    If multiple containers have the same name they will all be processed.
+    Top level keys beginning with '_' are ignored, so they could for example
+    be used as YAML anchors.
+
+    Definitions can be applied to projects, datasets, or datasets in projects.
+
+    'renderdef' is a rendering definition as passed to 'set', except that
+    'version' is omitted.
+
+    Example:
+
+    projects:
+      # Apply renderdef to all Images in Project A
+      - name: Project A
+        renderdef:
+          channels:
+            ...
+
+      # Apply renderdef to all Images in Dataset 1 in Project B
+      - name: Project B
+        datasets:
+          - name: Dataset 1
+            renderdef:
+              channels:
+                ...
+
+    datasets:
+      # Apply the channel_group1 renderdef anchor to all Images in Dataset 2
+      - name: Dataset 2
+        renderdef:
+          channels:
+            *channel_group1
+
+    _anchors: &channel_group1
+      1:
+        active: true
+        label: Plk1
+
+"""
+
 TEST_HELP = """Test that underlying pixel data is available
 
     The syntax for specifying objects is: <object>:<id>
@@ -372,6 +419,7 @@ def _configure(self, parser):
         info = parser.add(sub, self.info, INFO_HELP)
         copy = parser.add(sub, self.copy, COPY_HELP)
         set_cmd = parser.add(sub, self.set, SET_HELP)
+        batchset = parser.add(sub, self.batchset, BATCHSET_HELP)
         edit = parser.add(sub, self.edit, EDIT_HELP)
         test = parser.add(sub, self.test, TEST_HELP)
 
@@ -386,7 +434,10 @@ def _configure(self, parser):
             x.add_argument("object", type=render_type, help=tgt_help,
                            nargs="+")
 
-        for x in (copy, set_cmd, edit):
+        batchset.add_argument("path", help=(
+            "Directory containing subdirectories with renderdefs."))
+
+        for x in (copy, set_cmd, edit, batchset):
             x.add_argument(
                 "--skipthumbs", help="Do not regenerate thumbnails "
                                      "immediately", action="store_true")
@@ -400,13 +451,14 @@ def _configure(self, parser):
         copy.add_argument("target", type=render_type, help=tgt_help,
                           nargs="+")
 
-        for x in (set_cmd, edit):
+        for x in (set_cmd, edit, batchset):
             x.add_argument(
                 "--disable", help="Disable non specified channels ",
                 action="store_true")
             x.add_argument(
                 "--ignore-errors", help="Do not error on mismatching"
                 " rendering settings", action="store_true")
+        for x in (set_cmd, edit):
             x.add_argument(
                 "channels",
                 help="Local file or OriginalFile:ID which specifies the "
@@ -430,6 +482,11 @@ def _lookup(self, gateway, type, oid):
             self.ctx.die(110, "No such %s: %s" % (type, oid))
         return obj
 
+    def _lookup_name(self, gateway, type, name):
+        gateway.SERVICE_OPTS.setOmeroGroup('-1')
+        objs = gateway.getObjects(type, attributes={'name': name})
+        return list(objs)
+
     def render_images(self, gateway, object, batch=100):
         """
         Get the images.
@@ -612,6 +669,8 @@ def _load_rendering_settings(self, source, session=None):
             self.ctx.dbg(e)
             self.ctx.die(103, "Could not read %s" % source)
 
+        validate_renderdef(data)
+
         if 'channels' not in data:
             self.ctx.die(104, "ERROR: No channels found in %s" % source)
 
@@ -667,21 +726,91 @@ def set(self, args):
         """ Implements the 'set' command """
         data = self._load_rendering_settings(
             args.channels, session=self.client.getSession())
+        self._set(data, args.object,
+                  args.ignore_errors, args.disable, args.skipthumbs)
+
+    def _get_batchset_targets(self, otype, descriptor):
+        """ Gets targets and renderdefs for the 'batchset' command """
+        def require_one(required, keys):
+            if sum((r in keys) for r in required) != 1:
+                raise ValueError(
+                    'Exactly one of {} required'.format(required))
+
+        descriptor_keys = list(descriptor.keys())
+
+        require_one(['name'], descriptor_keys)
+        if otype == 'Project':
+            require_one(['datasets', 'renderdef'], descriptor_keys)
+        else:
+            require_one(['renderdef'], descriptor_keys)
+
+        parents = self._lookup_name(self.gateway, otype, descriptor['name'])
+        if not parents:
+            raise Exception(f"No match for Project: {descriptor['name']}")
+        for parent in parents:
+            if 'renderdef' in descriptor_keys:
+                yield parent, descriptor['renderdef']
+            else:
+                dataset_renderdefs = dict(
+                    (d['name'], d) for d in descriptor['datasets'])
+                expected_names = set(dataset_renderdefs.keys())
+                for dataset in parent.listChildren():
+                    if dataset.name in dataset_renderdefs:
+                        for target in self._get_batchset_targets(
+                                'Dataset', dataset_renderdefs[dataset.name]):
+                            yield target
+                        try:
+                            expected_names.remove(dataset.name)
+                        except KeyError:
+                            # Multiple containers have the same name
+                            pass
+                if expected_names:
+                    raise Exception(
+                        'No match for datasets: {}'.format(expected_names))
+
+    @gateway_required
+    def batchset(self, args):
+        """ Implements the 'batchset' command """
+        try:
+            batch_data = pydict_text_io.load(
+                args.path, session=self.client.getSession())
+        except Exception as e:
+            self.ctx.dbg(e)
+            self.ctx.die(103, "Could not read %s" % args.path)
+        validate_renderdef_batch(batch_data)
+
+        for key, containers in batch_data.items():
+            if key.startswith('_'):
+                continue
+            if key not in {'projects', 'datasets'}:
+                raise NotImplementedError(
+                    'Invalid batchset container: {}'.format(key))
+            for container in containers:
+                for c, renderdef in self._get_batchset_targets(
+                        f'{key[0].upper()}{key[1:-1]}', container):
+                    self.ctx.out('Applying settings to '
+                                 f'{c.OMERO_CLASS}:{c.id} {c.name}')
+                    self._set(
+                        renderdef, c._obj,
+                        args.ignore_errors, args.disable, args.skipthumbs)
+
+    def _set(self, data, target, ignore_errors, disable, skipthumbs):
+        """ Utility method to apply rendering settings """
         (namedict, cindices, rangelist, colourlist, minmaxlist) = \
             self._read_channels(data)
         greyscale = data.get('greyscale', None)
         if greyscale is not None:
             self.ctx.dbg('greyscale=%s' % greyscale)
 
         iids = []
-        for img in self.render_images(self.gateway, args.object, batch=1):
+        for img in self.render_images(self.gateway, target, batch=1):
             iids.append(img.id)
 
             (def_z, def_t) = self._read_default_planes(
-                img, data, ignore_errors=args.ignore_errors)
+                img, data, ignore_errors=ignore_errors)
 
             active_channels = []
-            if not args.disable:
+            if not disable:
                 # Calling set_active_channels will disable channels which
                 # are not specified.
                 # Need to reset ALL active channels after set_active_channels()
@@ -730,7 +859,7 @@ def set(self, args):
                 img.saveDefaults()
                 self.ctx.dbg(
                     "Updated rendering settings for Image:%s" % img.id)
-                if not args.skipthumbs:
+                if not skipthumbs:
                     self._generate_thumbs([img])
             except Exception as e:
                 self.ctx.err('ERROR: %s' % e)
@@ -739,7 +868,7 @@ def set(self, args):
 
         if not iids:
             self.ctx.die(113, "ERROR: No images found for %s %d" %
-                         (args.object.__class__.__name__, args.object.id._val))
+                         (target.__class__.__name__, target.id._val))
 
         if namedict:
             self._update_channel_names(self.gateway, iids, namedict)

src/omero_render/__init__.py

@@ -0,0 +1,2 @@
+from .schema import validate_renderdef, validate_renderdef_batch
+__all__ = ['validate_renderdef', 'validate_renderdef_batch']

src/omero_render/renderdef-batch-schema.yaml

@@ -0,0 +1,115 @@
+---
+
+$schema: https://json-schema.org/schema#
+$id: https://github.com/ome/omero-cli-render#renderdef-batch
+title: jsonschema for batch rendering configuration
+type: object
+additionalProperties: false
+# Top level objects beginning with _ will be ignored
+patternProperties:
+  "^_":
+      type:
+        object
+
+# Python jsonschema doesn't support $ref so use YAML anchors instead
+definitions:
+  channel: &channel
+    type: object
+    additionalProperties: false
+    properties:
+      active:
+        title: Active channel
+        type: boolean
+      color:
+        title: Channel color as HTML RGB triplet
+        type: string
+      label:
+        title: Channel name
+        type: string
+      start:
+        title: Start of rendering window, optional (needs end)
+        type: number
+      end:
+        title: End of rendering window, optional (needs start)
+        type: number
+      min:
+        title: Minimum pixel intensity, optional (needs max)
+        type: number
+      max:
+        title: Maximum pixel intensity, optional (needs min)
+        type: number
+
+  renderdef: &renderdef
+    type: object
+    additionalProperties: false
+    required:
+      - channels
+    properties:
+      channels:
+        title: Dictionary of channels
+        type: object
+        additionalProperties: false
+        patternProperties:
+          "^[0-9]+$":
+            title: Channel index, 1-based
+            type: object
+            properties: *channel
+              # $ref: "#/definitions/channel"
+      greyscale:
+        title: Greyscale rendering, optional
+        type: boolean
+      z:
+        title: Default Z plane index, 1-based, optional
+        type: integer
+      t:
+        title: Default T plane index, 1-based, optional
+        type: integer
+      version:
+        title: Version of the renderdef specification
+        type: integer
+
+  dataset: &dataset
+    type: object
+    additionalProperties: false
+    required:
+      - name
+      - renderdef
+    properties:
+      name:
+        title: Name of Dataset
+        type: string
+      renderdef: *renderdef
+        # $ref: "#/definitions/renderdef"
+
+  project: &project
+    type: object
+    additionalProperties: false
+    required:
+      - name
+    properties:
+      name:
+        title: Name of Project
+        type: string
+      datasets:
+        title: List of Datasets
+        type: array
+        minItems: 1
+        items: *dataset
+          # $ref: "#/definitions/dataset"
+      renderdef: *renderdef
+        # $ref: "#/definitions/renderdef"
+
+properties:
+  projects:
+    title: List of Projects
+    type: array
+    minItems: 1
+    items: *project
+      # $ref: "#/definitions/project"
+
+  datasets:
+    title: List of Datasets
+    type: array
+    minItems: 1
+    items: *dataset
+      # $ref: "#/definitions/dataset"