JPEG-2000 compression based refactoring

.github/workflows/workflow.yml

@@ -14,7 +14,6 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - '3.6'
           - '3.7'
           - '3.8'
           - '3.9'

appveyor.yml

@@ -1,18 +1,20 @@
 image: ubuntu1804
 
-install:
-  - sudo apt-get install -y python3-setuptools python3-wheel twine
-
 build: off
 
+install:
+  - sudo apt-get install -y python3-pip
+
 build_script:
+  - pip3 install --user -U pip setuptools wheel flake8
+  - export PATH="$(python3 -m site --user-base)/bin:${PATH}"
   - python3 setup.py build
 
 test_script:
-  - python3 setup.py flake8
+  - flake8 .
 
 after_test:
-  - python3 setup.py bdist_wheel bdist_egg
+  - python3 setup.py bdist_wheel
 
 artifacts:
   - path: dist/*

isyntax2raw/__init__.py

@@ -19,6 +19,14 @@
 import softwarerenderbackend
 import zarr
 
+from numcodecs.abc import Codec
+from numcodecs.compat import \
+    ensure_bytes, \
+    ensure_contiguous_ndarray, \
+    ndarray_copy
+from numcodecs.registry import register_codec
+import imagecodecs
+
 from datetime import datetime
 from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait
 from threading import BoundedSemaphore
@@ -81,7 +89,7 @@ class WriteTiles(object):
 
     def __init__(
         self, tile_width, tile_height, resolutions, max_workers,
-        batch_size, fill_color, nested, input_path, output_path
+        batch_size, fill_color, nested, input_path, output_path, psnr
     ):
         self.tile_width = tile_width
         self.tile_height = tile_height
@@ -92,6 +100,7 @@ def __init__(
         self.nested = nested
         self.input_path = input_path
         self.slide_directory = output_path
+        self.psnr = psnr
 
         render_context = softwarerendercontext.SoftwareRenderContext()
         render_backend = softwarerenderbackend.SoftwareRenderBackend()
@@ -388,6 +397,18 @@ def wait_any(self, regions):
         else:
             return self.pixel_engine.wait_any(regions)
 
+    def write_image_metadata(self, resolutions, series):
+        multiscales = [{
+            'metadata': {
+                'method': 'pixelengine',
+                'version': str(self.pixel_engine.version)
+            },
+            'version': '0.2',
+            'datasets': [{'path': str(v)} for v in resolutions]
+        }]
+        z = self.zarr_group["%d" % series]
+        z.attrs['multiscales'] = multiscales
+
     def write_metadata_json(self, metadata_file):
         '''write metadata to a JSON file'''
 
@@ -491,9 +512,10 @@ def write_image_type(self, image_type, series):
             tile = self.zarr_group["%d/0" % series]
             tile.attrs['image type'] = image_type
             for channel in range(0, 3):
-                band = np.array(img.getdata(band=channel))
-                band.shape = (height, width)
-                tile[0, 0, channel] = band
+                data = np.array(img.getdata())
+                data.shape = (height, width, 3)
+                tile[0, 0, :] = data
+            self.write_image_metadata(range(1), series)
 
             log.info("wrote %s image" % image_type)
 
@@ -512,21 +534,15 @@ def create_tile_directory(self, series, resolution, width, height):
 
         # important to explicitly set the chunk size to 1 for non-XY dims
         # setting to None may cause all planes to be chunked together
-        # ordering is TZCYX and hard-coded since Z and T are not present
+        # ordering is TZYXC (interleaved) and hard-coded since Z and T
+        # are not present
         self.zarr_group.create_dataset(
             "%s/%s" % (str(series), str(resolution)),
-            shape=(1, 1, 3, height, width),
-            chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B'
+            shape=(1, 1, height, width, 3),
+            chunks=(1, 1, self.tile_height, self.tile_width, 3), dtype='B',
+            compressor=j2k(self.psnr)
         )
 
-    def make_planar(self, pixels, tile_width, tile_height):
-        r = pixels[0::3]
-        g = pixels[1::3]
-        b = pixels[2::3]
-        for v in (r, g, b):
-            v.shape = (tile_height, tile_width)
-        return np.array([r, g, b])
-
     def write_pyramid(self):
         '''write the slide's pyramid as a set of tiles'''
         pe_in = self.pixel_engine["in"]
@@ -547,11 +563,9 @@ def write_tile(
             x_end = x_start + tile_width
             y_end = y_start + tile_height
             try:
-                # Zarr has a single n-dimensional array representation on
-                # disk (not interleaved RGB)
-                pixels = self.make_planar(pixels, tile_width, tile_height)
+                pixels.shape = (tile_height, tile_width, 3)
                 z = self.zarr_group["0/%d" % resolution]
-                z[0, 0, :, y_start:y_end, x_start:x_end] = pixels
+                z[0, 0, y_start:y_end, x_start:x_end, :] = pixels
             except Exception:
                 log.error(
                     "Failed to write tile [:, %d:%d, %d:%d]" % (
@@ -656,6 +670,7 @@ def write_tile(
                                 x_start, y_start, width, height
                             ))
             wait(jobs, return_when=ALL_COMPLETED)
+            self.write_image_metadata(resolutions, 0)
 
     def create_patch_list(
         self, dim_ranges, tiles, tile_size, tile_directory
@@ -705,3 +720,36 @@ def create_patch_list(
                 # order to identify the patches returned asynchronously
                 patch_ids.append((x, y))
         return patches, patch_ids
+
+
+class j2k(Codec):
+    """Codec providing j2k compression via imagecodecs.
+    Parameters
+    ----------
+    psnr : int
+        Compression peak signal noise ratio.
+    """
+
+    codec_id = "j2k"
+
+    def __init__(self, psnr=50):
+        self.psnr = psnr
+        assert (self.psnr > 0 and self.psnr <= 100
+                and isinstance(self.psnr, int))
+        super().__init__()
+
+    def encode(self, buf):
+        return imagecodecs.jpeg2k_encode(np.squeeze(buf), level=self.psnr)
+
+    def decode(self, buf, out=None):
+        buf = ensure_bytes(buf)
+        decoded = imagecodecs.jpeg2k_decode(buf)
+        if out is not None:
+            out_view = ensure_contiguous_ndarray(out)
+            ndarray_copy(decoded, out_view)
+        else:
+            out = decoded
+        return out
+
+
+register_codec(j2k)

isyntax2raw/cli/isyntax2raw.py

@@ -64,16 +64,20 @@ def cli():
     "--debug", is_flag=True,
     help="enable debugging",
 )
+@click.option(
+    "--psnr", default=50, show_default=True,
+    help="JPEG-2000 compression PSNR"
+)
 @click.argument("input_path")
 @click.argument("output_path")
 def write_tiles(
     tile_width, tile_height, resolutions, max_workers, batch_size,
-    fill_color, nested, debug, input_path, output_path
+    fill_color, nested, debug, input_path, output_path, psnr
 ):
     setup_logging(debug)
     with WriteTiles(
         tile_width, tile_height, resolutions, max_workers,
-        batch_size, fill_color, nested, input_path, output_path
+        batch_size, fill_color, nested, input_path, output_path, psnr
     ) as wt:
         wt.write_metadata()
         wt.write_label_image()

isyntax2raw/resources/ome_template.xml

@@ -7,8 +7,8 @@
     <Description>${image['description']}</Description>
     <InstrumentRef ID="Instrument:0"/>
     <ObjectiveSettings ID="Objective:0:0"/>
-    <Pixels BigEndian="false" DimensionOrder="XYCZT" ID="Pixels:0"
-            Interleaved="false"
+    <Pixels BigEndian="false" DimensionOrder="XYZCT" ID="Pixels:0"
+            Interleaved="true"
             PhysicalSizeX="${image['pixels']['physicalSizeX']}"
             PhysicalSizeXUnit="µm"
             PhysicalSizeY="${image['pixels']['physicalSizeY']}"
@@ -27,8 +27,8 @@
     <AcquisitionDate>${image['acquisitionDate']}</AcquisitionDate>
     <InstrumentRef ID="Instrument:0"/>
     <ObjectiveSettings ID="Objective:0:0"/>
-    <Pixels BigEndian="false" DimensionOrder="XYCZT" ID="Pixels:1"
-            Interleaved="false" SignificantBits="8" SizeC="3"
+    <Pixels BigEndian="false" DimensionOrder="XYZCT" ID="Pixels:1"
+            Interleaved="true" SignificantBits="8" SizeC="3"
             SizeT="1" SizeX="${label['pixels']['sizeX']}" SizeY="${label['pixels']['sizeY']}"
             SizeZ="1" Type="uint8">
       <Channel ID="Channel:1:0" SamplesPerPixel="3">
@@ -41,8 +41,8 @@
     <AcquisitionDate>${image['acquisitionDate']}</AcquisitionDate>
     <InstrumentRef ID="Instrument:0"/>
     <ObjectiveSettings ID="Objective:0:0"/>
-    <Pixels BigEndian="false" DimensionOrder="XYCZT" ID="Pixels:2"
-            Interleaved="false" SignificantBits="8" SizeC="3"
+    <Pixels BigEndian="false" DimensionOrder="XYZCT" ID="Pixels:2"
+            Interleaved="true" SignificantBits="8" SizeC="3"
             SizeT="1"
             SizeX="${macro['pixels']['sizeX']}"
             SizeY="${macro['pixels']['sizeY']}"

setup.py

@@ -62,7 +62,7 @@ def read(fname):
 
 setup(name='isyntax2raw',
       version=version.getVersion(),
-      python_requires='>=3.6',
+      python_requires='>=3.7',
       description='iSyntax to raw format converter',
       long_description=read('README.md'),
       long_description_content_type='text/markdown',
@@ -85,6 +85,7 @@ def read(fname):
           'numpy==1.17.3',
           'zarr==2.8.1',
           'kajiki==0.8.2',
+          'imagecodecs==2021.8.26',
           'fsspec>=0.9.0',
       ],
       tests_require=[