Update in-place experiment writing to use the new WriterContract API …

…in slicedimage 4.0.0 (#1447)

Test plan: in-place experiment writing tests pass.

REQUIREMENTS-STRICT.txt

@@ -61,7 +61,7 @@ semantic-version==2.6.0
 Send2Trash==1.5.0
 showit==1.1.4
 six==1.12.0
-slicedimage==3.1.2
+slicedimage==4.0.0
 sympy==1.4
 terminado==0.8.2
 testpath==0.4.2

REQUIREMENTS.txt

@@ -10,7 +10,7 @@ scikit-image>=0.14.0
 scikit-learn
 scipy
 showit >= 1.1.4
-slicedimage==3.1.2
+slicedimage==4.0.0
 scikit-learn
 sympy
 tqdm

starfish/REQUIREMENTS-STRICT.txt

@@ -61,7 +61,7 @@ semantic-version==2.6.0
 Send2Trash==1.5.0
 showit==1.1.4
 six==1.12.0
-slicedimage==3.1.2
+slicedimage==4.0.0
 sympy==1.4
 terminado==0.8.2
 testpath==0.4.2

starfish/core/experiment/builder/__init__.py

@@ -1,6 +1,7 @@
 import functools
 import json
 import os
+import warnings
 from dataclasses import astuple, dataclass
 from pathlib import Path
 from typing import (
@@ -24,6 +25,7 @@
     Tile,
     TileSet,
     Writer,
+    WriterContract,
 )
 
 from starfish import FieldOfView
@@ -48,18 +50,6 @@ class TileIdentifier:
     zplane_label: int
 
 
-def _tile_opener(toc_path: Path, tile: Tile, file_ext: str) -> BinaryIO:
-    base = toc_path.parent / toc_path.stem
-    return open(
-        f"{os.fspath(base)}-Z{tile.indices[Axes.ZPLANE]}-"
-        f"R{tile.indices[Axes.ROUND]}-C{tile.indices[Axes.CH]}.{file_ext}",
-        "wb")
-
-
-def _fov_path_generator(parent_toc_path: Path, toc_name: str) -> Path:
-    return parent_toc_path.parent / "{}-{}.json".format(parent_toc_path.stem, toc_name)
-
-
 def build_irregular_image(
         tile_identifiers: Iterable[TileIdentifier],
         image_fetcher: TileFetcher,
@@ -212,6 +202,7 @@ def write_irregular_experiment_json(
         default_shape: Optional[Mapping[Axes, int]]=None,
         fov_path_generator: Callable[[Path, str], Path] = None,
         tile_opener: Optional[Callable[[Path, Tile, str], BinaryIO]] = None,
+        writer_contract: Optional[WriterContract] = None,
 ) -> None:
     """
     Build and returns a top-level experiment description with the following characteristics:
@@ -235,19 +226,28 @@ def write_irregular_experiment_json(
     fov_path_generator : Optional[Callable[[Path, str], Path]]
         Generates the path for a FOV's json file.  If one is not provided, the default generates
         the FOV's json file at the same level as the top-level json file for an image.    If this is
-        not provided, a reasonable default will be provided.
+        not provided, a reasonable default will be provided.  If this is provided, writer_contract
+        should not be provided.
     tile_opener : Optional[Callable[[Path, Tile, str], BinaryIO]]
         Callable that gets invoked with the following arguments: 1. the directory of the experiment
         that is being constructed, 2. the tile that is being written, and 3. the file extension
         that the tile should be written with.  The callable is expected to return an open file
-        handle.  If this is not provided, a reasonable default will be provided.
+        handle.  If this is not provided, a reasonable default will be provided.  If this is
+        provided, writer_contract should not be provided.
+    writer_contract : Optional[WriterContract]
+        Contract for specifying how the slicedimage image is to be laid out.  If this is provided,
+        fov_path_generator and tile_opener should not be provided.
     """
     if postprocess_func is None:
         postprocess_func = lambda doc: doc
-    if fov_path_generator is None:
-        fov_path_generator = _fov_path_generator
-    if tile_opener is None:
-        tile_opener = _tile_opener
+    if fov_path_generator is not None or tile_opener is not None:
+        warnings.warn(
+            "`fov_path_generator` and `tile_opener` options for writing experiment files is "
+            "deprecated.  Use `writer_contract` instead.",
+            DeprecationWarning)
+        if writer_contract is not None:
+            raise ValueError(
+                "Cannot specify both `writer_contract` and `fov_path_generator` or `tile_opener`")
 
     experiment_doc: Dict[str, Any] = {
         'version': str(CURRENT_VERSION),
@@ -261,10 +261,11 @@ def write_irregular_experiment_json(
 
         Writer.write_to_path(
             image,
-            os.path.join(path, f"{image_type}.json"),
+            Path(path) / f"{image_type}.json",
             pretty=True,
             partition_path_generator=fov_path_generator,
             tile_opener=tile_opener,
+            writer_contract=writer_contract,
             tile_format=tile_format,
         )
         experiment_doc['images'][image_type] = f"{image_type}.json"
@@ -302,6 +303,7 @@ def write_experiment_json(
         dimension_order: Sequence[Axes]=(Axes.ZPLANE, Axes.ROUND, Axes.CH),
         fov_path_generator: Optional[Callable[[Path, str], Path]] = None,
         tile_opener: Optional[Callable[[Path, Tile, str], BinaryIO]] = None,
+        writer_contract: Optional[WriterContract] = None,
 ) -> None:
     """
     Build and returns a top-level experiment description with the following characteristics:
@@ -346,15 +348,20 @@ def write_experiment_json(
           (ROUND=1, CH=0, Z=1)
           (ROUND=1, CH=1, Z=1)
         (default = (Axes.Z, Axes.ROUND, Axes.CH))
-    fov_path_generator : Callable[[Path, str], Path]
+    fov_path_generator : Optional[Callable[[Path, str], Path]]
         Generates the path for a FOV's json file.  If one is not provided, the default generates
-        the FOV's json file at the same level as the top-level json file for an image.  If this is
-        not provided, a reasonable default will be provided.
-    tile_opener : Callable[[Path, Tile, str], BinaryIO]
+        the FOV's json file at the same level as the top-level json file for an image.    If this is
+        not provided, a reasonable default will be provided.  If this is provided, writer_contract
+        should not be provided.
+    tile_opener : Optional[Callable[[Path, Tile, str], BinaryIO]]
         Callable that gets invoked with the following arguments: 1. the directory of the experiment
         that is being constructed, 2. the tile that is being written, and 3. the file extension
         that the tile should be written with.  The callable is expected to return an open file
-        handle.  If this is not provided, a reasonable default will be provided.
+        handle.  If this is not provided, a reasonable default will be provided.  If this is
+        provided, writer_contract should not be provided.
+    writer_contract : Optional[WriterContract]
+        Contract for specifying how the slicedimage image is to be laid out.  If this is provided,
+        fov_path_generator and tile_opener should not be provided.
     """
     all_tile_fetcher: MutableMapping[str, TileFetcher] = {}
     if aux_tile_fetcher is not None:
@@ -393,4 +400,5 @@ def write_experiment_json(
         default_shape=default_shape,
         fov_path_generator=fov_path_generator,
         tile_opener=tile_opener,
+        writer_contract=writer_contract,
     )

starfish/core/experiment/builder/inplace.py

@@ -1,64 +1,39 @@
 """To build experiments in-place, a few things must be done:
 
-1. Call enable_inplace_mode().
-2. Call write_experiment_json with tile_opener=inplace_tile_opener.
-3. The TileFetcher should return an instance of a InplaceTileFetcher.
-
-Please note that enabling in-place experiment construction should be only done in an isolated script
-dedicated to constructing an experiment, as it modifies some existing code paths.
+1. Call write_experiment_json with writer_contract=InplaceWriterContract().
+2. The TileFetcher should return an instance of a InplaceTileFetcher.
 """
 
 
 import abc
-import io
-import sys
+import warnings
 from pathlib import Path
-from typing import BinaryIO
+from typing import Mapping, Optional
 
 import numpy as np
-from slicedimage import Tile
+from slicedimage import ImageFormat, Tile
+from slicedimage.io import WriterContract
 
 from starfish.core.types import Axes
 from .providers import FetchedTile
 
 
-def sha256_get(tile_self):
-    return tile_self.provider.sha256
-
+class InplaceWriterContract(WriterContract):
+    def tile_url_generator(self, tileset_url: str, tile: Tile, ext: str) -> str:
+        return f"file://{tile.provider.filepath}"
 
-def sha256_set(tile_self, value):
-    pass
+    def write_tile(
+            self,
+            tile_url: str,
+            tile: Tile,
+            tile_format: ImageFormat,
+            backend_config: Optional[Mapping] = None,
+    ) -> str:
+        return tile.provider.sha256
 
 
 def enable_inplace_mode():
-    Tile.sha256 = property(sha256_get, sha256_set)
-
-
-def inplace_tile_opener(toc_path: Path, tile: Tile, file_ext: str) -> BinaryIO:
-    return DevNull(tile.provider.filepath)
-
-
-class DevNull(io.BytesIO):
-    """A class meant to mimic an open(filepath, 'wb') operation but
-    prevents any actual writing, reading, or seeking.
-    This class is an ugly hack to prevent the slicedimage
-    Writer.generate_partition_document() function from needlessly creating
-    a copy of image data.
-    See: https://docs.python.org/3/library/io.html
-    See also: cpython/Lib/_pyio.py
-    """
-    def __init__(self, filepath: str, *args, **kwargs):
-        super(DevNull, self).__init__(*args, **kwargs)
-        self.name = filepath
-
-    def read(self, size=-1):
-        raise NotImplementedError()
-
-    def write(self, b):
-        return sys.getsizeof(b)
-
-    def seek(self, pos, whence=0):
-        raise NotImplementedError()
+    warnings.warn("`enable_inplace_mode()` is no longer necessary.", DeprecationWarning)
 
 
 class InplaceFetchedTile(FetchedTile):

starfish/core/experiment/builder/structured_formatter.py

@@ -6,8 +6,6 @@
 from dataclasses import dataclass
 from pathlib import Path
 from typing import (
-    BinaryIO,
-    Callable,
     cast,
     FrozenSet,
     Mapping,
@@ -20,11 +18,11 @@
 from warnings import warn
 
 import numpy as np
-from slicedimage import ImageFormat, Tile
+from slicedimage import ImageFormat, WriterContract
 
 from starfish.core.types import Axes, Coordinates, CoordinateValue, Number
 from . import TileIdentifier, write_irregular_experiment_json
-from .inplace import enable_inplace_mode, inplace_tile_opener, InplaceFetchedTile
+from .inplace import InplaceFetchedTile, InplaceWriterContract
 from .providers import FetchedTile, TileFetcher
 
 FILENAME_CRE = re.compile(
@@ -130,17 +128,16 @@ def format_structured_dataset(
     }
 
     if in_place:
-        enable_inplace_mode()
-        tile_opener: Optional[Callable[[Path, Tile, str], BinaryIO]] = inplace_tile_opener
+        writer_contract: Optional[WriterContract] = InplaceWriterContract()
     else:
-        tile_opener = None
+        writer_contract = None
 
     write_irregular_experiment_json(
         output_experiment_dir_str,
         tile_format,
         image_tile_identifiers=image_tile_identifiers,
         tile_fetchers=tile_fetchers,
-        tile_opener=tile_opener,
+        writer_contract=writer_contract,
     )
 
 

starfish/core/experiment/builder/test/inplace_script.py

@@ -10,7 +10,7 @@
 
 from starfish.core.experiment.builder import FetchedTile, TileFetcher, write_experiment_json
 from starfish.core.experiment.builder.inplace import (
-    enable_inplace_mode, inplace_tile_opener, InplaceFetchedTile
+    InplaceFetchedTile, InplaceWriterContract,
 )
 from starfish.core.experiment.experiment import Experiment, FieldOfView
 from starfish.core.types import Axes, Coordinates, CoordinateValue
@@ -80,8 +80,6 @@ def add_codebook(experiment_json_doc):
         experiment_json_doc['codebook'] = "codebook.json"
         return experiment_json_doc
 
-    enable_inplace_mode()
-
     write_experiment_json(
         path=os.fspath(image_dir),
         fov_count=num_fovs,
@@ -95,8 +93,7 @@ def add_codebook(experiment_json_doc):
         },
         postprocess_func=add_codebook,
         default_shape=SHAPE,
-        fov_path_generator=fov_path_generator,
-        tile_opener=inplace_tile_opener,
+        writer_contract=InplaceWriterContract(),
     )