patch iterator spawn fix

DIAGNijmegen · Jul 11, 2024 · f112521 · f112521
1 parent 2c50b67
commit f112521
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 61 deletions.
diff --git a/docs/source/userguide/notebooks/components/downloaddata.ipynb b/docs/source/userguide/notebooks/components/downloaddata.ipynb
@@ -13,32 +13,15 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: gdown in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (4.6.0)\n",
-      "Requirement already satisfied: six in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from gdown) (1.16.0)\n",
-      "Requirement already satisfied: beautifulsoup4 in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from gdown) (4.11.1)\n",
-      "Requirement already satisfied: tqdm in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from gdown) (4.64.1)\n",
-      "Requirement already satisfied: filelock in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from gdown) (3.9.0)\n",
-      "Requirement already satisfied: requests[socks] in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from gdown) (2.28.1)\n",
-      "Requirement already satisfied: soupsieve>1.2 in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from beautifulsoup4->gdown) (2.3.2.post1)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from requests[socks]->gdown) (2022.12.7)\n",
-      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from requests[socks]->gdown) (1.26.13)\n",
-      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from requests[socks]->gdown) (2.1.1)\n",
-      "Requirement already satisfied: idna<4,>=2.5 in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from requests[socks]->gdown) (3.4)\n",
-      "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /home/mart/miniconda3/envs/wholeslidedata/lib/python3.9/site-packages (from requests[socks]->gdown) (1.7.1)\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "!pip install gdown\n",
+    "# !pip install gdown\n",
     "import gdown\n",
     "import sys\n",
     "from pathlib import Path\n",
     "\n",
+    "from downloaddata import WSA_DOWNLOAD_LINK, WSA_NAME, WSI_DOWNLOAD_LINK, WSI_NAME, WSM_DOWNLOAD_LINK, WSM_NAME\n",
+    "\n",
     "def _download(output_folder, download_link, name):\n",
     "    output_path = Path(output_folder) / name\n",
     "    if not output_path.exists():\n",
@@ -59,31 +42,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Downloading...\n",
-      "From: https://drive.google.com/uc?id=1noRtbC5fxBlnO7YnvktjIDhFI61PdOSB\n",
-      "To: /tmp/TCGA-21-5784-01Z-00-DX1.tif\n",
-      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 363M/363M [00:09<00:00, 39.7MB/s]\n",
-      "Downloading...\n",
-      "From: https://drive.google.com/uc?id=1jkTp0IJHHpmLd1yDO1L3KRFJgm0STh0d\n",
-      "To: /tmp/TCGA-21-5784-01Z-00-DX1.xml\n",
-      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9.67k/9.67k [00:00<00:00, 11.7MB/s]\n",
-      "Downloading...\n",
-      "From: https://drive.google.com/uc?id=1nLdKzSLq79mon1RCevgEmG59E8Oq9JhZ\n",
-      "To: /tmp/TCGA-21-5784-01Z-00-DX1_tb_mask.tif\n",
-      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 318k/318k [00:00<00:00, 10.6MB/s]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "\n",
-    "WSI_DOWNLOAD_LINK = \"https://drive.google.com/uc?id=1NefnQu3e0l4WR7Xb809gpVORXrKpJb0q\"\n",
+    "WSI_DOWNLOAD_LINK = \"https://drive.google.com/uc?id=1a-77TNRlSoPbRlNI0VtlHG3d7xTSpcwB\"\n",
     "WSI_NAME = Path(\"TCGA-21-5784-01Z-00-DX1.tif\")\n",
     "\n",
     "WSA_DOWNLOAD_LINK = \"https://drive.google.com/uc?id=1jkTp0IJHHpmLd1yDO1L3KRFJgm0STh0d\"\n",
@@ -99,20 +62,13 @@
     "    \n",
     "download_segmentation_data()\n"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "wholeslidedata",
    "language": "python",
-   "name": "python3"
+   "name": "wholeslidedata"
   },
   "language_info": {
    "codemirror_mode": {
@@ -124,7 +80,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.0"
+   "version": "3.8.17"
   }
  },
  "nbformat": 4,

diff --git a/downloaddata.py b/downloaddata.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import gdown
 
-WSI_DOWNLOAD_LINK = "https://drive.google.com/uc?id=1NefnQu3e0l4WR7Xb809gpVORXrKpJb0q"
+WSI_DOWNLOAD_LINK = "https://drive.google.com/uc?id=1a-77TNRlSoPbRlNI0VtlHG3d7xTSpcwB"
 WSI_NAME = Path("TCGA-21-5784-01Z-00-DX1.tif")
 
 WSA_DOWNLOAD_LINK = "https://drive.google.com/uc?id=1jkTp0IJHHpmLd1yDO1L3KRFJgm0STh0d"

diff --git a/scripts/save_image_at_spacing.py b/scripts/save_image_at_spacing.py
@@ -77,6 +77,7 @@ def write_image_at_spacing(image, spacing, output_path):
         pyramid=True,
         tile=True,
         compression="jpeg",
+        Q=98,
         xres=1000.0 / spacing[0],
         yres=1000.0 / spacing[1],
         bigtiff=True,

diff --git a/tests/patch_iterator_test_spawn.py b/tests/patch_iterator_test_spawn.py
@@ -0,0 +1,18 @@
+from pytest_cov.embed import cleanup_on_sigterm
+
+from wholeslidedata.iterators.patchiterator import (PatchConfiguration,
+                                                    create_patch_iterator)
+
+cleanup_on_sigterm()
+
+
+def test_patch_iterator():
+    with create_patch_iterator(
+        image_path="/tmp/TCGA-21-5784-01Z-00-DX1.tif",
+        mask_path="/tmp/TCGA-21-5784-01Z-00-DX1_tb_mask.tif",
+        patch_configuration=PatchConfiguration(spacings=(32,)),
+        backend='asap',
+        context='spawn',
+    ) as patch_iterator:
+        for patch in patch_iterator:
+            pass
diff --git a/wholeslidedata/buffer/patchcommander.py b/wholeslidedata/buffer/patchcommander.py
@@ -1,4 +1,5 @@
 from abc import abstractmethod
+import math
 from multiprocessing import Queue
 from pathlib import Path
 from concurrentbuffer.commander import Commander
@@ -43,14 +44,28 @@ def __init__(
         self._x_dims, self._y_dims = wsi.shapes[0][:2]
         self._level_0_spacing = wsi.spacings[0]
         wsi.close()
-
+        wsi = None
+        del wsi
+
         self._info_queue = Queue()
-        self._n_messages = None
         self._messages = []
-        self.reset()
 
     def __len__(self):
-        return self._n_messages
+        offset_y = self._patch_configuration.offset[1]
+        offset_x = self._patch_configuration.offset[0]
+        y_dims = self._y_dims
+        x_dims = self._x_dims
+
+        step_row = int(self._patch_configuration.patch_shape[0] * self._ratio) - int(
+            self._patch_configuration.overlap[0] * self._ratio
+        )
+        step_col = int(self._patch_configuration.patch_shape[1] * self._ratio) - int(
+            self._patch_configuration.overlap[1] * self._ratio
+        )
+        num_outer_iterations = math.ceil((y_dims - offset_y) / step_row)
+        num_inner_iterations = math.ceil((x_dims - offset_x) / step_col)
+
+        return num_outer_iterations * num_inner_iterations
 
     @property
     def shapes(self):
@@ -60,13 +75,15 @@ def shapes(self):
     def info_queue(self):
         return self._info_queue
 
+    def build(self):
+        self.reset()
+
     @abstractmethod
     def get_patch_messages() -> list:
         ...
 
     def reset(self):
         messages = self.get_patch_messages()
-        self._n_messages = len(messages)
         self._messages = iter(messages)
 
     def create_message(self) -> dict:
@@ -87,7 +104,7 @@ def get_patch_messages(self):
             self._patch_configuration.overlap[1] * self._ratio
         )
 
-        if self._mask_path is not None:
+        if self._mask_path is not None and self._mask is None:
             self._mask = WholeSlideImage(self._mask_path, backend=self._backend, auto_resample=True)
         for row in range(self._patch_configuration.offset[1], self._y_dims, step_row):
             for col in range(self._patch_configuration.offset[0], self._x_dims, step_col):