From 731ae2a7eeb18e8abc28e2efbbc00edacb80b0e2 Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:23:53 -0600 Subject: [PATCH 01/11] Update Dockerfile for ray-ml --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0af09e7..2eed7be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,14 +19,14 @@ RUN python -m venv venv \ # We force this so there is no error even if the dll does not exist. && rm -f libopenslide-0.dll -FROM python:3.8-slim +FROM rayproject/ray-ml:latest-gpu ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +RUN sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends \ libopenslide0 \ libtk8.6 \ procps \ - && rm -rf /var/lib/apt/lists/* + && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt/HistoQC COPY --from=builder /opt/HistoQC/ . ENV PATH="/opt/HistoQC/venv/bin:$PATH" From 3e90985bb5e131d54520d0b8d93b527ca9157c37 Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:46:47 -0600 Subject: [PATCH 02/11] Update Dockerfile for running histoqc on ray-ml --- Dockerfile | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2eed7be..457913d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,24 @@ -# Dockerfile for HistoQC. -# -# This Dockerfile uses two stages. In the first, the project's python dependencies are -# installed. This requires a C compiler. In the second stage, the HistoQC directory and -# the python environment are copied over. We do not require a C compiler in the second -# stage, and so we can use a slimmer base image. - -FROM python:3.8 AS builder -ARG DEBIAN_FRONTEND=noninteractive -WORKDIR /opt/HistoQC -COPY . . -# Create virtual environment for this project. This makes it easier to copy the Python -# installation into the second stage of the build. -ENV PATH="/opt/HistoQC/venv/bin:$PATH" -RUN python -m venv venv \ - && python -m pip install --no-cache-dir setuptools wheel \ - && python -m pip install --no-cache-dir -r requirements.txt \ - && python -m pip install --no-cache-dir . \ - # We force this so there is no error even if the dll does not exist. - && rm -f libopenslide-0.dll - FROM rayproject/ray-ml:latest-gpu + ARG DEBIAN_FRONTEND=noninteractive + +USER root + RUN sudo apt-get update \ && sudo apt-get install -y --no-install-recommends \ libopenslide0 \ libtk8.6 \ procps \ && sudo rm -rf /var/lib/apt/lists/* + +USER ray + +COPY ./requirements.txt /opt/HistoQC/requirements.txt + WORKDIR /opt/HistoQC -COPY --from=builder /opt/HistoQC/ . -ENV PATH="/opt/HistoQC/venv/bin:$PATH" -WORKDIR /data +RUN pip install -r requirements.txt + +ADD . /opt/HistoQC/ + +CMD [ "python", "-m", "histoqc", "--help" ] From e3265c7536afd40cc5f62c5f18c5a93a554875b4 Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:16:26 -0600 Subject: [PATCH 03/11] Update Dockerfile --- Dockerfile | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 457913d..4593a4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,26 @@ -FROM rayproject/ray-ml:latest-gpu +# Dockerfile for HistoQC. +# This Dockerfile uses two stages. In the first, the project's python dependencies are +# installed. This requires a C compiler. In the second stage, the HistoQC directory and +# the python environment are copied over. We do not require a C compiler in the second +# stage, and so we can use a slimmer base image. +FROM python:3.8 AS builder +ARG DEBIAN_FRONTEND=noninteractive +WORKDIR /opt/HistoQC +COPY . . +# Create virtual environment for this project. This makes it easier to copy the Python +# installation into the second stage of the build. +ENV PATH="/opt/HistoQC/venv/bin:$PATH" +RUN python -m venv venv \ + && python -m pip install --no-cache-dir setuptools wheel \ + && python -m pip install --no-cache-dir -r requirements.txt \ + && python -m pip install --no-cache-dir . \ + # We force this so there is no error even if the dll does not exist. + && rm -f libopenslide-0.dll + +FROM rayproject/ray-ml:latest-py38-cu118 ARG DEBIAN_FRONTEND=noninteractive - USER root - RUN sudo apt-get update \ && sudo apt-get install -y --no-install-recommends \ libopenslide0 \ @@ -11,14 +28,13 @@ RUN sudo apt-get update \ procps \ && sudo rm -rf /var/lib/apt/lists/* -USER ray - -COPY ./requirements.txt /opt/HistoQC/requirements.txt - WORKDIR /opt/HistoQC +COPY --from=builder /opt/HistoQC/ . -RUN pip install -r requirements.txt +RUN pip install . -ADD . /opt/HistoQC/ +USER ray +WORKDIR /data +# CMD ["bash"] CMD [ "python", "-m", "histoqc", "--help" ] From b9735865ba1d8775908c9d6a70a7ec584a65a1a8 Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:18:32 -0600 Subject: [PATCH 04/11] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4593a4d..50e58f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN python -m venv venv \ # We force this so there is no error even if the dll does not exist. && rm -f libopenslide-0.dll -FROM rayproject/ray-ml:latest-py38-cu118 +FROM rayproject/ray-ml:latest-gpu ARG DEBIAN_FRONTEND=noninteractive USER root RUN sudo apt-get update \ From af2a9baeef37affca05f008fb8d0f90ee82e2f61 Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:21:03 -0600 Subject: [PATCH 05/11] Update Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 50e58f4..f1aaff7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,12 +21,12 @@ RUN python -m venv venv \ FROM rayproject/ray-ml:latest-gpu ARG DEBIAN_FRONTEND=noninteractive USER root -RUN sudo apt-get update \ - && sudo apt-get install -y --no-install-recommends \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ libopenslide0 \ libtk8.6 \ procps \ - && sudo rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* WORKDIR /opt/HistoQC COPY --from=builder /opt/HistoQC/ . From 597b9147fa390dceac74d36f4be02836da2720c2 Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Tue, 12 Mar 2024 00:54:39 -0500 Subject: [PATCH 06/11] Update setup.cfg to add histoqc.import_wrapper package --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 1237c87..28c8f8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,7 @@ packages = histoqc.config histoqc.data histoqc.ui + histoqc.import_wrapper python_requires = >=3.7 install_requires = importlib_resources; python_version < '3.9' From 8f9b213e613f2ee64c7db92d344e47740f5f05c0 Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Tue, 12 Mar 2024 00:56:02 -0500 Subject: [PATCH 07/11] Update setup.py to add histoqc.import_wrapper files --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3ac2793..277cbee 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,11 @@ for f in glob.iglob(f"histoqc/data/{folder}/**/*", recursive=True) if os.path.isfile(f) ] - +import_wrapper_files = [ + os.path.relpath(f, "histoqc/import_wrapper_files/") + for f in glob.iglob(f"histoqc/import_wrapper_files/**/*", recursive=True) + if os.path.isfile(f) +] setup( use_scm_version={ # duplicated config from pyproject.toml; keep in sync @@ -28,5 +32,6 @@ 'histoqc.config': ['*.ini'], 'histoqc.data': data_files, 'histoqc.ui': ui_files, + 'histoqc.import_wrapper': import_wrapper_files } ) From b619f8bca504bf2803449917a6d8e3fd584ce60a Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Tue, 12 Mar 2024 00:56:57 -0500 Subject: [PATCH 08/11] Update __init__.py to import openslide --- histoqc/import_wrapper/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/histoqc/import_wrapper/__init__.py b/histoqc/import_wrapper/__init__.py index e69de29..78ffef2 100644 --- a/histoqc/import_wrapper/__init__.py +++ b/histoqc/import_wrapper/__init__.py @@ -0,0 +1 @@ +from .openslide import openslide From c77b94c0409d90fdbd4e6628a4c85c3d63d6a24b Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Tue, 12 Mar 2024 00:57:34 -0500 Subject: [PATCH 09/11] Update BaseImage.py --- histoqc/BaseImage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/histoqc/BaseImage.py b/histoqc/BaseImage.py index 6e656e1..42305ab 100644 --- a/histoqc/BaseImage.py +++ b/histoqc/BaseImage.py @@ -1,5 +1,5 @@ import logging -import os +import os, re import numpy as np import zlib, dill from distutils.util import strtobool From dc85fd0143b090e404a73e03ba23c3fb9ead1bbd Mon Sep 17 00:00:00 2001 From: nanli-emory <40213692+nanli-emory@users.noreply.github.com> Date: Tue, 12 Mar 2024 00:59:40 -0500 Subject: [PATCH 10/11] Update Dockerfile --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1aaff7..8abe828 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ # installed. This requires a C compiler. In the second stage, the HistoQC directory and # the python environment are copied over. We do not require a C compiler in the second # stage, and so we can use a slimmer base image. - FROM python:3.8 AS builder ARG DEBIAN_FRONTEND=noninteractive WORKDIR /opt/HistoQC @@ -30,11 +29,12 @@ RUN apt-get update \ WORKDIR /opt/HistoQC COPY --from=builder /opt/HistoQC/ . +# install +RUN pip install -r requirements.txt \ + && pip install . -RUN pip install . - -USER ray WORKDIR /data # CMD ["bash"] -CMD [ "python", "-m", "histoqc", "--help" ] +CMD [ "python", "-m", "histoqc", "-c", "/data/config_v2.1.ini", "/data/*.svs" ] + From fc5edc6ebc59e519c6a07b4a9f099b48cc777940 Mon Sep 17 00:00:00 2001 From: nanli-emory Date: Tue, 12 Mar 2024 16:23:36 -0400 Subject: [PATCH 11/11] remove first stage build --- Dockerfile | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8abe828..8110d1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,4 @@ # Dockerfile for HistoQC. -# This Dockerfile uses two stages. In the first, the project's python dependencies are -# installed. This requires a C compiler. In the second stage, the HistoQC directory and -# the python environment are copied over. We do not require a C compiler in the second -# stage, and so we can use a slimmer base image. -FROM python:3.8 AS builder -ARG DEBIAN_FRONTEND=noninteractive -WORKDIR /opt/HistoQC -COPY . . -# Create virtual environment for this project. This makes it easier to copy the Python -# installation into the second stage of the build. -ENV PATH="/opt/HistoQC/venv/bin:$PATH" -RUN python -m venv venv \ - && python -m pip install --no-cache-dir setuptools wheel \ - && python -m pip install --no-cache-dir -r requirements.txt \ - && python -m pip install --no-cache-dir . \ - # We force this so there is no error even if the dll does not exist. - && rm -f libopenslide-0.dll - FROM rayproject/ray-ml:latest-gpu ARG DEBIAN_FRONTEND=noninteractive USER root @@ -28,13 +10,14 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* WORKDIR /opt/HistoQC -COPY --from=builder /opt/HistoQC/ . +COPY . . # install -RUN pip install -r requirements.txt \ - && pip install . +RUN pip install --no-cache-dir setuptools wheel \ + && pip install --no-cache-dir -r requirements.txt \ + && pip install --no-cache-dir . \ + && rm -f libopenslide-0.dll WORKDIR /data -# CMD ["bash"] -CMD [ "python", "-m", "histoqc", "-c", "/data/config_v2.1.ini", "/data/*.svs" ] +CMD ["bash"]