Skip to content

Commit

Permalink
Merge pull request #86 from sameeul/update_docker_maven_path
Browse files Browse the repository at this point in the history 
Update docker containers to address singularity path issue for maven repo
  • Loading branch information
@sameeul
sameeul authored Jul 19, 2024
2 parents 2ad3b27 + 37b9454 commit 6834b16
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 92 deletions.
56 changes: 32 additions & 24 deletions docker/base/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,29 +27,37 @@ RUN apt update && \
# Copy the source
COPY . /bfio_src

# Apply m2 repo patch
RUN git apply --directory=bfio_src /bfio_src/docker/m2_repo.patch

# Install
RUN pip3 install /bfio_src --no-cache-dir && \
rm -rf /bfio_src

# create a non-root user named polusai
RUN groupadd polusai && \
useradd --no-log-init -r -m -g polusai polusai && \
usermod -aG sudo polusai

ENV HOME=/home/polusai
RUN chmod -R o+rx ${HOME}
# NOTE: By default, CWL adds --user=... to the docker run commands.
# https://github.com/common-workflow-language/cwltool/blob/7b8f174828e3c07b3067e4f88568124f7c878d2d/cwltool/docker.py#L362-L366
# Since we do not control the userid and/or groupid on the
# host machine(s) where this docker image will be executed,
# we must make HOME readable and executable for all users! Otherwise:
# /usr/bin/python3: Error while finding module specification for 'polus.plugins...'
# (ModuleNotFoundError: No module named 'polus')

RUN chown -R polusai ${EXEC_DIR} && \
chown -R polusai ${DATA_DIR} && \
chmod -R u+rwx ${EXEC_DIR} && \
chmod -R u+rwx ${DATA_DIR}

USER polusai
RUN pip3 install /bfio_src --no-cache-dir

#Apply jgo copy patch
RUN git apply --directory=usr/local/lib/python3.10/dist-packages /bfio_src/docker/jgo_copy.patch

# clean up
RUN rm -rf /bfio_src


# Update Maven local repository path
RUN mkdir -p /usr/share/maven/conf
RUN echo '<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" ' \
'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ' \
'xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 ' \
'http://maven.apache.org/xsd/settings-1.0.0.xsd">' \
'<localRepository>/etc/m2</localRepository>' \
'</settings>' > /usr/share/maven/conf/settings.xml

# Ensure the /etc/m2 directory exists
RUN mkdir -p /etc/m2

# Set the correct permissions
RUN chmod -R 777 ${EXEC_DIR} && \
chmod -R 777 ${DATA_DIR} && \
chmod -R 777 /etc/m2

RUN python3 -c 'import bfio; bfio.start()'

# Set the correct permissions again, for sanity
RUN chmod -R 777 /etc/m2
85 changes: 49 additions & 36 deletions docker/imagej/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,21 @@
# Get Linux
FROM maven:3.8.6-jdk-11-slim

# Copy the source
COPY . /bfio_src
# Polus platform file extension
ENV POLUS_IMG_EXT=".ome.tif"
ENV POLUS_TAB_EXT=".csv"

# Container log level
ENV POLUS_LOG="INFO"

# Data and code directories
ENV EXEC_DIR="/opt/executables"
ENV DATA_DIR="/data"

RUN mkdir ${EXEC_DIR} && \
mkdir ${DATA_DIR} && \
mkdir /bfio_src


# Install Python
RUN apt update && \
Expand All @@ -16,49 +29,49 @@ RUN apt update && \

RUN ln -sf /usr/bin/python3.9 /usr/bin/python3


# Copy the source
COPY . /bfio_src

# Apply m2 repo patch
RUN git apply --directory=bfio_src /bfio_src/docker/m2_repo.patch

# Install
RUN pip3 install /bfio_src --no-cache-dir && \
rm -rf /bfio_src
RUN pip3 install /bfio_src --no-cache-dir

#Apply jgo copy patch
RUN git apply --directory=usr/local/lib/python3.9/dist-packages /bfio_src/docker/jgo_copy.patch

# Install packages
ENV LIBRARY_PATH=/lib:/usr/lib
RUN pip3 install cython --no-cache-dir && \
pip3 install pyimagej==1.4.1 --no-cache-dir

# Polus platform file extension
ENV POLUS_IMG_EXT=".ome.tif"
ENV POLUS_TAB_EXT=".csv"
#Apply pyimagej patch
RUN git apply --directory=usr/local/lib/python3.9/dist-packages /bfio_src/docker/pyimagej_m2repo.patch

# Container log level
ENV POLUS_LOG="INFO"
# clean up
RUN rm -rf /bfio_src

# Data and code directories
ENV EXEC_DIR="/opt/executables"
ENV DATA_DIR="/data"
# Update Maven local repository path
RUN mkdir -p /usr/share/maven/conf
RUN echo '<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" ' \
'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ' \
'xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 ' \
'http://maven.apache.org/xsd/settings-1.0.0.xsd">' \
'<localRepository>/etc/m2</localRepository>' \
'</settings>' > /usr/share/maven/conf/settings.xml

# Ensure the /etc/m2 directory exists
RUN mkdir -p /etc/m2

# Set the correct permissions
RUN chmod -R 777 ${EXEC_DIR} && \
chmod -R 777 ${DATA_DIR} && \
chmod -R 777 /etc/m2

RUN mkdir ${EXEC_DIR} && \
mkdir ${DATA_DIR}

# create a non-root user named polusai
RUN groupadd polusai && \
useradd --no-log-init -r -m -g polusai polusai && \
usermod -aG sudo polusai

ENV HOME=/home/polusai
RUN chmod -R o+rx ${HOME}
# NOTE: By default, CWL adds --user=... to the docker run commands.
# https://github.com/common-workflow-language/cwltool/blob/7b8f174828e3c07b3067e4f88568124f7c878d2d/cwltool/docker.py#L362-L366
# Since we do not control the userid and/or groupid on the
# host machine(s) where this docker image will be executed,
# we must make HOME readable and executable for all users! Otherwise:
# /usr/bin/python3: Error while finding module specification for 'polus.plugins...'
# (ModuleNotFoundError: No module named 'polus')

RUN chown -R polusai ${EXEC_DIR} && \
chown -R polusai ${DATA_DIR} && \
chmod -R u+rwx ${EXEC_DIR} && \
chmod -R u+rwx ${DATA_DIR}

USER polusai
RUN python3 -c 'import bfio; bfio.start()'
RUN python3 -c 'import imagej; ij = imagej.init(); print(ij.getVersion())'

# Set the correct permissions again, for sanity
RUN chmod -R 777 /etc/m2
13 changes: 13 additions & 0 deletions docker/jgo_copy.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff --git a/jgo/jgo.py b/jgo/jgo.py
index bd2b328..e2a824a 100644
--- a/jgo/jgo.py
+++ b/jgo/jgo.py
@@ -710,7 +710,7 @@ def resolve_dependencies(
jar_file_in_workspace,
link_type=link_type,
)
- except FileExistsError:
+ except (FileExistsError, shutil.SameFileError):
# Do not throw exception if target file exists.
pass
pathlib.Path(build_success_file).touch(exist_ok=True)
12 changes: 12 additions & 0 deletions docker/m2_repo.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff --git a/src/bfio/utils.py b/src/bfio/utils.py
index 954d682..cb88332 100644
--- a/src/bfio/utils.py
+++ b/src/bfio/utils.py
@@ -38,6 +38,7 @@ try:

global JAR_VERSION
scyjava.config.endpoints.append("ome:formats-gpl:7.3.0")
+ scyjava.config.set_m2_repo("/etc/m2/")
scyjava.start_jvm()
import loci

12 changes: 12 additions & 0 deletions docker/pyimagej_m2repo.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff --git a/imagej/__init__.py b/imagej/__init__.py
index 18aa9a8..74524ce 100644
--- a/imagej/__init__.py
+++ b/imagej/__init__.py
@@ -1421,6 +1421,7 @@ def _create_jvm(
sj.config.endpoints.extend(original_endpoints)

try:
+ sj.config.set_m2_repo("/etc/m2/")
sj.start_jvm()
except subprocess.CalledProcessError as e:
# Check to see if initialization failed due to "un-managed"
72 changes: 40 additions & 32 deletions docker/tensorflow/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,6 @@
# Get Linux
FROM tensorflow/tensorflow:2.13.0-gpu

# Install Java and Maven
RUN apt update && \
apt install software-properties-common -y && \
apt install git maven openjdk-8-jre -y && \
apt autoremove -y && \
rm -rf /var/lib/apt/lists/*

# Polus platform file extension
ENV POLUS_IMG_EXT=".ome.tif"
ENV POLUS_TAB_EXT=".csv"
Expand All @@ -20,36 +13,51 @@ ENV EXEC_DIR="/opt/executables"
ENV DATA_DIR="/data"

RUN mkdir ${EXEC_DIR} && \
mkdir ${DATA_DIR}
mkdir ${DATA_DIR} && \
mkdir /bfio_src

# Install Java and Maven
RUN apt update && \
apt install software-properties-common -y && \
apt install git maven openjdk-8-jre -y && \
apt autoremove -y && \
rm -rf /var/lib/apt/lists/*

RUN pip3 install --upgrade pip --no-cache-dir

# Copy the source
COPY . /bfio_src

# Apply m2 repo patch
RUN git apply --directory=bfio_src /bfio_src/docker/m2_repo.patch

# Install
RUN pip3 install /bfio_src --no-cache-dir && \
rm -rf /bfio_src

# create a non-root user named polusai
RUN groupadd polusai && \
useradd --no-log-init -r -m -g polusai polusai && \
usermod -aG sudo polusai

ENV HOME=/home/polusai
RUN chmod -R o+rx ${HOME}
# NOTE: By default, CWL adds --user=... to the docker run commands.
# https://github.com/common-workflow-language/cwltool/blob/7b8f174828e3c07b3067e4f88568124f7c878d2d/cwltool/docker.py#L362-L366
# Since we do not control the userid and/or groupid on the
# host machine(s) where this docker image will be executed,
# we must make HOME readable and executable for all users! Otherwise:
# /usr/bin/python3: Error while finding module specification for 'polus.plugins...'
# (ModuleNotFoundError: No module named 'polus')

RUN chown -R polusai ${EXEC_DIR} && \
chown -R polusai ${DATA_DIR} && \
chmod -R u+rwx ${EXEC_DIR} && \
chmod -R u+rwx ${DATA_DIR}

USER polusai
RUN pip3 install /bfio_src --no-cache-dir

#Apply jgo copy patch
RUN git apply --directory=usr/local/lib/python3.8/dist-packages /bfio_src/docker/jgo_copy.patch

# clean up
RUN rm -rf /bfio_src

# Update Maven local repository path
RUN mkdir -p /usr/share/maven/conf
RUN echo '<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" ' \
'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ' \
'xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 ' \
'http://maven.apache.org/xsd/settings-1.0.0.xsd">' \
'<localRepository>/etc/m2</localRepository>' \
'</settings>' > /usr/share/maven/conf/settings.xml

# Ensure the /etc/m2 directory exists
RUN mkdir -p /etc/m2

# Set the correct permissions
RUN chmod -R 777 ${EXEC_DIR} && \
chmod -R 777 ${DATA_DIR} && \
chmod -R 777 /etc/m2

RUN python3 -c 'import bfio; bfio.start()'

# Set the correct permissions again, for sanity
RUN chmod -R 777 /etc/m2

0 comments on commit 6834b16

Please sign in to comment.