From 0e3bb01fb37aaff8ebbf4779041f849166b5cefe Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 14 Nov 2023 14:09:09 -0800 Subject: [PATCH 01/10] remove some python packages --- deployments/biology/image/environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployments/biology/image/environment.yml b/deployments/biology/image/environment.yml index c0bae3c90..8100ad2b8 100644 --- a/deployments/biology/image/environment.yml +++ b/deployments/biology/image/environment.yml @@ -9,7 +9,7 @@ dependencies: - nb_conda_kernels=2.3.1 # Packages from bioconda for IB134L -- bwa=0.7.12 +# - bwa=0.7.12 - samtools=1.3.1 - mafft=7.471 - emboss=6.6.0 @@ -23,8 +23,8 @@ dependencies: - fastqc=0.11.9 - phyml=3.3.20200621 - sra-tools=2.11 -- hisat2=2.2.1 -- subread=2.0.1 +# - hisat2=2.2.1 +# - subread=2.0.1 - plink=1.90b6.21 - syncthing==1.18.6 From 618fc9152e31bdf7b598b92f125bf07aee0bc1b0 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 14 Nov 2023 14:09:26 -0800 Subject: [PATCH 02/10] time the R installation and rm-ing of temp files --- deployments/biology/image/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index b82678a63..544d7c6e6 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -186,8 +186,8 @@ RUN r -e "install.packages('IRkernel', version='1.2')" && \ # Install R packages, cleanup temp package download location COPY install.R /tmp/install.R -RUN r /tmp/install.R && \ - rm -rf /tmp/downloaded_packages/ /tmp/*.rds +RUN time r /tmp/install.R +RUN time rm -rf /tmp/downloaded_packages/ /tmp/*.rds # install bio1b packages COPY bio1b-packages.bash /tmp/bio1b-packages.bash From d87475bc1bc548bbf4d9fb6adbc58be055f70834 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 14 Nov 2023 14:12:06 -0800 Subject: [PATCH 03/10] time python package installs --- deployments/biology/image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index 544d7c6e6..b60af9456 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -169,7 +169,7 @@ USER ${NB_USER} COPY environment.yml /tmp/ COPY infra-requirements.txt /tmp/ -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && mamba clean -afy +RUN time mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && mamba clean -afy RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ jupyter nbextensions_configurator enable --sys-prefix From c5d5dcaab575968d390c6ff50616c12e2e2a634b Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 14 Nov 2023 14:37:24 -0800 Subject: [PATCH 04/10] install time, add quotes --- deployments/biology/image/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index b60af9456..77a4f049f 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -59,7 +59,8 @@ RUN apt-get update -qq --yes && \ wget \ vim \ tini \ - locales > /dev/null + locales \ + time > /dev/null RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ locale-gen @@ -169,7 +170,7 @@ USER ${NB_USER} COPY environment.yml /tmp/ COPY infra-requirements.txt /tmp/ -RUN time mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && mamba clean -afy +RUN time "mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml" && mamba clean -afy RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ jupyter nbextensions_configurator enable --sys-prefix @@ -186,8 +187,8 @@ RUN r -e "install.packages('IRkernel', version='1.2')" && \ # Install R packages, cleanup temp package download location COPY install.R /tmp/install.R -RUN time r /tmp/install.R -RUN time rm -rf /tmp/downloaded_packages/ /tmp/*.rds +RUN time "r /tmp/install.R" +RUN time "rm -rf /tmp/downloaded_packages/ /tmp/*.rds" # install bio1b packages COPY bio1b-packages.bash /tmp/bio1b-packages.bash From b18c6248ca72cba1e2f9481d2cc7ee9c721d5188 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 14 Nov 2023 15:18:10 -0800 Subject: [PATCH 05/10] shell gymnastics ftl --- deployments/biology/image/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index 77a4f049f..fc4f7dbbc 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -170,7 +170,8 @@ USER ${NB_USER} COPY environment.yml /tmp/ COPY infra-requirements.txt /tmp/ -RUN time "mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml" && mamba clean -afy +RUN echo "/srv/conda/condabin/mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN mamba clean -afy RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ jupyter nbextensions_configurator enable --sys-prefix @@ -187,8 +188,8 @@ RUN r -e "install.packages('IRkernel', version='1.2')" && \ # Install R packages, cleanup temp package download location COPY install.R /tmp/install.R -RUN time "r /tmp/install.R" -RUN time "rm -rf /tmp/downloaded_packages/ /tmp/*.rds" +RUN echo "/usr/bin/r /tmp/install.R" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN rm -rf /tmp/downloaded_packages/ /tmp/*.rds # install bio1b packages COPY bio1b-packages.bash /tmp/bio1b-packages.bash From d28e05f4d1fee487378ec9441a9822faf2d36eb3 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 14 Nov 2023 15:24:04 -0800 Subject: [PATCH 06/10] moar shell gymnastics ftl --- deployments/biology/image/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index fc4f7dbbc..c03ec8ea9 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -183,8 +183,8 @@ COPY rsession.conf /etc/rstudio/rsession.conf COPY file-locks /etc/rstudio/file-locks # Install IRKernel -RUN r -e "install.packages('IRkernel', version='1.2')" && \ - r -e "IRkernel::installspec(prefix='${CONDA_DIR}')" +RUN /usr/bin/r -e "install.packages('IRkernel', version='1.2')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN /usr/bin/r -e "IRkernel::installspec(prefix='${CONDA_DIR}')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash # Install R packages, cleanup temp package download location COPY install.R /tmp/install.R From 2f63e414646e314d6d2557fea2cca2ef5cc3dcfb Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 14 Nov 2023 15:25:37 -0800 Subject: [PATCH 07/10] moar shell gymnastics ftl le sigh --- deployments/biology/image/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index c03ec8ea9..0bd9e1ce5 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -183,8 +183,8 @@ COPY rsession.conf /etc/rstudio/rsession.conf COPY file-locks /etc/rstudio/file-locks # Install IRKernel -RUN /usr/bin/r -e "install.packages('IRkernel', version='1.2')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN /usr/bin/r -e "IRkernel::installspec(prefix='${CONDA_DIR}')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN echo /usr/bin/r -e "install.packages('IRkernel', version='1.2')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN echo /usr/bin/r -e "IRkernel::installspec(prefix='${CONDA_DIR}')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash # Install R packages, cleanup temp package download location COPY install.R /tmp/install.R From 21af23b5b1c06fdf865f5c4712d4a7a7e1f28f30 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 15 Nov 2023 10:16:13 -0800 Subject: [PATCH 08/10] and i quote, help me escape from this bash hell --- deployments/biology/image/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index 0bd9e1ce5..f7ee84fce 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -183,8 +183,8 @@ COPY rsession.conf /etc/rstudio/rsession.conf COPY file-locks /etc/rstudio/file-locks # Install IRKernel -RUN echo /usr/bin/r -e "install.packages('IRkernel', version='1.2')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN echo /usr/bin/r -e "IRkernel::installspec(prefix='${CONDA_DIR}')" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN echo "/usr/bin/r -e \"install.packages('IRkernel', version='1.2')\"" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN echo "/usr/bin/r -e \"IRkernel::installspec(prefix='${CONDA_DIR}')\"" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash # Install R packages, cleanup temp package download location COPY install.R /tmp/install.R From 30fc0d7f55be72d15d830ecc7b8f4d39d5b0db9a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 15 Nov 2023 12:06:41 -0800 Subject: [PATCH 09/10] time ALL the things that take time --- deployments/biology/image/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index f7ee84fce..7810472fd 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -171,9 +171,9 @@ COPY environment.yml /tmp/ COPY infra-requirements.txt /tmp/ RUN echo "/srv/conda/condabin/mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN mamba clean -afy -RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ - jupyter nbextensions_configurator enable --sys-prefix +RUN echo "/srv/conda/condabin/mamba clean -afy" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN echo "jupyter contrib nbextensions install --sys-prefix --symlink" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN echo "jupyter nbextensions_configurator enable --sys-prefix" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash # Set CRAN mirror to rspm before we install anything COPY Rprofile.site /usr/lib/R/etc/Rprofile.site @@ -193,7 +193,7 @@ RUN rm -rf /tmp/downloaded_packages/ /tmp/*.rds # install bio1b packages COPY bio1b-packages.bash /tmp/bio1b-packages.bash -RUN bash /tmp/bio1b-packages.bash +RUN echo "bash /tmp/bio1b-packages.bash" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash # install ib134L packages COPY ib134-packages.bash /tmp/ib134-packages.bash @@ -201,6 +201,6 @@ RUN bash /tmp/ib134-packages.bash # install ccb293 packages COPY ccb293-packages.bash /tmp/ccb293-packages.bash -RUN bash /tmp/ccb293-packages.bash +RUN echo "bash /tmp/ccb293-packages.bash" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash ENTRYPOINT ["tini", "--"] From 60f52574f61ce2c261a8e24062d6faed2e9ff80a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 15 Nov 2023 13:19:10 -0800 Subject: [PATCH 10/10] remove timing code --- deployments/biology/image/Dockerfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile index 7810472fd..4a3abcc8d 100644 --- a/deployments/biology/image/Dockerfile +++ b/deployments/biology/image/Dockerfile @@ -59,8 +59,7 @@ RUN apt-get update -qq --yes && \ wget \ vim \ tini \ - locales \ - time > /dev/null + locales > /dev/null RUN echo "${LC_ALL} UTF-8" > /etc/locale.gen && \ locale-gen @@ -170,10 +169,11 @@ USER ${NB_USER} COPY environment.yml /tmp/ COPY infra-requirements.txt /tmp/ -RUN echo "/srv/conda/condabin/mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN echo "/srv/conda/condabin/mamba clean -afy" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN echo "jupyter contrib nbextensions install --sys-prefix --symlink" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN echo "jupyter nbextensions_configurator enable --sys-prefix" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ + mamba clean -afy + +RUN jupyter contrib nbextensions install --sys-prefix --symlink && \ + jupyter nbextensions_configurator enable --sys-prefix # Set CRAN mirror to rspm before we install anything COPY Rprofile.site /usr/lib/R/etc/Rprofile.site @@ -183,17 +183,17 @@ COPY rsession.conf /etc/rstudio/rsession.conf COPY file-locks /etc/rstudio/file-locks # Install IRKernel -RUN echo "/usr/bin/r -e \"install.packages('IRkernel', version='1.2')\"" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN echo "/usr/bin/r -e \"IRkernel::installspec(prefix='${CONDA_DIR}')\"" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN r -e "install.packages('IRkernel', version='1.2')" && \ + r -e "IRkernel::installspec(prefix='${CONDA_DIR}')" # Install R packages, cleanup temp package download location COPY install.R /tmp/install.R -RUN echo "/usr/bin/r /tmp/install.R" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash -RUN rm -rf /tmp/downloaded_packages/ /tmp/*.rds +RUN r /tmp/install.R && \ + rm -rf /tmp/downloaded_packages/ /tmp/*.rds # install bio1b packages COPY bio1b-packages.bash /tmp/bio1b-packages.bash -RUN echo "bash /tmp/bio1b-packages.bash" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN bash /tmp/bio1b-packages.bash # install ib134L packages COPY ib134-packages.bash /tmp/ib134-packages.bash @@ -201,6 +201,6 @@ RUN bash /tmp/ib134-packages.bash # install ccb293 packages COPY ccb293-packages.bash /tmp/ccb293-packages.bash -RUN echo "bash /tmp/ccb293-packages.bash" | /usr/bin/time -f "User\t%U\nSys\t%S\nReal\t%E\nCPU\t%P" /usr/bin/bash +RUN bash /tmp/ccb293-packages.bash ENTRYPOINT ["tini", "--"]