From 0bc327762e53af1641e74a7290e766b5763bd7f0 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 19 Jul 2024 10:16:03 +0100 Subject: [PATCH] Tar home build --- Dockerfile | 79 ++++++++++--------- before-notebook.d/00_untar_home.sh | 16 ++++ before-notebook.d/70_prepare-qe-executable.sh | 16 ---- before-notebook.d/71_install-qeapp.sh | 21 ----- 4 files changed, 59 insertions(+), 73 deletions(-) create mode 100644 before-notebook.d/00_untar_home.sh delete mode 100644 before-notebook.d/70_prepare-qe-executable.sh delete mode 100644 before-notebook.d/71_install-qeapp.sh diff --git a/Dockerfile b/Dockerfile index c53456900..5e3574b8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,48 +1,55 @@ # syntax=docker/dockerfile:1 FROM ghcr.io/astral-sh/uv:0.2.18 AS uv -FROM ghcr.io/aiidalab/full-stack:2024.1019 - -# Copy whole repo and pre-install the dependencies and app to the tmp folder. -# In the before notebook scripts the app will be re-installed by moving it to the app folder. -ENV PREINSTALL_APP_FOLDER=${CONDA_DIR}/aiidalab-qe -COPY --chown=${NB_UID}:${NB_GID} . ${PREINSTALL_APP_FOLDER} +FROM ghcr.io/aiidalab/full-stack:2024.1021 USER ${NB_USER} -# Using uv to speed up installation, per docs: +ENV QE_VERSION="7.2" + +# 1. Install Quantum Espresso into a conda environment +# (we do this here for better caching during Docker build) +RUN mamba create -p /opt/conda/envs/quantum-espresso --yes \ + qe=${QE_VERSION} && \ + mkdir -p /home/${NB_USER}/.conda/envs && \ + ln -s /opt/conda/envs/quantum-espresso /home/${NB_USER}/.conda/envs/quantum-espresso-${QE_VERSION} && \ + mamba clean --all -f -y + +ENV QE_APP_FOLDER=${AIIDALAB_APPS}/quantum-espresso +# 2. Copy files needed for installing stuff, the rest is copied at the end for better caching +COPY --chown=${NB_UID}:${NB_GID} src/ ${QE_APP_FOLDER}/src +COPY --chown=${NB_UID}:${NB_GID} setup.cfg pyproject.toml *yaml README.md ${QE_APP_FOLDER} + +WORKDIR "${QE_APP_FOLDER}" +# 3.Install python dependencies +# Use uv instead of pip to speed up installation, per docs: # https://github.com/astral-sh/uv/blob/main/docs/guides/docker.md#using-uv-temporarily -# Use the same constraint file as PIP +# Use the same constraint file as pip ENV UV_CONSTRAINT=${PIP_CONSTRAINT} -RUN --mount=from=uv,source=/uv,target=/bin/uv \ - cd ${PREINSTALL_APP_FOLDER} && \ - # Remove all untracked files and directories. For example the setup lock flag file. - git clean -fx && \ - # It is important to install from `aiidalab install` to mimic the exact installation operation as - # from the app store. - # The command wil first install the dependencies from list by parsing setup config files, - # (for `aiidalab/aiidalab<23.03.2` the `setup.py` should be in the root folder of the app https://github.com/aiidalab/aiidalab/pull/382). - # and then the app and restart the daemon in the end. - # But since the aiida profile not yet exists, the daemon restart will fail but it is not a problem. - # Because we only need the dependencies to be installed. - # aiidalab install --yes --python ${CONDA_DIR}/bin/python "quantum-espresso@file://${PREINSTALL_APP_FOLDER}" && \ - # However, have to use `pip install` explicitly because `aiidalab install` call `pip install --user` which will install the app to `/home/${NB_USER}/.local`. - # It won't cause issue for docker but for k8s deployment the home folder is not bind mounted to the pod and the dependencies won't be found. (see issue in `jupyter/docker-stacks` https://github.com/jupyter/docker-stacks/issues/815) - uv pip install --system --no-cache . && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" +RUN --mount=from=uv,source=/uv,target=/bin/uv \ + uv pip install --system --no-cache . && \ + rm -rf build/ src/aiidalab_qe.egg-info/ + +# 4. Prepare AiiDA profile and localhost computer +# 5. Install the QE pseudopotentials and codes +RUN bash /usr/local/bin/before-notebook.d/20_start-postgresql.sh && \ + bash /usr/local/bin/before-notebook.d/40_prepare-aiida.sh && \ + python -m aiidalab_qe install-qe && \ + python -m aiidalab_qe install-pseudos && \ + verdi daemon stop && \ + mamba run -n aiida-core-services pg_ctl stop -ENV QE_VERSION="7.2" -RUN mamba create -p /opt/conda/envs/quantum-espresso --yes \ - qe=${QE_VERSION} \ - && mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" -# Download the QE pseudopotentials to the folder for afterware installation. -ENV PSEUDO_FOLDER=${CONDA_DIR}/pseudo -RUN mkdir -p ${PSEUDO_FOLDER} && \ - python -m aiidalab_qe download-pseudos --dest ${PSEUDO_FOLDER} +# 6. Copy the whole repo +COPY --chown=${NB_UID}:${NB_GID} . ${QE_APP_FOLDER} +# Remove all untracked files and directories. +RUN git clean -dffx || true -COPY before-notebook.d/* /usr/local/bin/before-notebook.d/ +USER root +COPY ./before-notebook.d/* /usr/local/bin/before-notebook.d/ +RUN fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" WORKDIR "/home/${NB_USER}" +RUN tar -cf /opt/home.tar . + +USER ${NB_USER} diff --git a/before-notebook.d/00_untar_home.sh b/before-notebook.d/00_untar_home.sh new file mode 100644 index 000000000..5cf11cf8b --- /dev/null +++ b/before-notebook.d/00_untar_home.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +home="/home/${NB_USER}" + +if [[ ! -d ${home} ]]; then + echo "Directory $home does not exist!" + exit 1 +fi + +# Untar home archive file to restore home directory if it is empty +if [[ $(ls -A ${home} | wc -l) = "0" ]];then + if [ -f /opt/home.tar.gz ]; then + echo "Extracting /opt/home.tar to /home/${NB_USER}" + tar -xf /opt/home.tar -C /home/${NB_USER} + fi +fi diff --git a/before-notebook.d/70_prepare-qe-executable.sh b/before-notebook.d/70_prepare-qe-executable.sh deleted file mode 100644 index 41ba2d391..000000000 --- a/before-notebook.d/70_prepare-qe-executable.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -e - -# Debugging. -set -x - -# Copy quantum espresso env to user space. -mkdir -p /home/${NB_USER}/.conda/envs -if [ ! -d /home/${NB_USER}/.conda/envs/quantum-espresso-${QE_VERSION} ]; then - ln -s /opt/conda/envs/quantum-espresso /home/${NB_USER}/.conda/envs/quantum-espresso-${QE_VERSION} - - # Install qe so the progress bar not shown in the notebook when first time using app. - echo "Installing qe." - python -m aiidalab_qe install-qe -else - echo "Quantum ESPRESSO app is already installed." -fi diff --git a/before-notebook.d/71_install-qeapp.sh b/before-notebook.d/71_install-qeapp.sh deleted file mode 100644 index 849d92849..000000000 --- a/before-notebook.d/71_install-qeapp.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -e - -# Debugging. -set -x - -# Install qeapp if it is not already installed. -if aiidalab list | grep -q quantum-espresso; then - echo "Quantum ESPRESSO app is already installed." -else - echo "Installing Quantum ESPRESSO app." - # Install by move the repo folder that is already in the image. - mv ${PREINSTALL_APP_FOLDER} /home/${NB_USER}/apps/quantum-espresso -fi - -# Install the pseudo libraries if not already installed. -if aiida-pseudo list | grep -q "no pseudo potential families"; then - echo "Installing pseudo potential families." - python -m aiidalab_qe install-pseudos --source ${PSEUDO_FOLDER} -else - echo "Pseudo potential families are already installed." -fi