From 6cfb093b7dd91272258023d6d45e92da47986ddc Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 1 Aug 2023 19:22:25 +0200 Subject: [PATCH] Implement pre-installed QE components for demo server (#402) A new image is brought into the stack which includes the pre-installed Quantum ESPRESSO from conda forge to be run from localhost. The dependencies are installed from Dockerfile while the `aiidalab-qe` need to be installed the first time container started since the app should be installed in the user home which only created after the container started. Similar for the QE from conda forge, the Dockerfile creates the virtual env and installs the QE to the system folder and when the container started the first time it is copied to the user's home if the folder not existed. This image is designed as the image for the AiiDAlab demo server in the materials cloud so that new users can have a quick taste of AiiDAlab and run simulations with QeApp. --- .github/workflows/docker.yml | 36 +++++++++++++------ build.json | 6 ++++ docker-bake.hcl | 26 ++++++++++++-- dodo.py | 7 +--- stack/docker-compose.base-with-services.yml | 2 +- stack/docker-compose.base.yml | 2 +- stack/docker-compose.full-stack.yml | 2 +- stack/docker-compose.lab.yml | 2 +- stack/docker-compose.qe.yml | 19 ++++++++++ stack/qe/Dockerfile | 32 +++++++++++++++++ .../70_prepare-qe-executable.sh | 10 ++++++ .../qe/before-notebook.d/71_install-qeapp.sh | 12 +++++++ tests/conftest.py | 5 +++ tests/test-qe.py | 9 +++++ 14 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 stack/docker-compose.qe.yml create mode 100644 stack/qe/Dockerfile create mode 100644 stack/qe/before-notebook.d/70_prepare-qe-executable.sh create mode 100644 stack/qe/before-notebook.d/71_install-qeapp.sh create mode 100644 tests/test-qe.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 744b1e18..1916ac31 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -90,6 +90,14 @@ jobs: runsOn: ubuntu-latest needs: [amd64-base-with-services, amd64-lab] + amd64-qe: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + image: qe + architecture: amd64 + runsOn: ubuntu-latest + needs: [amd64-full-stack] + arm64-base: uses: ./.github/workflows/docker-build-test-upload.yml with: @@ -121,11 +129,19 @@ jobs: runsOn: buildjet-2vcpu-ubuntu-2204-arm needs: [arm64-base-with-services, arm64-lab] + arm64-qe: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + image: qe + architecture: arm64 + runsOn: buildjet-2vcpu-ubuntu-2204-arm + needs: [arm64-full-stack] + amd64-push-ghcr: uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["base", "base-with-services", "lab", "full-stack", "qe"] with: architecture: amd64 image: ${{ matrix.image }} @@ -133,13 +149,13 @@ jobs: secrets: REGISTRY_USERNAME: ${{ github.actor }} REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack] + needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack, amd64-qe] arm64-push-ghcr: uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["base", "base-with-services", "lab", "full-stack", "qe"] with: architecture: arm64 image: ${{ matrix.image }} @@ -147,13 +163,13 @@ jobs: secrets: REGISTRY_USERNAME: ${{ github.actor }} REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack] + needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack, arm64-qe] merge-tags-ghcr: uses: ./.github/workflows/docker-merge-tags.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["base", "base-with-services", "lab", "full-stack", "qe"] with: image: ${{ matrix.image }} registry: ghcr.io @@ -167,7 +183,7 @@ jobs: uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["base", "base-with-services", "lab", "full-stack", "qe"] with: architecture: amd64 image: ${{ matrix.image }} @@ -175,14 +191,14 @@ jobs: secrets: REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }} - needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack] + needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack, amd64-qe] arm64-push-dockerhub: if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["base", "base-with-services", "lab", "full-stack", "qe"] with: architecture: arm64 image: ${{ matrix.image }} @@ -190,14 +206,14 @@ jobs: secrets: REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }} - needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack] + needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack, arm64-qe] merge-tags-dockerhub: if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) uses: ./.github/workflows/docker-merge-tags.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["base", "base-with-services", "lab", "full-stack", "qe"] with: image: ${{ matrix.image }} registry: docker.io diff --git a/build.json b/build.json index 0a62732d..582fd44b 100644 --- a/build.json +++ b/build.json @@ -14,6 +14,12 @@ }, "AIIDALAB_HOME_VERSION": { "default": "23.03.1" + }, + "AIIDALAB_QE_VERSION": { + "default": "23.04.6" + }, + "QE_VERSION": { + "default": "7.2" } } } diff --git a/docker-bake.hcl b/docker-bake.hcl index f4aa2431..a01ecd6a 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -17,6 +17,12 @@ variable "AIIDALAB_VERSION" { variable "AIIDALAB_HOME_VERSION" { } +variable "AIIDALAB_QE_VERSION" { +} + +variable "QE_VERSION" { +} + variable "JUPYTER_BASE_IMAGE" { default = "jupyter/minimal-notebook:python-${PYTHON_VERSION}" } @@ -34,14 +40,13 @@ variable "PLATFORMS" { } variable "TARGETS" { - default = ["base", "base-with-services", "lab", "full-stack"] + default = ["base", "base-with-services", "lab", "full-stack", "qe"] } function "tags" { params = [image] result = [ "${REGISTRY}${ORGANIZATION}/${image}:newly-build", - "${REGISTRY}${ORGANIZATION}/${image}:latest", ] } @@ -49,6 +54,10 @@ group "default" { targets = "${TARGETS}" } +target "qe-meta" { + tags = tags("qe") +} + target "base-meta" { tags = tags("base") } @@ -105,3 +114,16 @@ target "full-stack" { } platforms = "${PLATFORMS}" } + +target "qe" { + inherits = ["qe-meta"] + context = "stack/qe" + contexts = { + full-stack = "target:full-stack" + } + args = { + "AIIDALAB_QE_VERSION" = "${AIIDALAB_QE_VERSION}" + "QE_VERSION" = "${QE_VERSION}" + } + platforms = "${PLATFORMS}" +} diff --git a/dodo.py b/dodo.py index 4a2030cb..a75f779f 100644 --- a/dodo.py +++ b/dodo.py @@ -70,11 +70,6 @@ def task_build(): def generate_version_override( version, registry, targets, architecture, organization ): - if len(targets) > 4: - # Workaround of issue of doit, which rather than override the default value, it appends - # https://github.com/pydoit/doit/issues/436 - targets = targets[4:] - platforms = [f"linux/{architecture}"] Path("docker-bake.override.json").write_text( @@ -107,7 +102,7 @@ def generate_version_override( "long": "targets", "short": "t", "type": list, - "default": ["base", "base-with-services", "lab", "full-stack"], + "default": [], "help": "Specify the target to build.", }, ], diff --git a/stack/docker-compose.base-with-services.yml b/stack/docker-compose.base-with-services.yml index 1beb491b..9074a8e7 100644 --- a/stack/docker-compose.base-with-services.yml +++ b/stack/docker-compose.base-with-services.yml @@ -4,7 +4,7 @@ version: '3.4' services: aiidalab: - image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/base-with-services}:${VERSION:-latest} + image: ${REGISTRY:-}${BASE_WITH_SERVICES_IMAGE:-aiidalab/base-with-services}:${VERSION:-newly-build} environment: TZ: Europe/Zurich DOCKER_STACKS_JUPYTER_CMD: notebook diff --git a/stack/docker-compose.base.yml b/stack/docker-compose.base.yml index 83750f49..3888e526 100644 --- a/stack/docker-compose.base.yml +++ b/stack/docker-compose.base.yml @@ -25,7 +25,7 @@ services: - aiida-rmq-data:/var/lib/rabbitmq/ aiidalab: - image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/base}:${VERSION-:latest} + image: ${REGISTRY:-}${BASE_IMAGE:-aiidalab/base}:${VERSION:-newly-build} environment: RMQHOST: messaging TZ: Europe/Zurich diff --git a/stack/docker-compose.full-stack.yml b/stack/docker-compose.full-stack.yml index 5ac60f12..b21f7538 100644 --- a/stack/docker-compose.full-stack.yml +++ b/stack/docker-compose.full-stack.yml @@ -4,7 +4,7 @@ version: '3.4' services: aiidalab: - image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/full-stack}:${VERSION:-latest} + image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/full-stack}:${VERSION:-newly-build} environment: TZ: Europe/Zurich DOCKER_STACKS_JUPYTER_CMD: notebook diff --git a/stack/docker-compose.lab.yml b/stack/docker-compose.lab.yml index ad21c463..5cc6f765 100644 --- a/stack/docker-compose.lab.yml +++ b/stack/docker-compose.lab.yml @@ -25,7 +25,7 @@ services: - aiida-rmq-data:/var/lib/rabbitmq/ aiidalab: - image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab}:${VERSION:-latest} + image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab}:${VERSION:-newly-build} environment: RMQHOST: messaging TZ: Europe/Zurich diff --git a/stack/docker-compose.qe.yml b/stack/docker-compose.qe.yml new file mode 100644 index 00000000..13c4f987 --- /dev/null +++ b/stack/docker-compose.qe.yml @@ -0,0 +1,19 @@ +--- +version: '3.4' + +services: + + aiidalab: + image: ${REGISTRY:-}${QE_IMAGE:-aiidalab/qe}:${VERSION:-newly-bulid} + environment: + TZ: Europe/Zurich + DOCKER_STACKS_JUPYTER_CMD: notebook + SETUP_DEFAULT_AIIDA_PROFILE: 'true' + AIIDALAB_DEFAULT_APPS: '' + volumes: + - aiidalab-home-folder:/home/jovyan + ports: + - "0.0.0.0:${AIIDALAB_PORT:-}:8888" + +volumes: + aiidalab-home-folder: diff --git a/stack/qe/Dockerfile b/stack/qe/Dockerfile new file mode 100644 index 00000000..acda6738 --- /dev/null +++ b/stack/qe/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 +FROM full-stack + +USER root + +USER ${NB_USER} + +# Install the quantum-espresso app. +ARG AIIDALAB_QE_VERSION +ENV AIIDALAB_QE_VERSION ${AIIDALAB_QE_VERSION} + +# Install aiidalab-qe, which actually for all the dependencies. +# so that it will be fairly fast when container is started. +RUN git clone https://github.com/aiidalab/aiidalab-qe && \ + cd aiidalab-qe && \ + git checkout v${AIIDALAB_QE_VERSION} && \ + pip install --quiet --no-cache-dir "./" && \ + fix-permissions "./" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +ARG QE_VERSION +ENV QE_VERSION ${QE_VERSION} +RUN mamba create -p /opt/conda/envs/quantum-espresso --yes \ + qe=${QE_VERSION} \ + && mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +COPY before-notebook.d/* /usr/local/bin/before-notebook.d/ + +WORKDIR "/home/${NB_USER}" diff --git a/stack/qe/before-notebook.d/70_prepare-qe-executable.sh b/stack/qe/before-notebook.d/70_prepare-qe-executable.sh new file mode 100644 index 00000000..e64a188c --- /dev/null +++ b/stack/qe/before-notebook.d/70_prepare-qe-executable.sh @@ -0,0 +1,10 @@ +#!/bin/bash -e + +# Debugging. +set -x + +# Copy quantum espresso env to user space. +mkdir -p /home/${NB_USER}/.conda/envs +if [ ! -d /home/${NB_USER}/.conda/envs/quantum-espresso-${QE_VERSION} ]; then + ln -s /opt/conda/envs/quantum-espresso /home/${NB_USER}/.conda/envs/quantum-espresso-${QE_VERSION} +fi diff --git a/stack/qe/before-notebook.d/71_install-qeapp.sh b/stack/qe/before-notebook.d/71_install-qeapp.sh new file mode 100644 index 00000000..1d0e018e --- /dev/null +++ b/stack/qe/before-notebook.d/71_install-qeapp.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +# Debugging. +set -x + +# Install qeapp if it is not already installed. +if aiidalab list | grep -q quantum-espresso; then + echo "Quantum ESPRESSO app is already installed." +else + echo "Installing Quantum ESPRESSO app." + aiidalab install --yes quantum-espresso==${AIIDALAB_QE_VERSION} +fi diff --git a/tests/conftest.py b/tests/conftest.py index 5524cc44..e8cf4799 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,3 +92,8 @@ def aiidalab_version(_build_config): @pytest.fixture(scope="session") def aiidalab_home_version(_build_config): return _build_config["AIIDALAB_HOME_VERSION"]["default"] + + +@pytest.fixture(scope="session") +def qe_version(_build_config): + return _build_config["QE_VERSION"]["default"] diff --git a/tests/test-qe.py b/tests/test-qe.py new file mode 100644 index 00000000..d83814af --- /dev/null +++ b/tests/test-qe.py @@ -0,0 +1,9 @@ +def test_pw_executable_exist(aiidalab_exec, qe_version): + """Test that pw.x executable exists in the conda environment""" + output = ( + aiidalab_exec(f"mamba run -n quantum-espresso-{qe_version} which pw.x") + .decode() + .strip() + ) + + assert output == f"/home/jovyan/.conda/envs/quantum-espresso-{qe_version}/bin/pw.x"