Implement pre-installed QE components for demo server (#402)

A new image is brought into the stack which includes the pre-installed Quantum ESPRESSO from conda forge to be run from localhost. The dependencies are installed from Dockerfile while the `aiidalab-qe` need to be installed the first time container started since the app should be installed in the user home which only created after the container started. Similar for the QE from conda forge, the Dockerfile creates the virtual env and installs the QE to the system folder and when the container started the first time it is copied to the user's home if the folder not existed. This image is designed as the image for the AiiDAlab demo server in the materials cloud so that new users can have a quick taste of AiiDAlab and run simulations with QeApp.
aiidalab · Aug 1, 2023 · 6cfb093 · 6cfb093
1 parent c1886dc
commit 6cfb093
Show file tree

Hide file tree

Showing 14 changed files with 148 additions and 22 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -90,6 +90,14 @@ jobs:
             runsOn: ubuntu-latest
         needs: [amd64-base-with-services, amd64-lab]
 
+    amd64-qe:
+        uses: ./.github/workflows/docker-build-test-upload.yml
+        with:
+            image: qe
+            architecture: amd64
+            runsOn: ubuntu-latest
+        needs: [amd64-full-stack]
+
     arm64-base:
         uses: ./.github/workflows/docker-build-test-upload.yml
         with:
@@ -121,39 +129,47 @@ jobs:
             runsOn: buildjet-2vcpu-ubuntu-2204-arm
         needs: [arm64-base-with-services, arm64-lab]
 
+    arm64-qe:
+        uses: ./.github/workflows/docker-build-test-upload.yml
+        with:
+            image: qe
+            architecture: arm64
+            runsOn: buildjet-2vcpu-ubuntu-2204-arm
+        needs: [arm64-full-stack]
+
     amd64-push-ghcr:
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack"]
+                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
         with:
             architecture: amd64
             image: ${{ matrix.image }}
             registry: ghcr.io
         secrets:
             REGISTRY_USERNAME: ${{ github.actor }}
             REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack]
+        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack, amd64-qe]
 
     arm64-push-ghcr:
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack"]
+                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
         with:
             architecture: arm64
             image: ${{ matrix.image }}
             registry: ghcr.io
         secrets:
             REGISTRY_USERNAME: ${{ github.actor }}
             REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack]
+        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack, arm64-qe]
 
     merge-tags-ghcr:
         uses: ./.github/workflows/docker-merge-tags.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack"]
+                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
         with:
             image: ${{ matrix.image }}
             registry: ghcr.io
@@ -167,37 +183,37 @@ jobs:
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack"]
+                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
         with:
             architecture: amd64
             image: ${{ matrix.image }}
             registry: docker.io
         secrets:
             REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
             REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }}
-        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack]
+        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack, amd64-qe]
 
     arm64-push-dockerhub:
         if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack"]
+                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
         with:
             architecture: arm64
             image: ${{ matrix.image }}
             registry: docker.io
         secrets:
             REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
             REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }}
-        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack]
+        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack, arm64-qe]
 
     merge-tags-dockerhub:
         if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
         uses: ./.github/workflows/docker-merge-tags.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack"]
+                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
         with:
             image: ${{ matrix.image }}
             registry: docker.io

diff --git a/build.json b/build.json
@@ -14,6 +14,12 @@
     },
     "AIIDALAB_HOME_VERSION": {
       "default": "23.03.1"
+    },
+    "AIIDALAB_QE_VERSION": {
+      "default": "23.04.6"
+    },
+    "QE_VERSION": {
+      "default": "7.2"
     }
   }
 }
diff --git a/docker-bake.hcl b/docker-bake.hcl
@@ -17,6 +17,12 @@ variable "AIIDALAB_VERSION" {
 variable "AIIDALAB_HOME_VERSION" {
 }
 
+variable "AIIDALAB_QE_VERSION" {
+}
+
+variable "QE_VERSION" {
+}
+
 variable "JUPYTER_BASE_IMAGE" {
   default = "jupyter/minimal-notebook:python-${PYTHON_VERSION}"
 }
@@ -34,21 +40,24 @@ variable "PLATFORMS" {
 }
 
 variable "TARGETS" {
-  default = ["base", "base-with-services", "lab", "full-stack"]
+  default = ["base", "base-with-services", "lab", "full-stack", "qe"]
 }
 
 function "tags" {
   params = [image]
   result = [
     "${REGISTRY}${ORGANIZATION}/${image}:newly-build",
-    "${REGISTRY}${ORGANIZATION}/${image}:latest",
   ]
 }
 
 group "default" {
   targets = "${TARGETS}"
 }
 
+target "qe-meta" {
+  tags = tags("qe")
+}
+
 target "base-meta" {
   tags = tags("base")
 }
@@ -105,3 +114,16 @@ target "full-stack" {
   }
   platforms = "${PLATFORMS}"
 }
+
+target "qe" {
+  inherits = ["qe-meta"]
+  context = "stack/qe"
+  contexts = {
+    full-stack = "target:full-stack"
+  }
+  args = {
+    "AIIDALAB_QE_VERSION"    = "${AIIDALAB_QE_VERSION}"
+    "QE_VERSION"             = "${QE_VERSION}"
+  }
+  platforms = "${PLATFORMS}"
+}
diff --git a/dodo.py b/dodo.py
@@ -70,11 +70,6 @@ def task_build():
     def generate_version_override(
         version, registry, targets, architecture, organization
     ):
-        if len(targets) > 4:
-            # Workaround of issue of doit, which rather than override the default value, it appends
-            # https://github.com/pydoit/doit/issues/436
-            targets = targets[4:]
-
         platforms = [f"linux/{architecture}"]
 
         Path("docker-bake.override.json").write_text(
@@ -107,7 +102,7 @@ def generate_version_override(
                 "long": "targets",
                 "short": "t",
                 "type": list,
-                "default": ["base", "base-with-services", "lab", "full-stack"],
+                "default": [],
                 "help": "Specify the target to build.",
             },
         ],

diff --git a/stack/docker-compose.base-with-services.yml b/stack/docker-compose.base-with-services.yml
@@ -4,7 +4,7 @@ version: '3.4'
 services:
 
     aiidalab:
-        image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/base-with-services}:${VERSION:-latest}
+        image: ${REGISTRY:-}${BASE_WITH_SERVICES_IMAGE:-aiidalab/base-with-services}:${VERSION:-newly-build}
         environment:
             TZ: Europe/Zurich
             DOCKER_STACKS_JUPYTER_CMD: notebook

diff --git a/stack/docker-compose.base.yml b/stack/docker-compose.base.yml
@@ -25,7 +25,7 @@ services:
             - aiida-rmq-data:/var/lib/rabbitmq/
 
     aiidalab:
-        image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/base}:${VERSION-:latest}
+        image: ${REGISTRY:-}${BASE_IMAGE:-aiidalab/base}:${VERSION:-newly-build}
         environment:
             RMQHOST: messaging
             TZ: Europe/Zurich

diff --git a/stack/docker-compose.full-stack.yml b/stack/docker-compose.full-stack.yml
@@ -4,7 +4,7 @@ version: '3.4'
 services:
 
     aiidalab:
-        image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/full-stack}:${VERSION:-latest}
+        image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/full-stack}:${VERSION:-newly-build}
         environment:
             TZ: Europe/Zurich
             DOCKER_STACKS_JUPYTER_CMD: notebook

diff --git a/stack/docker-compose.lab.yml b/stack/docker-compose.lab.yml
@@ -25,7 +25,7 @@ services:
             - aiida-rmq-data:/var/lib/rabbitmq/
 
     aiidalab:
-        image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab}:${VERSION:-latest}
+        image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab}:${VERSION:-newly-build}
         environment:
             RMQHOST: messaging
             TZ: Europe/Zurich

diff --git a/stack/docker-compose.qe.yml b/stack/docker-compose.qe.yml
@@ -0,0 +1,19 @@
+---
+version: '3.4'
+
+services:
+
+    aiidalab:
+        image: ${REGISTRY:-}${QE_IMAGE:-aiidalab/qe}:${VERSION:-newly-bulid}
+        environment:
+            TZ: Europe/Zurich
+            DOCKER_STACKS_JUPYTER_CMD: notebook
+            SETUP_DEFAULT_AIIDA_PROFILE: 'true'
+            AIIDALAB_DEFAULT_APPS: ''
+        volumes:
+            - aiidalab-home-folder:/home/jovyan
+        ports:
+            - "0.0.0.0:${AIIDALAB_PORT:-}:8888"
+
+volumes:
+    aiidalab-home-folder:
diff --git a/stack/qe/Dockerfile b/stack/qe/Dockerfile
@@ -0,0 +1,32 @@
+# syntax=docker/dockerfile:1
+FROM full-stack
+
+USER root
+
+USER ${NB_USER}
+
+# Install the quantum-espresso app.
+ARG AIIDALAB_QE_VERSION
+ENV AIIDALAB_QE_VERSION ${AIIDALAB_QE_VERSION}
+
+# Install aiidalab-qe, which actually for all the dependencies.
+# so that it will be fairly fast when container is started.
+RUN git clone https://github.com/aiidalab/aiidalab-qe && \
+     cd aiidalab-qe && \
+     git checkout v${AIIDALAB_QE_VERSION} && \
+     pip install --quiet --no-cache-dir "./" && \
+     fix-permissions "./" && \
+     fix-permissions "${CONDA_DIR}" && \
+     fix-permissions "/home/${NB_USER}"
+
+ARG QE_VERSION
+ENV QE_VERSION ${QE_VERSION}
+RUN mamba create -p /opt/conda/envs/quantum-espresso --yes \
+        qe=${QE_VERSION} \
+     && mamba clean --all -f -y && \
+     fix-permissions "${CONDA_DIR}" && \
+     fix-permissions "/home/${NB_USER}"
+
+COPY before-notebook.d/* /usr/local/bin/before-notebook.d/
+
+WORKDIR "/home/${NB_USER}"
diff --git a/stack/qe/before-notebook.d/70_prepare-qe-executable.sh b/stack/qe/before-notebook.d/70_prepare-qe-executable.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+# Debugging.
+set -x
+
+# Copy quantum espresso env to user space.
+mkdir -p /home/${NB_USER}/.conda/envs
+if [ ! -d /home/${NB_USER}/.conda/envs/quantum-espresso-${QE_VERSION} ]; then
+  ln -s /opt/conda/envs/quantum-espresso /home/${NB_USER}/.conda/envs/quantum-espresso-${QE_VERSION}
+fi
diff --git a/stack/qe/before-notebook.d/71_install-qeapp.sh b/stack/qe/before-notebook.d/71_install-qeapp.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+# Debugging.
+set -x
+
+# Install qeapp if it is not already installed.
+if aiidalab list | grep -q quantum-espresso; then
+    echo "Quantum ESPRESSO app is already installed."
+else
+    echo "Installing Quantum ESPRESSO app."
+    aiidalab install --yes quantum-espresso==${AIIDALAB_QE_VERSION}
+fi
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -92,3 +92,8 @@ def aiidalab_version(_build_config):
 @pytest.fixture(scope="session")
 def aiidalab_home_version(_build_config):
     return _build_config["AIIDALAB_HOME_VERSION"]["default"]
+
+
+@pytest.fixture(scope="session")
+def qe_version(_build_config):
+    return _build_config["QE_VERSION"]["default"]
diff --git a/tests/test-qe.py b/tests/test-qe.py
@@ -0,0 +1,9 @@
+def test_pw_executable_exist(aiidalab_exec, qe_version):
+    """Test that pw.x executable exists in the conda environment"""
+    output = (
+        aiidalab_exec(f"mamba run -n quantum-espresso-{qe_version} which pw.x")
+        .decode()
+        .strip()
+    )
+
+    assert output == f"/home/jovyan/.conda/envs/quantum-espresso-{qe_version}/bin/pw.x"