Devops: docker build runner macOS-ARM64 (#408)

The buildjet arm64 runner has only three-month trials, after that
we need to pay to use it. The self-hosted runner is deployed on
the macOS-arm64 machine located in PSI.

.github/actions/create-dev-env/action.yml

@@ -15,13 +15,21 @@ runs:
     # See: https://github.com/actions/setup-python/issues/108
     # python3 is manually preinstalled in the arm64 VM self-hosted runner
         - name: Set Up Python 🐍
+          if: ${{ inputs.architecture == 'amd64' }}
           uses: actions/setup-python@v4
           with:
               python-version: 3.x
-          if: ${{ inputs.architecture == 'amd64' }}
 
         - name: Install Dev Dependencies 📦
+          if: ${{ inputs.architecture == 'amd64' }}
           run: |
               pip install --upgrade pip
               pip install --upgrade -r requirements-dev.txt
           shell: bash
+
+        - name: Install Dev Dependencies 📦
+          if: ${{ inputs.architecture == 'arm64' }}
+          run: |
+              pip install --upgrade pip
+              pip install --upgrade -r requirements-dev-arm64.txt
+          shell: bash

.github/actions/load-image/action.yml

@@ -19,13 +19,13 @@ runs:
           uses: actions/download-artifact@v3
           with:
               name: ${{ inputs.image }}-${{ inputs.architecture }}
-              path: /tmp/
+              path: /tmp/aiidalab
         - name: Load downloaded image to docker 📥
           run: |
-              docker load --input /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar
+              docker load --input /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar
               docker image ls --all
           shell: bash
         - name: Delete the file 🗑️
-          run: rm -f /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar
+          run: rm -f /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar
           shell: bash
           if: always()

.github/workflows/docker-build-test-upload.yml

@@ -32,6 +32,18 @@ jobs:
               with:
                   architecture: ${{ inputs.architecture }}
 
+            # Self-hosted runners share a state (whole VM) between runs
+            # Also, they might have running or stopped containers,
+            # which are not cleaned up by `docker system prun`
+            - name: Reset docker state and cleanup artifacts 🗑️
+              if: ${{ inputs.platform != 'x86_64' }}
+              run: |
+                  docker kill $(docker ps --quiet) || true
+                  docker rm $(docker ps --all --quiet) || true
+                  docker system prune --all --force
+                  rm -rf /tmp/aiidalab/
+              shell: bash
+
             - name: Build image 🛠
               run: doit build --target ${{ inputs.image }} --arch ${{ inputs.architecture }} --organization ${{ env.OWNER }}
               env:
@@ -44,12 +56,14 @@ jobs:
               shell: bash
 
             - name: Save image as a tar for later use 💾
-              run: docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar
+              run: |
+                  mkdir -p /tmp/aiidalab/
+                  docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar
               shell: bash
 
             - name: Upload image as artifact 💾
               uses: actions/upload-artifact@v3
               with:
                   name: ${{ inputs.image }}-${{ inputs.architecture }}
-                  path: /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar
+                  path: /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar
                   retention-days: 3

.github/workflows/docker-merge-tags.yml

@@ -35,12 +35,12 @@ jobs:
               uses: actions/download-artifact@v3
               with:
                   name: ${{ inputs.registry }}-${{ inputs.image }}-amd64-tags
-                  path: /tmp/
+                  path: /tmp/aiidalab
             - name: Download arm64 tags file 📥
               uses: actions/download-artifact@v3
               with:
                   name: ${{ inputs.registry }}-${{ inputs.image }}-arm64-tags
-                  path: /tmp/
+                  path: /tmp/aiidalab
 
             - name: Login to Container Registry 🔑
               uses: docker/login-action@v2
@@ -51,13 +51,13 @@ jobs:
 
             - name: Merge tags for the images of different arch 🔀
               run: |
-                  for arch_tag in $(cat /tmp/${{ inputs.image }}-amd64-tags.txt); do
+                  for arch_tag in $(cat /tmp/aiidalab/${{ inputs.image }}-amd64-tags.txt); do
                     tag=$(echo $arch_tag | sed "s/:amd64-/:/")
                     docker manifest create $tag --amend $arch_tag
                     docker manifest push $tag
                   done
 
-                  for arch_tag in $(cat /tmp/${{ inputs.image }}-arm64-tags.txt); do
+                  for arch_tag in $(cat /tmp/aiidalab/${{ inputs.image }}-arm64-tags.txt); do
                     tag=$(echo $arch_tag | sed "s/:arm64-/:/")
                     docker manifest create $tag --amend $arch_tag
                     docker manifest push $tag

.github/workflows/docker-push.yml

@@ -81,13 +81,14 @@ jobs:
                       docker push ${arch_tag}
 
                       # write tag to file
-                      echo ${arch_tag} >> /tmp/${{ inputs.image }}-${{ inputs.architecture }}-tags.txt
+                      mkdir -p /tmp/aiidalab/
+                      echo ${arch_tag} >> /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}-tags.txt
                   done
               shell: bash
 
             - name: Upload tags file 📤
               uses: actions/upload-artifact@v3
               with:
                   name: ${{ inputs.registry }}-${{ inputs.image }}-${{ inputs.architecture }}-tags
-                  path: /tmp/${{ inputs.image }}-${{ inputs.architecture }}-tags.txt
+                  path: /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}-tags.txt
                   retention-days: 3

.github/workflows/docker.yml

@@ -90,86 +90,70 @@ jobs:
             runsOn: ubuntu-latest
         needs: [amd64-base-with-services, amd64-lab]
 
-    amd64-qe:
-        uses: ./.github/workflows/docker-build-test-upload.yml
-        with:
-            image: qe
-            architecture: amd64
-            runsOn: ubuntu-latest
-        needs: [amd64-full-stack]
-
     arm64-base:
         uses: ./.github/workflows/docker-build-test-upload.yml
         with:
             image: base
             architecture: arm64
-            runsOn: buildjet-2vcpu-ubuntu-2204-arm
+            runsOn: ARM64
 
     arm64-lab:
         uses: ./.github/workflows/docker-build-test-upload.yml
         with:
             image: lab
             architecture: arm64
-            runsOn: buildjet-2vcpu-ubuntu-2204-arm
+            runsOn: ARM64
         needs: [arm64-base]
 
     arm64-base-with-services:
         uses: ./.github/workflows/docker-build-test-upload.yml
         with:
             image: base-with-services
             architecture: arm64
-            runsOn: buildjet-2vcpu-ubuntu-2204-arm
+            runsOn: ARM64
         needs: [arm64-base]
 
     arm64-full-stack:
         uses: ./.github/workflows/docker-build-test-upload.yml
         with:
             image: full-stack
             architecture: arm64
-            runsOn: buildjet-2vcpu-ubuntu-2204-arm
+            runsOn: ARM64
         needs: [arm64-base-with-services, arm64-lab]
 
-    arm64-qe:
-        uses: ./.github/workflows/docker-build-test-upload.yml
-        with:
-            image: qe
-            architecture: arm64
-            runsOn: buildjet-2vcpu-ubuntu-2204-arm
-        needs: [arm64-full-stack]
-
     amd64-push-ghcr:
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
+                image: ["base", "base-with-services", "lab", "full-stack"]
         with:
             architecture: amd64
             image: ${{ matrix.image }}
             registry: ghcr.io
         secrets:
             REGISTRY_USERNAME: ${{ github.actor }}
             REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack, amd64-qe]
+        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack]
 
     arm64-push-ghcr:
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
+                image: ["base", "base-with-services", "lab", "full-stack"]
         with:
             architecture: arm64
             image: ${{ matrix.image }}
             registry: ghcr.io
         secrets:
             REGISTRY_USERNAME: ${{ github.actor }}
             REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack, arm64-qe]
+        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack]
 
     merge-tags-ghcr:
         uses: ./.github/workflows/docker-merge-tags.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
+                image: ["base", "base-with-services", "lab", "full-stack"]
         with:
             image: ${{ matrix.image }}
             registry: ghcr.io
@@ -183,37 +167,37 @@ jobs:
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
+                image: ["base", "base-with-services", "lab", "full-stack"]
         with:
             architecture: amd64
             image: ${{ matrix.image }}
             registry: docker.io
         secrets:
             REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
             REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }}
-        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack, amd64-qe]
+        needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack]
 
     arm64-push-dockerhub:
         if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
         uses: ./.github/workflows/docker-push.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
+                image: ["base", "base-with-services", "lab", "full-stack"]
         with:
             architecture: arm64
             image: ${{ matrix.image }}
             registry: docker.io
         secrets:
             REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
             REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }}
-        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack, arm64-qe]
+        needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack]
 
     merge-tags-dockerhub:
         if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
         uses: ./.github/workflows/docker-merge-tags.yml
         strategy:
             matrix:
-                image: ["base", "base-with-services", "lab", "full-stack", "qe"]
+                image: ["base", "base-with-services", "lab", "full-stack"]
         with:
             image: ${{ matrix.image }}
             registry: docker.io

aarch64-runner/README.md

@@ -0,0 +1,76 @@
+# Self-hosted runners
+
+For building `aarch64` images, we use self-hosted GitHub runners.
+The runner is hosted on the apple silicon machine in PSI.
+
+Configure your runner:
+
+1. Run under `root`:
+
+   Run this with caution. It can also be run manually step by step. See [setup.sh](setup.sh) for details.
+
+   ```bash
+   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/main/HEAD/aarch64-runner/setup.sh)"
+   ```
+
+   This will perform the initial runner setup and create a user `runner-user`.
+
+2. Run under `root`, Start docker service, we use [`colima`](https://github.com/abiosoft/colima) as the container runtime:
+
+   ```bash
+   colima start
+   ```
+
+   This command needs to be run every time after reboot. *(Optional: make it auto start on boot)*
+
+3. Setup new GitHub Runner under `runner-user` using [GitHub Instructions](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners).
+   **Do not `./run.sh` yet**.
+   **In the first step, use folder `actions-runner-aiidalab` to distinguish from the other runners.**
+
+4. Run under `runner-user`, install the runner as a service:
+
+   ```bash
+   cd /Users/runner-user/actions-runner-aiidalab/ && ./svc.sh install
+   ```
+   This will create the plist file for the runner service, it is not able to run it with the non-gui user.
+   As shown in the [issue](https://github.com/actions/runner/issues/1056#issuecomment-1237426462), real services start on boot, not on login so on macOS this means the service needs to be a `LaunchDaemon` and not a `LaunchAgent`.
+
+   In case the python path is not correct, change the `runsvc.sh` file to the correct path.
+   Since we use `colima` as the container runtime, the docker sock is located at `unix://$HOME/.colima/default/docker.sock`.
+   Change the `runsvc.sh` file to (notice we add two export lines so the runner can find the correct python and docker sock):
+
+   ```bash
+   #!/bin/bash
+
+   # convert SIGTERM signal to SIGINT
+   # for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
+   trap 'kill -INT $PID' TERM INT
+
+   if [ -f ".path" ]; then
+      # configure
+      export PATH=`cat .path`
+      eval "$(/opt/homebrew/bin/brew shellenv)"
+      export PATH="/opt/homebrew/bin:$PATH"
+      export DOCKER_HOST="unix://$HOME/.colima/default/docker.sock
+      echo ".path=${PATH}"
+   fi
+
+   nodever=${GITHUB_ACTIONS_RUNNER_FORCED_NODE_VERSION:-node16}
+
+   # insert anything to setup env when running as a service
+   # run the host process which keep the listener alive
+   ./externals/$nodever/bin/node ./bin/RunnerService.js &
+   PID=$!
+   wait $PID
+   trap - TERM INT
+   wait $PID
+   ```
+
+   Then, move the plist file to the correct location and load the service:
+   ```bash
+   sudo mv /Users/runner-user/Library/LaunchAgents/actions.runner.*.plist /Library/LaunchDaemons/
+   sudo chown root:wheel /Library/LaunchDaemons/actions.runner.*.plist
+   sudo /bin/launchctl load /Library/LaunchDaemons/actions.runner.aiidalab.Jusong-MacBook-Air.plist
+   ```
+
+5. Reboot the VM to apply all updates and run GitHub runner.