Run all benchmarks

.github/workflows/publish_images.yml

@@ -30,18 +30,34 @@ on:
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
 
 # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
 jobs:
   build-and-push-image:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+          target: 
+            - python
+          python_version:
+            - '3.9'
+            - '3.10'
+            - '3.11'
+            #- '3.12'
+          include:
+            - target: hannah
+              python_version: 3.12 # THis is ignored
+
+
     # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
     permissions:
       contents: read
       packages: write
       attestations: write
       #
+
+
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -53,11 +69,16 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Set image name
+        id: set_image_name
+        run: |
+          if [ "${{ matrix.target }}" = "python" ]; then
+            echo "image_name=${{ matrix.target }}_${{ matrix.python_version }}" >> $GITHUB_ENV
+          else
+            echo "image_name=${{ matrix.target }}" >> $GITHUB_ENV
+          fi
+      
+
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
@@ -67,5 +88,9 @@ jobs:
         with:
           context: .
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ env.REGISTRY }}/${{ github.repository }}_${{ env.image_name }}:latest
+          labels: org.opencontainers.image.source=${{ github.repositoryUrl }}, org.opencontainers.image.version=${{ github.sha }}
+          build-args: |
+            python_version=${{ matrix.python_version }}
+            target=${{ matrix.target }}
+            

.gitlab-ci.yml

@@ -27,39 +27,16 @@ variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   POETRY_HOME: "$CI_PROJECT_DIR/.poetry"
   POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pypoetry"
+  POETRY_VIRTUALENVS_CREATE: false
   GIT_SUBMODULE_STRATEGY: recursive
   DEBIAN_FRONTEND: "noninteractive"
 
-cache:
-  key: "$CI_JOB_NAME"
-
-before_script:
-  - apt update
-  - apt -y install git mesa-utils python3-dev libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential python-is-python3
-
-
-  # Install poetry using recommended method
-  - curl -sSL https://install.python-poetry.org/ | python3
-  - export PATH=${POETRY_HOME}/bin:${PATH}
-
-  # install and manage dependencies with poetry
-  - poetry config installer.max-workers 10
-  - poetry install --all-extras
-
-  # Show limits
-  - ulimit -a
-
 test_prebuilt_docker:
   stage: test
-  image: cgerum/hannah:latest
-  before_script:
-    - export POETRY_HOME=/root/.local
-    - export PATH=${POETRY_HOME}/bin:${PATH}
-    - poetry config installer.max-workers 10
-    - poetry install -E vision
-    - ulimit -a
+  image: ghcr.io/ekut-es/hannah_hannah:latest
   script:
     - set -e
+    - poetry install --all-extras
     - poetry run python3 -m pytest  -v --cov=hannah test hannah
   tags:
     - docker
@@ -70,49 +47,28 @@ test_prebuilt_docker:
   allow_failure: false
   dependencies: []
 
-
 test_python_39:
   stage: test
-  image: python:3.9
+  image: ghcr.io/ekut-es/hannah_python_3.9:latest
   script:
     - set -e
+    - poetry config installer.max-workers 10
+    - poetry install --all-extras
     - poetry run python3 -m pytest  -v --cov=hannah test hannah --integration
   tags:
     - docker
   interruptible: true
+  dependencies: []
   rules:
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-  dependencies: []
-  cache:
-    paths:
-      - .cache/
-
-#test_tvm:
-#  stage: test
-#  image: ubuntu:22.04
-#  script:
-#    - set -e
-#    - apt-get install -y llvm-dev libclang-dev g++ python-is-python3
-#    - poetry config installer.max-workers 10
-#    - poetry run external/hannah-tvm/scripts/install_full.sh
-#    - poetry install -E tvm -E vision
-#    - poetry run python -m pytest  -v --cov=hannah --cov external/hannah-tvm test --integration
-#  tags:
-#    - docker
-#  interruptible: true
-#  allow_failure: false
-#  rules:
-#    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-#  cache:
-#    paths:
-#      - .cache/
 
 test_python_310:
   stage: test
-  image: python:3.10
+  image: ghcr.io/ekut-es/hannah_python_3.10:latest
   script:
     - set -e
+    - poetry config installer.max-workers 10
+    - poetry install --all-extras
     - "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
     - export PYTHONPATH=$PWD
     - export COVERAGE_PROCESS_START=$PWD/.coveragerc
@@ -130,18 +86,16 @@ test_python_310:
         coverage_format: cobertura
         path: coverage.xml
   dependencies: []
-  cache:
-    paths:
-      - .cache/
   rules:
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-
+
 test_python_311:
   stage: test
-  image: python:3.11
+  image: ghcr.io/ekut-es/hannah_python_3.11:latest
   script:
     - set -e
+    - poetry config installer.max-workers 10
+    - poetry install --all-extras
     - "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
     - export PYTHONPATH=$PWD
     - export COVERAGE_PROCESS_START=$PWD/.coveragerc
@@ -159,16 +113,12 @@ test_python_311:
         coverage_format: cobertura
         path: coverage.xml
   dependencies: []
-  cache:
-    paths:
-      - .cache/
   rules:
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-
+
 test_python_312:
   stage: test
-  image: python:3.12
+  image: ghcr.io/ekut-es/hannah_python_3.11:latest
   script:
     - set -e
     - "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
@@ -180,7 +130,7 @@ test_python_312:
     - poetry run coverage report
     - poetry run coverage xml
   tags:
-    - gpu
+    - docker
   interruptible: true
   artifacts:
     reports:
@@ -193,11 +143,11 @@ test_python_312:
       - .cache/
   rules:
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
 
 build_docs:
   stage: test
-  image: ubuntu:22.04
+  image: ubuntu:20.04
   variables:
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   cache:
@@ -219,7 +169,6 @@ build_docs:
   # Empty dependencies to disable artifacts passing
   dependencies: []
 
-
 run_sca:
   stage: test
   image: cgerum/hannah:latest
@@ -239,27 +188,6 @@ run_sca:
   interruptible: true
   allow_failure: true
 
-#ml_cloud_integration:
-#  stage: build
-#  image: ubuntu:latest
-#  dependencies: []
-#  before_script:
-#    - apt update
-#    - apt -y install openssh-client
-#    - eval $(ssh-agent -s)
-#    - mkdir -p ~/.ssh
-#    - echo "$ML_CLOUD_KEY" > ~/.ssh/ml_cloud.key
-#    - chmod -R  0600 ~/.ssh
-#    - ssh-add ~/.ssh/ml_cloud.key
-#    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-#  script:
-#    - set -e
-#    - scp scripts/ml_cloud_ci.sh [email protected]:/home/bringmann/cgerum05/ci/ml_cloud_ci.sh
-#    - ssh [email protected]  /home/bringmann/cgerum05/ci/ml_cloud_ci.sh
-#  rules:
-#    - if: $CI_PIPELINE_SOURCE == "schedule"
-
-
 deploy to github:
   stage: build
   image: ubuntu:latest

Dockerfile

@@ -17,16 +17,30 @@
 ## limitations under the License.
 ##
 
+# Target for python/mlonmcu/python
 ARG target="hannah"
 
-FROM ubuntu:22.04 as hannah
+# ARG for Python version, defaulting to 3.12 only used if python target is selected
+ARG python_version=3.12
 
+FROM ubuntu:22.04 as hannah
 FROM tumeda/mlonmcu-bench:latest as mlonmcu
+FROM python:${python_version} as python
 
 FROM ${target}
 
+# These need to be set here again, as the FROM directive resets the ARGs on use
+ARG python_version
+ARG target
+
+ENV POETRY_CACHE_DIR="/tmp/poetry_cache"
 
-RUN  apt-get update -y && apt-get -y install git mesa-utils python3-pip python3-dev libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential python-is-python3
+
+RUN  if [ "$target" = "hannah" ] || [ "$target" = "mlonmcu" ]; then\
+        apt-get update -y && apt-get -y install git mesa-utils python3 python3-pip python3-dev libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential python-is-python3; \
+      else \
+        apt-get update -y && apt-get -y install git mesa-utils  libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential;  \
+      fi
 
 
 # Install poetry using recommended method
@@ -40,4 +54,6 @@ COPY poetry.lock pyproject.toml /deps/
 
 # Install dependencies
 RUN poetry config virtualenvs.create false \
-  && poetry install --no-interaction --no-ansi --all-extras --no-root
+  && poetry install --no-interaction --no-ansi --all-extras --no-root \
+  && rm -rf $POETRY_CACHE_DIR
+