Skip to content

Commit

Permalink
Merge branch 'f/testcontainers' into 'main'
Browse files Browse the repository at this point in the history
Improve testing speed with test containers

See merge request es/ai/hannah/hannah!401
  • Loading branch information
cgerum committed Aug 23, 2024
2 parents e9530ee + 81342ac commit 92ad0c1
Show file tree
Hide file tree
Showing 5 changed files with 460 additions and 462 deletions.
41 changes: 33 additions & 8 deletions .github/workflows/publish_images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,34 @@ on:
# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}

# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
build-and-push-image:
runs-on: ubuntu-latest
strategy:
matrix:
target:
- python
python_version:
- '3.9'
- '3.10'
- '3.11'
#- '3.12'
include:
- target: hannah
python_version: 3.12 # THis is ignored


# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
permissions:
contents: read
packages: write
attestations: write
#



steps:
- name: Checkout repository
uses: actions/checkout@v4
Expand All @@ -53,11 +69,16 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Set image name
id: set_image_name
run: |
if [ "${{ matrix.target }}" = "python" ]; then
echo "image_name=${{ matrix.target }}_${{ matrix.python_version }}" >> $GITHUB_ENV
else
echo "image_name=${{ matrix.target }}" >> $GITHUB_ENV
fi

# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
Expand All @@ -67,5 +88,9 @@ jobs:
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
tags: ${{ env.REGISTRY }}/${{ github.repository }}_${{ env.image_name }}:latest
labels: org.opencontainers.image.source=${{ github.repositoryUrl }}, org.opencontainers.image.version=${{ github.sha }}
build-args: |
python_version=${{ matrix.python_version }}
target=${{ matrix.target }}
97 changes: 14 additions & 83 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,39 +27,16 @@ variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
POETRY_HOME: "$CI_PROJECT_DIR/.poetry"
POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pypoetry"
POETRY_VIRTUALENVS_CREATE: false
GIT_SUBMODULE_STRATEGY: recursive
DEBIAN_FRONTEND: "noninteractive"

cache:
key: "$CI_JOB_NAME"

before_script:
- apt update
- apt -y install git mesa-utils python3-dev libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential python-is-python3


# Install poetry using recommended method
- curl -sSL https://install.python-poetry.org/ | python3
- export PATH=${POETRY_HOME}/bin:${PATH}

# install and manage dependencies with poetry
- poetry config installer.max-workers 10
- poetry install --all-extras

# Show limits
- ulimit -a

test_prebuilt_docker:
stage: test
image: cgerum/hannah:latest
before_script:
- export POETRY_HOME=/root/.local
- export PATH=${POETRY_HOME}/bin:${PATH}
- poetry config installer.max-workers 10
- poetry install -E vision
- ulimit -a
image: ghcr.io/ekut-es/hannah_hannah:latest
script:
- set -e
- poetry install -E vision
- poetry run python3 -m pytest -v --cov=hannah test hannah
tags:
- docker
Expand All @@ -68,48 +45,28 @@ test_prebuilt_docker:
- merge_requests
dependencies: []


test_python_39:
stage: test
image: python:3.9
image: ghcr.io/ekut-es/hannah_python_3.9:latest
script:
- set -e
- poetry config installer.max-workers 10
- poetry install -E vision
- poetry run python3 -m pytest -v --cov=hannah test hannah --integration
tags:
- docker
interruptible: true
dependencies: []
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
dependencies: []
cache:
paths:
- .cache/

#test_tvm:
# stage: test
# image: ubuntu:22.04
# script:
# - set -e
# - apt-get install -y llvm-dev libclang-dev g++ python-is-python3
# - poetry config installer.max-workers 10
# - poetry run external/hannah-tvm/scripts/install_full.sh
# - poetry install -E tvm -E vision
# - poetry run python -m pytest -v --cov=hannah --cov external/hannah-tvm test --integration
# tags:
# - docker
# interruptible: true
# allow_failure: false
# rules:
# - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# cache:
# paths:
# - .cache/

test_python_310:
stage: test
image: python:3.10
image: ghcr.io/ekut-es/hannah_python_3.10:latest
script:
- set -e
- poetry config installer.max-workers 10
- poetry install -E vision
- "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
- export PYTHONPATH=$PWD
- export COVERAGE_PROCESS_START=$PWD/.coveragerc
Expand All @@ -127,17 +84,16 @@ test_python_310:
coverage_format: cobertura
path: coverage.xml
dependencies: []
cache:
paths:
- .cache/
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

test_python_311:
stage: test
image: python:3.11
image: ghcr.io/ekut-es/hannah_python_3.11:latest
script:
- set -e
- poetry config installer.max-workers 10
- poetry install -E vision
- "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
- export PYTHONPATH=$PWD
- export COVERAGE_PROCESS_START=$PWD/.coveragerc
Expand All @@ -155,16 +111,12 @@ test_python_311:
coverage_format: cobertura
path: coverage.xml
dependencies: []
cache:
paths:
- .cache/
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH


build_docs:
stage: test
image: ubuntu:22.04
image: ubuntu:20.04
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
Expand Down Expand Up @@ -206,27 +158,6 @@ run_sca:
interruptible: true
allow_failure: true

#ml_cloud_integration:
# stage: build
# image: ubuntu:latest
# dependencies: []
# before_script:
# - apt update
# - apt -y install openssh-client
# - eval $(ssh-agent -s)
# - mkdir -p ~/.ssh
# - echo "$ML_CLOUD_KEY" > ~/.ssh/ml_cloud.key
# - chmod -R 0600 ~/.ssh
# - ssh-add ~/.ssh/ml_cloud.key
# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
# script:
# - set -e
# - scp scripts/ml_cloud_ci.sh [email protected]:/home/bringmann/cgerum05/ci/ml_cloud_ci.sh
# - ssh [email protected] /home/bringmann/cgerum05/ci/ml_cloud_ci.sh
# rules:
# - if: $CI_PIPELINE_SOURCE == "schedule"


deploy to github:
stage: build
image: ubuntu:latest
Expand Down
6 changes: 6 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,9 @@ repos:
args: [--fix, --exit-non-zero-on-fix]
# Formatter
- id: ruff-format
- repo: https://github.com/python-poetry/poetry
rev: '1.8'
hooks:
- id: poetry-check
# - id: poetry-lock
# args: ['--no-update']
22 changes: 19 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,30 @@
## limitations under the License.
##

# Target for python/mlonmcu/python
ARG target="hannah"

FROM ubuntu:22.04 as hannah
# ARG for Python version, defaulting to 3.12 only used if python target is selected
ARG python_version=3.12

FROM ubuntu:22.04 as hannah
FROM tumeda/mlonmcu-bench:latest as mlonmcu
FROM python:${python_version} as python

FROM ${target}

# These need to be set here again, as the FROM directive resets the ARGs on use
ARG python_version
ARG target

ENV POETRY_CACHE_DIR="/tmp/poetry_cache"

RUN apt-get update -y && apt-get -y install git mesa-utils python3-pip python3-dev libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential python-is-python3

RUN if [ "$target" = "hannah" ] || [ "$target" = "mlonmcu" ]; then\
apt-get update -y && apt-get -y install git mesa-utils python3 python3-pip python3-dev libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential python-is-python3; \
else \
apt-get update -y && apt-get -y install git mesa-utils libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential; \
fi


# Install poetry using recommended method
Expand All @@ -40,4 +54,6 @@ COPY poetry.lock pyproject.toml /deps/

# Install dependencies
RUN poetry config virtualenvs.create false \
&& poetry install --no-interaction --no-ansi --all-extras --no-root
&& poetry install --no-interaction --no-ansi --all-extras --no-root \
&& rm -rf $POETRY_CACHE_DIR

Loading

0 comments on commit 92ad0c1

Please sign in to comment.