Skip to content
Algorithm Validation Tests

Fix for worker celery logger #1976

Sign in to view logs

Fix for worker celery logger

Fix for worker celery logger #1976

Workflow file for this run

.github/workflows/algorithm_validation_tests.yml at 7eaa056
name: Algorithm Validation Tests
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
run_tests_in_one_node:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.3.2 # TODO https://github.com/pgjones/hypercorn/issues/102
virtualenvs-create: true
virtualenvs-in-project: true
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Load MONETDB cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/monetdb
key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
restore-keys: |
${{ runner.os }}-buildx-3-monetdb-
- name: Build MONETDB docker image
uses: docker/build-push-action@v3
with:
context: .
file: monetdb/Dockerfile
push: false
load: true
tags: madgik/exareme2_db:testing
cache-from: type=local,src=/tmp/.buildx-cache/monetdb
cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb
- name: Load RABBITMQ cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/rabbitmq
key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}}
restore-keys: |
${{ runner.os }}-buildx-rabbitmq-
- name: Build RABBITMQ docker image
uses: docker/build-push-action@v3
with:
context: .
file: rabbitmq/Dockerfile
push: false
load: true
tags: madgik/exareme2_rabbitmq:testing
cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move Docker images cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Copy .deployment.toml file
run: cp ./tests/algorithm_validation_tests/one_node_deployment_template.toml .deployment.toml
- name: Create config files
run: poetry run inv create-configs
- name: Deploy all api
run: poetry run inv deploy --no-install-dep --no-start-all
- name: Load data into DBs
run: poetry run inv load-data
- name: Start api after data loading
run: |
poetry run inv start-worker --all
poetry run inv start-controller --detached
- name: Controller logs
run: cat /tmp/exareme2/controller.out
- name: Globalworker logs
run: cat /tmp/exareme2/globalworker.out
- name: Localworker logs
run: cat /tmp/exareme2/localworker1.out
- name: Controller logs (post run)
uses: webiny/[email protected]
with:
run: cat /tmp/exareme2/controller.out
- name: Globalworker logs (post run)
uses: webiny/[email protected]
with:
run: cat /tmp/exareme2/globalworker.out
- name: Localworker logs (post run)
uses: webiny/[email protected]
with:
run: cat /tmp/exareme2/localworker1.out
- name: Run Flower algorithm validation tests
run: poetry run pytest tests/algorithm_validation_tests/flower/ --verbosity=4 -n 2 --reruns 6 --reruns-delay 5
- name: Run Exareme2 algorithm validation tests
run: poetry run pytest tests/algorithm_validation_tests/exareme2 --verbosity=4 -n 16 --reruns 3 --reruns-delay 5
run_tests_in_five_nodes:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.3.2 # TODO https://github.com/pgjones/hypercorn/issues/102
virtualenvs-create: true
virtualenvs-in-project: true
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Load MONETDB cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/monetdb
key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
restore-keys: |
${{ runner.os }}-buildx-3-monetdb-
- name: Build MONETDB docker image
uses: docker/build-push-action@v3
with:
context: .
file: monetdb/Dockerfile
push: false
load: true
tags: madgik/exareme2_db:testing
cache-from: type=local,src=/tmp/.buildx-cache/monetdb
cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb
- name: Load RABBITMQ cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/rabbitmq
key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}}
restore-keys: |
${{ runner.os }}-buildx-rabbitmq-
- name: Build RABBITMQ docker image
uses: docker/build-push-action@v3
with:
context: .
file: rabbitmq/Dockerfile
push: false
load: true
tags: madgik/exareme2_rabbitmq:testing
cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move Docker images cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Copy .deployment.toml file
run: cp ./tests/algorithm_validation_tests/five_node_deployment_template.toml .deployment.toml
- name: Create config files
run: poetry run inv create-configs
- name: Deploy all api
run: poetry run inv deploy --no-install-dep --no-start-all
- name: Load data into DBs
run: poetry run inv load-data
- name: Start api after data loading
run: |
poetry run inv start-worker --all
poetry run inv start-controller --detached
- name: Controller logs
run: cat /tmp/exareme2/controller.out
- name: Globalworker logs
run: cat /tmp/exareme2/globalworker.out
- name: Localworker logs
run: cat /tmp/exareme2/localworker1.out
- name: Controller logs (post run)
uses: webiny/[email protected]
with:
run: cat /tmp/exareme2/controller.out
- name: Globalworker logs (post run)
uses: webiny/[email protected]
with:
run: cat /tmp/exareme2/globalworker.out
- name: Localworker logs (post run)
uses: webiny/[email protected]
with:
run: cat /tmp/exareme2/localworker1.out
- name: Run Exareme2 algorithm validation tests
run: poetry run pytest tests/algorithm_validation_tests/exareme2/ --verbosity=4 -n 16 -k "input1 and not input1-" # run tests 10-19
- name: Run Flower algorithm validation tests
run: poetry run pytest tests/algorithm_validation_tests/flower/test_logistic_regression.py -n 2 --verbosity=4 --reruns 6 --reruns-delay 5