Logs refactoring to fit ELASTIC stack #1972

Workflow file for this run

.github/workflows/algorithm_validation_tests.yml at ad94560

	name: Algorithm Validation Tests

	on:
	push:
	branches:
	- master
	pull_request:
	branches:
	- master

	jobs:
	run_tests_in_one_node:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository
	uses: actions/checkout@v3

	- name: Set up python
	uses: actions/setup-python@v4
	with:
	python-version: 3.8

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: 1.3.2 # TODO https://github.com/pgjones/hypercorn/issues/102
	virtualenvs-create: true
	virtualenvs-in-project: true

	- name: Load cached venv
	id: cached-poetry-dependencies
	uses: actions/cache@v3
	with:
	path: .venv
	key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}

	- name: Install dependencies
	if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
	run: poetry install --no-interaction --no-root

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2

	- name: Load MONETDB cached image
	uses: actions/cache@v3
	with:
	path: /tmp/.buildx-cache/monetdb
	key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
	restore-keys: \|
	${{ runner.os }}-buildx-3-monetdb-

	- name: Build MONETDB docker image
	uses: docker/build-push-action@v3
	with:
	context: .
	file: monetdb/Dockerfile
	push: false
	load: true
	tags: madgik/exareme2_db:testing
	cache-from: type=local,src=/tmp/.buildx-cache/monetdb
	cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb

	- name: Load RABBITMQ cached image
	uses: actions/cache@v3
	with:
	path: /tmp/.buildx-cache/rabbitmq
	key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}}
	restore-keys: \|
	${{ runner.os }}-buildx-rabbitmq-

	- name: Build RABBITMQ docker image
	uses: docker/build-push-action@v3
	with:
	context: .
	file: rabbitmq/Dockerfile
	push: false
	load: true
	tags: madgik/exareme2_rabbitmq:testing
	cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
	cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq

	# Temp fix
	# https://github.com/docker/build-push-action/issues/252
	# https://github.com/moby/buildkit/issues/1896
	- name: Move Docker images cache
	run: \|
	rm -rf /tmp/.buildx-cache
	mv /tmp/.buildx-cache-new /tmp/.buildx-cache

	- name: Copy .deployment.toml file
	run: cp ./tests/algorithm_validation_tests/one_node_deployment_template.toml .deployment.toml

	- name: Create config files
	run: poetry run inv create-configs

	- name: Deploy all api
	run: poetry run inv deploy --no-install-dep --no-start-all

	- name: Load data into DBs
	run: poetry run inv load-data

	- name: Start api after data loading
	run: \|
	poetry run inv start-worker --all
	poetry run inv start-controller --detached

	- name: Controller logs
	run: cat /tmp/exareme2/controller.out

	- name: Globalworker logs
	run: cat /tmp/exareme2/globalworker.out

	- name: Localworker logs
	run: cat /tmp/exareme2/localworker1.out

	- name: Controller logs (post run)
	uses: webiny/[email protected]
	with:
	run: cat /tmp/exareme2/controller.out

	- name: Globalworker logs (post run)
	uses: webiny/[email protected]
	with:
	run: cat /tmp/exareme2/globalworker.out

	- name: Localworker logs (post run)
	uses: webiny/[email protected]
	with:
	run: cat /tmp/exareme2/localworker1.out

	- name: Run Flower algorithm validation tests
	run: poetry run pytest tests/algorithm_validation_tests/flower/ --verbosity=4 -n 2 --reruns 6 --reruns-delay 5

	- name: Run Exareme2 algorithm validation tests
	run: poetry run pytest tests/algorithm_validation_tests/exareme2 --verbosity=4 -n 16 --reruns 3 --reruns-delay 5

	run_tests_in_five_nodes:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository
	uses: actions/checkout@v3

	- name: Set up python
	uses: actions/setup-python@v4
	with:
	python-version: 3.8

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: 1.3.2 # TODO https://github.com/pgjones/hypercorn/issues/102
	virtualenvs-create: true
	virtualenvs-in-project: true

	- name: Load cached venv
	id: cached-poetry-dependencies
	uses: actions/cache@v3
	with:
	path: .venv
	key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}

	- name: Install dependencies
	if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
	run: poetry install --no-interaction --no-root

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2

	- name: Load MONETDB cached image
	uses: actions/cache@v3
	with:
	path: /tmp/.buildx-cache/monetdb
	key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
	restore-keys: \|
	${{ runner.os }}-buildx-3-monetdb-

	- name: Build MONETDB docker image
	uses: docker/build-push-action@v3
	with:
	context: .
	file: monetdb/Dockerfile
	push: false
	load: true
	tags: madgik/exareme2_db:testing
	cache-from: type=local,src=/tmp/.buildx-cache/monetdb
	cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb

	- name: Load RABBITMQ cached image
	uses: actions/cache@v3
	with:
	path: /tmp/.buildx-cache/rabbitmq
	key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}}
	restore-keys: \|
	${{ runner.os }}-buildx-rabbitmq-

	- name: Build RABBITMQ docker image
	uses: docker/build-push-action@v3
	with:
	context: .
	file: rabbitmq/Dockerfile
	push: false
	load: true
	tags: madgik/exareme2_rabbitmq:testing
	cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
	cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq

	# Temp fix
	# https://github.com/docker/build-push-action/issues/252
	# https://github.com/moby/buildkit/issues/1896
	- name: Move Docker images cache
	run: \|
	rm -rf /tmp/.buildx-cache
	mv /tmp/.buildx-cache-new /tmp/.buildx-cache

	- name: Copy .deployment.toml file
	run: cp ./tests/algorithm_validation_tests/five_node_deployment_template.toml .deployment.toml

	- name: Create config files
	run: poetry run inv create-configs

	- name: Deploy all api
	run: poetry run inv deploy --no-install-dep --no-start-all

	- name: Load data into DBs
	run: poetry run inv load-data

	- name: Start api after data loading
	run: \|
	poetry run inv start-worker --all
	poetry run inv start-controller --detached

	- name: Controller logs
	run: cat /tmp/exareme2/controller.out

	- name: Globalworker logs
	run: cat /tmp/exareme2/globalworker.out

	- name: Localworker logs
	run: cat /tmp/exareme2/localworker1.out

	- name: Controller logs (post run)
	uses: webiny/[email protected]
	with:
	run: cat /tmp/exareme2/controller.out

	- name: Globalworker logs (post run)
	uses: webiny/[email protected]
	with:
	run: cat /tmp/exareme2/globalworker.out

	- name: Localworker logs (post run)
	uses: webiny/[email protected]
	with:
	run: cat /tmp/exareme2/localworker1.out

	- name: Run Exareme2 algorithm validation tests
	run: poetry run pytest tests/algorithm_validation_tests/exareme2/ --verbosity=4 -n 16 -k "input1 and not input1-" # run tests 10-19

	- name: Run Flower algorithm validation tests
	run: poetry run pytest tests/algorithm_validation_tests/flower/test_logistic_regression.py -n 2 --verbosity=4 --reruns 6 --reruns-delay 5

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Logs refactoring to fit ELASTIC stack #1972

Workflow file

Logs refactoring to fit ELASTIC stack #1972

Jobs

Run details

Workflow file for this run