Skip to content

Commit

Permalink
[tox] create tox task for k8s bigquery sync scenario test
Browse files Browse the repository at this point in the history
  • Loading branch information
khoaguin committed Sep 20, 2024
1 parent a9266aa commit 4abda81
Showing 1 changed file with 44 additions and 0 deletions.
44 changes: 44 additions & 0 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ envlist =
syft.test.security
syft.test.unit
syft.test.scenario
stack.test.scenario.sync.k8s
syft.test.notebook
syft.test.notebook.scenario
syft.test.notebook.scenario.sync
Expand Down Expand Up @@ -1564,3 +1565,46 @@ commands_post =
bash -c "CLUSTER_NAME=${DATASITE_CLUSTER_NAME} tox -e dev.k8s.destroy || true"
bash -c 'rm -f ${MIGRATION_DATA_DIR}/migration.blob'
bash -c 'rm -f ${MIGRATION_DATA_DIR}/migration.yaml'


[testenv:stack.test.scenario.sync.k8s]
description = Big Query Scenario Tests with Syncing (Level 0)over k8s
deps =
-e{toxinidir}/packages/syft[dev,data_science]
pytest-asyncio
pytest-timeout
anyio
unsync
changedir = {toxinidir}/tests/scenarios
allowlist_externals =
tox
bash
setenv =
CLUSTER_NAME_HIGH = {env:CLUSTER_NAME_HIGH:bigquery-high}
CLUSTER_NAME_LOW = {env:CLUSTER_NAME_LOW:bigquery-low}
CLUSTER_HTTP_PORT_HIGH={env:CLUSTER_HTTP_PORT_HIGH:8080}
CLUSTER_HTTP_PORT_LOW={env:CLUSTER_HTTP_PORT_LOW:8081}
commands =
# destroy high and low side big query clusters
bash -c "echo Running highlow with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE DEV_MODE=$DEV_MODE TEST_NOTEBOOK_PATHS=$TEST_NOTEBOOK_PATHS; date"
bash -c 'CLUSTER_NAME_HIGH=${CLUSTER_NAME_HIGH} CLUSTER_NAME_LOW=${CLUSTER_NAME_LOW} tox -e dev.k8s.destroy.datasite.highlow'
bash -c "k3d registry delete k3d-registry.localhost || true"
bash -c "docker volume rm k3d-${CLUSTER_NAME_HIGH}-images --force || true"
bash -c "docker volume rm k3d-${CLUSTER_NAME_LOW}-images --force || true"

# Now create high and low side big query clusters
bash -c '\
export CLUSTER_NAME_HIGH=${CLUSTER_NAME_HIGH} \
CLUSTER_NAME_LOW=${CLUSTER_NAME_LOW} \
CLUSTER_PORT_HIGH=${CLUSTER_PORT_HIGH} \
CLUSTER_PORT_LOW=${CLUSTER_PORT_LOW} && \
tox -e dev.k8s.launch.datasite.highlow'

# Run level 0 tests with k8s
pytest -s --disable-warnings bigquery/level_0_test.py

# Clean up again
bash -c 'CLUSTER_NAME_HIGH=${CLUSTER_NAME_HIGH} CLUSTER_NAME_LOW=${CLUSTER_NAME_LOW} tox -e dev.k8s.destroy.datasite.highlow'
bash -c "k3d registry delete k3d-registry.localhost || true"
bash -c "docker volume rm k3d-${CLUSTER_NAME_HIGH}-images --force || true"
bash -c "docker volume rm k3d-${CLUSTER_NAME_LOW}-images --force || true"

0 comments on commit 4abda81

Please sign in to comment.