Integrate ELK stack #1968
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Production Env Tests | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| jobs: | |
| run_tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: Set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.3.2 # TODO https://github.com/pgjones/hypercorn/issues/102 | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Load MONETDB cached image | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache/monetdb | |
| key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-3-monetdb- | |
| - name: Build MONETDB docker image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: monetdb/Dockerfile | |
| push: false | |
| load: true | |
| tags: madgik/exareme2_db:dev | |
| cache-from: type=local,src=/tmp/.buildx-cache/monetdb | |
| cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb | |
| - name: Load MIPDB container cached image | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache/mipdb | |
| key: ${{ runner.os }}-buildx-mipdb-${{hashFiles('mipdb/**')}} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-mipdb- | |
| - name: Build MIPDB container docker image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: mipdb/Dockerfile | |
| push: false | |
| load: true | |
| tags: madgik/exareme2_mipdb:dev | |
| cache-from: type=local,src=/tmp/.buildx-cache/mipdb | |
| cache-to: type=local,dest=/tmp/.buildx-cache-new/mipdb | |
| - name: Load RABBITMQ cached image | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache/rabbitmq | |
| key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-rabbitmq- | |
| - name: Build RABBITMQ docker image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: rabbitmq/Dockerfile | |
| push: false | |
| load: true | |
| tags: madgik/exareme2_rabbitmq:dev | |
| cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq | |
| cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq | |
| - name: Load WORKER service cached image | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache/worker | |
| key: ${{ runner.os }}-buildx-worker-${{hashFiles('exareme2/**')}} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-worker- | |
| - name: Build WORKER service docker image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: exareme2/worker/Dockerfile | |
| push: false | |
| load: true | |
| tags: madgik/exareme2_worker:dev | |
| cache-from: type=local,src=/tmp/.buildx-cache/worker | |
| cache-to: type=local,dest=/tmp/.buildx-cache-new/worker | |
| - name: Load CONTROLLER service cached image | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/.buildx-cache/controller | |
| key: ${{ runner.os }}-buildx-controller-${{hashFiles('exareme2/**')}} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-controller- | |
| - name: Build CONTROLLER service docker image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: exareme2/controller/Dockerfile | |
| push: false | |
| load: true | |
| tags: madgik/exareme2_controller:dev | |
| cache-from: type=local,src=/tmp/.buildx-cache/controller | |
| cache-to: type=local,dest=/tmp/.buildx-cache-new/controller | |
| # Temp fix | |
| # https://github.com/docker/build-push-action/issues/252 | |
| # https://github.com/moby/buildkit/issues/1896 | |
| - name: Move Docker images cache | |
| run: | | |
| rm -rf /tmp/.buildx-cache | |
| mv /tmp/.buildx-cache-new /tmp/.buildx-cache | |
| - name: Create k8s Kind Cluster | |
| uses: helm/[email protected] | |
| with: | |
| cluster_name: kind | |
| config: tests/prod_env_tests/deployment_configs/kind_configuration/kind_cluster.yaml | |
| - name: Install Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| version: 3.9.1 | |
| id: install | |
| - name: Taint Nodes | |
| run: | | |
| kubectl taint nodes master node-role.kubernetes.io/control-plane- | |
| kubectl label node master master=true | |
| kubectl label node localworker1 worker=true | |
| kubectl label node localworker2 worker=true | |
| - name: Get container disk space | |
| run: df -h | |
| - name: Free up space, by removing dotnet, android and haskell unused libs. | |
| run: | | |
| rm -rf /usr/share/dotnet | |
| rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/lib/android | |
| - name: Get container disk space | |
| run: df -h | |
| - name: Load docker images to kind containers and delete them locally | |
| run: | | |
| kind load docker-image madgik/exareme2_worker:dev | |
| docker image rm madgik/exareme2_worker:dev | |
| kind load docker-image madgik/exareme2_controller:dev --nodes kind-control-plane | |
| docker image rm madgik/exareme2_controller:dev | |
| kind load docker-image madgik/exareme2_db:dev | |
| docker image rm madgik/exareme2_db:dev | |
| kind load docker-image madgik/exareme2_mipdb:dev | |
| docker image rm madgik/exareme2_mipdb:dev | |
| kind load docker-image madgik/exareme2_rabbitmq:dev | |
| docker image rm madgik/exareme2_rabbitmq:dev | |
| - name: Get container disk space | |
| run: df -h | |
| - name: Copy prod_env_tests values.yaml | |
| run: cp -r tests/prod_env_tests/deployment_configs/kubernetes_values.yaml kubernetes/values.yaml | |
| - name: Print Helm Templates | |
| run: helm template kubernetes/ | |
| - name: Deploy Helm | |
| run: helm install exareme2 kubernetes/ --debug | |
| - name: Wait for pods to get healthy | |
| run: timeout 300 bash -c 'while true; do if kubectl get pods --no-headers | awk '\''{if ($2 != "1/1" && $2 != "2/2" && $2 != "3/3" && $2 != "4/4") exit 1;}'\''; then echo "All pods are ready!"; break; else kubectl get pods -o wide; sleep 20; fi done' | |
| - name: Load data models into localworkers and globalworker | |
| run: | | |
| LOCALWORKER1=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker1") | .metadata.name') | |
| LOCALWORKER2=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker2") | .metadata.name') | |
| GLOBALWORKER=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r ".items[0].metadata.name") | |
| for POD in $LOCALWORKER1 $LOCALWORKER2 $GLOBALWORKER; do | |
| kubectl exec $POD -c db-importer -- sh -c 'mipdb init' | |
| for model in dementia_v_0_1 tbi_v_0_1 longitudinal_dementia_v_0_1; do | |
| kubectl exec $POD -c db-importer -- sh -c "mipdb add-data-model /opt/data/${model}/CDEsMetadata.json" | |
| done | |
| done | |
| - name: Load Dataset CSVs into Localworkers and Globalworker | |
| run: | | |
| LOCALWORKER1=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker1") | .metadata.name') | |
| LOCALWORKER2=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker2") | .metadata.name') | |
| GLOBALWORKER=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r ".items[0].metadata.name") | |
| for model in dementia_v_0_1 tbi_v_0_1 longitudinal_dementia_v_0_1; do | |
| for filepath in $(kubectl exec $GLOBALWORKER -c db-importer -- ls /opt/data/${model}); do | |
| filepath=/opt/data/${model}/${filepath} | |
| if [[ $filepath == *test.csv ]]; then | |
| echo "Loading file: $filepath at $GLOBALWORKER" | |
| kubectl exec $GLOBALWORKER -c db-importer -- mipdb add-dataset $filepath -d ${model%_v_*} -v 0.1 | |
| elif [[ $filepath == *.csv ]]; then | |
| filename=$(basename $filepath) | |
| suffix=$(echo $filename | grep -o '[0-9]*' | tail -1) | |
| if (( suffix % 2 == 0 )); then | |
| POD_NAME=$LOCALWORKER2 | |
| else | |
| POD_NAME=$LOCALWORKER1 | |
| fi | |
| echo "Loading file: $filepath at $POD_NAME" | |
| kubectl exec $POD_NAME -c db-importer -- mipdb add-dataset $filepath -d ${model%_v_*} -v 0.1 | |
| fi | |
| done | |
| done | |
| - name: Controller logs | |
| run: kubectl logs -l app=exareme2-controller --tail -1 | |
| - name: Globalnode logs | |
| run: kubectl logs -l nodeType=globalworker -c worker --tail -1 | |
| - name: Localnode logs | |
| run: kubectl logs -l nodeType=localworker -c worker --tail -1 | |
| - name: Controller logs (post run) | |
| uses: webiny/[email protected] | |
| with: | |
| run: kubectl logs -l app=exareme2-controller --tail -1 | |
| - name: Globalnode logs (post run) | |
| uses: webiny/[email protected] | |
| with: | |
| run: kubectl logs -l nodeType=globalworker -c worker --tail -1 | |
| - name: Localnode logs (post run) | |
| uses: webiny/[email protected] | |
| with: | |
| run: kubectl logs -l nodeType=localworker -c worker --tail -1 | |
| - name: Run Worker Landscape Aggregator update | |
| run: curl -X POST "http://172.17.0.1:5000/wla" | |
| - name: Run Healthcheck | |
| run: curl "http://172.17.0.1:5000/healthcheck" | |
| - name: Update WorkerLandscapeAggregator | |
| run: curl -X POST "http://172.17.0.1:5000/wla" | |
| - name: Run production env tests | |
| run: poetry run pytest tests/prod_env_tests --verbosity=4 |