From 982c7c474fc66e438879a1e1859d3e7a33679f4f Mon Sep 17 00:00:00 2001 From: Kostas Filippopolitis Date: Mon, 15 Jul 2024 15:44:28 +0300 Subject: [PATCH] Update the prod_env_tests.yml --- .github/workflows/prod_env_tests.yml | 42 ++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/prod_env_tests.yml b/.github/workflows/prod_env_tests.yml index ba1eb6e41..eefd26e14 100644 --- a/.github/workflows/prod_env_tests.yml +++ b/.github/workflows/prod_env_tests.yml @@ -221,7 +221,7 @@ jobs: GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/dementia_v_0_1/CDEsMetadata.json' - - name: Load dementia dataset csvs into localworkers and globalworker + - name: Load dementia dataset csvs into localworkers run: | for suffix in {0..9}; do if [ $suffix -eq 0 ] || [ $suffix -eq 3 ] || [ $suffix -eq 6 ]; then @@ -230,10 +230,19 @@ jobs: POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name") else POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') + for dataset in edsd ppmi desd-synthdata; do + if [[ ! $dataset$suffix.csv == *test.csv ]]; then + kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/dementia_v_0_1/${dataset}${suffix}.csv -d dementia -v 0.1" + fi + done fi - for dataset in edsd ppmi desd-synthdata; do - kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/dementia_v_0_1/${dataset}${suffix}.csv -d dementia -v 0.1" - done + done + + - name: Load dementia test dataset csvs into globalworker + run: | + GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') + for dataset in edsd ppmi desd-synthdata; do + kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/dementia_v_0_1/${dataset}-test.csv -d dementia -v 0.1" done - name: Load tbi data model into localworkers and globalworker @@ -245,7 +254,7 @@ jobs: GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/tbi_v_0_1/CDEsMetadata.json' - - name: Load tbi dataset csvs into localworkers and globalworker + - name: Load tbi dataset csvs into localworkers run: | for suffix in {0..9}; do if [ $suffix -eq 0 ] || [ $suffix -eq 3 ] || [ $suffix -eq 6 ]; then @@ -254,10 +263,17 @@ jobs: POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name") else POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') + if [[ ! dummy_tbi$suffix.csv == *test.csv ]]; then + kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/tbi_v_0_1/dummy_tbi${suffix}.csv -d tbi -v 0.1" + fi fi - kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/tbi_v_0_1/dummy_tbi${suffix}.csv -d tbi -v 0.1" done + - name: Load tbi test dataset csvs into globalworker + run: | + GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') + kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/tbi_v_0_1/dummy_tbi-test.csv -d tbi -v 0.1" + - name: Load longitudinal dementia data model into localworkers and globalworker run: | for i in 0 1; do @@ -267,7 +283,7 @@ jobs: GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/longitudinal_dementia_v_0_1/CDEsMetadata.json' - - name: Load longitudinal dementia datasets into localworkers and globalworker + - name: Load longitudinal dementia datasets into localworkers run: | for suffix in {0..2}; do if [ $suffix -eq 0 ]; then @@ -276,11 +292,19 @@ jobs: POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name") else POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') + if [[ ! longitudinal_dementia$suffix.csv == *test.csv ]]; then + kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/longitudinal_dementia_v_0_1/longitudinal_dementia${suffix}.csv -d longitudinal_dementia -v 0.1" + fi fi - kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/longitudinal_dementia_v_0_1/longitudinal_dementia${suffix}.csv -d longitudinal_dementia -v 0.1" done - - name: Controller logs + - name: Load longitudinal dementia test dataset csvs into globalworker + run: | + GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name') + kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/longitudinal_dementia_v_0_1/longitudinal_dementia-test.csv -d longitudinal_dementia -v 0.1" + + + - name: Controller logs run: kubectl logs -l app=exareme2-controller --tail -1 - name: Globalnode logs