Update the prod_env_tests.yml

madgik · Jul 15, 2024 · 982c7c4 · 982c7c4
1 parent c122b0f
commit 982c7c4
Showing 1 changed file with 33 additions and 9 deletions.
diff --git a/.github/workflows/prod_env_tests.yml b/.github/workflows/prod_env_tests.yml
@@ -221,7 +221,7 @@ jobs:
           GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
           kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/dementia_v_0_1/CDEsMetadata.json'
 
-      - name: Load dementia dataset csvs into localworkers and globalworker
+      - name: Load dementia dataset csvs into localworkers
         run: |
           for suffix in {0..9}; do
             if [ $suffix -eq 0 ] || [ $suffix -eq 3 ] || [ $suffix -eq 6 ]; then
@@ -230,10 +230,19 @@ jobs:
               POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name")
             else
               POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
+              for dataset in edsd ppmi desd-synthdata; do
+                if [[ ! $dataset$suffix.csv == *test.csv ]]; then
+                  kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/dementia_v_0_1/${dataset}${suffix}.csv -d dementia -v 0.1"
+                fi
+              done
             fi
-            for dataset in edsd ppmi desd-synthdata; do
-              kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/dementia_v_0_1/${dataset}${suffix}.csv -d dementia -v 0.1"
-            done
+          done
+
+      - name: Load dementia test dataset csvs into globalworker
+        run: |
+          GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
+          for dataset in edsd ppmi desd-synthdata; do
+            kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/dementia_v_0_1/${dataset}-test.csv -d dementia -v 0.1"
           done
 
       - name: Load tbi data model into localworkers and globalworker
@@ -245,7 +254,7 @@ jobs:
           GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
           kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/tbi_v_0_1/CDEsMetadata.json'
 
-      - name: Load tbi dataset csvs into localworkers and globalworker
+      - name: Load tbi dataset csvs into localworkers
         run: |
           for suffix in {0..9}; do
             if [ $suffix -eq 0 ] || [ $suffix -eq 3 ] || [ $suffix -eq 6 ]; then
@@ -254,10 +263,17 @@ jobs:
               POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name")
             else
               POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
+              if [[ ! dummy_tbi$suffix.csv == *test.csv ]]; then
+                kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/tbi_v_0_1/dummy_tbi${suffix}.csv -d tbi -v 0.1"
+              fi
             fi
-            kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/tbi_v_0_1/dummy_tbi${suffix}.csv -d tbi -v 0.1"
           done
 
+      - name: Load tbi test dataset csvs into globalworker
+        run: |
+          GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
+          kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/tbi_v_0_1/dummy_tbi-test.csv -d tbi -v 0.1"
+
       - name: Load longitudinal dementia data model into localworkers and globalworker
         run: |
           for i in 0 1; do
@@ -267,7 +283,7 @@ jobs:
           GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
           kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/longitudinal_dementia_v_0_1/CDEsMetadata.json'
 
-      - name: Load longitudinal dementia datasets into localworkers and globalworker
+      - name: Load longitudinal dementia datasets into localworkers
         run: |
           for suffix in {0..2}; do
             if [ $suffix -eq 0 ]; then
@@ -276,11 +292,19 @@ jobs:
               POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name")
             else
               POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
+              if [[ ! longitudinal_dementia$suffix.csv == *test.csv ]]; then
+                kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/longitudinal_dementia_v_0_1/longitudinal_dementia${suffix}.csv -d longitudinal_dementia -v 0.1"
+              fi
             fi
-            kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/longitudinal_dementia_v_0_1/longitudinal_dementia${suffix}.csv -d longitudinal_dementia -v 0.1"
           done
 
-      - name: Controller logs
+      - name: Load longitudinal dementia test dataset csvs into globalworker
+        run: |
+          GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
+          kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/longitudinal_dementia_v_0_1/longitudinal_dementia-test.csv -d longitudinal_dementia -v 0.1"
+
+
+        - name: Controller logs
         run: kubectl logs -l app=exareme2-controller --tail -1
 
       - name: Globalnode logs