[FEGA usecase] Merge ingest & verify from sda-pipeline into sda

.github/integration/sda-s3-integration.yml

@@ -47,7 +47,7 @@ services:
           "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms",
         ]
       interval: 10s
-      timeout: 2s
+      timeout: 5s
       retries: 6
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
     ports:
@@ -98,6 +98,7 @@ services:
     environment:
       - BROKER_PASSWORD=inbox
       - BROKER_USER=inbox
+      - BROKER_ROUTINGKEY=inbox
       - DB_PASSWORD=inbox
       - DB_USER=inbox
     restart: always
@@ -108,6 +109,56 @@ services:
       - "18000:8000"
       - "18001:8001"
 
+  ingest:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-ingest ]
+    container_name: ingest
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=ingest
+      - BROKER_USER=ingest
+      - BROKER_QUEUE=ingest
+      - BROKER_ROUTINGKEY=archived
+      - DB_PASSWORD=ingest
+      - DB_USER=ingest
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
+  verify:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-verify ]
+    container_name: verify
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=verify
+      - BROKER_USER=verify
+      - BROKER_QUEUE=archived
+      - BROKER_ROUTINGKEY=verified
+      - DB_PASSWORD=verify
+      - DB_USER=verify
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
   oidc:
     container_name: oidc
     command:
@@ -142,8 +193,12 @@ services:
     depends_on:
       credentials:
         condition: service_completed_successfully
+      ingest:
+        condition: service_started
       s3inbox:
         condition: service_started
+      verify:
+        condition: service_started
     environment:
       - PGPASSWORD=rootpasswd
       - STORAGETYPE=s3

.github/integration/sda/config.yaml

@@ -1,6 +1,17 @@
 log:
   format: "json"
+  level: "debug"
+archive:
+  type: s3
+  url: "http://s3"
+  port: 9000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "archive"
+  region: "us-east-1"
 inbox:
+  type: s3
   url: "http://s3"
   port: 9000
   readypath: "/minio/health/ready"
@@ -16,7 +27,7 @@ broker:
   password: ""
   vhost: "/sda"
   exchange: "sda"
-  routingKey: "inbox"
+  routingKey: ""
   ssl: "false"
 
 db:
@@ -27,6 +38,9 @@ db:
   database: "sda"
   sslmode: "disable"
 
+c4gh:
+  filePath: /shared/c4gh.sec.pem
+  passphrase: "c4ghpass"
 
 server:
   cert: ""

.github/integration/tests/run_scripts.sh

@@ -4,7 +4,7 @@ set -e
 apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
 apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client > /dev/null
 
-find "$1"/*.sh 2>/dev/null |  sort -t/ -k3 -n | while read -r runscript; do
+for runscript in "$1"/*.sh; do
     echo "Executing test script $runscript"
     bash -x "$runscript"
 done

.github/integration/tests/sda/20_ingest-verify_test.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+set -e
+
+cd shared || true
+
+for file in NA12878.bam NA12878_20k_b37.bam; do
+    ENC_SHA=$(sha256sum "$file.c4gh" | cut -d' ' -f 1)
+    ENC_MD5=$(md5sum "$file.c4gh" | cut -d' ' -f 1)
+
+    ## get correlation id from upload message
+    CORRID=$(
+        curl -s -X POST \
+            -H "content-type:application/json" \
+            -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \
+            -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id
+    )
+
+    ## publish message to trigger ingestion
+    properties=$(
+        jq -c -n \
+            --argjson delivery_mode 2 \
+            --arg correlation_id "$CORRID" \
+            --arg content_encoding UTF-8 \
+            --arg content_type application/json \
+            '$ARGS.named'
+    )
+
+    encrypted_checksums=$(
+        jq -c -n \
+            --arg sha256 "$ENC_SHA" \
+            --arg md5 "$ENC_MD5" \
+            '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))'
+    )
+
+    ingest_payload=$(
+        jq -r -c -n \
+            --arg type ingest \
+            --arg user [email protected] \
+            --arg filepath test_dummy.org/"$file.c4gh" \
+            --argjson encrypted_checksums "$encrypted_checksums" \
+            '$ARGS.named|@base64'
+    )
+
+    ingest_body=$(
+        jq -c -n \
+            --arg vhost sda \
+            --arg name sda \
+            --argjson properties "$properties" \
+            --arg routing_key "ingest" \
+            --arg payload_encoding base64 \
+            --arg payload "$ingest_payload" \
+            '$ARGS.named'
+    )
+
+    curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
+        -H 'Content-Type: application/json;charset=UTF-8' \
+        -d "$ingest_body"
+done
+
+echo "waiting for verify to complete"
+RETRY_TIMES=0
+until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/verified/ | jq -r '.messages_ready')" -eq 2 ]; do
+    echo "waiting for verify to complete"
+    RETRY_TIMES=$((RETRY_TIMES + 1))
+    if [ "$RETRY_TIMES" -eq 30 ]; then
+        echo "::error::Time out while waiting for verify to complete"
+        exit 1
+    fi
+    sleep 2
+done