Skip to content

Commit

Permalink
[Actions][Integration test] expand sda test suite
Browse files Browse the repository at this point in the history
Add test for ingesting and verifying an uploaded file.
  • Loading branch information
jbygdell committed Sep 15, 2023
1 parent ca620f5 commit 2bf326c
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 4 deletions.
57 changes: 56 additions & 1 deletion .github/integration/sda-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ services:
"rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms",
]
interval: 10s
timeout: 2s
timeout: 5s
retries: 6
image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
ports:
Expand Down Expand Up @@ -98,6 +98,7 @@ services:
environment:
- BROKER_PASSWORD=inbox
- BROKER_USER=inbox
- BROKER_ROUTINGKEY=inbox
- DB_PASSWORD=inbox
- DB_USER=inbox
restart: always
Expand All @@ -108,6 +109,56 @@ services:
- "18000:8000"
- "18001:8001"

ingest:
image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
command: [ sda-ingest ]
container_name: ingest
depends_on:
credentials:
condition: service_completed_successfully
minio:
condition: service_healthy
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
environment:
- BROKER_PASSWORD=ingest
- BROKER_USER=ingest
- BROKER_QUEUE=ingest
- BROKER_ROUTINGKEY=archived
- DB_PASSWORD=ingest
- DB_USER=ingest
restart: always
volumes:
- ./sda/config.yaml:/config.yaml
- shared:/shared

verify:
image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
command: [ sda-verify ]
container_name: verify
depends_on:
credentials:
condition: service_completed_successfully
minio:
condition: service_healthy
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
environment:
- BROKER_PASSWORD=verify
- BROKER_USER=verify
- BROKER_QUEUE=archived
- BROKER_ROUTINGKEY=verified
- DB_PASSWORD=verify
- DB_USER=verify
restart: always
volumes:
- ./sda/config.yaml:/config.yaml
- shared:/shared

oidc:
container_name: oidc
command:
Expand Down Expand Up @@ -142,8 +193,12 @@ services:
depends_on:
credentials:
condition: service_completed_successfully
ingest:
condition: service_started
s3inbox:
condition: service_started
verify:
condition: service_started
environment:
- PGPASSWORD=rootpasswd
image: python:3.10-slim-bullseye
Expand Down
16 changes: 15 additions & 1 deletion .github/integration/sda/config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
log:
format: "json"
level: "debug"
archive:
type: s3
url: "http://s3"
port: 9000
readypath: "/minio/health/ready"
accessKey: "access"
secretKey: "secretKey"
bucket: "archive"
region: "us-east-1"
inbox:
type: s3
url: "http://s3"
port: 9000
readypath: "/minio/health/ready"
Expand All @@ -16,7 +27,7 @@ broker:
password: ""
vhost: "/sda"
exchange: "sda"
routingKey: "inbox"
routingKey: ""
ssl: "false"

db:
Expand All @@ -27,6 +38,9 @@ db:
database: "sda"
sslmode: "disable"

c4gh:
filePath: /shared/c4gh.sec.pem
passphrase: "c4ghpass"

server:
cert: ""
Expand Down
2 changes: 1 addition & 1 deletion .github/integration/tests/run_scripts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ set -e
apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client > /dev/null

find "$1"/*.sh 2>/dev/null | sort -t/ -k3 -n | while read -r runscript; do
for runscript in "$1"/*.sh; do
echo "Executing test script $runscript"
bash -x "$runscript"
done
3 changes: 2 additions & 1 deletion .github/integration/tests/sda/10_upload_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,15 @@ for t in curl jq postgresql-client; do
fi
done

pip -q install --upgrade pip
pip -q install s3cmd

cd shared || true

for file in NA12878.bam NA12878_20k_b37.bam; do
curl -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
if [ ! -f "$file.c4gh" ]; then
/shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
fi
s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/
done
Expand Down
70 changes: 70 additions & 0 deletions .github/integration/tests/sda/20_ingest-verify_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/sh
set -e

cd shared || true

for file in NA12878.bam NA12878_20k_b37.bam; do
ENC_SHA=$(sha256sum "$file.c4gh" | cut -d' ' -f 1)
ENC_MD5=$(md5sum "$file.c4gh" | cut -d' ' -f 1)

## get correlation id from upload message
CORRID=$(
curl -s -X POST \
-H "content-type:application/json" \
-u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \
-d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id
)

## publish message to trigger ingestion
properties=$(
jq -c -n \
--argjson delivery_mode 2 \
--arg correlation_id "$CORRID" \
--arg content_encoding UTF-8 \
--arg content_type application/json \
'$ARGS.named'
)

encrypted_checksums=$(
jq -c -n \
--arg sha256 "$ENC_SHA" \
--arg md5 "$ENC_MD5" \
'$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))'
)

ingest_payload=$(
jq -r -c -n \
--arg type ingest \
--arg user [email protected] \
--arg filepath test_dummy.org/"$file.c4gh" \
--argjson encrypted_checksums "$encrypted_checksums" \
'$ARGS.named|@base64'
)

ingest_body=$(
jq -c -n \
--arg vhost sda \
--arg name sda \
--argjson properties "$properties" \
--arg routing_key "ingest" \
--arg payload_encoding base64 \
--arg payload "$ingest_payload" \
'$ARGS.named'
)

curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
-H 'Content-Type: application/json;charset=UTF-8' \
-d "$ingest_body"
done

echo "waiting for verify to complete"
RETRY_TIMES=0
until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/verified/ | jq -r '.messages_ready')" -eq 2 ]; do
echo "waiting for verify to complete"
RETRY_TIMES=$((RETRY_TIMES + 1))
if [ "$RETRY_TIMES" -eq 30 ]; then
echo "::error::Time out while waiting for verify to complete"
exit 1
fi
sleep 2
done

0 comments on commit 2bf326c

Please sign in to comment.