Feature/api keys

.github/integration/scripts/make_db_credentials.sh

@@ -4,7 +4,7 @@ set -e
 apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
 apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client >/dev/null
 
-for n in download finalize inbox ingest mapper sync verify; do
+for n in api download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h migrate -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"

.github/integration/scripts/make_sda_credentials.sh

@@ -14,7 +14,7 @@ apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl post
 pip install --upgrade pip > /dev/null
 pip install aiohttp Authlib joserfc requests > /dev/null
 
-for n in download finalize inbox ingest mapper sync verify; do
+for n in api download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
     psql -U postgres -h postgres -d sda -c "GRANT base TO $n;"
@@ -92,4 +92,4 @@ if [ ! -f "/shared/grpcurl" ]; then
     echo "downloading grpcurl"
     latest_grpculr=$(curl --retry 100 -sL https://api.github.com/repos/fullstorydev/grpcurl/releases/latest | jq -r '.name' | sed -e 's/v//')
     curl --retry 100 -s -L "https://github.com/fullstorydev/grpcurl/releases/download/v${latest_grpculr}/grpcurl_${latest_grpculr}_linux_x86_64.tar.gz" | tar -xz -C /shared/ && chmod +x /shared/grpcurl
-fi
+fi

.github/integration/sda-posix-integration.yml

@@ -182,7 +182,7 @@ services:
     container_name: tester
     command:
       - "bash"
-      - "/tests/sda/10_upload_test.sh"
+      - "/tests/sda/10.1_upload_test.sh"
     depends_on:
       inbox:
         condition: service_started

.github/integration/sda-s3-integration.yml

@@ -289,11 +289,10 @@ services:
       rabbitmq:
         condition: service_healthy
     environment:
-      - BROKER_PASSWORD=ingest
-      - BROKER_USER=ingest
-      - BROKER_ROUTINGKEY=ingest
-      - DB_PASSWORD=download
-      - DB_USER=download
+      - BROKER_PASSWORD=api
+      - BROKER_USER=api
+      - DB_PASSWORD=api
+      - DB_USER=api
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
     ports:
       - "8090:8080"

.github/integration/tests/sda/01_install_dependencies.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+# install tools if missing
+for t in curl expect jq openssh-client postgresql-client xxd; do
+    if [ ! "$(command -v $t)" ]; then
+        if [ "$(id -u)" != 0 ]; then
+            echo "$t is missing, unable to install it"
+            exit 1
+        fi
+
+        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
+        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
+    fi
+done

.github/integration/tests/sda/09_healthchecks.sh

@@ -1,20 +1,6 @@
 #!/bin/sh
 set -e
 
-# install tools if missing
-for t in curl  jq ; do
-    if [ ! "$(command -v $t)" ]; then
-        if [ "$(id -u)" != 0 ]; then
-            echo "$t is missing, unable to install it"
-            exit 1
-        fi
-
-        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
-        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
-    fi
-done
-
-
 # Test the s3inbox's healthchecks, GET /health and HEAD /
 response="$(curl -s -k -LI "http://s3inbox:8000" -o /dev/null -w "%{http_code}\n")"
 if [ "$response" != "200" ]; then

.github/integration/tests/sda/10.1_upload_test.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+set -e
+
+if [ -z "$STORAGETYPE" ]; then
+    echo "STORAGETYPE not set, exiting"
+    exit 1
+fi
+
+if [ "$STORAGETYPE" = "s3" ]; then
+    exit 0
+fi
+
+for t in curl jq openssh-client postgresql-client; do
+    if [ ! "$(command -v $t)" ]; then
+        if [ "$(id -u)" != 0 ]; then
+            echo "$t is missing, unable to install it"
+            exit 1
+        fi
+
+        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
+        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
+    fi
+done
+
+cd shared || true
+
+## verify that messages exists in MQ
+URI=http://rabbitmq:15672
+if [ -n "$PGSSLCERT" ]; then
+    URI=https://rabbitmq:15671
+fi
+## empty all queues ##
+for q in accession archived backup completed inbox ingest mappings verified; do
+    curl -s -k -u guest:guest -X DELETE "$URI/api/queues/sda/$q/contents"
+done
+## truncate database
+psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;"
+
+if [ "$STORAGETYPE" = "posix" ]; then
+    for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
+        echo "downloading $file"
+        curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
+        if [ ! -f "$file.c4gh" ]; then
+            yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
+        fi
+
+        sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF
+    put "${file}"
+    dir
+    ls -al
+    exit
+EOF
+    done
+
+    ## reupload a file under a different name
+    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF
+    put NA12878.bam.c4gh NB12878.bam.c4gh
+    dir
+    ls -al
+    exit
+EOF
+
+    ## reupload a file with the same name
+    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF
+    put NA12878.bam.c4gh
+    dir
+    ls -al
+    exit
+EOF
+
+fi
+
+echo "waiting for upload to complete"
+RETRY_TIMES=0
+until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do
+    echo "waiting for upload to complete"
+    RETRY_TIMES=$((RETRY_TIMES + 1))
+    if [ "$RETRY_TIMES" -eq 30 ]; then
+        echo "::error::Time out while waiting for upload to complete"
+        exit 1
+    fi
+    sleep 2
+done
+
+
+echo "files uploaded successfully"

.github/integration/tests/sda/10_upload_test.sh

@@ -6,19 +6,6 @@ if [ -z "$STORAGETYPE" ]; then
     exit 1
 fi
 
-# install tools if missing
-for t in curl expect jq openssh-client postgresql-client; do
-    if [ ! "$(command -v $t)" ]; then
-        if [ "$(id -u)" != 0 ]; then
-            echo "$t is missing, unable to install it"
-            exit 1
-        fi
-
-        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
-        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
-    fi
-done
-
 cd shared || true
 
 ## verify that messages exists in MQ
@@ -33,57 +20,22 @@ done
 ## truncate database
 psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;"
 
-if [ "$STORAGETYPE" = "posix" ]; then
-    for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
-        echo "downloading $file"
-        curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
-        if [ ! -f "$file.c4gh" ]; then
-            yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
-        fi
-
-        sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF
-    put "${file}"
-    dir
-    ls -al
-    exit
-EOF
-    done
-
-    ## reupload a file under a different name
-    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF
-    put NA12878.bam.c4gh NB12878.bam.c4gh
-    dir
-    ls -al
-    exit
-EOF
-
-    ## reupload a file with the same name
-    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF
-    put NA12878.bam.c4gh
-    dir
-    ls -al
-    exit
-EOF
-
-fi
+pip -q install s3cmd
 
-if [ "$STORAGETYPE" = "s3" ]; then
-    pip -q install s3cmd
+for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
+    curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
+    if [ ! -f "$file.c4gh" ]; then
+        yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
+    fi
+    s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/
+done
 
-    for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
-        curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
-        if [ ! -f "$file.c4gh" ]; then
-            yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
-        fi
-        s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/
-    done
+## reupload a file under a different name
+s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh
 
-    ## reupload a file under a different name
-    s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh
+## reupload a file with the same name
+s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/
 
-    ## reupload a file with the same name
-    s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/
-fi
 
 echo "waiting for upload to complete"
 RETRY_TIMES=0
@@ -97,51 +49,50 @@ until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messag
     sleep 2
 done
 
-if [ "$STORAGETYPE" = "s3" ]; then
-    num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
-    if [ "$num_rows" -ne 5 ]; then
-        echo "database queries for register_files failed, expected 5 got $num_rows"
-        exit 1
-    fi
+num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
+if [ "$num_rows" -ne 5 ]; then
+    echo "database queries for register_files failed, expected 5 got $num_rows"
+    exit 1
+fi
 
-    num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
-    if [ "$num_log_rows" -ne 12 ]; then
-        echo "database queries for file_event_logs failed, expected 12 got $num_log_rows"
-        exit 1
-    fi
+num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
+if [ "$num_log_rows" -ne 12 ]; then
+    echo "database queries for file_event_logs failed, expected 12 got $num_log_rows"
+    exit 1
+fi
 
-    ## test with token from OIDC service
-    echo "testing with OIDC token"
-    newToken=$(curl http://oidc:8080/tokens | jq '.[0]')
-    cp s3cfg oidc_s3cfg
-    sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg
+## test with token from OIDC service
+echo "testing with OIDC token"
+newToken=$(curl http://oidc:8080/tokens | jq '.[0]')
+cp s3cfg oidc_s3cfg
+sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg
 
-    s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh
+s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh
 
-    ## verify that messages exists in MQ
+## verify that messages exists in MQ
+echo "waiting for upload to complete"
+RETRY_TIMES=0
+until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do
     echo "waiting for upload to complete"
-    RETRY_TIMES=0
-    until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do
-        echo "waiting for upload to complete"
-        RETRY_TIMES=$((RETRY_TIMES + 1))
-        if [ "$RETRY_TIMES" -eq 30 ]; then
-            echo "::error::Time out while waiting for upload to complete"
-            exit 1
-        fi
-        sleep 2
-    done
-
-    num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
-    if [ "$num_rows" -ne 6 ]; then
-        echo "database queries for register_files failed, expected 6 got $num_rows"
+    RETRY_TIMES=$((RETRY_TIMES + 1))
+    if [ "$RETRY_TIMES" -eq 30 ]; then
+        echo "::error::Time out while waiting for upload to complete"
         exit 1
     fi
+    sleep 2
+done
 
-    num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
-    if [ "$num_log_rows" -ne 14 ]; then
-        echo "database queries for file_event_logs failed, expected 14 got $num_log_rows"
-        exit 1
-    fi
+num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
+if [ "$num_rows" -ne 6 ]; then
+    echo "database queries for register_files failed, expected 6 got $num_rows"
+    exit 1
 fi
 
+num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
+if [ "$num_log_rows" -ne 14 ]; then
+    echo "database queries for file_event_logs failed, expected 14 got $num_log_rows"
+    exit 1
+fi
+
+
 echo "files uploaded successfully"

.github/integration/tests/sda/11_api-getfiles_test.sh

@@ -3,11 +3,39 @@ set -e
 
 # Test the API files endpoint
 token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
-curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token"
-response="$(curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')"
+response="$(curl -s -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')"
 if [ "$response" != "uploaded" ]; then
 	echo "API returned incorrect value, expected ready got: $response"
 	exit 1
 fi
 
+# test inserting a c4gh public key hash
+payload=$(
+	jq -c -n \
+		--arg description "this is the key description" \
+		--arg pubkey "$( base64 -w0 /shared/c4gh.pub.pem)" \
+		'$ARGS.named'
+)
+
+resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")"
+if [ "$resp" != "200" ]; then
+	echo "Error when adding a public key hash, expected 200 got: $resp"
+	exit 1
+fi
+
+# again to verify we get an error
+resp="$(curl -s -k -L  -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")"
+if [ "$resp" != "409" ]; then
+	echo "Error when adding a public key hash, expected 409 got: $resp"
+	exit 1
+fi
+
+manual_hash=$(sed -n '2p' /shared/c4gh.pub.pem | base64 -d -w0 | xxd -c64 -ps)
+
+db_hash=$(psql -U postgres -h postgres -d sda -At -c "SELECT key_hash FROM sda.encryption_keys WHERE description = 'this is the key description';")
+if [ "$db_hash" != "$manual_hash" ]; then
+	echo "wrong hash in the database, expected $manual_hash got $db_hash"
+	exit 1
+fi
+
 echo "api test completed successfully"