-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1084 from neicnordic/feature/api-keys
Feature/api keys
- Loading branch information
Showing
17 changed files
with
534 additions
and
131 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| #!/bin/sh | ||
| set -e | ||
|
|
||
| # install tools if missing | ||
| for t in curl expect jq openssh-client postgresql-client xxd; do | ||
| if [ ! "$(command -v $t)" ]; then | ||
| if [ "$(id -u)" != 0 ]; then | ||
| echo "$t is missing, unable to install it" | ||
| exit 1 | ||
| fi | ||
|
|
||
| apt-get -o DPkg::Lock::Timeout=60 update >/dev/null | ||
| apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| #!/bin/sh | ||
| set -e | ||
|
|
||
| if [ -z "$STORAGETYPE" ]; then | ||
| echo "STORAGETYPE not set, exiting" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ "$STORAGETYPE" = "s3" ]; then | ||
| exit 0 | ||
| fi | ||
|
|
||
| for t in curl jq openssh-client postgresql-client; do | ||
| if [ ! "$(command -v $t)" ]; then | ||
| if [ "$(id -u)" != 0 ]; then | ||
| echo "$t is missing, unable to install it" | ||
| exit 1 | ||
| fi | ||
|
|
||
| apt-get -o DPkg::Lock::Timeout=60 update >/dev/null | ||
| apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null | ||
| fi | ||
| done | ||
|
|
||
| cd shared || true | ||
|
|
||
| ## verify that messages exists in MQ | ||
| URI=http://rabbitmq:15672 | ||
| if [ -n "$PGSSLCERT" ]; then | ||
| URI=https://rabbitmq:15671 | ||
| fi | ||
| ## empty all queues ## | ||
| for q in accession archived backup completed inbox ingest mappings verified; do | ||
| curl -s -k -u guest:guest -X DELETE "$URI/api/queues/sda/$q/contents" | ||
| done | ||
| ## truncate database | ||
| psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;" | ||
|
|
||
| if [ "$STORAGETYPE" = "posix" ]; then | ||
| for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do | ||
| echo "downloading $file" | ||
| curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" | ||
| if [ ! -f "$file.c4gh" ]; then | ||
| yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" | ||
| fi | ||
|
|
||
| sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF | ||
| put "${file}" | ||
| dir | ||
| ls -al | ||
| exit | ||
| EOF | ||
| done | ||
|
|
||
| ## reupload a file under a different name | ||
| sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF | ||
| put NA12878.bam.c4gh NB12878.bam.c4gh | ||
| dir | ||
| ls -al | ||
| exit | ||
| EOF | ||
|
|
||
| ## reupload a file with the same name | ||
| sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF | ||
| put NA12878.bam.c4gh | ||
| dir | ||
| ls -al | ||
| exit | ||
| EOF | ||
|
|
||
| fi | ||
|
|
||
| echo "waiting for upload to complete" | ||
| RETRY_TIMES=0 | ||
| until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do | ||
| echo "waiting for upload to complete" | ||
| RETRY_TIMES=$((RETRY_TIMES + 1)) | ||
| if [ "$RETRY_TIMES" -eq 30 ]; then | ||
| echo "::error::Time out while waiting for upload to complete" | ||
| exit 1 | ||
| fi | ||
| sleep 2 | ||
| done | ||
|
|
||
|
|
||
| echo "files uploaded successfully" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,19 +6,6 @@ if [ -z "$STORAGETYPE" ]; then | |
| exit 1 | ||
| fi | ||
|
|
||
| # install tools if missing | ||
| for t in curl expect jq openssh-client postgresql-client; do | ||
| if [ ! "$(command -v $t)" ]; then | ||
| if [ "$(id -u)" != 0 ]; then | ||
| echo "$t is missing, unable to install it" | ||
| exit 1 | ||
| fi | ||
|
|
||
| apt-get -o DPkg::Lock::Timeout=60 update >/dev/null | ||
| apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null | ||
| fi | ||
| done | ||
|
|
||
| cd shared || true | ||
|
|
||
| ## verify that messages exists in MQ | ||
|
|
@@ -33,57 +20,22 @@ done | |
| ## truncate database | ||
| psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;" | ||
|
|
||
| if [ "$STORAGETYPE" = "posix" ]; then | ||
| for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do | ||
| echo "downloading $file" | ||
| curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" | ||
| if [ ! -f "$file.c4gh" ]; then | ||
| yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" | ||
| fi | ||
|
|
||
| sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF | ||
| put "${file}" | ||
| dir | ||
| ls -al | ||
| exit | ||
| EOF | ||
| done | ||
|
|
||
| ## reupload a file under a different name | ||
| sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF | ||
| put NA12878.bam.c4gh NB12878.bam.c4gh | ||
| dir | ||
| ls -al | ||
| exit | ||
| EOF | ||
|
|
||
| ## reupload a file with the same name | ||
| sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o [email protected] -P 2222 inbox <<-EOF | ||
| put NA12878.bam.c4gh | ||
| dir | ||
| ls -al | ||
| exit | ||
| EOF | ||
|
|
||
| fi | ||
| pip -q install s3cmd | ||
|
|
||
| if [ "$STORAGETYPE" = "s3" ]; then | ||
| pip -q install s3cmd | ||
| for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do | ||
| curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" | ||
| if [ ! -f "$file.c4gh" ]; then | ||
| yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" | ||
| fi | ||
| s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/ | ||
| done | ||
|
|
||
| for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do | ||
| curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" | ||
| if [ ! -f "$file.c4gh" ]; then | ||
| yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" | ||
| fi | ||
| s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/ | ||
| done | ||
| ## reupload a file under a different name | ||
| s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh | ||
|
|
||
| ## reupload a file under a different name | ||
| s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh | ||
| ## reupload a file with the same name | ||
| s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ | ||
|
|
||
| ## reupload a file with the same name | ||
| s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ | ||
| fi | ||
|
|
||
| echo "waiting for upload to complete" | ||
| RETRY_TIMES=0 | ||
|
|
@@ -97,51 +49,50 @@ until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messag | |
| sleep 2 | ||
| done | ||
|
|
||
| if [ "$STORAGETYPE" = "s3" ]; then | ||
| num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") | ||
| if [ "$num_rows" -ne 5 ]; then | ||
| echo "database queries for register_files failed, expected 5 got $num_rows" | ||
| exit 1 | ||
| fi | ||
| num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") | ||
| if [ "$num_rows" -ne 5 ]; then | ||
| echo "database queries for register_files failed, expected 5 got $num_rows" | ||
| exit 1 | ||
| fi | ||
|
|
||
| num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") | ||
| if [ "$num_log_rows" -ne 12 ]; then | ||
| echo "database queries for file_event_logs failed, expected 12 got $num_log_rows" | ||
| exit 1 | ||
| fi | ||
| num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") | ||
| if [ "$num_log_rows" -ne 12 ]; then | ||
| echo "database queries for file_event_logs failed, expected 12 got $num_log_rows" | ||
| exit 1 | ||
| fi | ||
|
|
||
| ## test with token from OIDC service | ||
| echo "testing with OIDC token" | ||
| newToken=$(curl http://oidc:8080/tokens | jq '.[0]') | ||
| cp s3cfg oidc_s3cfg | ||
| sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg | ||
| ## test with token from OIDC service | ||
| echo "testing with OIDC token" | ||
| newToken=$(curl http://oidc:8080/tokens | jq '.[0]') | ||
| cp s3cfg oidc_s3cfg | ||
| sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg | ||
|
|
||
| s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh | ||
| s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh | ||
|
|
||
| ## verify that messages exists in MQ | ||
| ## verify that messages exists in MQ | ||
| echo "waiting for upload to complete" | ||
| RETRY_TIMES=0 | ||
| until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do | ||
| echo "waiting for upload to complete" | ||
| RETRY_TIMES=0 | ||
| until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do | ||
| echo "waiting for upload to complete" | ||
| RETRY_TIMES=$((RETRY_TIMES + 1)) | ||
| if [ "$RETRY_TIMES" -eq 30 ]; then | ||
| echo "::error::Time out while waiting for upload to complete" | ||
| exit 1 | ||
| fi | ||
| sleep 2 | ||
| done | ||
|
|
||
| num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") | ||
| if [ "$num_rows" -ne 6 ]; then | ||
| echo "database queries for register_files failed, expected 6 got $num_rows" | ||
| RETRY_TIMES=$((RETRY_TIMES + 1)) | ||
| if [ "$RETRY_TIMES" -eq 30 ]; then | ||
| echo "::error::Time out while waiting for upload to complete" | ||
| exit 1 | ||
| fi | ||
| sleep 2 | ||
| done | ||
|
|
||
| num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") | ||
| if [ "$num_log_rows" -ne 14 ]; then | ||
| echo "database queries for file_event_logs failed, expected 14 got $num_log_rows" | ||
| exit 1 | ||
| fi | ||
| num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") | ||
| if [ "$num_rows" -ne 6 ]; then | ||
| echo "database queries for register_files failed, expected 6 got $num_rows" | ||
| exit 1 | ||
| fi | ||
|
|
||
| num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") | ||
| if [ "$num_log_rows" -ne 14 ]; then | ||
| echo "database queries for file_event_logs failed, expected 14 got $num_log_rows" | ||
| exit 1 | ||
| fi | ||
|
|
||
|
|
||
| echo "files uploaded successfully" | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.