diff --git a/sda-download/.github/integration/setup/common/10_services.sh b/sda-download/.github/integration/setup/common/10_services.sh index 1680ff917..52af643e8 100644 --- a/sda-download/.github/integration/setup/common/10_services.sh +++ b/sda-download/.github/integration/setup/common/10_services.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e # Build containers docker build -t neicnordic/sda-download:latest . || exit 1 diff --git a/sda-download/.github/integration/setup/common/1_keys.sh b/sda-download/.github/integration/setup/common/1_keys.sh index 2e8f08653..3a3bcdb08 100644 --- a/sda-download/.github/integration/setup/common/1_keys.sh +++ b/sda-download/.github/integration/setup/common/1_keys.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e cd dev_utils || exit 1 diff --git a/sda-download/.github/integration/setup/common/20_tools.sh b/sda-download/.github/integration/setup/common/20_tools.sh index f134a6081..a4d4e5042 100644 --- a/sda-download/.github/integration/setup/common/20_tools.sh +++ b/sda-download/.github/integration/setup/common/20_tools.sh @@ -1,4 +1,8 @@ #!/bin/bash +set -e + +C4GH_VERSION="$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')" +curl --retry 100 -sL https://github.com/neicnordic/crypt4gh/releases/download/"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ && + sudo chmod +x /usr/bin/crypt4gh -pip3 install crypt4gh sudo apt install -y jq s3cmd diff --git a/sda-download/.github/integration/setup/common/23_create_db_entries.sh b/sda-download/.github/integration/setup/common/23_create_db_entries.sh index 95172e88c..14085487d 100755 --- a/sda-download/.github/integration/setup/common/23_create_db_entries.sh +++ b/sda-download/.github/integration/setup/common/23_create_db_entries.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e cd dev_utils || exit 1 diff --git a/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh b/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh index 5a58fc2f9..ac2d6b7a7 100644 --- a/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh +++ b/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh @@ -1,6 +1,5 @@ #!/bin/bash - -pip3 install s3cmd +set -e cd dev_utils || exit 1 diff --git a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh index 8d34eb95d..d270a3e6b 100644 --- a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh +++ b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh @@ -1,6 +1,6 @@ #!/bin/bash -pip3 install s3cmd +sudo apt install -y s3cmd cd dev_utils || exit 1 diff --git a/sda-download/.github/integration/tests/common/30_check_db.sh b/sda-download/.github/integration/tests/common/30_check_db.sh index 30423f939..4a2509093 100644 --- a/sda-download/.github/integration/tests/common/30_check_db.sh +++ b/sda-download/.github/integration/tests/common/30_check_db.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e cd dev_utils || exit 1 diff --git a/sda-download/.github/integration/tests/common/50_check_endpoint.sh b/sda-download/.github/integration/tests/common/50_check_endpoint.sh index da754245d..27257662f 100755 --- a/sda-download/.github/integration/tests/common/50_check_endpoint.sh +++ b/sda-download/.github/integration/tests/common/50_check_endpoint.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e cd dev_utils || exit 1 @@ -56,11 +57,11 @@ echo "got correct response when POST method used" # ------------------ # Test good token -token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') +token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') ## Test datasets endpoint -check_dataset=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]') +check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]') if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found" @@ -72,7 +73,7 @@ echo "expected dataset found" ## Test datasets/files endpoint -check_files=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') +check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') if [ "$check_files" != "urn:neic:001-002" ]; then echo "file with id urn:neic:001-002 not found" @@ -88,9 +89,9 @@ echo "expected file found" C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g') export C4GH_PASSPHRASE -crypt4gh decrypt --sk c4gh.sec.pem < dummy_data.c4gh > old-file.txt +crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt cmp --silent old-file.txt test-download.txt @@ -102,7 +103,7 @@ else exit 1 fi -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt @@ -115,7 +116,7 @@ else exit 1 fi -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt @@ -128,7 +129,7 @@ else exit 1 fi -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt dd if=old-file.txt ibs=1 skip=70000 count=70000 > old-part3.txt @@ -162,7 +163,7 @@ echo "got correct response when token has no permissions" # Test token with untrusted sources # for this test we attach a list of trusted sources -token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[2]') +token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[2]') ## Test datasets endpoint diff --git a/sda-download/.github/integration/tests/common/70_check_download.sh b/sda-download/.github/integration/tests/common/70_check_download.sh index 795325a47..3708d4f72 100644 --- a/sda-download/.github/integration/tests/common/70_check_download.sh +++ b/sda-download/.github/integration/tests/common/70_check_download.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e if [ "$STORAGETYPE" = s3notls ]; then exit 0 @@ -7,7 +8,7 @@ fi cd dev_utils || exit 1 # get a token, set up variables -token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') +token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') dataset="https://doi.example/ty009.sfrrss/600.45asasga" file="dummy_data" expected_size=1048605 @@ -15,7 +16,7 @@ C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g') export C4GH_PASSPHRASE # download decrypted full file, check file size -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam file_size=$(stat -c %s full1.bam) # Get the size of the file if [ "$file_size" -ne "$expected_size" ]; then @@ -24,9 +25,11 @@ if [ "$file_size" -ne "$expected_size" ]; then fi # test that start, end=0 returns the whole file -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam if ! cmp --silent full1.bam full2.bam; then echo "Full decrypted files, with and without coordinates, are different" exit 1 fi + +echo "OK" \ No newline at end of file diff --git a/sda-download/.github/integration/tests/common/80_check_reencrypt.sh b/sda-download/.github/integration/tests/common/80_check_reencrypt.sh index d57102067..8e98ad327 100644 --- a/sda-download/.github/integration/tests/common/80_check_reencrypt.sh +++ b/sda-download/.github/integration/tests/common/80_check_reencrypt.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e if [ "$STORAGETYPE" = s3notls ]; then exit 0 @@ -7,7 +8,7 @@ fi cd dev_utils || exit 1 # Get a token, set up variables -token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') +token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') if [ -z "$token" ]; then echo "Failed to obtain token" @@ -19,7 +20,7 @@ file="dummy_data" expected_size=1048605 # Download unencrypted full file, check file size -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam if [ ! -f "full1.bam" ]; then echo "Failed to download full1.bam" @@ -36,7 +37,7 @@ fi # Test reencrypt the file header with the client public key clientkey=$(base64 -w0 client.pub.pem) reencryptedFile=reencrypted.bam.c4gh -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file" --output $reencryptedFile +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file" --output $reencryptedFile if [ ! -f "$reencryptedFile" ]; then echo "Failed to download re-encrypted file" exit 1 @@ -48,13 +49,15 @@ if [ "$file_size" -ne "$expected_encrypted_size" ]; then echo "Incorrect file size for the re-encrypted file, should be $expected_encrypted_size but is $file_size" exit 1 fi - # Decrypt the reencrypted file and compare it with the original unencrypted file export C4GH_PASSPHRASE="strongpass" # passphrase for the client crypt4gh key -if ! crypt4gh decrypt --sk client.sec.pem < $reencryptedFile > full2.bam; then +crypt4gh decrypt -s client.sec.pem -f $reencryptedFile +if [ ! -f "${reencryptedFile%.c4gh}" ] ; then echo "Failed to decrypt re-encrypted file with the client's private key" exit 1 fi +mv "${reencryptedFile%.c4gh}" full2.bam + if ! cmp --silent full1.bam full2.bam; then echo "Decrypted version of $reencryptedFile and the original unencrypted file, are different" @@ -63,7 +66,7 @@ fi # download reencrypted partial file, check file size partReencryptedFile=part1.bam.c4gh -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile file_size=$(stat -c %s $partReencryptedFile) # Get the size of the file part_expected_size=65688 @@ -72,7 +75,8 @@ if [ "$file_size" -ne "$part_expected_size" ]; then exit 1 fi -if ! crypt4gh decrypt --sk client.sec.pem < $partReencryptedFile > part1.bam; then +crypt4gh decrypt -s client.sec.pem -f $partReencryptedFile +if [ ! -f "${partReencryptedFile%.c4gh}" ] ; then echo "Re-encrypted partial file could not be decrypted" exit 1 fi @@ -106,3 +110,5 @@ resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-P if [ "$resp" -ne 500 ]; then echo "Incorrect response with missing public key, expected 500 got $resp" fi + +echo "OK" \ No newline at end of file