cleanup test scripts

- use neic crypt4gh
- do not use pip
- silent curls

sda-download/.github/integration/setup/common/10_services.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 # Build containers
 docker build -t neicnordic/sda-download:latest . || exit 1

sda-download/.github/integration/setup/common/1_keys.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/setup/common/20_tools.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+set -e
+
+C4GH_VERSION="$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')"
+curl --retry 100 -sL https://github.com/neicnordic/crypt4gh/releases/download/"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ &&
+        sudo chmod +x /usr/bin/crypt4gh
 
-pip3 install crypt4gh
 sudo apt install -y jq s3cmd

sda-download/.github/integration/setup/common/23_create_db_entries.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-
-pip3 install s3cmd
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
+set -e
 
-pip3 install s3cmd
+sudo apt install -y s3cmd
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/tests/common/30_check_db.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/tests/common/50_check_endpoint.sh

@@ -56,11 +56,11 @@ echo "got correct response when POST method used"
 # ------------------
 # Test good token
 
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
 
 ## Test datasets endpoint
 
-check_dataset=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]')
+check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]')
 
 if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then
     echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found"
@@ -72,7 +72,7 @@ echo "expected dataset found"
 
 ## Test datasets/files endpoint
 
-check_files=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')
+check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')
 
 if [ "$check_files" != "urn:neic:001-002" ]; then
     echo "file with id urn:neic:001-002 not found"
@@ -88,10 +88,9 @@ echo "expected file found"
 C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g')
 export C4GH_PASSPHRASE
 
-crypt4gh decrypt --sk c4gh.sec.pem < dummy_data.c4gh > old-file.txt
-
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt
+crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt
 
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt
 
 cmp --silent old-file.txt test-download.txt
 status=$?
@@ -102,7 +101,7 @@ else
     exit 1
 fi
 
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt
 
 dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt
 
@@ -115,7 +114,7 @@ else
     exit 1
 fi
 
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt
 
 dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt
 
@@ -128,7 +127,7 @@ else
     exit 1
 fi
 
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt
 
 dd if=old-file.txt ibs=1 skip=70000 count=70000 > old-part3.txt
 
@@ -162,7 +161,7 @@ echo "got correct response when token has no permissions"
 # Test token with untrusted sources
 # for this test we attach a list of trusted sources
 
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[2]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[2]')
 
 ## Test datasets endpoint
 

sda-download/.github/integration/tests/common/60_check_s3_endpoint.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-
 TLS="True"
 [[ "$STORAGETYPE" == "s3notls" ]] && TLS="False"
 

sda-download/.github/integration/tests/common/70_check_download.sh

@@ -7,15 +7,15 @@ fi
 cd dev_utils || exit 1
 
 # get a token, set up variables
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
 dataset="https://doi.example/ty009.sfrrss/600.45asasga"
 file="dummy_data"
 expected_size=1048605
 C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g')
 export C4GH_PASSPHRASE
 
 # download decrypted full file,  check file size
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
 file_size=$(stat -c %s full1.bam)  # Get the size of the file
 
 if [ "$file_size" -ne "$expected_size" ]; then
@@ -24,9 +24,11 @@ if [ "$file_size" -ne "$expected_size" ]; then
 fi
 
 # test that start, end=0 returns the whole file
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam
 
 if ! cmp --silent full1.bam full2.bam; then
     echo "Full decrypted files, with and without coordinates, are different"
     exit 1
 fi
+
+echo "OK"

sda-download/.github/integration/tests/common/80_check_reencrypt.sh

@@ -7,7 +7,7 @@ fi
 cd dev_utils || exit 1
 
 # Get a token, set up variables
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
 
 if [ -z "$token" ]; then
     echo "Failed to obtain token"
@@ -19,15 +19,9 @@ file="dummy_data"
 expected_size=1048605
 
 # Download unencrypted full file,  check file size
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
-
-if [ ! -f "full1.bam" ]; then
-    echo "Failed to download full1.bam"
-    exit 1
-fi
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
 
 file_size=$(stat -c %s full1.bam)  # Get the size of the file
-
 if [ "$file_size" -ne "$expected_size" ]; then
     echo "Incorrect file size for downloaded file"
     exit 1
@@ -36,11 +30,7 @@ fi
 # Test reencrypt the file header with the client public key 
 clientkey=$(base64 -w0 client.pub.pem)
 reencryptedFile=reencrypted.bam.c4gh
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file" --output $reencryptedFile
-if [ ! -f "$reencryptedFile" ]; then
-    echo "Failed to download re-encrypted file"
-    exit 1
-fi
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file" --output $reencryptedFile
 
 expected_encrypted_size=1049205
 file_size=$(stat -c %s $reencryptedFile)
@@ -51,10 +41,13 @@ fi
 
 # Decrypt the reencrypted file and compare it with the original unencrypted file
 export C4GH_PASSPHRASE="strongpass" # passphrase for the client crypt4gh key
-if ! crypt4gh decrypt --sk client.sec.pem < $reencryptedFile > full2.bam; then
+crypt4gh decrypt -s client.sec.pem -f $reencryptedFile
+if [ ! -f "${reencryptedFile%.c4gh}" ] ; then
     echo "Failed to decrypt re-encrypted file with the client's private key"
     exit 1
 fi
+mv "${reencryptedFile%.c4gh}" full2.bam
+
 
 if ! cmp --silent full1.bam full2.bam; then
     echo "Decrypted version of $reencryptedFile and the original unencrypted file, are different"
@@ -63,7 +56,7 @@ fi
 
 # download reencrypted partial file, check file size
 partReencryptedFile=part1.bam.c4gh
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile 
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile
 file_size=$(stat -c %s $partReencryptedFile)  # Get the size of the file
 part_expected_size=65688
 
@@ -72,7 +65,8 @@ if [ "$file_size" -ne "$part_expected_size" ]; then
     exit 1
 fi
 
-if ! crypt4gh decrypt --sk client.sec.pem < $partReencryptedFile > part1.bam; then
+crypt4gh decrypt -s client.sec.pem -f $partReencryptedFile
+if [ ! -f "${partReencryptedFile%.c4gh}" ] ; then
     echo "Re-encrypted partial file could not be decrypted"
     exit 1
 fi
@@ -106,3 +100,5 @@ resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-P
 if [ "$resp" -ne 500 ]; then
     echo "Incorrect response with missing public key, expected 500 got $resp"
 fi
+
+echo "OK"

sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh

@@ -41,11 +41,11 @@ echo "got correct response when POST method used"
 # ------------------
 # Test good token
 
-token=$(curl "http://localhost:8000/tokens" | jq -r  '.[0]')
+token=$(curl -s "http://localhost:8000/tokens" | jq -r  '.[0]')
 
 ## Test datasets endpoint
 
-check_dataset=$(curl -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets | jq -r '.[0]')
+check_dataset=$(curl -s -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets | jq -r '.[0]')
 
 if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then
     echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found"
@@ -57,7 +57,7 @@ echo "expected dataset found"
 
 ## Test datasets/files endpoint
 
-check_files=$(curl -H "Authorization: Bearer $token" "http://localhost:8080/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')
+check_files=$(curl -s -H "Authorization: Bearer $token" "http://localhost:8080/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')
 
 if [ "$check_files" != "urn:neic:001-002" ]; then
     echo "file with id urn:neic:001-002 not found"
@@ -73,9 +73,9 @@ echo "expected file found"
 C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g')
 export C4GH_PASSPHRASE
 
-crypt4gh decrypt --sk c4gh.sec.pem < dummy_data.c4gh > old-file.txt
+crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt
 
-curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002" --output test-download.txt
+curl -s -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002" --output test-download.txt
 
 
 cmp --silent old-file.txt test-download.txt
@@ -86,7 +86,7 @@ else
     echo "Files are different"
 fi
 
-curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt
+curl -s -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt
 
 dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt
 
@@ -99,7 +99,7 @@ else
     exit 1
 fi
 
-curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt
+curl -s -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt
 
 dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt
 
@@ -115,7 +115,7 @@ fi
 # ------------------
 # Test get visas failed
 
-token=$(curl "http://localhost:8000/tokens" | jq -r  '.[1]')
+token=$(curl -s "http://localhost:8000/tokens" | jq -r  '.[1]')
 
 ## Test datasets endpoint
 
@@ -133,11 +133,11 @@ echo "got correct response when token has no permissions"
 # Test token with untrusted sources
 # for this test we don't attach a list of trusted sources
 
-token=$(curl "http://localhost:8000/tokens" | jq -r  '.[2]')
+token=$(curl -s "http://localhost:8000/tokens" | jq -r  '.[2]')
 
 ## Test datasets endpoint
 
-check_dataset=$(curl -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets | jq -r '.[0]')
+check_dataset=$(curl -s -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets | jq -r '.[0]')
 
 if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then
     echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found"