Skip to content

Commit

Permalink
cleanup test scripts
Browse files Browse the repository at this point in the history
- use neic crypt4gh
- do not use pip
- silent curls
  • Loading branch information
aaperis committed Dec 4, 2024
1 parent b0aa964 commit 5ad5768
Show file tree
Hide file tree
Showing 10 changed files with 40 additions and 23 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

# Build containers
docker build -t neicnordic/sda-download:latest . || exit 1
Expand Down
1 change: 1 addition & 0 deletions sda-download/.github/integration/setup/common/1_keys.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

cd dev_utils || exit 1

Expand Down
6 changes: 5 additions & 1 deletion sda-download/.github/integration/setup/common/20_tools.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
#!/bin/bash
set -e

C4GH_VERSION="$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')"
curl --retry 100 -sL https://github.com/neicnordic/crypt4gh/releases/download/"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ &&
sudo chmod +x /usr/bin/crypt4gh

pip3 install crypt4gh
sudo apt install -y jq s3cmd
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

cd dev_utils || exit 1

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#!/bin/bash

pip3 install s3cmd
set -e

cd dev_utils || exit 1

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

pip3 install s3cmd
sudo apt install -y s3cmd

cd dev_utils || exit 1

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

cd dev_utils || exit 1

Expand Down
19 changes: 10 additions & 9 deletions sda-download/.github/integration/tests/common/50_check_endpoint.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

cd dev_utils || exit 1

Expand Down Expand Up @@ -56,11 +57,11 @@ echo "got correct response when POST method used"
# ------------------
# Test good token

token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]')
token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]')

## Test datasets endpoint

check_dataset=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]')
check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]')

if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then
echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found"
Expand All @@ -72,7 +73,7 @@ echo "expected dataset found"

## Test datasets/files endpoint

check_files=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')
check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')

if [ "$check_files" != "urn:neic:001-002" ]; then
echo "file with id urn:neic:001-002 not found"
Expand All @@ -88,9 +89,9 @@ echo "expected file found"
C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g')
export C4GH_PASSPHRASE

crypt4gh decrypt --sk c4gh.sec.pem < dummy_data.c4gh > old-file.txt
crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt

curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt


cmp --silent old-file.txt test-download.txt
Expand All @@ -102,7 +103,7 @@ else
exit 1
fi

curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt

dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt

Expand All @@ -115,7 +116,7 @@ else
exit 1
fi

curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt

dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt

Expand All @@ -128,7 +129,7 @@ else
exit 1
fi

curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt

dd if=old-file.txt ibs=1 skip=70000 count=70000 > old-part3.txt

Expand Down Expand Up @@ -162,7 +163,7 @@ echo "got correct response when token has no permissions"
# Test token with untrusted sources
# for this test we attach a list of trusted sources

token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[2]')
token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[2]')

## Test datasets endpoint

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

if [ "$STORAGETYPE" = s3notls ]; then
exit 0
Expand All @@ -7,15 +8,15 @@ fi
cd dev_utils || exit 1

# get a token, set up variables
token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]')
token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]')
dataset="https://doi.example/ty009.sfrrss/600.45asasga"
file="dummy_data"
expected_size=1048605
C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g')
export C4GH_PASSPHRASE

# download decrypted full file, check file size
curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
file_size=$(stat -c %s full1.bam) # Get the size of the file

if [ "$file_size" -ne "$expected_size" ]; then
Expand All @@ -24,9 +25,11 @@ if [ "$file_size" -ne "$expected_size" ]; then
fi

# test that start, end=0 returns the whole file
curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam

if ! cmp --silent full1.bam full2.bam; then
echo "Full decrypted files, with and without coordinates, are different"
exit 1
fi

echo "OK"
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -e

if [ "$STORAGETYPE" = s3notls ]; then
exit 0
Expand All @@ -7,7 +8,7 @@ fi
cd dev_utils || exit 1

# Get a token, set up variables
token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]')
token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]')

if [ -z "$token" ]; then
echo "Failed to obtain token"
Expand All @@ -19,7 +20,7 @@ file="dummy_data"
expected_size=1048605

# Download unencrypted full file, check file size
curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam

if [ ! -f "full1.bam" ]; then
echo "Failed to download full1.bam"
Expand All @@ -36,7 +37,7 @@ fi
# Test reencrypt the file header with the client public key
clientkey=$(base64 -w0 client.pub.pem)
reencryptedFile=reencrypted.bam.c4gh
curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file" --output $reencryptedFile
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file" --output $reencryptedFile
if [ ! -f "$reencryptedFile" ]; then
echo "Failed to download re-encrypted file"
exit 1
Expand All @@ -48,13 +49,15 @@ if [ "$file_size" -ne "$expected_encrypted_size" ]; then
echo "Incorrect file size for the re-encrypted file, should be $expected_encrypted_size but is $file_size"
exit 1
fi

# Decrypt the reencrypted file and compare it with the original unencrypted file
export C4GH_PASSPHRASE="strongpass" # passphrase for the client crypt4gh key
if ! crypt4gh decrypt --sk client.sec.pem < $reencryptedFile > full2.bam; then
crypt4gh decrypt -s client.sec.pem -f $reencryptedFile
if [ ! -f "${reencryptedFile%.c4gh}" ] ; then
echo "Failed to decrypt re-encrypted file with the client's private key"
exit 1
fi
mv "${reencryptedFile%.c4gh}" full2.bam


if ! cmp --silent full1.bam full2.bam; then
echo "Decrypted version of $reencryptedFile and the original unencrypted file, are different"
Expand All @@ -63,7 +66,7 @@ fi

# download reencrypted partial file, check file size
partReencryptedFile=part1.bam.c4gh
curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile
curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3-encrypted/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile
file_size=$(stat -c %s $partReencryptedFile) # Get the size of the file
part_expected_size=65688

Expand All @@ -72,7 +75,8 @@ if [ "$file_size" -ne "$part_expected_size" ]; then
exit 1
fi

if ! crypt4gh decrypt --sk client.sec.pem < $partReencryptedFile > part1.bam; then
crypt4gh decrypt -s client.sec.pem -f $partReencryptedFile
if [ ! -f "${partReencryptedFile%.c4gh}" ] ; then
echo "Re-encrypted partial file could not be decrypted"
exit 1
fi
Expand Down Expand Up @@ -106,3 +110,5 @@ resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-P
if [ "$resp" -ne 500 ]; then
echo "Incorrect response with missing public key, expected 500 got $resp"
fi

echo "OK"

0 comments on commit 5ad5768

Please sign in to comment.