diff --git a/.github/integration/scripts/make_db_credentials.sh b/.github/integration/scripts/make_db_credentials.sh index 5d2e40e7d..bef674cb2 100644 --- a/.github/integration/scripts/make_db_credentials.sh +++ b/.github/integration/scripts/make_db_credentials.sh @@ -4,7 +4,7 @@ set -e apt-get -o DPkg::Lock::Timeout=60 update > /dev/null apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client >/dev/null -for n in download finalize inbox ingest mapper sync verify; do +for n in api download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" psql -U postgres -h migrate -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 36fa29f8e..3b4289313 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -14,7 +14,7 @@ apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl post pip install --upgrade pip > /dev/null pip install aiohttp Authlib joserfc requests > /dev/null -for n in download finalize inbox ingest mapper sync verify; do +for n in api download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" psql -U postgres -h postgres -d sda -c "GRANT base TO $n;" @@ -92,4 +92,4 @@ if [ ! -f "/shared/grpcurl" ]; then echo "downloading grpcurl" latest_grpculr=$(curl --retry 100 -sL https://api.github.com/repos/fullstorydev/grpcurl/releases/latest | jq -r '.name' | sed -e 's/v//') curl --retry 100 -s -L "https://github.com/fullstorydev/grpcurl/releases/download/v${latest_grpculr}/grpcurl_${latest_grpculr}_linux_x86_64.tar.gz" | tar -xz -C /shared/ && chmod +x /shared/grpcurl -fi \ No newline at end of file +fi diff --git a/.github/integration/sda-posix-integration.yml b/.github/integration/sda-posix-integration.yml index fe88f0563..1433096cc 100644 --- a/.github/integration/sda-posix-integration.yml +++ b/.github/integration/sda-posix-integration.yml @@ -182,7 +182,7 @@ services: container_name: tester command: - "bash" - - "/tests/sda/10_upload_test.sh" + - "/tests/sda/10.1_upload_test.sh" depends_on: inbox: condition: service_started diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 71f141090..bedb2577e 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -289,11 +289,10 @@ services: rabbitmq: condition: service_healthy environment: - - BROKER_PASSWORD=ingest - - BROKER_USER=ingest - - BROKER_ROUTINGKEY=ingest - - DB_PASSWORD=download - - DB_USER=download + - BROKER_PASSWORD=api + - BROKER_USER=api + - DB_PASSWORD=api + - DB_USER=api image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8090:8080" diff --git a/.github/integration/tests/sda/01_install_dependencies.sh b/.github/integration/tests/sda/01_install_dependencies.sh new file mode 100644 index 000000000..9020696c1 --- /dev/null +++ b/.github/integration/tests/sda/01_install_dependencies.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -e + +# install tools if missing +for t in curl expect jq openssh-client postgresql-client xxd; do + if [ ! "$(command -v $t)" ]; then + if [ "$(id -u)" != 0 ]; then + echo "$t is missing, unable to install it" + exit 1 + fi + + apt-get -o DPkg::Lock::Timeout=60 update >/dev/null + apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null + fi +done diff --git a/.github/integration/tests/sda/09_healthchecks.sh b/.github/integration/tests/sda/09_healthchecks.sh index f1dc369c8..c58c84c8e 100644 --- a/.github/integration/tests/sda/09_healthchecks.sh +++ b/.github/integration/tests/sda/09_healthchecks.sh @@ -1,20 +1,6 @@ #!/bin/sh set -e -# install tools if missing -for t in curl jq ; do - if [ ! "$(command -v $t)" ]; then - if [ "$(id -u)" != 0 ]; then - echo "$t is missing, unable to install it" - exit 1 - fi - - apt-get -o DPkg::Lock::Timeout=60 update >/dev/null - apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null - fi -done - - # Test the s3inbox's healthchecks, GET /health and HEAD / response="$(curl -s -k -LI "http://s3inbox:8000" -o /dev/null -w "%{http_code}\n")" if [ "$response" != "200" ]; then diff --git a/.github/integration/tests/sda/10.1_upload_test.sh b/.github/integration/tests/sda/10.1_upload_test.sh new file mode 100644 index 000000000..d7fcea9c6 --- /dev/null +++ b/.github/integration/tests/sda/10.1_upload_test.sh @@ -0,0 +1,86 @@ +#!/bin/sh +set -e + +if [ -z "$STORAGETYPE" ]; then + echo "STORAGETYPE not set, exiting" + exit 1 +fi + +if [ "$STORAGETYPE" = "s3" ]; then + exit 0 +fi + +for t in curl jq openssh-client postgresql-client; do + if [ ! "$(command -v $t)" ]; then + if [ "$(id -u)" != 0 ]; then + echo "$t is missing, unable to install it" + exit 1 + fi + + apt-get -o DPkg::Lock::Timeout=60 update >/dev/null + apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null + fi +done + +cd shared || true + +## verify that messages exists in MQ +URI=http://rabbitmq:15672 +if [ -n "$PGSSLCERT" ]; then + URI=https://rabbitmq:15671 +fi +## empty all queues ## +for q in accession archived backup completed inbox ingest mappings verified; do + curl -s -k -u guest:guest -X DELETE "$URI/api/queues/sda/$q/contents" +done +## truncate database +psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;" + +if [ "$STORAGETYPE" = "posix" ]; then + for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do + echo "downloading $file" + curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" + if [ ! -f "$file.c4gh" ]; then + yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" + fi + + sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF + put "${file}" + dir + ls -al + exit +EOF + done + + ## reupload a file under a different name + sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF + put NA12878.bam.c4gh NB12878.bam.c4gh + dir + ls -al + exit +EOF + + ## reupload a file with the same name + sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF + put NA12878.bam.c4gh + dir + ls -al + exit +EOF + +fi + +echo "waiting for upload to complete" +RETRY_TIMES=0 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for upload to complete" + exit 1 + fi + sleep 2 +done + + +echo "files uploaded successfully" diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh index 266f73c7b..e20939cf4 100644 --- a/.github/integration/tests/sda/10_upload_test.sh +++ b/.github/integration/tests/sda/10_upload_test.sh @@ -6,19 +6,6 @@ if [ -z "$STORAGETYPE" ]; then exit 1 fi -# install tools if missing -for t in curl expect jq openssh-client postgresql-client; do - if [ ! "$(command -v $t)" ]; then - if [ "$(id -u)" != 0 ]; then - echo "$t is missing, unable to install it" - exit 1 - fi - - apt-get -o DPkg::Lock::Timeout=60 update >/dev/null - apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null - fi -done - cd shared || true ## verify that messages exists in MQ @@ -33,57 +20,22 @@ done ## truncate database psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;" -if [ "$STORAGETYPE" = "posix" ]; then - for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do - echo "downloading $file" - curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" - if [ ! -f "$file.c4gh" ]; then - yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" - fi - - sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF - put "${file}" - dir - ls -al - exit -EOF - done - - ## reupload a file under a different name - sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF - put NA12878.bam.c4gh NB12878.bam.c4gh - dir - ls -al - exit -EOF - - ## reupload a file with the same name - sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF - put NA12878.bam.c4gh - dir - ls -al - exit -EOF - -fi +pip -q install s3cmd -if [ "$STORAGETYPE" = "s3" ]; then - pip -q install s3cmd +for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do + curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" + if [ ! -f "$file.c4gh" ]; then + yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" + fi + s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/ +done - for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do - curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" - if [ ! -f "$file.c4gh" ]; then - yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" - fi - s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/ - done +## reupload a file under a different name +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh - ## reupload a file under a different name - s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh +## reupload a file with the same name +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ - ## reupload a file with the same name - s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ -fi echo "waiting for upload to complete" RETRY_TIMES=0 @@ -97,51 +49,50 @@ until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messag sleep 2 done -if [ "$STORAGETYPE" = "s3" ]; then - num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") - if [ "$num_rows" -ne 5 ]; then - echo "database queries for register_files failed, expected 5 got $num_rows" - exit 1 - fi +num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") +if [ "$num_rows" -ne 5 ]; then + echo "database queries for register_files failed, expected 5 got $num_rows" + exit 1 +fi - num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") - if [ "$num_log_rows" -ne 12 ]; then - echo "database queries for file_event_logs failed, expected 12 got $num_log_rows" - exit 1 - fi +num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") +if [ "$num_log_rows" -ne 12 ]; then + echo "database queries for file_event_logs failed, expected 12 got $num_log_rows" + exit 1 +fi - ## test with token from OIDC service - echo "testing with OIDC token" - newToken=$(curl http://oidc:8080/tokens | jq '.[0]') - cp s3cfg oidc_s3cfg - sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg +## test with token from OIDC service +echo "testing with OIDC token" +newToken=$(curl http://oidc:8080/tokens | jq '.[0]') +cp s3cfg oidc_s3cfg +sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg - s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh +s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh - ## verify that messages exists in MQ +## verify that messages exists in MQ +echo "waiting for upload to complete" +RETRY_TIMES=0 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do echo "waiting for upload to complete" - RETRY_TIMES=0 - until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do - echo "waiting for upload to complete" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - echo "::error::Time out while waiting for upload to complete" - exit 1 - fi - sleep 2 - done - - num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") - if [ "$num_rows" -ne 6 ]; then - echo "database queries for register_files failed, expected 6 got $num_rows" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for upload to complete" exit 1 fi + sleep 2 +done - num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") - if [ "$num_log_rows" -ne 14 ]; then - echo "database queries for file_event_logs failed, expected 14 got $num_log_rows" - exit 1 - fi +num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") +if [ "$num_rows" -ne 6 ]; then + echo "database queries for register_files failed, expected 6 got $num_rows" + exit 1 fi +num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") +if [ "$num_log_rows" -ne 14 ]; then + echo "database queries for file_event_logs failed, expected 14 got $num_log_rows" + exit 1 +fi + + echo "files uploaded successfully" diff --git a/.github/integration/tests/sda/11_api-getfiles_test.sh b/.github/integration/tests/sda/11_api-getfiles_test.sh index fb168da50..a99bb9eaa 100644 --- a/.github/integration/tests/sda/11_api-getfiles_test.sh +++ b/.github/integration/tests/sda/11_api-getfiles_test.sh @@ -3,11 +3,39 @@ set -e # Test the API files endpoint token="$(curl http://oidc:8080/tokens | jq -r '.[0]')" -curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token" -response="$(curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')" +response="$(curl -s -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')" if [ "$response" != "uploaded" ]; then echo "API returned incorrect value, expected ready got: $response" exit 1 fi +# test inserting a c4gh public key hash +payload=$( + jq -c -n \ + --arg description "this is the key description" \ + --arg pubkey "$( base64 -w0 /shared/c4gh.pub.pem)" \ + '$ARGS.named' +) + +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")" +if [ "$resp" != "200" ]; then + echo "Error when adding a public key hash, expected 200 got: $resp" + exit 1 +fi + +# again to verify we get an error +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")" +if [ "$resp" != "409" ]; then + echo "Error when adding a public key hash, expected 409 got: $resp" + exit 1 +fi + +manual_hash=$(sed -n '2p' /shared/c4gh.pub.pem | base64 -d -w0 | xxd -c64 -ps) + +db_hash=$(psql -U postgres -h postgres -d sda -At -c "SELECT key_hash FROM sda.encryption_keys WHERE description = 'this is the key description';") +if [ "$db_hash" != "$manual_hash" ]; then + echo "wrong hash in the database, expected $manual_hash got $db_hash" + exit 1 +fi + echo "api test completed successfully" diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index 5cb5c9807..1eadb112b 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -158,10 +158,31 @@ GRANT SELECT ON local_ega_ebi.file_dataset TO download; -------------------------------------------------------------------------------- +CREATE ROLE api; + +GRANT USAGE ON SCHEMA sda TO api; +GRANT SELECT ON sda.files TO api; +GRANT SELECT ON sda.file_dataset TO api; +GRANT SELECT ON sda.checksums TO api; +GRANT SELECT ON sda.file_event_log TO api; +GRANT SELECT ON sda.encryption_keys TO api; +GRANT SELECT ON sda.datasets TO api; +GRANT SELECT ON sda.dataset_event_log TO api; +GRANT INSERT ON sda.encryption_keys TO api; +GRANT UPDATE ON sda.encryption_keys TO api; + +-- legacy schema +GRANT USAGE ON SCHEMA local_ega TO api; +GRANT USAGE ON SCHEMA local_ega_ebi TO api; +GRANT SELECT ON local_ega.files TO api; +GRANT SELECT ON local_ega_ebi.file TO api; +GRANT SELECT ON local_ega_ebi.file_dataset TO api; + +-------------------------------------------------------------------------------- -- lega_in permissions -GRANT base, ingest, verify, finalize, sync TO lega_in; +GRANT base, ingest, verify, finalize, sync, api TO lega_in; -- lega_out permissions -GRANT mapper, download TO lega_out; +GRANT mapper, download, api TO lega_out; -GRANT base TO download, inbox, ingest, finalize, mapper, verify +GRANT base TO api, download, inbox, ingest, finalize, mapper, verify diff --git a/postgresql/migratedb.d/13.sql b/postgresql/migratedb.d/13.sql new file mode 100644 index 000000000..13f07a047 --- /dev/null +++ b/postgresql/migratedb.d/13.sql @@ -0,0 +1,58 @@ +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 12; + changes VARCHAR := 'Create API user'; +BEGIN +-- No explicit transaction handling here, this all happens in a transaction +-- automatically + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + -- Temporary function for creating roles if they do not already exist. + CREATE FUNCTION create_role_if_not_exists(role_name NAME) RETURNS void AS $created$ + BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = role_name) THEN + RAISE NOTICE 'Role "%" already exists. Skipping.', role_name; + ELSE + BEGIN + EXECUTE format('CREATE ROLE %I', role_name); + EXCEPTION + WHEN duplicate_object THEN + RAISE NOTICE 'Role "%" was just created by a concurrent transaction. Skipping.', role_name; + END; + END IF; + END; + $created$ LANGUAGE plpgsql; + + PERFORM create_role_if_not_exists('api'); + GRANT USAGE ON SCHEMA sda TO api; + GRANT SELECT ON sda.files TO api; + GRANT SELECT ON sda.file_event_log TO api; + GRANT SELECT ON sda.file_dataset TO api; + GRANT SELECT ON sda.checksums TO api; + GRANT SELECT ON sda.datasets TO api; + GRANT SELECT ON sda.dataset_event_log TO api; + GRANT SELECT, INSERT, UPDATE ON sda.encryption_keys TO api; + GRANT USAGE ON SCHEMA local_ega TO api; + GRANT USAGE ON SCHEMA local_ega_ebi TO api; + GRANT SELECT ON local_ega.files TO api; + GRANT SELECT ON local_ega_ebi.file TO api; + GRANT SELECT ON local_ega_ebi.file_dataset TO api; + + GRANT base TO api, download, inbox, ingest, finalize, mapper, verify; + + -- Drop temporary user creation function + DROP FUNCTION create_role_if_not_exists; + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 4dde5b1fe..b0efe1476 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -1,8 +1,11 @@ package main import ( + "bytes" "context" "crypto/tls" + "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -15,6 +18,7 @@ import ( "github.com/gin-gonic/gin" "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" @@ -23,9 +27,11 @@ import ( log "github.com/sirupsen/logrus" ) -var Conf *config.Config -var err error -var auth *userauth.ValidateFromToken +var ( + Conf *config.Config + err error + auth *userauth.ValidateFromToken +) func main() { Conf, err = config.NewConfig("api") @@ -80,6 +86,7 @@ func setup(config *config.Config) *http.Server { r.POST("/file/accession", isAdmin(), setAccession) // assign accession ID to a file r.POST("/dataset/create", isAdmin(), createDataset) // maps a set of files to a dataset r.POST("/dataset/release/*dataset", isAdmin(), releaseDataset) // Releases a dataset to be accessible + r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash) // Adds a key hash to the database r.GET("/users", isAdmin(), listActiveUsers) // Lists all users r.GET("/users/:username/files", isAdmin(), listUserFiles) // Lists all unmapped files for a user } @@ -93,7 +100,7 @@ func setup(config *config.Config) *http.Server { TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), ReadHeaderTimeout: 20 * time.Second, ReadTimeout: 5 * time.Minute, - WriteTimeout: 20 * time.Second, + WriteTimeout: 2 * time.Minute, } return srv @@ -414,3 +421,83 @@ func listUserFiles(c *gin.Context) { c.Writer.Header().Set("Content-Type", "application/json") c.JSON(200, files) } + +// addC4ghHash handles the addition of a hashed public key to the database. +// It expects a JSON payload containing the base64 encoded public key and its description. +// If the JSON payload is invalid, it responds with a 400 Bad Request status. +// If the hash is already in the database, it responds with a 409 Conflict status +// If the database insertion fails, it responds with a 500 Internal Server Error status. +// On success, it responds with a 200 OK status. +func addC4ghHash(c *gin.Context) { + var c4gh schema.C4ghPubKey + if err := c.BindJSON(&c4gh); err != nil { + c.AbortWithStatusJSON( + http.StatusBadRequest, + gin.H{ + "error": "json decoding : " + err.Error(), + "status": http.StatusBadRequest, + }, + ) + + log.Errorf("Invalid JSON payload: %v", err) + + return + } + + b64d, err := base64.StdEncoding.DecodeString(c4gh.PubKey) + if err != nil { + c.AbortWithStatusJSON( + http.StatusBadRequest, + gin.H{ + "error": "base64 decoding : " + err.Error(), + "status": http.StatusBadRequest, + }, + ) + + log.Errorf("Invalid JSON payload: %v", err) + + return + } + + pubKey, err := keys.ReadPublicKey(bytes.NewReader(b64d)) + if err != nil { + c.AbortWithStatusJSON( + http.StatusBadRequest, + gin.H{ + "error": "not a public key : " + err.Error(), + "status": http.StatusBadRequest, + }, + ) + + log.Errorf("Invalid JSON payload: %v", err) + + return + } + + err = Conf.API.DB.AddKeyHash(hex.EncodeToString(pubKey[:]), c4gh.Description) + if err != nil { + if strings.Contains(err.Error(), "key hash already exists") { + c.AbortWithStatusJSON( + http.StatusConflict, + gin.H{ + "error": err.Error(), + "status": http.StatusConflict, + }, + ) + log.Error("Key hash already exists") + } else { + c.AbortWithStatusJSON( + http.StatusInternalServerError, + gin.H{ + "error": err.Error(), + "status": http.StatusInternalServerError, + }, + ) + log.Errorf("Database insertion failed: %v", err) + } + + return + } + + c.Status(http.StatusOK) +} diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 80854d83a..fa96b4aeb 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -119,6 +119,23 @@ Admin endpoints are only available to a set of whitelisted users specified in th - `401` Token user is not in the list of admins. - `500` Internal error due to DB failure. +- `/c4gh-keys/add` + - accepts `POST` requests with the hex hash of the key and its description + - registers the key hash in the database. + + - Error codes + - `200` Query execute ok. + - `400` Error due to bad payload. + - `401` Token user is not in the list of admins. + - `409` Key hash already exists in the database. + - `500` Internal error due to DB failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /PATH/TO/c4gh.pub)"'", "description": "this is the key description"}' https://HOSTNAME/c4gh-keys/add + ``` + #### Configure Admin users The users that should have administrative access can be set in two ways: diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 67eff0872..9332893ae 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -25,6 +25,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" + "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" log "github.com/sirupsen/logrus" @@ -1281,3 +1282,95 @@ func (suite *TestSuite) TestListUserFiles() { assert.NoError(suite.T(), err, "failed to list users from DB") assert.Equal(suite.T(), 2, len(files)) } + +func (suite *TestSuite) TestAddC4ghHash() { + gin.SetMode(gin.ReleaseMode) + assert.NoError(suite.T(), setupJwtAuth()) + Conf.API.Admins = []string{"dummy"} + + r := gin.Default() + r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash) + ts := httptest.NewServer(r) + defer ts.Close() + + client := &http.Client{} + assert.NoError(suite.T(), setupJwtAuth()) + + // Create a valid request body + keyhash := schema.C4ghPubKey{ + PubKey: "LS0tLS1CRUdJTiBDUllQVDRHSCBQVUJMSUMgS0VZLS0tLS0KdWxGRUF6SmZZNEplUEVDZWd3YmJrVVdLNnZ2SE9SWStqMTRGdVpWVnYwND0KLS0tLS1FTkQgQ1JZUFQ0R0ggUFVCTElDIEtFWS0tLS0tCg==", + Description: "Test key description", + } + body, err := json.Marshal(keyhash) + assert.NoError(suite.T(), err) + + req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) + assert.NoError(suite.T(), err) + req.Header.Add("Authorization", "Bearer "+suite.Token) + req.Header.Add("Content-Type", "application/json") + + resp, err := client.Do(req) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + defer resp.Body.Close() + + // Isert pubkey again and expect error + resp2, err := client.Do(req) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), http.StatusConflict, resp2.StatusCode) + defer resp2.Body.Close() +} + +func (suite *TestSuite) TestAddC4ghHash_emptyJson() { + gin.SetMode(gin.ReleaseMode) + assert.NoError(suite.T(), setupJwtAuth()) + Conf.API.Admins = []string{"dummy"} + + r := gin.Default() + r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash) + ts := httptest.NewServer(r) + defer ts.Close() + + client := &http.Client{} + assert.NoError(suite.T(), setupJwtAuth()) + + // Create an invalid request body + body := []byte(`{"invalid_json"}`) + + req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) + assert.NoError(suite.T(), err) + req.Header.Add("Authorization", "Bearer "+suite.Token) + req.Header.Add("Content-Type", "application/json") + + resp, err := client.Do(req) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + defer resp.Body.Close() +} + +func (suite *TestSuite) TestAddC4ghHash_notBase64() { + gin.SetMode(gin.ReleaseMode) + assert.NoError(suite.T(), setupJwtAuth()) + Conf.API.Admins = []string{"dummy"} + + r := gin.Default() + r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash) + ts := httptest.NewServer(r) + defer ts.Close() + + client := &http.Client{} + assert.NoError(suite.T(), setupJwtAuth()) + + // Create an invalid request body + body := []byte(`{"pubkey": "asdkjsahfd=", "decription": ""}`) + + req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) + assert.NoError(suite.T(), err) + req.Header.Add("Authorization", "Bearer "+suite.Token) + req.Header.Add("Content-Type", "application/json") + + resp, err := client.Do(req) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + defer resp.Body.Close() +} diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 11abc15e2..f9cf2eb66 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "math" + "strings" "time" log "github.com/sirupsen/logrus" @@ -671,7 +672,7 @@ func (dbs *SDAdb) GetCorrID(user, path string) (string, error) { // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { corrID, err = dbs.getCorrID(user, path) - if err == nil { + if err == nil || strings.Contains(err.Error(), "sql: no rows in result set") { break } time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) @@ -748,3 +749,36 @@ func (dbs *SDAdb) getDatasetStatus(datasetID string) (string, error) { return status, nil } + +// AddKeyHash adds a key hash and key description in the encryption_keys table +func (dbs *SDAdb) AddKeyHash(keyHash, keyDescription string) error { + var err error + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + err = dbs.addKeyHash(keyHash, keyDescription) + if err == nil || strings.Contains(err.Error(), "key hash already exists") { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return err +} + +func (dbs *SDAdb) addKeyHash(keyHash, keyDescription string) error { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const query = "INSERT INTO sda.encryption_keys(key_hash, description) VALUES($1, $2) ON CONFLICT DO NOTHING;" + + result, err := db.Exec(query, keyHash, keyDescription) + if err != nil { + return err + } + + if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { + return errors.New("key hash already exists or no rows were updated") + } + + return nil +} diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 5ce59d4b6..2b2235736 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -593,7 +593,13 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { assert.Equal(suite.T(), fileID, corrID) checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999} + fileInfo := FileInfo{ + fmt.Sprintf("%x", sha256.New().Sum(nil)), + 1234, + filePath, + checksum, + 999, + } err = db.SetArchived(fileInfo, fileID, corrID) if err != nil { suite.FailNow("failed to mark file as Archived") @@ -634,3 +640,20 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { assert.NoError(suite.T(), err, "got (%v) when no error weas expected") assert.Equal(suite.T(), "deprecated", status) } + +func (suite *DatabaseTests) TestAddKeyHash() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // Test registering a new key and its description + keyHex := `cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23` + keyDescription := "this is a test key" + err = db.AddKeyHash(keyHex, keyDescription) + assert.NoError(suite.T(), err, "failed to register key in database") + + // Verify that the key was added + var exists bool + err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.encryption_keys WHERE key_hash=$1 AND description=$2)", keyHex, keyDescription).Scan(&exists) + assert.NoError(suite.T(), err, "failed to verify key hash existence") + assert.True(suite.T(), exists, "key hash was not added to the database") +} diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go index c01440469..503e7e308 100644 --- a/sda/internal/schema/schema.go +++ b/sda/internal/schema/schema.go @@ -179,3 +179,8 @@ type SyncMetadata struct { type Metadata struct { Metadata interface{} } + +type C4ghPubKey struct { + PubKey string `json:"pubkey"` + Description string `json:"description"` +}