diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 96e48cc36..ff402dc7e 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -5,16 +5,18 @@ labels: bug --- -**Describe the bug** +## Describe the bug -**Steps to reproduce** +## Steps to reproduce -**Expected behavior** +## Expected behavior - [ ] - [ ] Tests verifying the fix are added -**Additional context** +## Additional context -**Estimation of size**: small/medium/big +## Estimation of size +small/medium/big -**Estimation of priority**: low/medium/high +## Estimation of priority +low/medium/high diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md index 82273c659..59d23b5a0 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md @@ -4,15 +4,21 @@ about: Suggest an idea for this project --- -**Please describe the feature** +## Please describe the feature + As a [type of user], I want [an action] so that [a benefit/a value]. -**Acceptance criteria** +## Acceptance criteria + - [ ] - [ ] Tests verifying the changes are added -**Additional context** +## Additional context + +## Estimation of size + +small/medium/big -**Estimation of size**: small/medium/big +## Estimation of priority -**Estimation of priority**: low/medium/high +low/medium/high diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b1680c226..9a3333b6b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,8 +1,8 @@ -**Related issue(s) and PR(s)** +## Related issue(s) and PR(s) This PR closes [issue number]. -**Description** +## Description -**How to test** +## How to test diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 9bcb2f8cb..0bfb3f1f5 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -101,4 +101,113 @@ updates: interval: weekly open-pull-requests-limit: 10 reviewers: - - "neicnordic/sensitive-data-development-collaboration" \ No newline at end of file + - "neicnordic/sensitive-data-development-collaboration" + +## release v1 branch +### Docker + - package-ecosystem: docker + target-branch: release_v1 + directory: "/postgresql" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/rabbitmq" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda-doa" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda-download" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda-sftp-inbox" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + +### JAVA + - package-ecosystem: maven + target-branch: release_v1 + directory: "/sda-doa" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily + + - package-ecosystem: maven + target-branch: release_v1 + directory: "/sda-sftp-inbox" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily +### GO + - package-ecosystem: gomod + target-branch: release_v1 + directory: "/sda-download" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily + + - package-ecosystem: gomod + target-branch: release_v1 + directory: "/sda" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 688ea7dea..59ecdff77 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -52,6 +52,7 @@ services: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" + - "5672:5672" restart: always volumes: - rabbitmq_data:/var/lib/rabbitmq @@ -220,6 +221,8 @@ services: depends_on: credentials: condition: service_completed_successfully + extra_hosts: + - "localhost:host-gateway" healthcheck: test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] interval: 10s @@ -330,8 +333,12 @@ services: - AUTH_RESIGNJWT=false - OIDC_ID=XC56EL11xx - OIDC_SECRET=wHPVQaYXmdDHg + - OIDC_PROVIDER=http://localhost:8080 + - OIDC_REDIRECTURL=http://localhost:8889/oidc/login - DB_PASSWORD=auth - DB_USER=auth + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8889:8080" @@ -369,6 +376,8 @@ services: condition: service_started reencrypt: condition: service_started + extra_hosts: + - "localhost:host-gateway" environment: - PGPASSWORD=rootpasswd - STORAGETYPE=s3 diff --git a/.github/integration/sda/oidc.py b/.github/integration/sda/oidc.py index c7969b136..a6b387af9 100644 --- a/.github/integration/sda/oidc.py +++ b/.github/integration/sda/oidc.py @@ -51,7 +51,7 @@ def _generate_token() -> Tuple: # See available claims here: http://www.iana.org/assignments/jwt/jwt.xhtml # the important claim is the "authorities" header = { - "jku": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "jku": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "alg": "ES256", "typ": "JWT", "kid": ec_key1.thumbprint() @@ -61,7 +61,7 @@ def _generate_token() -> Tuple: "aud": ["aud1", "aud2"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", @@ -71,21 +71,21 @@ def _generate_token() -> Tuple: "aud": ["aud2", "aud3"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } empty_payload = { "sub": "requester@demo.org", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 99999999999, "iat": 1547794655, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } # Craft passports passport_terms = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "AcceptedTermsAndPolicies", @@ -100,7 +100,7 @@ def _generate_token() -> Tuple: } # passport for dataset permissions 1 passport_dataset1 = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -165,12 +165,12 @@ def _generate_token() -> Tuple: async def fixed_response(request: web.Request) -> web.Response: global HTTP_PROTOCOL WELL_KNOWN = { - "issuer": f"{HTTP_PROTOCOL}://oidc:8080", - "authorization_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/authorize", - "registration_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/register", - "token_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/token", - "userinfo_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/userinfo", - "jwks_uri": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "issuer": f"{HTTP_PROTOCOL}://localhost:8080", + "authorization_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/authorize", + "registration_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/register", + "token_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/token", + "userinfo_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/userinfo", + "jwks_uri": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "response_types_supported": [ "code", "id_token", diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index a4a4ca394..a76d6c849 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -30,6 +30,11 @@ "path": "/file/accession", "action": "POST" }, + { + "role": "submission", + "path": "/file/*", + "action": "DELETE" + }, { "role": "submission", "path": "/users", diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 8478afd21..cecf5a86c 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -1,11 +1,13 @@ #!/bin/sh set -e +cd shared || true token="$(curl http://oidc:8080/tokens | jq -r '.[0]')" +# Upload a file and make sure it's listed result="$(curl -sk -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq '. | length')" if [ "$result" -ne 2 ]; then echo "wrong number of files returned for user test@dummy.org" - echo "expected 4 got $result" + echo "expected 2 got $result" exit 1 fi @@ -35,4 +37,122 @@ resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer if [ "$resp" != "404" ]; then echo "Error when starting re-verification of missing dataset, expected 404 got: $resp" exit 1 +fi + +# Reupload a file under a different name, test to delete it +s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh + +echo "waiting for upload to complete" +URI=http://rabbitmq:15672 +if [ -n "$PGSSLCERT" ]; then + URI=https://rabbitmq:15671 +fi +RETRY_TIMES=0 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 4 ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for upload to complete" + exit 1 + fi + sleep 2 +done + +# get the fileId of the new file +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" + +output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) +if [ -z "$output" ] ; then + echo "Uploaded file not in inbox" + exit 1 +fi +# delete it +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +if [ "$resp" != "200" ]; then + echo "Error when deleting the file, expected 200 got: $resp" + exit 1 +fi + +last_event=$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id='$fileid' order by started_at desc limit 1;") + +if [ "$last_event" != "disabled" ]; then + echo "The file $fileid does not have the expected las event 'disabled', but $last_event." +fi + +output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) +if [ -n "$output" ] ; then + echo "Deleted file is still in inbox" + exit 1 +fi + +# Try to delete an unknown file +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/badfileid")" +if [ "$resp" != "404" ]; then + echo "Error when deleting the file, expected error got: $resp" + exit 1 +fi + + +# Try to delete file of other user +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/requester@demo.org/files" | jq -r '.[0]| .fileID')" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +if [ "$resp" != "404" ]; then + echo "Error when deleting the file, expected 404 got: $resp" + exit 1 +fi + + +# Re-upload the file and use the api to ingest it, then try to delete it +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NE12878.bam.c4gh + +URI=http://rabbitmq:15672 +if [ -n "$PGSSLCERT" ]; then + URI=https://rabbitmq:15671 +fi +RETRY_TIMES=0 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 3 ]; then + echo "::error::Time out while waiting for upload to complete" + #exit 1 + break + fi + sleep 2 +done + +# Ingest it +new_payload=$( +jq -c -n \ + --arg filepath "test_dummy.org/NE12878.bam.c4gh" \ + --arg user "test@dummy.org" \ + '$ARGS.named' +) + +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$new_payload" "http://api:8080/file/ingest")" +if [ "$resp" != "200" ]; then + echo "Error when requesting to ingesting file, expected 200 got: $resp" + exit 1 +fi + +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +# wait for the fail to get the correct status +RETRY_TIMES=0 + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_events e where e.title in (select event from sda.file_event_log where file_id = '$fileid' order by started_at desc limit 1);")" -gt 30 ]; do + echo "waiting for ingest to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for ingest to complete" + exit 1 + fi + sleep 2 +done + +# Try to delete file not in inbox +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +if [ "$resp" != "404" ]; then + echo "Error when deleting the file, expected 404 got: $resp" + exit 1 fi \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 25ea74ffd..5358e3bcb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.0.7 + uses: codecov/codecov-action@v5.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.0.7 + uses: codecov/codecov-action@v5.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.0.7 + uses: codecov/codecov-action@v5.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..be6bdc999 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,101 @@ +# Run services with `go run` + +This section explains how to run some of the services using `go run` instead of the Docker setup to facilitate development. + +## Running `sda-download` with `go run` + +- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: + +```sh +make integrationtest-sda-s3-run +``` + +- Change to the folder `sda-download` and start the `sda-download` service using: + +```sh +CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go +``` + +- Check if `sda-download` works as expected using: + +```sh +curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health +``` + +If successful, the curl command should output the HTTP code `200`. + +You can further check the endpoint `/metadata/datasets` using: + +```sh +token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets +``` + +If successful, the curl command should output a JSON body containing: + +```json +["EGAD74900000101"] +``` + +## Running other SDA services with `go run` + +Running any of the SDA services located in the `sda` subfolder requires that the service specific credentials and RabbitMQ configurations are set as ENVs. Here, we'll use `ingest` as an example. + +- Bring up all SDA services with the S3 backend by running the following command in the root folder of the repository: + +```sh +make sda-s3-up +``` + +- When the previous command is finished, bring down the `ingest` service using: + +```sh +docker stop ingest +``` + +- Copy keys and other information from the shared folder of the container using: + +```sh +docker cp verify:/shared /tmp/ +``` + +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the auto generated files that will be required. + +- Change to the folder `sda` and start the `ingest` service using: + +```sh +export BROKER_PASSWORD=ingest +export BROKER_USER=ingest +export BROKER_QUEUE=ingest +export BROKER_ROUTINGKEY=archived +export DB_PASSWORD=ingest +export DB_USER=ingest +CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go +``` + +- Check if the `ingest` service works as expected by following these steps + +```sh +# create a test file +seq 10 > /tmp/t1.txt + +# update the s3cmd config file +sed -i '/host_/s/s3inbox:8000/localhost:18000/g' /tmp/shared/s3cfg + +# upload /tmp/t1.txt to s3inbox by sda-cli +sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt + +# use sda-admin to check if t1.txt has been uploaded +export API_HOST=http://localhost:8090 +export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'uploaded' + +# register the Crypt4GH key +curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add + +# use sda-admin to ingest the file t1.txt +sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org + +# verify that t1.txt has been ingested using sda-admin +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'verified' +``` diff --git a/GETTINGSTARTED.md b/GETTINGSTARTED.md deleted file mode 100644 index cb86ff5de..000000000 --- a/GETTINGSTARTED.md +++ /dev/null @@ -1,93 +0,0 @@ -## Getting Started developing components of the SDA stack - -Should one wish to engage in the development of the SDA stack itself, the prerequisite is the installation of [Go](https://www.golang.org/) on the respective machine. -The recommended version can be checked by running: - -```sh -$ make go-version-check -... -``` - -In preparation for local development, it is essential to verify the proper installation of Go, including the establishment of a [GOPATH](https://golang.org/doc/code.html#GOPATH). Confirm that $GOPATH/bin is included in the system's path, as certain distributions may package outdated versions of build tools. Subsequently, proceed to clone the repository. SDA employs [Go Modules](https://github.com/golang/go/wiki/Modules), and it is advisable to perform the cloning operation outside the GOPATH. Following this, obtain any necessary build tools by initializing the environment through bootstrapping: - -```sh -$ make bootstrap -... -``` - -### Makefile options - -The Makefile is primarily designed to be an aid during development work. - -#### Building the containers - -To build all containers for the SDA stack: - -```sh -$ make build-all -... -``` - -To build the container for a specific component replace `all` with the folder name: - -```sh -$ make build- -... -``` - -#### Running the integration tests - -This will build the container and run the integration test for the PostgreSQL container. The same test will run on every PR in github: - -```sh -$ make integrationtest-postgres -... -``` - -This will build the RabbitMQ and SDA containers and run the integration test for the RabbitMQ container. The same test will run on every PR in github: - -```sh -$ make integrationtest-rabbitmq -... -``` - -This will build all containers and run the integration tests for the SDA stack. The same test will run on every PR in github: - -```sh -$ make integrationtest-sda -... -``` - -#### Linting the GO code - -To run golangci-lint for all go components: - -```sh -$ make lint-all -... -``` - -To run golangci-lint for a specific component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`): - -```sh -$ make lint- -... -``` - -#### Running the static code tests - -For the go code this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a maven container. - -To run the static code tests for all components: - -```sh -$ make test-all -... -``` - -To run the static code tests for a specific component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`, `sda-sftp-inbox`): - -```sh -$ make test- -... -``` diff --git a/Makefile b/Makefile index 51791cd7e..b3217c3f7 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ bootstrap: go-version-check docker-version-check fi @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ sh -s -- -b $$(go env GOPATH)/bin - GO111MODULE=off go get golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports@latest # build containers build-all: build-postgresql build-rabbitmq build-sda build-sda-download build-sda-sftp-inbox build-sda-admin @@ -49,9 +49,10 @@ go-version-check: ( $${GO_VERSION_ARR[1]} -lt $${GO_VERSION_REQ[1]} ||\ ( $${GO_VERSION_ARR[1]} -eq $${GO_VERSION_REQ[1]} && $${GO_VERSION_ARR[2]} -lt $${GO_VERSION_REQ[2]} )))\ ]]; then\ - echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}.";\ - exit 1;\ - fi; + echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}."; \ + exit 1; \ + fi; \ + echo "GO version: $${GO_VERSION}." docker-version-check: @DOCKER_VERSION=$$(docker version -f "{{.Server.Version}}" | cut -d'.' -f 1); \ @@ -66,8 +67,26 @@ docker-version-check: fi; \ if [ ! $$(docker buildx version | cut -d' ' -f 2) ]; then \ echo "Docker buildx does not exist can't continue"; \ - fi; + exit 1;\ + fi; \ + echo "Docker version: $${DOCKER_VERSION}."; \ + echo "Docker Compose version: $${DOCKER_COMPOSE_VERSION}." +# bring up the services +sda-s3-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml up -d +sda-posix-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml up -d +sda-sync-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml up -d + +# bring down the services +sda-s3-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans +sda-posix-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans +sda-sync-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans # run intrgration tests, same as being run in Github Actions during a PR integrationtest-postgres: build-postgresql @@ -77,7 +96,7 @@ integrationtest-rabbitmq: build-rabbitmq build-sda @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml down -v --remove-orphans -integrationtest-sda: build_all +integrationtest-sda: build-all @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test diff --git a/README.md b/README.md index fcd1f9ff2..4538ca0df 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,157 @@ # Sensitive Data Archive -`SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/) It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a isolated Sensitive Data Archive. +The `SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/). It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a standalone Sensitive Data Archive. -For more information about the different components see the readme files in the respecive folders. -For more information on how to start developing read [Getting Started developing components of the SDA stack](/GETTINGSTARTED.md). \ No newline at end of file +For more information about the different components, please refer to the README files in their respective folders. + +## How to run the SDA stack +The following instructions outline the steps to set up and run the `SDA` services for development and testing using Docker. These steps are based on the provided [Makefile](./Makefile) commands. + +### Prerequisites +Ensure you have the following installed on your system: + +- [`Go`](https://www.golang.org/): The required version is specified in the `sda` Dockerfile. Verify using + ```sh + $ make go-version-check + ``` + +- Docker: Version 24 or higher. Verify using + ```sh + $ make docker-version-check + ``` +- Docker Compose: Version 2 or higher. For Linux, ensure the [Compose plugin](https://docs.docker.com/compose/install/linux/) is installed. + +In preparation for local development, it is essential to verify that `$GOPATH/bin` is part of the system PATH, as certain distributions may package outdated versions of build tools. SDA uses [Go Modules](https://github.com/golang/go/wiki/Modules), and it is advisable to clone the repository outside the `GOPATH`. After cloning, initialize the environment and obtain necessary build tools using the bootstrap command: + +```sh +$ make bootstrap +``` + +### Build Docker images + +Build the required Docker images for all SDA services: + +```sh +$ make build-all +``` + +You can also build images for individual services by replacing `all` with the folder name (`postgresql`, `rabbitmq`, `sda`, `sda-download`, `sda-sftp-inbox`), for example + +```sh +$ make build-sda +``` + +To build the `sda-admin` CLI tool: + +```sh +$ make build-sda-admin +``` + +### Running the services + +#### Start services with Docker Compose +The following command will bring up all services using the Docker Compose file [sda-s3-integration.yml](.github/integration/sda-s3-integration.yml) (configured for S3 as the storage backend): + +```sh +$ make sda-s3-up +``` + +#### Shut down all services and clean up resources +The following command will shut down all services and clean up all related resources: + +```sh +$ make sda-s3-down +``` + +For the setup with POSIX as the storage backend, use +`make sda-posix-up` and `make sda-posix-down` to start and shut down services. + +For the setup including the [`sync`](https://github.com/neicnordic/sda-sync) service, use `make sda-sync-up` and `make sda-sync-down` to start and shut down services. + +### Running the integration tests +This will build all required images, bring up the services, run the integration test, and then shut down services and clean up resources. The same test runs on every pull request (PR) in GitHub. + +- Integration test for the database: + ```sh + make integrationtest-postgres + ``` +- Integration test for RabbitMQ: + ```sh + make integrationtest-rabbitmq + ``` +- Integration test for all SDA setups (including S3, POSIX and sync): + ```sh + make integrationtest-sda + ``` +- Integration test for SDA using POSIX as the storage backend: + ```sh + make integrationtest-sda-posix + ``` +- Integration test for SDA using S3 as the storage backend: + ```sh + make integrationtest-sda-s3 + ``` +- Integration test for SDA including the sync service: + ```sh + make integrationtest-sda-sync + ``` +#### Running the integration tests without shutting down the services +This will run the integration tests and keep the services running after the tests are finished. + +- Integration test for SDA using POSIX as the storage backend: + ```sh + make integrationtest-sda-posix-run + ``` +- Integration test for SDA using S3 as the storage backend: + ```sh + make integrationtest-sda-s3-run + ``` +- Integration test for SDA including the sync service: + ```sh + make integrationtest-sda-sync-run + ``` + +After that, you will need to shut down the services manually. + +- Shut down services for SDA using POSIX as the storage backend + ```sh + make integrationtest-sda-posix-down + ``` +- Shut down services for SDA using S3 as the storage backend + ```sh + make integrationtest-sda-s3-down + ``` +- Shut down services for SDA including the sync service: + ```sh + make integrationtest-sda-sync-down + ``` + +### Linting the Go code + +To run `golangci-lint` for all Go components: + +```sh +$ make lint-all +``` + +To run `golangci-lint` for a specific component, replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`), for example: + +```sh +$ make lint-sda +``` + +### Running the static code tests + +For Go code, this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a Maven container. + +To run the static code tests for all components: + +```sh +$ make test-all +``` + +To run the static code tests for a specific component, replace `all` with the folder name (`sda`, `sda-admin`, `sda-download`, `sda-sftp-inbox`), for example: + +```sh +$ make test-sda +``` \ No newline at end of file diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index ea68efe54..5d0486baf 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -147,6 +147,40 @@ spec: value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - name: DB_PORT value: {{ .Values.global.db.port | quote }} + - name: INBOX_TYPE + {{- if eq "s3" .Values.global.inbox.storageType }} + value: "s3" + - name: INBOX_BUCKET + value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + - name: INBOX_CACERT + value: {{ template "tlsPath" . }}/ca.crt + {{- end }} + - name: INBOX_REGION + value: {{ default "us-east-1" .Values.global.inbox.s3Region }} + - name: INBOX_URL + value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- if .Values.global.inbox.s3Port }} + - name: INBOX_PORT + value: {{ .Values.global.inbox.s3Port | quote }} + {{- end }} + {{- else }} + value: "posix" + - name: INBOX_LOCATION + value: "{{ .Values.global.inbox.path }}/" + {{- end }} + {{- if eq "s3" .Values.global.inbox.storageType }} + - name: INBOX_ACCESSKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3inbox-keys + key: s3InboxAccessKey + - name: INBOX_SECRETKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3inbox-keys + key: s3InboxSecretKey + {{- end }} {{- if .Values.global.log.format }} - name: LOG_FORMAT value: {{ .Values.global.log.format | quote }} @@ -204,6 +238,10 @@ spec: {{- if .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} + {{- end }} + {{- if eq "posix" .Values.global.inbox.storageType }} + - name: inbox + mountPath: {{ .Values.global.inbox.path | quote }} {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} @@ -238,4 +276,15 @@ spec: secretName: {{ required "An certificate issuer or a TLS secret name is required for api" .Values.api.tls.secretName }} {{- end }} {{- end }} + {{- if eq "posix" .Values.global.inbox.storageType }} + - name: inbox + {{- if .Values.global.inbox.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.global.inbox.existingClaim }} + {{- else }} + nfs: + server: {{ required "An inbox NFS server is required" .Values.global.inbox.nfsServer | quote }} + path: {{ if .Values.global.inbox.nfsPath }}{{ .Values.global.inbox.nfsPath | quote }}{{ else }}{{ "/" }}{{ end }} + {{- end }} + {{- end }} {{- end }} diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index cc27cf3df..abbe307f6 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -28,7 +28,8 @@ VALUES (0, now(), 'Created with version'), (11, now(), 'Grant select permission to download on dataset_event_log'), (12, now(), 'Add key hash'), (13, now(), 'Create API user'), - (14, now(), 'Create Auth user'); + (14, now(), 'Create Auth user'), + (15, now(), 'Give API user insert priviledge in logs table'); -- Datasets are used to group files, and permissions are set on the dataset -- level diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index aa8a423f4..0cc65072d 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -164,12 +164,13 @@ GRANT USAGE ON SCHEMA sda TO api; GRANT SELECT ON sda.files TO api; GRANT SELECT ON sda.file_dataset TO api; GRANT SELECT ON sda.checksums TO api; -GRANT SELECT ON sda.file_event_log TO api; +GRANT SELECT, INSERT ON sda.file_event_log TO api; GRANT SELECT ON sda.encryption_keys TO api; GRANT SELECT ON sda.datasets TO api; GRANT SELECT ON sda.dataset_event_log TO api; GRANT INSERT ON sda.encryption_keys TO api; GRANT UPDATE ON sda.encryption_keys TO api; +GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO api; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO api; diff --git a/postgresql/migratedb.d/14.sql b/postgresql/migratedb.d/14.sql index 24a9cff6e..f39a19af1 100644 --- a/postgresql/migratedb.d/14.sql +++ b/postgresql/migratedb.d/14.sql @@ -45,4 +45,4 @@ BEGIN RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; END IF; END -$$ \ No newline at end of file +$$ diff --git a/postgresql/migratedb.d/15.sql b/postgresql/migratedb.d/15.sql new file mode 100644 index 000000000..2849382c4 --- /dev/null +++ b/postgresql/migratedb.d/15.sql @@ -0,0 +1,22 @@ + +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 14; + changes VARCHAR := 'Give API user insert priviledge in logs table'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + GRANT INSERT ON sda.file_event_log TO api; + GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO api; + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index fe298aa87..9c25a9edd 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -79,7 +79,7 @@ io.minio minio - 8.5.13 + 8.5.14 net.logstash.logback diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 41d3875fd..9b6b9f0d2 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -6,9 +6,10 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* + chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 5ee4d5b0b..a3685c878 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -5,10 +5,10 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml new file mode 100644 index 000000000..d425077a8 --- /dev/null +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -0,0 +1,35 @@ +app: + serveUnencryptedData: true + port: 18080 + +log: + level: "debug" + format: "json" + +archive: + type: "s3" + # S3 backend + url: "http://localhost" + port: 19000 + accesskey: "access" + secretkey: "secretKey" + bucket: "archive" + region: "us-east-1" + chunksize: 32 + +grpc: + host: localhost + port: 50051 + +db: + host: "localhost" + port: 15432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +oidc: + # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri" + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index c8c2cdee3..338b9cf00 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "math" "net/http" "os" "os/signal" @@ -25,6 +26,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" ) @@ -100,6 +102,7 @@ func setup(config *config.Config) *http.Server { r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) // Adds a key hash to the database r.GET("/c4gh-keys/list", rbac(e), listC4ghHashes) // Lists key hashes in the database r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash + r.DELETE("/file/:username/:fileid", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file @@ -285,6 +288,63 @@ func ingestFile(c *gin.Context) { c.Status(http.StatusOK) } +// The deleteFile function deletes files from the inbox and marks them as +// discarded in the db. Files are identified by their ids and the user id. +func deleteFile(c *gin.Context) { + inbox, err := storage.NewBackend(Conf.Inbox) + if err != nil { + log.Error(err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + + submissionUser := c.Param("username") + log.Debug("submission user:", submissionUser) + + fileID := c.Param("fileid") + fileID = strings.TrimPrefix(fileID, "/") + log.Debug("submission file:", fileID) + if fileID == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, "file ID is required") + + return + } + + // Get the file path from the fileID and submission user + filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) + if err != nil { + log.Errorf("getting file from fileID failed, reason: (%v)", err) + c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") + + return + } + + var RetryTimes = 5 + for count := 1; count <= RetryTimes; count++ { + err = inbox.RemoveFile(filePath) + if err == nil { + break + } + log.Errorf("Remove file from inbox failed, reason: %v", err) + if count == 5 { + c.AbortWithStatusJSON(http.StatusInternalServerError, ("remove file from inbox failed")) + + return + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + if err := Conf.API.DB.UpdateFileEventLog(fileID, "disabled", fileID, "api", "{}", "{}"); err != nil { + log.Errorf("set status deleted failed, reason: (%v)", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + + c.Status(http.StatusOK) +} + func setAccession(c *gin.Context) { var accession schema.IngestionAccession if err := c.BindJSON(&accession); err != nil { @@ -477,6 +537,8 @@ func listActiveUsers(c *gin.Context) { c.JSON(http.StatusOK, users) } +// listUserFiles returns a list of files for a specific user +// If the file has status disabled, the file will be skipped func listUserFiles(c *gin.Context) { username := c.Param("username") username = strings.TrimPrefix(username, "/") diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 321d9a6e4..abccbb8c6 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -90,6 +90,24 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X PUT -d '{"accession_id": "my-id-01", "filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/accession ``` +- `/file/:username/:fileid` + - accepts `DELETE` requests + - marks the file as `disabled` in the database, and deletes it from the inbox. + - The file is identified by its id, returned by `users/:username/:files` + + - Response codes + - `200` Query execute ok. + - `400` File id not provided + - `401` Token user is not in the list of admins. + - `404` File not found + - `500` Internal error due to Inbox, DB or MQ failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -X DELETE https://HOSTNAME/file/user@demo.org/123abc + ``` + - `/dataset/create` - accepts `POST` requests with JSON data with the format: `{"accession_ids": ["", ""], "dataset_id": "", "user": ""}` - creates a dataset from the list of accession IDs and the dataset ID. diff --git a/sda/config_local.yaml b/sda/config_local.yaml new file mode 100644 index 000000000..81946299b --- /dev/null +++ b/sda/config_local.yaml @@ -0,0 +1,108 @@ +log: + format: "json" + level: "debug" +api: + rbacFile: ../.github/integration/sda/rbac.json + +archive: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "archive" + region: "us-east-1" + +auth: + cega: + authUrl: "http://localhost:8443/username/" + id: + secret: + infoText: "About service text" + infoURL: "http://example.org/about" + jwt: + issuer: "https://localhost:8888" + privateKey: "/tmp/shared/keys/jwt.key" + signatureAlg: ES256 + tokenTTL: 168 + publicFile: "/tmp/shared/c4gh.pub.pem" + resignJwt: + s3Inbox: "http://localhost:18000" + +backup: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "backup" + region: "us-east-1" + +inbox: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "inbox" + region: "us-east-1" + +broker: + host: "localhost" + port: "5672" + user: "" + password: "" + vhost: "/sda" + exchange: "sda" + routingKey: "" + ssl: "false" + +db: + host: "localhost" + port: "15432" + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +c4gh: + filePath: "/tmp/shared/c4gh.sec.pem" + passphrase: "c4ghpass" + syncPubKeyPath: "/tmp/shared/sync.pub.pem" + +oidc: + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" + +server: + cert: "" + key: "" + jwtpubkeypath: "/tmp/shared/keys/pub/" + jwtpubkeyurl: "http://oidc:8080/jwk" + +sync: + api: + password: "pass" + user: "user" + centerPrefix: "SYNC" + destination: + type: "s3" + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "sync" + region: "us-east-1" + remote: + host: "http://sync-api" + port: "8080" + password: "pass" + user: "user" + +schema: + type: "isolated" + path: "schemas/isolated" \ No newline at end of file diff --git a/sda/go.mod b/sda/go.mod index 9453a363a..744ca6109 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,8 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.47 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42 - github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/aws/smithy-go v1.22.1 github.com/casbin/casbin/v2 v2.102.0 github.com/coreos/go-oidc/v3 v3.11.0 @@ -31,9 +31,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.29.0 + golang.org/x/crypto v0.30.0 golang.org/x/oauth2 v0.24.0 - google.golang.org/grpc v1.68.0 + google.golang.org/grpc v1.68.1 google.golang.org/protobuf v1.35.2 ) @@ -158,8 +158,8 @@ require ( golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/sda/go.sum b/sda/go.sum index 833a839f4..59fa32994 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6 github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42 h1:vEnk9vtjJ62OO2wOhEmgKMZgNcn1w0aF7XCiNXO5rK0= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42/go.mod h1:GUOPbPJWRZsdt1OJ355upCrry4d3ZFgdX6rhT7gtkto= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 h1:iLdpkYZ4cXIQMO7ud+cqMWR1xK5ESbt1rvN77tRi1BY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43/go.mod h1:OgbsKPAswXDd5kxnR4vZov69p3oYjbvUyIRBAAV0y9o= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70R github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0 h1:HrHFR8RoS4l4EvodRMFcJMYQ8o3UhmALn2nbInXaxZA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= @@ -396,8 +396,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -445,16 +445,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -462,8 +462,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -481,8 +481,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= -google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index ae08188da..984ccf967 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -83,6 +83,7 @@ type APIConf struct { Session SessionConfig DB *database.SDAdb MQ *broker.AMQPBroker + INBOX storage.Backend } type SessionConfig struct { @@ -215,6 +216,14 @@ func NewConfig(app string) (*Config, error) { "db.password", "db.database", } + switch viper.GetString("inbox.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) + default: + return nil, fmt.Errorf("inbox.type not set") + } case "auth": requiredConfVars = []string{ "auth.s3Inbox", @@ -464,6 +473,8 @@ func NewConfig(app string) (*Config, error) { return nil, err } + c.configInbox() + err = c.configAPI() if err != nil { return nil, err @@ -922,6 +933,10 @@ func (c *Config) configSchemas() { } else { c.Broker.SchemasPath = "/schemas/isolated/" } + + if viper.IsSet("schema.path") { + c.Broker.SchemasPath = viper.GetString("schema.path") + } } // configS3Storage populates and returns a S3Conf from the diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 3b877cb1b..563fcca3f 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -65,6 +65,7 @@ func (suite *ConfigTestSuite) SetupTest() { viper.Set("inbox.accesskey", "testaccess") viper.Set("inbox.secretkey", "testsecret") viper.Set("inbox.bucket", "testbucket") + viper.Set("inbox.type", "s3") viper.Set("server.jwtpubkeypath", "testpath") viper.Set("log.level", "debug") } diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 651e097c0..eed386a8c 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -46,6 +46,7 @@ type SyncData struct { } type SubmissionFileInfo struct { + FileID string `json:"fileID"` InboxPath string `json:"inboxPath"` Status string `json:"fileStatus"` CreateAt string `json:"createAt"` diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 1bd45634e..a4064a2ae 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -63,6 +63,42 @@ func (dbs *SDAdb) getFileID(corrID string) (string, error) { return fileID, nil } +// GetInboxFilePathFromID checks if a file exists in the database for a given user and fileID +// and that is not yet archived +func (dbs *SDAdb) GetInboxFilePathFromID(submissionUser, fileID string) (string, error) { + var ( + err error + count int + filePath string + ) + + for count == 0 || (err != nil && count < RetryTimes) { + filePath, err = dbs.getInboxFilePathFromID(submissionUser, fileID) + count++ + } + + return filePath, err +} + +func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const getFilePath = "SELECT submission_file_path from sda.files where " + + "submission_user= $1 and id = $2 " + + "AND EXISTS (SELECT 1 FROM " + + "(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) " + + "as subquery WHERE event = 'uploaded')" + + var filePath string + err := db.QueryRow(getFilePath, submissionUser, fileID).Scan(&filePath) + if err != nil { + return "", err + } + + return filePath, nil +} + // UpdateFileEventLog updates the status in of the file in the database. // The message parameter is the rabbitmq message sent on file upload. func (dbs *SDAdb) UpdateFileEventLog(fileUUID, event, corrID, user, details, message string) error { @@ -672,7 +708,7 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { db := dbs.DB // select all files (that are not part of a dataset) of the user, each one annotated with its latest event - const query = "SELECT f.submission_file_path, e.event, f.created_at FROM sda.files f " + + const query = "SELECT f.id, f.submission_file_path, e.event, f.created_at FROM sda.files f " + "LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " + "AND f.id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id); " @@ -687,13 +723,15 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { for rows.Next() { // Read rows into struct fi := &SubmissionFileInfo{} - err := rows.Scan(&fi.InboxPath, &fi.Status, &fi.CreateAt) + err := rows.Scan(&fi.FileID, &fi.InboxPath, &fi.Status, &fi.CreateAt) if err != nil { return nil, err } - // Add instance of struct (file) to array - files = append(files, fi) + // Add instance of struct (file) to array if the status is not disabled + if fi.Status != "disabled" { + files = append(files, fi) + } } return files, nil diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index ac86cb02e..53c2a7a8a 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1187,3 +1187,27 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { assert.NoError(suite.T(), err, "failed to get accessions for a dataset") assert.Equal(suite.T(), []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}, accessions) } + +func (suite *DatabaseTests) TestGetInboxFilePathFromID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + user := "UserX" + filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) + fileID, err := db.RegisterFile(filePath, user) + if err != nil { + suite.FailNow("Failed to register file") + } + err = db.UpdateFileEventLog(fileID, "uploaded", fileID, "User-z", "{}", "{}") + if err != nil { + suite.FailNow("Failed to update file event log") + } + path, err := db.getInboxFilePathFromID(user, fileID) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), path, filePath) + + err = db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}") + assert.NoError(suite.T(), err) + _, err = db.getInboxFilePathFromID(user, fileID) + assert.Error(suite.T(), err) +}