From 93533213e57d6de98a77231eb3f5ea4c45d2ebf1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 6 Dec 2024 11:15:10 +0100 Subject: [PATCH 01/58] Run dependabot on release branch --- .github/dependabot.yaml | 111 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 9bcb2f8cb..0bfb3f1f5 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -101,4 +101,113 @@ updates: interval: weekly open-pull-requests-limit: 10 reviewers: - - "neicnordic/sensitive-data-development-collaboration" \ No newline at end of file + - "neicnordic/sensitive-data-development-collaboration" + +## release v1 branch +### Docker + - package-ecosystem: docker + target-branch: release_v1 + directory: "/postgresql" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/rabbitmq" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda-doa" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda-download" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + + - package-ecosystem: docker + target-branch: release_v1 + directory: "/sda-sftp-inbox" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + +### JAVA + - package-ecosystem: maven + target-branch: release_v1 + directory: "/sda-doa" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily + + - package-ecosystem: maven + target-branch: release_v1 + directory: "/sda-sftp-inbox" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily +### GO + - package-ecosystem: gomod + target-branch: release_v1 + directory: "/sda-download" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily + + - package-ecosystem: gomod + target-branch: release_v1 + directory: "/sda" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + schedule: + interval: daily From 8e4d9096c4c0cbd6467399ad2f62809eb2ac5088 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:02:32 +0000 Subject: [PATCH 02/58] Bump codecov/codecov-action from 5.0.7 to 5.1.1 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.0.7 to 5.1.1. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.0.7...v5.1.1) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 25ea74ffd..5358e3bcb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.0.7 + uses: codecov/codecov-action@v5.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.0.7 + uses: codecov/codecov-action@v5.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.0.7 + uses: codecov/codecov-action@v5.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt From 7954e4e3958007699cc7e444df510f41a69cdd39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:17:57 +0000 Subject: [PATCH 03/58] Bump io.minio:minio in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [io.minio:minio](https://github.com/minio/minio-java). Updates `io.minio:minio` from 8.5.13 to 8.5.14 - [Release notes](https://github.com/minio/minio-java/releases) - [Commits](https://github.com/minio/minio-java/compare/8.5.13...8.5.14) --- updated-dependencies: - dependency-name: io.minio:minio dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index fe298aa87..9c25a9edd 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -79,7 +79,7 @@ io.minio minio - 8.5.13 + 8.5.14 net.logstash.logback From 1e0f5211f198b7357e7ac8ebfe4c33b7a134b5ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 20:03:30 +0000 Subject: [PATCH 04/58] Bump the all-modules group in /sda with 4 updates Bumps the all-modules group in /sda with 4 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2), [golang.org/x/crypto](https://github.com/golang/crypto) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.42 to 1.17.43 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.42...credentials/v1.17.43) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.70.0 to 1.71.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.70.0...service/s3/v1.71.0) Updates `golang.org/x/crypto` from 0.29.0 to 0.30.0 - [Commits](https://github.com/golang/crypto/compare/v0.29.0...v0.30.0) Updates `google.golang.org/grpc` from 1.68.0 to 1.68.1 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.68.0...v1.68.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 12 ++++++------ sda/go.sum | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 9453a363a..744ca6109 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,8 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.47 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42 - github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/aws/smithy-go v1.22.1 github.com/casbin/casbin/v2 v2.102.0 github.com/coreos/go-oidc/v3 v3.11.0 @@ -31,9 +31,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.29.0 + golang.org/x/crypto v0.30.0 golang.org/x/oauth2 v0.24.0 - google.golang.org/grpc v1.68.0 + google.golang.org/grpc v1.68.1 google.golang.org/protobuf v1.35.2 ) @@ -158,8 +158,8 @@ require ( golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/sda/go.sum b/sda/go.sum index 833a839f4..59fa32994 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6 github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42 h1:vEnk9vtjJ62OO2wOhEmgKMZgNcn1w0aF7XCiNXO5rK0= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.42/go.mod h1:GUOPbPJWRZsdt1OJ355upCrry4d3ZFgdX6rhT7gtkto= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43 h1:iLdpkYZ4cXIQMO7ud+cqMWR1xK5ESbt1rvN77tRi1BY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.43/go.mod h1:OgbsKPAswXDd5kxnR4vZov69p3oYjbvUyIRBAAV0y9o= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70R github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0 h1:HrHFR8RoS4l4EvodRMFcJMYQ8o3UhmALn2nbInXaxZA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.70.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= @@ -396,8 +396,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -445,16 +445,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -462,8 +462,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -481,8 +481,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= -google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e987e8a53cca783f178baabe4452bf1fa5ec5aed Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Thu, 7 Nov 2024 16:30:53 +0100 Subject: [PATCH 05/58] [postgres] Add api user priviledges --- postgresql/initdb.d/01_main.sql | 3 ++- postgresql/initdb.d/04_grants.sql | 3 ++- postgresql/migratedb.d/14.sql | 2 +- postgresql/migratedb.d/15.sql | 22 ++++++++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 postgresql/migratedb.d/15.sql diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index cc27cf3df..abbe307f6 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -28,7 +28,8 @@ VALUES (0, now(), 'Created with version'), (11, now(), 'Grant select permission to download on dataset_event_log'), (12, now(), 'Add key hash'), (13, now(), 'Create API user'), - (14, now(), 'Create Auth user'); + (14, now(), 'Create Auth user'), + (15, now(), 'Give API user insert priviledge in logs table'); -- Datasets are used to group files, and permissions are set on the dataset -- level diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index aa8a423f4..0cc65072d 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -164,12 +164,13 @@ GRANT USAGE ON SCHEMA sda TO api; GRANT SELECT ON sda.files TO api; GRANT SELECT ON sda.file_dataset TO api; GRANT SELECT ON sda.checksums TO api; -GRANT SELECT ON sda.file_event_log TO api; +GRANT SELECT, INSERT ON sda.file_event_log TO api; GRANT SELECT ON sda.encryption_keys TO api; GRANT SELECT ON sda.datasets TO api; GRANT SELECT ON sda.dataset_event_log TO api; GRANT INSERT ON sda.encryption_keys TO api; GRANT UPDATE ON sda.encryption_keys TO api; +GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO api; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO api; diff --git a/postgresql/migratedb.d/14.sql b/postgresql/migratedb.d/14.sql index 24a9cff6e..f39a19af1 100644 --- a/postgresql/migratedb.d/14.sql +++ b/postgresql/migratedb.d/14.sql @@ -45,4 +45,4 @@ BEGIN RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; END IF; END -$$ \ No newline at end of file +$$ diff --git a/postgresql/migratedb.d/15.sql b/postgresql/migratedb.d/15.sql new file mode 100644 index 000000000..2849382c4 --- /dev/null +++ b/postgresql/migratedb.d/15.sql @@ -0,0 +1,22 @@ + +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 14; + changes VARCHAR := 'Give API user insert priviledge in logs table'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + GRANT INSERT ON sda.file_event_log TO api; + GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO api; + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ From e59d5f57d0c0692565c92703da28e18aa0738c8c Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Thu, 7 Nov 2024 16:32:29 +0100 Subject: [PATCH 06/58] [api] Add api configuration --- sda/internal/config/config.go | 12 ++++++++++++ sda/internal/config/config_test.go | 1 + 2 files changed, 13 insertions(+) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index ae08188da..3b5e149fc 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -83,6 +83,7 @@ type APIConf struct { Session SessionConfig DB *database.SDAdb MQ *broker.AMQPBroker + INBOX storage.Backend } type SessionConfig struct { @@ -214,6 +215,15 @@ func NewConfig(app string) (*Config, error) { "db.user", "db.password", "db.database", + "inbox.type", + } + switch viper.GetString("inbox.type") { + case S3: + requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) + case POSIX: + requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) + default: + return nil, fmt.Errorf("inbox.type not set") } case "auth": requiredConfVars = []string{ @@ -464,6 +474,8 @@ func NewConfig(app string) (*Config, error) { return nil, err } + c.configInbox() + err = c.configAPI() if err != nil { return nil, err diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 3b877cb1b..563fcca3f 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -65,6 +65,7 @@ func (suite *ConfigTestSuite) SetupTest() { viper.Set("inbox.accesskey", "testaccess") viper.Set("inbox.secretkey", "testsecret") viper.Set("inbox.bucket", "testbucket") + viper.Set("inbox.type", "s3") viper.Set("server.jwtpubkeypath", "testpath") viper.Set("log.level", "debug") } From 38c4bb8fce033b697c3609a80c5ccecaf4ed9cd1 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Thu, 7 Nov 2024 16:33:02 +0100 Subject: [PATCH 07/58] [api] Add fileID to api struct --- sda/internal/database/database.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 651e097c0..eed386a8c 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -46,6 +46,7 @@ type SyncData struct { } type SubmissionFileInfo struct { + FileID string `json:"fileID"` InboxPath string `json:"inboxPath"` Status string `json:"fileStatus"` CreateAt string `json:"createAt"` From f0beba177360473b245c1391513ace8a39ba6a84 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Thu, 7 Nov 2024 16:36:07 +0100 Subject: [PATCH 08/58] [api] Change list to return id and skip disabled --- sda/internal/database/db_functions.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 8d58503a1..15e5c9ddc 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -639,7 +639,7 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { db := dbs.DB // select all files (that are not part of a dataset) of the user, each one annotated with its latest event - const query = "SELECT f.submission_file_path, e.event, f.created_at FROM sda.files f " + + const query = "SELECT f.id, f.submission_file_path, e.event, f.created_at FROM sda.files f " + "LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " + "AND f.id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id); " @@ -654,13 +654,15 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { for rows.Next() { // Read rows into struct fi := &SubmissionFileInfo{} - err := rows.Scan(&fi.InboxPath, &fi.Status, &fi.CreateAt) + err := rows.Scan(&fi.FileID, &fi.InboxPath, &fi.Status, &fi.CreateAt) if err != nil { return nil, err } - // Add instance of struct (file) to array - files = append(files, fi) + // Add instance of struct (file) to array if the status is not disabled + if fi.Status != "disabled" { + files = append(files, fi) + } } return files, nil From fff7dee35bcd7e126d19717851643d7d2356ced2 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Thu, 7 Nov 2024 16:37:07 +0100 Subject: [PATCH 09/58] [api] Add disable file event log function --- sda/internal/database/db_functions.go | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 15e5c9ddc..ef8b2ea73 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -63,6 +63,37 @@ func (dbs *SDAdb) getFileID(corrID string) (string, error) { return fileID, nil } +// UserFileExists checks if a file exists in the database +// for a given user and fileID +func (dbs *SDAdb) GetFilePathFromID(submission_user, fileID string) (string, error) { + var ( + err error + count int + filePath string + ) + + for count == 0 || (err != nil && count < RetryTimes) { + filePath, err = dbs.getFilePathFromID(submission_user, fileID) + count++ + } + + return filePath, err +} +func (dbs *SDAdb) getFilePathFromID(submission_user, fileID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const getFilePath = "SELECT submission_file_path from sda.files where submission_user= $1 and id = $2;" + + var filePath string + err := db.QueryRow(getFilePath, submission_user, fileID).Scan(&filePath) + if err != nil { + return "", err + } + + return filePath, nil +} + // UpdateFileEventLog updates the status in of the file in the database. // The message parameter is the rabbitmq message sent on file upload. func (dbs *SDAdb) UpdateFileEventLog(fileUUID, event, corrID, user, details, message string) error { From 6759b94998d8396419c7bad25838574331de021f Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Thu, 7 Nov 2024 16:37:48 +0100 Subject: [PATCH 10/58] [api] Add delete file functionality --- sda/cmd/api/api.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index c8c2cdee3..94107cdd1 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "math" "net/http" "os" "os/signal" @@ -25,6 +26,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" ) @@ -54,6 +56,10 @@ func main() { if err != nil { log.Fatal(err) } + Conf.API.INBOX, err = storage.NewBackend(Conf.Inbox) + if err != nil { + log.Fatal(err) + } if err := setupJwtAuth(); err != nil { log.Fatalf("error when setting up JWT auth, reason %s", err.Error()) @@ -100,6 +106,7 @@ func setup(config *config.Config) *http.Server { r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) // Adds a key hash to the database r.GET("/c4gh-keys/list", rbac(e), listC4ghHashes) // Lists key hashes in the database r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash + r.DELETE("/file/:username/*file", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file @@ -285,6 +292,66 @@ func ingestFile(c *gin.Context) { c.Status(http.StatusOK) } +// The deleteFile function will take the user id and the file id +// because the user id won't be in the path of the file in the future +// therefore we need to make the list files, get the file id and then +// call the delete file +func deleteFile(c *gin.Context) { + + inbox, err := storage.NewBackend(Conf.Inbox) + if err != nil { + log.Fatal(err) + } + + submissionUser := c.Param("username") + log.Warn("submission user:", submissionUser) + + // TODO: Add check for the input file path + fileID := c.Param("file") + fileID = strings.TrimPrefix(fileID, "/") + log.Warn("submission file:", fileID) + if fileID == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, "file ID is required") + } + + filePath := "" + // Get the file path from the fileID and submission user + if filePath, err = Conf.API.DB.GetFilePathFromID(submissionUser, fileID); err != nil { + log.Errorf("getting file from fileID failed, reason: (%v)", err) + c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") + + return + } + + // Requires a filepath instead of fileID + // TODO: The remove fails randomly sometimes, maybe we should retry + var RetryTimes = 5 + for count := 1; count <= RetryTimes; count++ { + log.Warn("trying to remove file from inbox, try", count) + err = inbox.RemoveFile(filePath) + if err != nil { + log.Errorf("Remove file from inbox failed, reason: %v", err) + } + + // The GetFileSize returns zero in case of error after retrying a number of times + fileSize, _ := inbox.GetFileSize(filePath) + if fileSize == 0 { + break + } + + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + if err := Conf.API.DB.UpdateFileEventLog(fileID, "disabled", fileID, "api", "{}", "{}"); err != nil { + log.Errorf("set status deleted failed, reason: (%v)", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + + c.Status(http.StatusOK) +} + func setAccession(c *gin.Context) { var accession schema.IngestionAccession if err := c.BindJSON(&accession); err != nil { @@ -477,6 +544,8 @@ func listActiveUsers(c *gin.Context) { c.JSON(http.StatusOK, users) } +// listUserFiles returns a list of files for a specific user +// If the file has status disabled, the file will be skipped func listUserFiles(c *gin.Context) { username := c.Param("username") username = strings.TrimPrefix(username, "/") From 7eeaac399f325100c17fa6014e6f422f2c485bd5 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 18 Nov 2024 13:30:16 +0100 Subject: [PATCH 11/58] [api] add integration test --- .../tests/sda/60_api_admin_test.sh | 39 +++++++++++++++++++ sda/internal/config/config.go | 1 - 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 8478afd21..ee36724e8 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -1,5 +1,6 @@ #!/bin/sh set -e +cd shared || true token="$(curl http://oidc:8080/tokens | jq -r '.[0]')" result="$(curl -sk -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq '. | length')" @@ -35,4 +36,42 @@ resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer if [ "$resp" != "404" ]; then echo "Error when starting re-verification of missing dataset, expected 404 got: $resp" exit 1 +fi + +## reupload a file under a different name, test to delete it +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NC12878.bam.c4gh + +echo "waiting for upload to complete" +URI=http://rabbitmq:15672 +if [ -n "$PGSSLCERT" ]; then + URI=https://rabbitmq:15671 +fi +RETRY_TIMES=0 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -gt 0 ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 10 ]; then + echo "::error::Time out while waiting for upload to complete" + exit 1 + fi + echo "read now:" `curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"'` + sleep 2 +done + +# get the fileId of the new file +fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" +echo "Found uploaded file " $fileid + +# delete it +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +if [ "$resp" != "200" ]; then + echo "Error when deleting the file, expected 200 got: $resp" + exit 1 +fi + +last_event=$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id='$fileid' order by started_at desc limit 1;") +echo "last event was" $last_event + +if [[ "$last_event" != "disabled" ]]; then + echo "The file $fileid does not have the expected las event 'disabled', but $last_event." fi \ No newline at end of file diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 3b5e149fc..cdc1472c5 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -215,7 +215,6 @@ func NewConfig(app string) (*Config, error) { "db.user", "db.password", "db.database", - "inbox.type", } switch viper.GetString("inbox.type") { case S3: From f80f6354a06d3f006ea35abcbac1c4bbdcb87776 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 18 Nov 2024 14:59:31 +0100 Subject: [PATCH 12/58] [api delete] only search for unarchived files --- sda/cmd/api/api.go | 2 +- sda/internal/database/db_functions.go | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 94107cdd1..f972de131 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -316,7 +316,7 @@ func deleteFile(c *gin.Context) { filePath := "" // Get the file path from the fileID and submission user - if filePath, err = Conf.API.DB.GetFilePathFromID(submissionUser, fileID); err != nil { + if filePath, err = Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID); err != nil { log.Errorf("getting file from fileID failed, reason: (%v)", err) c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index ef8b2ea73..ac44b91a2 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -63,9 +63,9 @@ func (dbs *SDAdb) getFileID(corrID string) (string, error) { return fileID, nil } -// UserFileExists checks if a file exists in the database -// for a given user and fileID -func (dbs *SDAdb) GetFilePathFromID(submission_user, fileID string) (string, error) { +// GetInboxFilePathFromID checks if a file exists in the database for a given user and fileID +// and that is not yet archived +func (dbs *SDAdb) GetInboxFilePathFromID(submissionUser, fileID string) (string, error) { var ( err error count int @@ -73,17 +73,21 @@ func (dbs *SDAdb) GetFilePathFromID(submission_user, fileID string) (string, err ) for count == 0 || (err != nil && count < RetryTimes) { - filePath, err = dbs.getFilePathFromID(submission_user, fileID) + filePath, err = dbs.getInboxFilePathFromID(submissionUser, fileID) count++ } return filePath, err } -func (dbs *SDAdb) getFilePathFromID(submission_user, fileID string) (string, error) { +func (dbs *SDAdb) getInboxFilePathFromID(submission_user, fileID string) (string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFilePath = "SELECT submission_file_path from sda.files where submission_user= $1 and id = $2;" + const getFilePath = "SELECT submission_file_path from sda.files where " + + "submission_user= $1 and id = $2 " + + "AND EXISTS (SELECT 1 FROM " + + "(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) " + + "as subquery WHERE event in ('registered', 'uploaded', 'submitted', 'ingested'))" var filePath string err := db.QueryRow(getFilePath, submission_user, fileID).Scan(&filePath) From c1cfc269b3bb0350df550d4ed4de1c5310c3f23c Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 19 Nov 2024 11:52:49 +0100 Subject: [PATCH 13/58] [tests] api admin tests --- .../tests/sda/60_api_admin_test.sh | 109 +++++++++++++++--- 1 file changed, 90 insertions(+), 19 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index ee36724e8..83fda513e 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -3,10 +3,11 @@ set -e cd shared || true token="$(curl http://oidc:8080/tokens | jq -r '.[0]')" +# Upload a file and mke sure it's listed result="$(curl -sk -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq '. | length')" if [ "$result" -ne 2 ]; then echo "wrong number of files returned for user test@dummy.org" - echo "expected 4 got $result" + echo "expected 2 got $result" exit 1 fi @@ -38,40 +39,110 @@ if [ "$resp" != "404" ]; then exit 1 fi -## reupload a file under a different name, test to delete it -s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NC12878.bam.c4gh +# Reupload a file under a different name, test to delete it +s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh echo "waiting for upload to complete" URI=http://rabbitmq:15672 if [ -n "$PGSSLCERT" ]; then - URI=https://rabbitmq:15671 + URI=https://rabbitmq:15671 fi RETRY_TIMES=0 -until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -gt 0 ]; do - echo "waiting for upload to complete" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 10 ]; then - echo "::error::Time out while waiting for upload to complete" - exit 1 - fi - echo "read now:" `curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"'` - sleep 2 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 4 ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for upload to complete" + exit 1 + fi + sleep 2 done # get the fileId of the new file fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" -echo "Found uploaded file " $fileid # delete it resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "200" ]; then - echo "Error when deleting the file, expected 200 got: $resp" - exit 1 + echo "Error when deleting the file, expected 200 got: $resp" + exit 1 fi last_event=$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id='$fileid' order by started_at desc limit 1;") -echo "last event was" $last_event -if [[ "$last_event" != "disabled" ]]; then - echo "The file $fileid does not have the expected las event 'disabled', but $last_event." +if [ "$last_event" != "disabled" ]; then + echo "The file $fileid does not have the expected las event 'disabled', but $last_event." +fi + + +# Try to delete an unknown file +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/badfileid")" +if [ "$resp" != "404" ]; then + echo "Error when deleting the file, expected error got: $resp" + exit 1 +fi + + +# Try to delete file of other user +fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/requester@demo.org/files" | jq -r '.[0]| .fileID')" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +if [ "$resp" != "404" ]; then + echo "Error when deleting the file, expected 404 got: $resp" + exit 1 +fi + + +# Re-upload the file and use the api to ingest it, then try to delete it +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NE12878.bam.c4gh + +URI=http://rabbitmq:15672 +if [ -n "$PGSSLCERT" ]; then + URI=https://rabbitmq:15671 +fi +RETRY_TIMES=0 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 3 ]; then + echo "::error::Time out while waiting for upload to complete" + #exit 1 + break + fi + sleep 2 +done + +# Ingest it +new_payload=$( +jq -c -n \ + --arg filepath "test_dummy.org/NE12878.bam.c4gh" \ + --arg user "test@dummy.org" \ + '$ARGS.named' +) + +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$new_payload" "http://api:8080/file/ingest")" +if [ "$resp" != "200" ]; then + echo "Error when requesting to ingesting file, expected 200 got: $resp" + exit 1 +fi + +fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +# wait for the fail to get the correct status +RETRY_TIMES=0 + +until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_events e where e.title in (select event from sda.file_event_log where file_id = '$fileid' order by started_at desc limit 1);")" -gt 30 ]; do + echo "waiting for ingest to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for upload to complete" + exit 1 + fi + sleep 2 +done + +# Try to delete file not in inbox +fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +if [ "$resp" != "404" ]; then + echo "Error when deleting the file, expected 404 got: $resp" + exit 1 fi \ No newline at end of file From aff5092647382827e565beae08c7d088a390e2dd Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 18 Nov 2024 14:59:31 +0100 Subject: [PATCH 14/58] [api delete] only search for unarchived files --- sda/internal/database/db_functions.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index ac44b91a2..df1c4893b 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -79,7 +79,8 @@ func (dbs *SDAdb) GetInboxFilePathFromID(submissionUser, fileID string) (string, return filePath, err } -func (dbs *SDAdb) getInboxFilePathFromID(submission_user, fileID string) (string, error) { + +func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB @@ -90,7 +91,7 @@ func (dbs *SDAdb) getInboxFilePathFromID(submission_user, fileID string) (string "as subquery WHERE event in ('registered', 'uploaded', 'submitted', 'ingested'))" var filePath string - err := db.QueryRow(getFilePath, submission_user, fileID).Scan(&filePath) + err := db.QueryRow(getFilePath, submissionUser, fileID).Scan(&filePath) if err != nil { return "", err } From 7a3d42c83f64cd1c7674cd08aaf46a870129849a Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 19 Nov 2024 14:40:53 +0100 Subject: [PATCH 15/58] [tests] unit test for db function --- sda/internal/database/db_functions_test.go | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index ac86cb02e..11134d9a4 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1187,3 +1187,28 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { assert.NoError(suite.T(), err, "failed to get accessions for a dataset") assert.Equal(suite.T(), []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}, accessions) } + +func (suite *DatabaseTests) TestGetInboxFilePathFromID() { + + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + user := "UserX" + filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) + fileID, err := db.RegisterFile(filePath, user) + if err != nil { + suite.FailNow("Failed to register file") + } + err = db.UpdateFileEventLog(fileID, "uploaded", fileID, "User-z", "{}", "{}") + if err != nil { + suite.FailNow("Failed to update file event log") + } + path, err := db.getInboxFilePathFromID(user, fileID) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), path, filePath) + + err = db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}") + assert.NoError(suite.T(), err) + _, err = db.getInboxFilePathFromID(user, fileID) + assert.Error(suite.T(), err) +} From 637495723579de332b13182c40218c8430fdfd72 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 21 Nov 2024 08:24:10 +0100 Subject: [PATCH 16/58] [api] allow deletion of files marked with error --- sda/internal/database/db_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index df1c4893b..0edac5bad 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -88,7 +88,7 @@ func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, "submission_user= $1 and id = $2 " + "AND EXISTS (SELECT 1 FROM " + "(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) " + - "as subquery WHERE event in ('registered', 'uploaded', 'submitted', 'ingested'))" + "as subquery WHERE event in ('registered', 'uploaded', 'submitted', 'ingested', 'error'))" var filePath string err := db.QueryRow(getFilePath, submissionUser, fileID).Scan(&filePath) From 1fcad3a94ceb66dd8c6091ae578cbe2438e679dc Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 21 Nov 2024 08:40:54 +0100 Subject: [PATCH 17/58] [charts] add inbox conf to api --- charts/sda-svc/templates/api-deploy.yaml | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index ea68efe54..5d0486baf 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -147,6 +147,40 @@ spec: value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - name: DB_PORT value: {{ .Values.global.db.port | quote }} + - name: INBOX_TYPE + {{- if eq "s3" .Values.global.inbox.storageType }} + value: "s3" + - name: INBOX_BUCKET + value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + - name: INBOX_CACERT + value: {{ template "tlsPath" . }}/ca.crt + {{- end }} + - name: INBOX_REGION + value: {{ default "us-east-1" .Values.global.inbox.s3Region }} + - name: INBOX_URL + value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- if .Values.global.inbox.s3Port }} + - name: INBOX_PORT + value: {{ .Values.global.inbox.s3Port | quote }} + {{- end }} + {{- else }} + value: "posix" + - name: INBOX_LOCATION + value: "{{ .Values.global.inbox.path }}/" + {{- end }} + {{- if eq "s3" .Values.global.inbox.storageType }} + - name: INBOX_ACCESSKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3inbox-keys + key: s3InboxAccessKey + - name: INBOX_SECRETKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3inbox-keys + key: s3InboxSecretKey + {{- end }} {{- if .Values.global.log.format }} - name: LOG_FORMAT value: {{ .Values.global.log.format | quote }} @@ -204,6 +238,10 @@ spec: {{- if .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} + {{- end }} + {{- if eq "posix" .Values.global.inbox.storageType }} + - name: inbox + mountPath: {{ .Values.global.inbox.path | quote }} {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} @@ -238,4 +276,15 @@ spec: secretName: {{ required "An certificate issuer or a TLS secret name is required for api" .Values.api.tls.secretName }} {{- end }} {{- end }} + {{- if eq "posix" .Values.global.inbox.storageType }} + - name: inbox + {{- if .Values.global.inbox.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.global.inbox.existingClaim }} + {{- else }} + nfs: + server: {{ required "An inbox NFS server is required" .Values.global.inbox.nfsServer | quote }} + path: {{ if .Values.global.inbox.nfsPath }}{{ .Values.global.inbox.nfsPath | quote }}{{ else }}{{ "/" }}{{ end }} + {{- end }} + {{- end }} {{- end }} From 3855db4bcd3739357ca27e29ca6595725f809b40 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 21 Nov 2024 14:33:59 +0100 Subject: [PATCH 18/58] [api] add documentation --- sda/cmd/api/api.go | 2 +- sda/cmd/api/api.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index f972de131..4bd8bba69 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -324,7 +324,7 @@ func deleteFile(c *gin.Context) { } // Requires a filepath instead of fileID - // TODO: The remove fails randomly sometimes, maybe we should retry + // Note: The remove fails randomly sometimes var RetryTimes = 5 for count := 1; count <= RetryTimes; count++ { log.Warn("trying to remove file from inbox, try", count) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 321d9a6e4..5ef7aae8f 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -90,6 +90,24 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X PUT -d '{"accession_id": "my-id-01", "filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/accession ``` +- `/file/:username/*fileid` + - accepts `DELETE` requests + - marks the file as `disabled` in the database, and deletes it from the inbox. + - The file is identfied by its id, returned by `users/:username/:files` + + - Response codes + - `200` Query execute ok. + - `400` File id not provided + - `401` Token user is not in the list of admins. + - `404` File not found + - `500` Internal error due to Inbox, DB or MQ failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01"}' https://HOSTNAME/dataset/create + ``` + - `/dataset/create` - accepts `POST` requests with JSON data with the format: `{"accession_ids": ["", ""], "dataset_id": "", "user": ""}` - creates a dataset from the list of accession IDs and the dataset ID. From 8bc103787e34d80a68c44c3f7df782ba048f103d Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Wed, 27 Nov 2024 09:14:28 +0100 Subject: [PATCH 19/58] [api] typos etc --- .github/integration/tests/sda/60_api_admin_test.sh | 2 +- sda/cmd/api/api.go | 1 - sda/cmd/api/api.md | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 83fda513e..09e5caf02 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -3,7 +3,7 @@ set -e cd shared || true token="$(curl http://oidc:8080/tokens | jq -r '.[0]')" -# Upload a file and mke sure it's listed +# Upload a file and make sure it's listed result="$(curl -sk -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq '. | length')" if [ "$result" -ne 2 ]; then echo "wrong number of files returned for user test@dummy.org" diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 4bd8bba69..23920d779 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -306,7 +306,6 @@ func deleteFile(c *gin.Context) { submissionUser := c.Param("username") log.Warn("submission user:", submissionUser) - // TODO: Add check for the input file path fileID := c.Param("file") fileID = strings.TrimPrefix(fileID, "/") log.Warn("submission file:", fileID) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 5ef7aae8f..c736e50b4 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -105,7 +105,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th Example: ```bash - curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01"}' https://HOSTNAME/dataset/create + curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE https://HOSTNAME/file/user@demo.org/123abc ``` - `/dataset/create` From 533c64a6b3174eecc902e6b0a8aa1f81ffa7eab5 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 29 Nov 2024 08:52:02 +0100 Subject: [PATCH 20/58] [api] rephrase comment --- sda/cmd/api/api.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 23920d779..fa3ba2f7b 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -292,10 +292,8 @@ func ingestFile(c *gin.Context) { c.Status(http.StatusOK) } -// The deleteFile function will take the user id and the file id -// because the user id won't be in the path of the file in the future -// therefore we need to make the list files, get the file id and then -// call the delete file +// The deleteFile function deletes files from the inbox and marks them as +// discarded in the db. Files are identified by their ids and the user id. func deleteFile(c *gin.Context) { inbox, err := storage.NewBackend(Conf.Inbox) From aad252ff09e953b5a2b402ad75b59b0eaded69b5 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 29 Nov 2024 10:21:31 +0100 Subject: [PATCH 21/58] [tests] add rbac permissions for file deletion endpoint --- .github/integration/sda/rbac.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index a4a4ca394..70bb39ae5 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -30,6 +30,11 @@ "path": "/file/accession", "action": "POST" }, + { + "role": "admin", + "path": "/file/*", + "action": "(DELETE)" + }, { "role": "submission", "path": "/users", From 1c4d6c381303a53879edb2ce3ccbac1d5a245afc Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 2 Dec 2024 14:20:40 +0100 Subject: [PATCH 22/58] Apply suggestions from code review --- .github/integration/tests/sda/60_api_admin_test.sh | 10 ++++++++++ sda/cmd/api/api.go | 4 ++-- sda/cmd/api/api.md | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 09e5caf02..71f9046f9 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -61,6 +61,11 @@ done # get the fileId of the new file fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" +output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) +if [ -z "$output" ] ; then + echo "Uploaded file not in inbox" + exit 1 +fi # delete it resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "200" ]; then @@ -74,6 +79,11 @@ if [ "$last_event" != "disabled" ]; then echo "The file $fileid does not have the expected las event 'disabled', but $last_event." fi +output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) +if [ -n "$output" ] ; then + echo "Deleted file is still in inbox" + exit 1 +fi # Try to delete an unknown file resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/badfileid")" diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index fa3ba2f7b..6e0302454 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -302,11 +302,11 @@ func deleteFile(c *gin.Context) { } submissionUser := c.Param("username") - log.Warn("submission user:", submissionUser) + log.Debug("submission user:", submissionUser) fileID := c.Param("file") fileID = strings.TrimPrefix(fileID, "/") - log.Warn("submission file:", fileID) + log.Debug("submission file:", fileID) if fileID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, "file ID is required") } diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index c736e50b4..a2b8bb087 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -93,7 +93,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th - `/file/:username/*fileid` - accepts `DELETE` requests - marks the file as `disabled` in the database, and deletes it from the inbox. - - The file is identfied by its id, returned by `users/:username/:files` + - The file is identified by its id, returned by `users/:username/:files` - Response codes - `200` Query execute ok. From 59547e56debd94c3b249ac8a53314f4fc8ca0448 Mon Sep 17 00:00:00 2001 From: Malin Klang Date: Thu, 5 Dec 2024 12:14:32 +0100 Subject: [PATCH 23/58] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- .github/integration/sda/rbac.json | 4 ++-- .github/integration/tests/sda/60_api_admin_test.sh | 2 +- sda/cmd/api/api.go | 8 +++----- sda/cmd/api/api.md | 2 +- sda/internal/database/db_functions_test.go | 1 - 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 70bb39ae5..a76d6c849 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -31,9 +31,9 @@ "action": "POST" }, { - "role": "admin", + "role": "submission", "path": "/file/*", - "action": "(DELETE)" + "action": "DELETE" }, { "role": "submission", diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 71f9046f9..811706728 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -67,7 +67,7 @@ if [ -z "$output" ] ; then exit 1 fi # delete it -resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "200" ]; then echo "Error when deleting the file, expected 200 got: $resp" exit 1 diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 6e0302454..f06a85496 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -106,7 +106,7 @@ func setup(config *config.Config) *http.Server { r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) // Adds a key hash to the database r.GET("/c4gh-keys/list", rbac(e), listC4ghHashes) // Lists key hashes in the database r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash - r.DELETE("/file/:username/*file", rbac(e), deleteFile) // Delete a file from inbox + r.DELETE("/file/:username/:file", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file @@ -295,7 +295,6 @@ func ingestFile(c *gin.Context) { // The deleteFile function deletes files from the inbox and marks them as // discarded in the db. Files are identified by their ids and the user id. func deleteFile(c *gin.Context) { - inbox, err := storage.NewBackend(Conf.Inbox) if err != nil { log.Fatal(err) @@ -311,9 +310,9 @@ func deleteFile(c *gin.Context) { c.AbortWithStatusJSON(http.StatusBadRequest, "file ID is required") } - filePath := "" // Get the file path from the fileID and submission user - if filePath, err = Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID); err != nil { + filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) + if err != nil { log.Errorf("getting file from fileID failed, reason: (%v)", err) c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") @@ -321,7 +320,6 @@ func deleteFile(c *gin.Context) { } // Requires a filepath instead of fileID - // Note: The remove fails randomly sometimes var RetryTimes = 5 for count := 1; count <= RetryTimes; count++ { log.Warn("trying to remove file from inbox, try", count) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index a2b8bb087..bc966da11 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -90,7 +90,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X PUT -d '{"accession_id": "my-id-01", "filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/accession ``` -- `/file/:username/*fileid` +- `/file/:username/:fileid` - accepts `DELETE` requests - marks the file as `disabled` in the database, and deletes it from the inbox. - The file is identified by its id, returned by `users/:username/:files` diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 11134d9a4..53c2a7a8a 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1189,7 +1189,6 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { } func (suite *DatabaseTests) TestGetInboxFilePathFromID() { - db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) From 94e05a991fd141bac5d27690198e1d459bbbc2aa Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 5 Dec 2024 14:52:23 +0100 Subject: [PATCH 24/58] [api] only allow deletion of files marked as uploaded --- sda/internal/database/db_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 0edac5bad..ea7a39344 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -88,7 +88,7 @@ func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, "submission_user= $1 and id = $2 " + "AND EXISTS (SELECT 1 FROM " + "(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) " + - "as subquery WHERE event in ('registered', 'uploaded', 'submitted', 'ingested', 'error'))" + "as subquery WHERE event = 'uploaded')" var filePath string err := db.QueryRow(getFilePath, submissionUser, fileID).Scan(&filePath) From 145f021388e10ddae55b8693aca0f253537ab5eb Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 9 Dec 2024 08:54:39 +0100 Subject: [PATCH 25/58] [tests] remove unnecessary content-types --- .github/integration/tests/sda/60_api_admin_test.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 811706728..f05d472ea 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -59,7 +59,7 @@ until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messag done # get the fileId of the new file -fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) if [ -z "$output" ] ; then @@ -86,7 +86,7 @@ if [ -n "$output" ] ; then fi # Try to delete an unknown file -resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/badfileid")" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/badfileid")" if [ "$resp" != "404" ]; then echo "Error when deleting the file, expected error got: $resp" exit 1 @@ -94,8 +94,8 @@ fi # Try to delete file of other user -fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/requester@demo.org/files" | jq -r '.[0]| .fileID')" -resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/requester@demo.org/files" | jq -r '.[0]| .fileID')" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "404" ]; then echo "Error when deleting the file, expected 404 got: $resp" exit 1 @@ -135,7 +135,7 @@ if [ "$resp" != "200" ]; then exit 1 fi -fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" # wait for the fail to get the correct status RETRY_TIMES=0 @@ -150,8 +150,8 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_e done # Try to delete file not in inbox -fileid="$(curl -k -L -H "Authorization: Bearer $token" -H "Content-Type: application/json" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" -resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "404" ]; then echo "Error when deleting the file, expected 404 got: $resp" exit 1 From 9b9ec5131eac27916cdcf79c861559b3ad62c24b Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 9 Dec 2024 08:54:55 +0100 Subject: [PATCH 26/58] [api] refactor, apply PR suggestions --- sda/cmd/api/api.go | 15 ++++++++------- sda/cmd/api/api.md | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index f06a85496..9881429d7 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -56,10 +56,6 @@ func main() { if err != nil { log.Fatal(err) } - Conf.API.INBOX, err = storage.NewBackend(Conf.Inbox) - if err != nil { - log.Fatal(err) - } if err := setupJwtAuth(); err != nil { log.Fatalf("error when setting up JWT auth, reason %s", err.Error()) @@ -106,7 +102,7 @@ func setup(config *config.Config) *http.Server { r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) // Adds a key hash to the database r.GET("/c4gh-keys/list", rbac(e), listC4ghHashes) // Lists key hashes in the database r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash - r.DELETE("/file/:username/:file", rbac(e), deleteFile) // Delete a file from inbox + r.DELETE("/file/:username/:fileid", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file @@ -297,17 +293,22 @@ func ingestFile(c *gin.Context) { func deleteFile(c *gin.Context) { inbox, err := storage.NewBackend(Conf.Inbox) if err != nil { - log.Fatal(err) + log.Error(err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return } submissionUser := c.Param("username") log.Debug("submission user:", submissionUser) - fileID := c.Param("file") + fileID := c.Param("fileid") fileID = strings.TrimPrefix(fileID, "/") log.Debug("submission file:", fileID) if fileID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, "file ID is required") + + return } // Get the file path from the fileID and submission user diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index bc966da11..abccbb8c6 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -105,7 +105,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th Example: ```bash - curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X DELETE https://HOSTNAME/file/user@demo.org/123abc + curl -H "Authorization: Bearer $token" -X DELETE https://HOSTNAME/file/user@demo.org/123abc ``` - `/dataset/create` From 5640d72a46558d4466375407666b63d4c50c120a Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 9 Dec 2024 14:27:15 +0100 Subject: [PATCH 27/58] [api] fail if file cannot be deleted from inbox --- .../integration/tests/sda/60_api_admin_test.sh | 2 +- sda/cmd/api/api.go | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index f05d472ea..cecf5a86c 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -143,7 +143,7 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_e echo "waiting for ingest to complete" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then - echo "::error::Time out while waiting for upload to complete" + echo "::error::Time out while waiting for ingest to complete" exit 1 fi sleep 2 diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 9881429d7..338b9cf00 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -320,21 +320,18 @@ func deleteFile(c *gin.Context) { return } - // Requires a filepath instead of fileID var RetryTimes = 5 for count := 1; count <= RetryTimes; count++ { - log.Warn("trying to remove file from inbox, try", count) err = inbox.RemoveFile(filePath) - if err != nil { - log.Errorf("Remove file from inbox failed, reason: %v", err) - } - - // The GetFileSize returns zero in case of error after retrying a number of times - fileSize, _ := inbox.GetFileSize(filePath) - if fileSize == 0 { + if err == nil { break } + log.Errorf("Remove file from inbox failed, reason: %v", err) + if count == 5 { + c.AbortWithStatusJSON(http.StatusInternalServerError, ("remove file from inbox failed")) + return + } time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) } From ba9bb6f2cee853cc3d93ed03595a42a9933e8661 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 3 Dec 2024 18:23:13 +0000 Subject: [PATCH 28/58] fix: typo for build-all in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 51791cd7e..65f81a1fa 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ integrationtest-rabbitmq: build-rabbitmq build-sda @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml down -v --remove-orphans -integrationtest-sda: build_all +integrationtest-sda: build-all @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test From 8cb5812fab19c5c91a72fc6bc2a06f8a43fa8cf5 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 3 Dec 2024 18:24:17 +0000 Subject: [PATCH 29/58] fix: bootstrap in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 65f81a1fa..9228edbb4 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ bootstrap: go-version-check docker-version-check fi @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ sh -s -- -b $$(go env GOPATH)/bin - GO111MODULE=off go get golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports@latest # build containers build-all: build-postgresql build-rabbitmq build-sda build-sda-download build-sda-sftp-inbox build-sda-admin From cbd5440916cdcd2812b5a6d6d82ed50b110aeccb Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Wed, 4 Dec 2024 18:50:29 +0000 Subject: [PATCH 30/58] Add instructions to run the SDA stack in README --- GETTINGSTARTED.md | 93 ------------------------------------ README.md | 119 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 96 deletions(-) delete mode 100644 GETTINGSTARTED.md diff --git a/GETTINGSTARTED.md b/GETTINGSTARTED.md deleted file mode 100644 index cb86ff5de..000000000 --- a/GETTINGSTARTED.md +++ /dev/null @@ -1,93 +0,0 @@ -## Getting Started developing components of the SDA stack - -Should one wish to engage in the development of the SDA stack itself, the prerequisite is the installation of [Go](https://www.golang.org/) on the respective machine. -The recommended version can be checked by running: - -```sh -$ make go-version-check -... -``` - -In preparation for local development, it is essential to verify the proper installation of Go, including the establishment of a [GOPATH](https://golang.org/doc/code.html#GOPATH). Confirm that $GOPATH/bin is included in the system's path, as certain distributions may package outdated versions of build tools. Subsequently, proceed to clone the repository. SDA employs [Go Modules](https://github.com/golang/go/wiki/Modules), and it is advisable to perform the cloning operation outside the GOPATH. Following this, obtain any necessary build tools by initializing the environment through bootstrapping: - -```sh -$ make bootstrap -... -``` - -### Makefile options - -The Makefile is primarily designed to be an aid during development work. - -#### Building the containers - -To build all containers for the SDA stack: - -```sh -$ make build-all -... -``` - -To build the container for a specific component replace `all` with the folder name: - -```sh -$ make build- -... -``` - -#### Running the integration tests - -This will build the container and run the integration test for the PostgreSQL container. The same test will run on every PR in github: - -```sh -$ make integrationtest-postgres -... -``` - -This will build the RabbitMQ and SDA containers and run the integration test for the RabbitMQ container. The same test will run on every PR in github: - -```sh -$ make integrationtest-rabbitmq -... -``` - -This will build all containers and run the integration tests for the SDA stack. The same test will run on every PR in github: - -```sh -$ make integrationtest-sda -... -``` - -#### Linting the GO code - -To run golangci-lint for all go components: - -```sh -$ make lint-all -... -``` - -To run golangci-lint for a specific component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`): - -```sh -$ make lint- -... -``` - -#### Running the static code tests - -For the go code this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a maven container. - -To run the static code tests for all components: - -```sh -$ make test-all -... -``` - -To run the static code tests for a specific component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`, `sda-sftp-inbox`): - -```sh -$ make test- -... -``` diff --git a/README.md b/README.md index fcd1f9ff2..7d3132c46 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,119 @@ # Sensitive Data Archive -`SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/) It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a isolated Sensitive Data Archive. +The `SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/). It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a standalone Sensitive Data Archive. -For more information about the different components see the readme files in the respecive folders. -For more information on how to start developing read [Getting Started developing components of the SDA stack](/GETTINGSTARTED.md). \ No newline at end of file +For more information about the different components, please refer to the README files in their respective folders. + +## How to run the SDA stack +The following instructions outline the steps to set up and run the `SDA` services for development and testing using Docker. These steps are based on the provided [Makefile](./Makefile) commands. + +### Prerequisites +Ensure you have the following installed on your system: + +- [`Go`](https://www.golang.org/): The required version is specified in the `sda` Dockerfile. Verify using + ```sh + $ make go-version-check + ``` + +- Docker: Version 24 or higher. Verify using + ```sh + $ make docker-version-check + ``` +- Docker Compose: Version 2 or higher. For Linux, ensure the [Compose plugin](https://docs.docker.com/compose/install/linux/) is installed. + +In preparation for local development, it is essential to verify that `$GOPATH/bin` is part of the system PATH, as certain distributions may package outdated versions of build tools. SDA uses [Go Modules](https://github.com/golang/go/wiki/Modules), and it is advisable to clone the repository outside the `GOPATH`. After cloning, initialize the environment and obtain necessary build tools using the bootstrap command: + +```sh +$ make bootstrap +``` + +### Build Docker images + +Build the required Docker images for all SDA services: + +```sh +$ make build-all +``` + +You can also build images for individual services by replacing `all` with the folder name (`postgresql`, `rabbitmq`, `sda`, `sda-download`, `sda-sftp-inbox`, `sda-admin`), for example + +```sh +$ make build-sda +``` + +### Running the services + +#### Start services with Docker Compose +The following command will build all required images, bring up all services using the Docker Compose file [sda-s3-integration.yml](.github/integration/sda-s3-integration.yml) (configured for S3 as the storage method) and run the integration test: + +```sh +$ make integrationtest-sda-s3-run +``` + +#### Shut down all services and clean up resources +The following command will shut down all services and clean up all related resources: + +```sh +$ make integrationtest-sda-s3-down +``` + +For the setup with POSIX as the storage method, use +`make integrationtest-sda-posix-run` and `make integrationtest-sda-posix-down` to start and shut down services. For the setup including the [`sync`](https://github.com/neicnordic/sda-sync) service, use `make integrationtest-sda-sync-run` and `make integrationtest-sda-sync-down` to start and shut down services. + +#### Running the integration tests +This will build all required images, bring up the services, run the integration test, and then shut down services and clean up resources. The same test runs on every pull request (PR) in GitHub. + +- Integration test for the database: + ```sh + make integrationtest-postgres + ``` +- Integration test for RabbitMQ: + ```sh + make integrationtest-rabbitmq + ``` +- Integration test for all SDA setups (including S3, POSIX and sync): + ```sh + make integrationtest-sda + ``` +- Integration test for SDA using POSIX as the storage method: + ```sh + make integrationtest-sda-posix + ``` +- Integration test for SDA using S3 as the storage method: + ```sh + make integrationtest-sda-s3 + ``` +- Integration test for SDA including the sync service: + ```sh + make integrationtest-sda-sync + ``` + +### Linting the Go code + +To run `golangci-lint` for all Go components: + +```sh +$ make lint-all +``` + +To run `golangci-lint` for a specific component, replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`), for example: + +```sh +$ make lint-sda +``` + +### Running the static code tests + +For Go code, this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a Maven container. + +To run the static code tests for all components: + +```sh +$ make test-all +``` + +To run the static code tests for a specific component, replace `all` with the folder name (`sda`, `sda-admin`, `sda-download`, `sda-sftp-inbox`), for example: + +```sh +$ make test-sda +``` \ No newline at end of file From 5af56ec7ca7988fe5ced277d2dd42315bec91190 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Thu, 5 Dec 2024 12:30:24 +0000 Subject: [PATCH 31/58] move building of the sda-admin CLI to a separate section --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d3132c46..423fe88dc 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,18 @@ Build the required Docker images for all SDA services: $ make build-all ``` -You can also build images for individual services by replacing `all` with the folder name (`postgresql`, `rabbitmq`, `sda`, `sda-download`, `sda-sftp-inbox`, `sda-admin`), for example +You can also build images for individual services by replacing `all` with the folder name (`postgresql`, `rabbitmq`, `sda`, `sda-download`, `sda-sftp-inbox`), for example ```sh $ make build-sda ``` +To build the CLI for `sda-admin`: + +```sh +$ make build-sda-admin +``` + ### Running the services #### Start services with Docker Compose From 9fc89efb07b42e63a042fa4452c79d1221d9a6ea Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Fri, 6 Dec 2024 23:36:27 +0100 Subject: [PATCH 32/58] add commands that bring up and bring down services --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index 9228edbb4..728ad5894 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,21 @@ docker-version-check: echo "Docker buildx does not exist can't continue"; \ fi; +# bring up the services +sda-s3-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml up -d +sda-posix-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml up -d +sda-sync-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml up -d + +# bring down the services +sda-s3-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans +sda-posix-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans +sda-sync-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans # run intrgration tests, same as being run in Github Actions during a PR integrationtest-postgres: build-postgresql From 805942d0a01859beec0c05afd00b81a2f367dd7d Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Fri, 6 Dec 2024 23:37:01 +0100 Subject: [PATCH 33/58] update README --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 423fe88dc..14fe43c00 100644 --- a/README.md +++ b/README.md @@ -50,23 +50,25 @@ $ make build-sda-admin ### Running the services #### Start services with Docker Compose -The following command will build all required images, bring up all services using the Docker Compose file [sda-s3-integration.yml](.github/integration/sda-s3-integration.yml) (configured for S3 as the storage method) and run the integration test: +The following command will bring up all services using the Docker Compose file [sda-s3-integration.yml](.github/integration/sda-s3-integration.yml) (configured for S3 as the storage backend): ```sh -$ make integrationtest-sda-s3-run +$ make sda-s3-up ``` #### Shut down all services and clean up resources The following command will shut down all services and clean up all related resources: ```sh -$ make integrationtest-sda-s3-down +$ make sda-s3-down ``` -For the setup with POSIX as the storage method, use -`make integrationtest-sda-posix-run` and `make integrationtest-sda-posix-down` to start and shut down services. For the setup including the [`sync`](https://github.com/neicnordic/sda-sync) service, use `make integrationtest-sda-sync-run` and `make integrationtest-sda-sync-down` to start and shut down services. +For the setup with POSIX as the storage backend, use +`make sda-posix-up` and `make sda-posix-down` to start and shut down services. -#### Running the integration tests +For the setup including the [`sync`](https://github.com/neicnordic/sda-sync) service, use `make sda-sync-up` and `make sda-sync-down` to start and shut down services. + +### Running the integration tests This will build all required images, bring up the services, run the integration test, and then shut down services and clean up resources. The same test runs on every pull request (PR) in GitHub. - Integration test for the database: @@ -81,11 +83,11 @@ This will build all required images, bring up the services, run the integration ```sh make integrationtest-sda ``` -- Integration test for SDA using POSIX as the storage method: +- Integration test for SDA using POSIX as the storage backend: ```sh make integrationtest-sda-posix ``` -- Integration test for SDA using S3 as the storage method: +- Integration test for SDA using S3 as the storage backend: ```sh make integrationtest-sda-s3 ``` @@ -93,6 +95,36 @@ This will build all required images, bring up the services, run the integration ```sh make integrationtest-sda-sync ``` +#### Running the integration tests without shutting down the services +This will run the integration tests and keep the services running after the tests are finished. + +- Integration test for SDA using POSIX as the storage backend: + ```sh + make integrationtest-sda-posix-run + ``` +- Integration test for SDA using S3 as the storage backend: + ```sh + make integrationtest-sda-s3-run + ``` +- Integration test for SDA including the sync service: + ```sh + make integrationtest-sda-sync-run + ``` + +After that, you will need to shut down the services manually. + +- Shut down services for SDA using POSIX as the storage backend + ```sh + make integrationtest-sda-posix-down + ``` +- Shut down services for SDA using S3 as the storage backend + ```sh + make integrationtest-sda-s3-down + ``` +- Shut down services for SDA including the sync service: + ```sh + make integrationtest-sda-sync-down + ``` ### Linting the Go code From 6d5d496792ff4d9f4f8c6018dca3315165024c32 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 9 Dec 2024 15:55:16 +0100 Subject: [PATCH 34/58] print message on passed version check --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 728ad5894..78e4c86f6 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,10 @@ go-version-check: ( $${GO_VERSION_ARR[1]} -lt $${GO_VERSION_REQ[1]} ||\ ( $${GO_VERSION_ARR[1]} -eq $${GO_VERSION_REQ[1]} && $${GO_VERSION_ARR[2]} -lt $${GO_VERSION_REQ[2]} )))\ ]]; then\ - echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}.";\ - exit 1;\ + echo "Version check failed. SDA requires Go $${GO_VERSION_MIN} to build."; \ + exit 1; \ + else \ + echo "Version check passed. Installed GO version: $${GO_VERSION}."; \ fi; docker-version-check: @@ -66,7 +68,9 @@ docker-version-check: fi; \ if [ ! $$(docker buildx version | cut -d' ' -f 2) ]; then \ echo "Docker buildx does not exist can't continue"; \ - fi; + exit 1;\ + fi; \ + echo "Version check passed. Installed Docker version: $${DOCKER_VERSION} and Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; # bring up the services sda-s3-up: From 452f3f948e51d4657477893f525024686396cf7c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 9 Dec 2024 17:03:49 +0100 Subject: [PATCH 35/58] Update Makefile Co-authored-by: Joakim Bygdell --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 78e4c86f6..1e9a623da 100644 --- a/Makefile +++ b/Makefile @@ -49,11 +49,10 @@ go-version-check: ( $${GO_VERSION_ARR[1]} -lt $${GO_VERSION_REQ[1]} ||\ ( $${GO_VERSION_ARR[1]} -eq $${GO_VERSION_REQ[1]} && $${GO_VERSION_ARR[2]} -lt $${GO_VERSION_REQ[2]} )))\ ]]; then\ - echo "Version check failed. SDA requires Go $${GO_VERSION_MIN} to build."; \ + echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}."; \ exit 1; \ - else \ - echo "Version check passed. Installed GO version: $${GO_VERSION}."; \ - fi; + fi; \ + echo "GO version: $${GO_VERSION}."; docker-version-check: @DOCKER_VERSION=$$(docker version -f "{{.Server.Version}}" | cut -d'.' -f 1); \ @@ -70,7 +69,8 @@ docker-version-check: echo "Docker buildx does not exist can't continue"; \ exit 1;\ fi; \ - echo "Version check passed. Installed Docker version: $${DOCKER_VERSION} and Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; + echo "Docker version: $${DOCKER_VERSION}."; \ + echo "Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; # bring up the services sda-s3-up: From 06d83ad5dd3499ac8a94a3d46f514ce215f9a501 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:47:51 +0100 Subject: [PATCH 36/58] remove unnecessary trailing semicolon --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1e9a623da..b3217c3f7 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ go-version-check: echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}."; \ exit 1; \ fi; \ - echo "GO version: $${GO_VERSION}."; + echo "GO version: $${GO_VERSION}." docker-version-check: @DOCKER_VERSION=$$(docker version -f "{{.Server.Version}}" | cut -d'.' -f 1); \ @@ -70,7 +70,7 @@ docker-version-check: exit 1;\ fi; \ echo "Docker version: $${DOCKER_VERSION}."; \ - echo "Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; + echo "Docker Compose version: $${DOCKER_COMPOSE_VERSION}." # bring up the services sda-s3-up: From 0bac334e3f54af333a992115f8573aff3d3c0c1e Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 10 Dec 2024 12:54:31 +0100 Subject: [PATCH 37/58] refactor: sentence reformatting Co-authored-by: Joakim Bygdell --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 14fe43c00..4538ca0df 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ You can also build images for individual services by replacing `all` with the fo $ make build-sda ``` -To build the CLI for `sda-admin`: +To build the `sda-admin` CLI tool: ```sh $ make build-sda-admin From 7e1bea6d30d86b760c390acc62d0baafc9ad6d1b Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Tue, 10 Dec 2024 17:34:42 +0100 Subject: [PATCH 38/58] Update FEATURE_REQUEST.md Semantically proper use of markdown headings for headings. This also frees up the bold markup for use in the issue text if needed. --- .github/ISSUE_TEMPLATE/FEATURE_REQUEST.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md index 82273c659..59d23b5a0 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md @@ -4,15 +4,21 @@ about: Suggest an idea for this project --- -**Please describe the feature** +## Please describe the feature + As a [type of user], I want [an action] so that [a benefit/a value]. -**Acceptance criteria** +## Acceptance criteria + - [ ] - [ ] Tests verifying the changes are added -**Additional context** +## Additional context + +## Estimation of size + +small/medium/big -**Estimation of size**: small/medium/big +## Estimation of priority -**Estimation of priority**: low/medium/high +low/medium/high From 417c150ea210b02a1693a5972b02ba2a3433b7fd Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Tue, 10 Dec 2024 17:36:02 +0100 Subject: [PATCH 39/58] Update BUG_REPORT.md --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 96e48cc36..ff402dc7e 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -5,16 +5,18 @@ labels: bug --- -**Describe the bug** +## Describe the bug -**Steps to reproduce** +## Steps to reproduce -**Expected behavior** +## Expected behavior - [ ] - [ ] Tests verifying the fix are added -**Additional context** +## Additional context -**Estimation of size**: small/medium/big +## Estimation of size +small/medium/big -**Estimation of priority**: low/medium/high +## Estimation of priority +low/medium/high From 0371d83c21ee2594c049c4309fc1782f31af903f Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Tue, 10 Dec 2024 17:36:29 +0100 Subject: [PATCH 40/58] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b1680c226..9a3333b6b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,8 +1,8 @@ -**Related issue(s) and PR(s)** +## Related issue(s) and PR(s) This PR closes [issue number]. -**Description** +## Description -**How to test** +## How to test From 39beba68cd2ccbf5937ccc9f3fd3810d6a758699 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:00:50 +0100 Subject: [PATCH 41/58] add config file for running sda-download with go run --- .../dev_utils/config-notls_local.yaml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 sda-download/dev_utils/config-notls_local.yaml diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml new file mode 100644 index 000000000..11f9d768c --- /dev/null +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -0,0 +1,34 @@ +app: + serveUnencryptedData: true + +log: + level: "debug" + format: "json" + +archive: + type: "s3" + # S3 backend + url: "http://localhost" + port: 19000 + accesskey: "access" + secretkey: "secretKey" + bucket: "archive" + region: "us-east-1" + chunksize: 32 + +grpc: + host: localhost + port: 50051 + +db: + host: "localhost" + port: 15432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +oidc: + # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri" + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" From 5b163879e898d98e3433f43f60ead0ce4006d529 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:13:09 +0100 Subject: [PATCH 42/58] expose API port for rabbitmq --- .github/integration/sda-s3-integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 688ea7dea..a4a900c59 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -52,6 +52,7 @@ services: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" + - "15671:5672" restart: always volumes: - rabbitmq_data:/var/lib/rabbitmq From 9591f5f86efd84215f251bcf76fa688cae2b3163 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:14:17 +0100 Subject: [PATCH 43/58] add config file for running sda services (except download) with go run --- sda/config_local.yaml | 107 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 sda/config_local.yaml diff --git a/sda/config_local.yaml b/sda/config_local.yaml new file mode 100644 index 000000000..21fe9b399 --- /dev/null +++ b/sda/config_local.yaml @@ -0,0 +1,107 @@ +log: + format: "json" + level: "debug" +api: + rbacFile: ../.github/integration/sda/rbac.json + +archive: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "archive" + region: "us-east-1" + +auth: + cega: + authUrl: "http://localhost:8443/username/" + id: + secret: + infoText: "About service text" + infoURL: "http://example.org/about" + jwt: + issuer: "https://localhost:8888" + privateKey: "tmp/shared/keys/jwt.key" + signatureAlg: ES256 + tokenTTL: 168 + publicFile: "tmp/shared/c4gh.pub.pem" + resignJwt: + s3Inbox: "http://localhost:18000" + +backup: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "backup" + region: "us-east-1" + +inbox: + type: s3 + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "inbox" + region: "us-east-1" + +broker: + host: "localhost" + port: "15671" + user: "" + password: "" + vhost: "/sda" + exchange: "sda" + routingKey: "" + ssl: "false" + +db: + host: "localhost" + port: "15432" + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +c4gh: + filePath: "tmp/shared/c4gh.sec.pem" + passphrase: "c4ghpass" + syncPubKeyPath: "tmp/shared/sync.pub.pem" + +oidc: + configuration: + url: "http://localhost:8080/.well-known/openid-configuration" + +server: + cert: "" + key: "" + jwtpubkeypath: "tmp/shared/keys/pub/" + jwtpubkeyurl: "http://oidc:8080/jwk" + +sync: + api: + password: "pass" + user: "user" + centerPrefix: "SYNC" + destination: + type: "s3" + url: "http://localhost" + port: 19000 + readypath: "/minio/health/ready" + accessKey: "access" + secretKey: "secretKey" + bucket: "sync" + region: "us-east-1" + remote: + host: "http://sync-api" + port: "8080" + password: "pass" + user: "user" + +schema: + type: "isolated" \ No newline at end of file From 1fe05754961bab7a4fce549a9615aac40276d467 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:14:52 +0100 Subject: [PATCH 44/58] add development instructions using go run to run services --- DEVELOPMENT.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..fdd14b826 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,97 @@ +## Run services with `go run` + +This section explains how to run some of the services using `go run` instead of the Docker setup to facilitate development. + +### Running `sda-download` with `go run` +1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: +```sh +make integrationtest-sda-s3-run +``` + +2. Change to the folder `sda-download` and then create the credentials using: +```sh +pushd dev_utils +bash make_certs.sh +chmod 600 certs/*-key.* +popd +``` + +3. Update your `/etc/hosts` file: Add the following line to ensure the service can resolve OIDC locally: +``` +127.0.0.1 oidc +``` + +4. Start the `sda-download` service using: +```sh +APP_PORT=8553 CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go +``` + +5. Check if `sda-download` works as expected using: +```sh +token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +curl -H "Authorization: Bearer $token" http://localhost:8553/metadata/datasets +``` +If successful, the curl command should output a JSON body containing: +```json +["EGAD74900000101"] +``` + + +### Running other SDA services with `go run` +Running other SDA services located in the `sda` subfolder, such as `ingest` or `verify`, differs slightly from running the `sda-download` service. Here, we'll use `ingest` as an example. + +1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository (this step is the same as for `sda-download`): +```sh +make integrationtest-sda-s3-run +``` + +2. Bring down the `ingest` service using: +```sh +PR_NUMBER=$(/bin/date +%F) docker compose -f .github/integration/sda-s3-integration.yml stop ingest +``` + +3. Change to the folder `sda`, then copy keys and other information from the shared folder of the container using: +```sh +mkdir -p tmp +docker cp verify:/shared tmp/ +``` +This will copy all data from the container's `/shared` folder to `tmp/shared` on your local machine. + +4. Copy schemas to `/schemas` on the localhost using: +```sh +sudo rsync -arvz schemas/ /schemas/ +``` + +5. Start the `ingest` service using: +```sh +export BROKER_PASSWORD=ingest +export BROKER_USER=ingest +export BROKER_QUEUE=ingest +export BROKER_ROUTINGKEY=archived +export DB_PASSWORD=ingest +export DB_USER=ingest +CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go +``` + +6. Check if the `ingest` service works as expected by following these steps +```sh +# create a test file +seq 10 > tmp/t1.txt + +# update the s3cmd config file +sed -i '/host_/s/s3inbox:8000/localhost:18000/g' tmp/shared/s3cfg + +# upload tmp/t1.txt to s3inbox by sda-cli +sda-cli -config tmp/shared/s3cfg upload -encrypt-with-key tmp/shared/c4gh.pub.pem tmp/t1.txt + +# use sda-admin to check if t1.txt has been uploaded +export API_HOST=http://localhost:8090 +export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') +sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'uploaded' + +# use sda-admin to ingest the file t1.txt +sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org + +# verify that t1.txt has been uploaded using sda-admin +sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'verified' +``` From c8a2c4e1aabc573598b039f60579a1882a901736 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Sun, 8 Dec 2024 22:31:06 +0100 Subject: [PATCH 45/58] fix failed integration test with permission error on db-key.pem --- sda-download/dev_utils/compose-sda.yml | 7 ++++--- sda-download/dev_utils/compose.yml | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 41d3875fd..fad8ca2d5 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -6,13 +6,14 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* + chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 5ee4d5b0b..d43b3cdea 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -5,14 +5,14 @@ services: - -c - | cp /origcerts/* /certs - chown -R nobody.nobody /certs/* + chown -R nobody:nobody /certs/* chmod -R 644 /certs/* chmod -R og-rw /certs/*-key.pem - chown -R 70.70 /certs/db* + chown -R 70:70 /certs/db* ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs From abfbe1dccb296c1b5d67acb528d7831645bd0577 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 11:16:41 +0100 Subject: [PATCH 46/58] use other ports for rabbitmq --- .github/integration/sda-s3-integration.yml | 2 +- sda/config_local.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index a4a900c59..f8e357849 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -52,7 +52,7 @@ services: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" - - "15671:5672" + - "5672:5672" restart: always volumes: - rabbitmq_data:/var/lib/rabbitmq diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 21fe9b399..67008f95a 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -52,7 +52,7 @@ inbox: broker: host: "localhost" - port: "15671" + port: "5672" user: "" password: "" vhost: "/sda" From f89580662d2d69ba719d622543360a752441493b Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 11:18:08 +0100 Subject: [PATCH 47/58] remove trailing whitespace --- sda-download/dev_utils/compose-sda.yml | 2 +- sda-download/dev_utils/compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index fad8ca2d5..9b6b9f0d2 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -13,7 +13,7 @@ services: ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index d43b3cdea..a3685c878 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -12,7 +12,7 @@ services: ls -la /certs/ container_name: certfixer - image: alpine:latest + image: alpine:latest volumes: - ./certs:/origcerts - certs:/certs From 1ef45b71808721cfb83d18afd606dce646d58582 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 11:34:52 +0100 Subject: [PATCH 48/58] use the system /tmp instead of relative path tmp --- DEVELOPMENT.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index fdd14b826..7b2cb5a8e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -52,10 +52,9 @@ PR_NUMBER=$(/bin/date +%F) docker compose -f .github/integration/sda-s3-integra 3. Change to the folder `sda`, then copy keys and other information from the shared folder of the container using: ```sh -mkdir -p tmp -docker cp verify:/shared tmp/ +docker cp verify:/shared /tmp/ ``` -This will copy all data from the container's `/shared` folder to `tmp/shared` on your local machine. +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine. 4. Copy schemas to `/schemas` on the localhost using: ```sh @@ -76,13 +75,13 @@ CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go 6. Check if the `ingest` service works as expected by following these steps ```sh # create a test file -seq 10 > tmp/t1.txt +seq 10 > /tmp/t1.txt # update the s3cmd config file -sed -i '/host_/s/s3inbox:8000/localhost:18000/g' tmp/shared/s3cfg +sed -i '/host_/s/s3inbox:8000/localhost:18000/g' /tmp/shared/s3cfg -# upload tmp/t1.txt to s3inbox by sda-cli -sda-cli -config tmp/shared/s3cfg upload -encrypt-with-key tmp/shared/c4gh.pub.pem tmp/t1.txt +# upload /tmp/t1.txt to s3inbox by sda-cli +sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt # use sda-admin to check if t1.txt has been uploaded export API_HOST=http://localhost:8090 From 4b2c024ffcd24b7970d2352897b129527de809bf Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 15:27:43 +0100 Subject: [PATCH 49/58] add support for a customizable schemas path --- sda/internal/config/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index cdc1472c5..984ccf967 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -933,6 +933,10 @@ func (c *Config) configSchemas() { } else { c.Broker.SchemasPath = "/schemas/isolated/" } + + if viper.IsSet("schema.path") { + c.Broker.SchemasPath = viper.GetString("schema.path") + } } // configS3Storage populates and returns a S3Conf from the From 92f23fd23e3a50d7c1e4280d002acf9d4301b5d4 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Mon, 9 Dec 2024 15:29:04 +0100 Subject: [PATCH 50/58] update config and instructions for using custom schemas path --- DEVELOPMENT.md | 9 ++------- sda/config_local.yaml | 3 ++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7b2cb5a8e..bad3f628c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -56,12 +56,7 @@ docker cp verify:/shared /tmp/ ``` This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine. -4. Copy schemas to `/schemas` on the localhost using: -```sh -sudo rsync -arvz schemas/ /schemas/ -``` - -5. Start the `ingest` service using: +4. Start the `ingest` service using: ```sh export BROKER_PASSWORD=ingest export BROKER_USER=ingest @@ -72,7 +67,7 @@ export DB_USER=ingest CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go ``` -6. Check if the `ingest` service works as expected by following these steps +5. Check if the `ingest` service works as expected by following these steps ```sh # create a test file seq 10 > /tmp/t1.txt diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 67008f95a..332048b08 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -104,4 +104,5 @@ sync: user: "user" schema: - type: "isolated" \ No newline at end of file + type: "isolated" + path: "schemas/isolated" \ No newline at end of file From e32734df4e6a96a5c24264eb347a857071403be7 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:37:45 +0100 Subject: [PATCH 51/58] add port number in the config file --- sda-download/dev_utils/config-notls_local.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml index 11f9d768c..d425077a8 100644 --- a/sda-download/dev_utils/config-notls_local.yaml +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -1,5 +1,6 @@ app: serveUnencryptedData: true + port: 18080 log: level: "debug" From 8c74f9dd3a9ebd7d5f361d3a22a675f9052af442 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:38:51 +0100 Subject: [PATCH 52/58] update instructions based on reviewers comments --- DEVELOPMENT.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index bad3f628c..5b4512a17 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -23,13 +23,19 @@ popd 4. Start the `sda-download` service using: ```sh -APP_PORT=8553 CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go +CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go ``` 5. Check if `sda-download` works as expected using: ```sh +curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health +``` +If successful, the curl command should output the HTTP code `200`. + +You can further check the endpoint `/metadata/datasets` using: +```sh token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') -curl -H "Authorization: Bearer $token" http://localhost:8553/metadata/datasets +curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets ``` If successful, the curl command should output a JSON body containing: ```json @@ -40,7 +46,7 @@ If successful, the curl command should output a JSON body containing: ### Running other SDA services with `go run` Running other SDA services located in the `sda` subfolder, such as `ingest` or `verify`, differs slightly from running the `sda-download` service. Here, we'll use `ingest` as an example. -1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository (this step is the same as for `sda-download`): +1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: ```sh make integrationtest-sda-s3-run ``` From 5e2d09c0bab31a8fbf4524e5c521eddb3393789f Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:42:05 +0100 Subject: [PATCH 53/58] change from relative path tmp to /tmp --- sda/config_local.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 332048b08..81946299b 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -23,10 +23,10 @@ auth: infoURL: "http://example.org/about" jwt: issuer: "https://localhost:8888" - privateKey: "tmp/shared/keys/jwt.key" + privateKey: "/tmp/shared/keys/jwt.key" signatureAlg: ES256 tokenTTL: 168 - publicFile: "tmp/shared/c4gh.pub.pem" + publicFile: "/tmp/shared/c4gh.pub.pem" resignJwt: s3Inbox: "http://localhost:18000" @@ -69,9 +69,9 @@ db: sslmode: "disable" c4gh: - filePath: "tmp/shared/c4gh.sec.pem" + filePath: "/tmp/shared/c4gh.sec.pem" passphrase: "c4ghpass" - syncPubKeyPath: "tmp/shared/sync.pub.pem" + syncPubKeyPath: "/tmp/shared/sync.pub.pem" oidc: configuration: @@ -80,7 +80,7 @@ oidc: server: cert: "" key: "" - jwtpubkeypath: "tmp/shared/keys/pub/" + jwtpubkeypath: "/tmp/shared/keys/pub/" jwtpubkeyurl: "http://oidc:8080/jwk" sync: From 0b1261ef106844f9e9a8bdf75ac61aa1180a8d8a Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 12:13:02 +0100 Subject: [PATCH 54/58] remove instructions for creating credentials for notls --- DEVELOPMENT.md | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5b4512a17..86b92ffa6 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -8,25 +8,16 @@ This section explains how to run some of the services using `go run` instead of make integrationtest-sda-s3-run ``` -2. Change to the folder `sda-download` and then create the credentials using: -```sh -pushd dev_utils -bash make_certs.sh -chmod 600 certs/*-key.* -popd -``` - -3. Update your `/etc/hosts` file: Add the following line to ensure the service can resolve OIDC locally: -``` -127.0.0.1 oidc -``` - -4. Start the `sda-download` service using: +2. Change to the folder `sda-download` and start the `sda-download` service using: ```sh CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go ``` +> If the host `oidc` cannot be accessed on the local machine, update your `/etc/hosts` file by adding the following line to ensure the service can resolve `oidc` locally: +> ``` +> 127.0.0.1 oidc +> ``` -5. Check if `sda-download` works as expected using: +3. Check if `sda-download` works as expected using: ```sh curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health ``` From f1a79844a99b2dda8db4c3c5e46316ff51f45fb8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 10 Dec 2024 13:09:22 +0100 Subject: [PATCH 55/58] Rework oidc.py to work when calling from the users shell --- .github/integration/sda-s3-integration.yml | 8 ++++++++ .github/integration/sda/oidc.py | 24 +++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index f8e357849..59ecdff77 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -221,6 +221,8 @@ services: depends_on: credentials: condition: service_completed_successfully + extra_hosts: + - "localhost:host-gateway" healthcheck: test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] interval: 10s @@ -331,8 +333,12 @@ services: - AUTH_RESIGNJWT=false - OIDC_ID=XC56EL11xx - OIDC_SECRET=wHPVQaYXmdDHg + - OIDC_PROVIDER=http://localhost:8080 + - OIDC_REDIRECTURL=http://localhost:8889/oidc/login - DB_PASSWORD=auth - DB_USER=auth + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8889:8080" @@ -370,6 +376,8 @@ services: condition: service_started reencrypt: condition: service_started + extra_hosts: + - "localhost:host-gateway" environment: - PGPASSWORD=rootpasswd - STORAGETYPE=s3 diff --git a/.github/integration/sda/oidc.py b/.github/integration/sda/oidc.py index c7969b136..a6b387af9 100644 --- a/.github/integration/sda/oidc.py +++ b/.github/integration/sda/oidc.py @@ -51,7 +51,7 @@ def _generate_token() -> Tuple: # See available claims here: http://www.iana.org/assignments/jwt/jwt.xhtml # the important claim is the "authorities" header = { - "jku": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "jku": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "alg": "ES256", "typ": "JWT", "kid": ec_key1.thumbprint() @@ -61,7 +61,7 @@ def _generate_token() -> Tuple: "aud": ["aud1", "aud2"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", @@ -71,21 +71,21 @@ def _generate_token() -> Tuple: "aud": ["aud2", "aud3"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } empty_payload = { "sub": "requester@demo.org", - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "exp": 99999999999, "iat": 1547794655, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102", } # Craft passports passport_terms = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "AcceptedTermsAndPolicies", @@ -100,7 +100,7 @@ def _generate_token() -> Tuple: } # passport for dataset permissions 1 passport_dataset1 = { - "iss": "https://oidc:8080/", + "iss": "https://localhost:8080/", "sub": "requester@demo.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -165,12 +165,12 @@ def _generate_token() -> Tuple: async def fixed_response(request: web.Request) -> web.Response: global HTTP_PROTOCOL WELL_KNOWN = { - "issuer": f"{HTTP_PROTOCOL}://oidc:8080", - "authorization_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/authorize", - "registration_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/register", - "token_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/token", - "userinfo_endpoint": f"{HTTP_PROTOCOL}://oidc:8080/userinfo", - "jwks_uri": f"{HTTP_PROTOCOL}://oidc:8080/jwk", + "issuer": f"{HTTP_PROTOCOL}://localhost:8080", + "authorization_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/authorize", + "registration_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/register", + "token_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/token", + "userinfo_endpoint": f"{HTTP_PROTOCOL}://localhost:8080/userinfo", + "jwks_uri": f"{HTTP_PROTOCOL}://localhost:8080/jwk", "response_types_supported": [ "code", "id_token", From 2e1d482c2c06ddc50b1a1b0ed014484b2ca9ff95 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 15:41:21 +0100 Subject: [PATCH 56/58] update instructions --- DEVELOPMENT.md | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 86b92ffa6..437849d8e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,59 +1,68 @@ -## Run services with `go run` +# Run services with `go run` This section explains how to run some of the services using `go run` instead of the Docker setup to facilitate development. -### Running `sda-download` with `go run` -1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: +## Running `sda-download` with `go run` + +- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: + ```sh make integrationtest-sda-s3-run ``` -2. Change to the folder `sda-download` and start the `sda-download` service using: +- Change to the folder `sda-download` and start the `sda-download` service using: + ```sh CONFIGFILE=dev_utils/config-notls_local.yaml go run cmd/main.go ``` -> If the host `oidc` cannot be accessed on the local machine, update your `/etc/hosts` file by adding the following line to ensure the service can resolve `oidc` locally: -> ``` -> 127.0.0.1 oidc -> ``` -3. Check if `sda-download` works as expected using: +- Check if `sda-download` works as expected using: + ```sh curl -o /dev/null -s -w "%{http_code}\n" http://localhost:18080/health ``` + If successful, the curl command should output the HTTP code `200`. You can further check the endpoint `/metadata/datasets` using: + ```sh token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets ``` + If successful, the curl command should output a JSON body containing: + ```json ["EGAD74900000101"] ``` +## Running other SDA services with `go run` + +Running any of the SDA services located in the `sda` subfolder requires that the service specific credentials and RabbitMQ configurations are set as ENVs. Here, we'll use `ingest` as an example. -### Running other SDA services with `go run` -Running other SDA services located in the `sda` subfolder, such as `ingest` or `verify`, differs slightly from running the `sda-download` service. Here, we'll use `ingest` as an example. +- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: -1. Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: ```sh make integrationtest-sda-s3-run ``` -2. Bring down the `ingest` service using: +- When the previous command is finished, bring down the `ingest` service using: + ```sh -PR_NUMBER=$(/bin/date +%F) docker compose -f .github/integration/sda-s3-integration.yml stop ingest +docker stop ingest ``` -3. Change to the folder `sda`, then copy keys and other information from the shared folder of the container using: +- Copy keys and other information from the shared folder of the container using: + ```sh docker cp verify:/shared /tmp/ ``` -This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine. -4. Start the `ingest` service using: +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the required auto generated files that will be required. + +- Change to the folder `sda` and start the `ingest` service using: + ```sh export BROKER_PASSWORD=ingest export BROKER_USER=ingest @@ -64,7 +73,8 @@ export DB_USER=ingest CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go ``` -5. Check if the `ingest` service works as expected by following these steps +- Check if the `ingest` service works as expected by following these steps + ```sh # create a test file seq 10 > /tmp/t1.txt @@ -83,6 +93,6 @@ sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'u # use sda-admin to ingest the file t1.txt sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org -# verify that t1.txt has been uploaded using sda-admin +# verify that t1.txt has been ingested using sda-admin sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'verified' ``` From f3c1537c949f0f977f960fda4f3876de0d09e21b Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 21:56:51 +0100 Subject: [PATCH 57/58] update instructions for file ingestions --- DEVELOPMENT.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 437849d8e..c952962dc 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -41,10 +41,10 @@ If successful, the curl command should output a JSON body containing: Running any of the SDA services located in the `sda` subfolder requires that the service specific credentials and RabbitMQ configurations are set as ENVs. Here, we'll use `ingest` as an example. -- Bring up all SDA services with the S3 backend and populate them with test data by running the following command in the root folder of the repository: +- Bring up all SDA services with the S3 backend by running the following command in the root folder of the repository: ```sh -make integrationtest-sda-s3-run +make sda-s3-up ``` - When the previous command is finished, bring down the `ingest` service using: @@ -88,11 +88,14 @@ sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub. # use sda-admin to check if t1.txt has been uploaded export API_HOST=http://localhost:8090 export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') -sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'uploaded' +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'uploaded' + +# register the Crypt4GH key +curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add # use sda-admin to ingest the file t1.txt sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org # verify that t1.txt has been ingested using sda-admin -sda-admin file list -user test@dummy.org # file t1.txt should have fileStatus 'verified' +sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'verified' ``` From de39a95a781a09d5e73f19ea3a9dffe4c4931444 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 11 Dec 2024 11:07:19 +0100 Subject: [PATCH 58/58] readme: grammatical correction Co-authored-by: Joakim Bygdell --- DEVELOPMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c952962dc..be6bdc999 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -59,7 +59,7 @@ docker stop ingest docker cp verify:/shared /tmp/ ``` -This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the required auto generated files that will be required. +This will copy all data from the container's `/shared` folder to `/tmp/shared` on your local machine, so that we have access to all the auto generated files that will be required. - Change to the folder `sda` and start the `ingest` service using: