From ba9bb6f2cee853cc3d93ed03595a42a9933e8661 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 3 Dec 2024 18:23:13 +0000 Subject: [PATCH 01/10] fix: typo for build-all in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 51791cd7e..65f81a1fa 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ integrationtest-rabbitmq: build-rabbitmq build-sda @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml down -v --remove-orphans -integrationtest-sda: build_all +integrationtest-sda: build-all @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test From 8cb5812fab19c5c91a72fc6bc2a06f8a43fa8cf5 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 3 Dec 2024 18:24:17 +0000 Subject: [PATCH 02/10] fix: bootstrap in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 65f81a1fa..9228edbb4 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ bootstrap: go-version-check docker-version-check fi @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ sh -s -- -b $$(go env GOPATH)/bin - GO111MODULE=off go get golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports@latest # build containers build-all: build-postgresql build-rabbitmq build-sda build-sda-download build-sda-sftp-inbox build-sda-admin From cbd5440916cdcd2812b5a6d6d82ed50b110aeccb Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Wed, 4 Dec 2024 18:50:29 +0000 Subject: [PATCH 03/10] Add instructions to run the SDA stack in README --- GETTINGSTARTED.md | 93 ------------------------------------ README.md | 119 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 96 deletions(-) delete mode 100644 GETTINGSTARTED.md diff --git a/GETTINGSTARTED.md b/GETTINGSTARTED.md deleted file mode 100644 index cb86ff5de..000000000 --- a/GETTINGSTARTED.md +++ /dev/null @@ -1,93 +0,0 @@ -## Getting Started developing components of the SDA stack - -Should one wish to engage in the development of the SDA stack itself, the prerequisite is the installation of [Go](https://www.golang.org/) on the respective machine. -The recommended version can be checked by running: - -```sh -$ make go-version-check -... -``` - -In preparation for local development, it is essential to verify the proper installation of Go, including the establishment of a [GOPATH](https://golang.org/doc/code.html#GOPATH). Confirm that $GOPATH/bin is included in the system's path, as certain distributions may package outdated versions of build tools. Subsequently, proceed to clone the repository. SDA employs [Go Modules](https://github.com/golang/go/wiki/Modules), and it is advisable to perform the cloning operation outside the GOPATH. Following this, obtain any necessary build tools by initializing the environment through bootstrapping: - -```sh -$ make bootstrap -... -``` - -### Makefile options - -The Makefile is primarily designed to be an aid during development work. - -#### Building the containers - -To build all containers for the SDA stack: - -```sh -$ make build-all -... -``` - -To build the container for a specific component replace `all` with the folder name: - -```sh -$ make build- -... -``` - -#### Running the integration tests - -This will build the container and run the integration test for the PostgreSQL container. The same test will run on every PR in github: - -```sh -$ make integrationtest-postgres -... -``` - -This will build the RabbitMQ and SDA containers and run the integration test for the RabbitMQ container. The same test will run on every PR in github: - -```sh -$ make integrationtest-rabbitmq -... -``` - -This will build all containers and run the integration tests for the SDA stack. The same test will run on every PR in github: - -```sh -$ make integrationtest-sda -... -``` - -#### Linting the GO code - -To run golangci-lint for all go components: - -```sh -$ make lint-all -... -``` - -To run golangci-lint for a specific component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`): - -```sh -$ make lint- -... -``` - -#### Running the static code tests - -For the go code this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a maven container. - -To run the static code tests for all components: - -```sh -$ make test-all -... -``` - -To run the static code tests for a specific component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`, `sda-sftp-inbox`): - -```sh -$ make test- -... -``` diff --git a/README.md b/README.md index fcd1f9ff2..7d3132c46 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,119 @@ # Sensitive Data Archive -`SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/) It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a isolated Sensitive Data Archive. +The `SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/). It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a standalone Sensitive Data Archive. -For more information about the different components see the readme files in the respecive folders. -For more information on how to start developing read [Getting Started developing components of the SDA stack](/GETTINGSTARTED.md). \ No newline at end of file +For more information about the different components, please refer to the README files in their respective folders. + +## How to run the SDA stack +The following instructions outline the steps to set up and run the `SDA` services for development and testing using Docker. These steps are based on the provided [Makefile](./Makefile) commands. + +### Prerequisites +Ensure you have the following installed on your system: + +- [`Go`](https://www.golang.org/): The required version is specified in the `sda` Dockerfile. Verify using + ```sh + $ make go-version-check + ``` + +- Docker: Version 24 or higher. Verify using + ```sh + $ make docker-version-check + ``` +- Docker Compose: Version 2 or higher. For Linux, ensure the [Compose plugin](https://docs.docker.com/compose/install/linux/) is installed. + +In preparation for local development, it is essential to verify that `$GOPATH/bin` is part of the system PATH, as certain distributions may package outdated versions of build tools. SDA uses [Go Modules](https://github.com/golang/go/wiki/Modules), and it is advisable to clone the repository outside the `GOPATH`. After cloning, initialize the environment and obtain necessary build tools using the bootstrap command: + +```sh +$ make bootstrap +``` + +### Build Docker images + +Build the required Docker images for all SDA services: + +```sh +$ make build-all +``` + +You can also build images for individual services by replacing `all` with the folder name (`postgresql`, `rabbitmq`, `sda`, `sda-download`, `sda-sftp-inbox`, `sda-admin`), for example + +```sh +$ make build-sda +``` + +### Running the services + +#### Start services with Docker Compose +The following command will build all required images, bring up all services using the Docker Compose file [sda-s3-integration.yml](.github/integration/sda-s3-integration.yml) (configured for S3 as the storage method) and run the integration test: + +```sh +$ make integrationtest-sda-s3-run +``` + +#### Shut down all services and clean up resources +The following command will shut down all services and clean up all related resources: + +```sh +$ make integrationtest-sda-s3-down +``` + +For the setup with POSIX as the storage method, use +`make integrationtest-sda-posix-run` and `make integrationtest-sda-posix-down` to start and shut down services. For the setup including the [`sync`](https://github.com/neicnordic/sda-sync) service, use `make integrationtest-sda-sync-run` and `make integrationtest-sda-sync-down` to start and shut down services. + +#### Running the integration tests +This will build all required images, bring up the services, run the integration test, and then shut down services and clean up resources. The same test runs on every pull request (PR) in GitHub. + +- Integration test for the database: + ```sh + make integrationtest-postgres + ``` +- Integration test for RabbitMQ: + ```sh + make integrationtest-rabbitmq + ``` +- Integration test for all SDA setups (including S3, POSIX and sync): + ```sh + make integrationtest-sda + ``` +- Integration test for SDA using POSIX as the storage method: + ```sh + make integrationtest-sda-posix + ``` +- Integration test for SDA using S3 as the storage method: + ```sh + make integrationtest-sda-s3 + ``` +- Integration test for SDA including the sync service: + ```sh + make integrationtest-sda-sync + ``` + +### Linting the Go code + +To run `golangci-lint` for all Go components: + +```sh +$ make lint-all +``` + +To run `golangci-lint` for a specific component, replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`), for example: + +```sh +$ make lint-sda +``` + +### Running the static code tests + +For Go code, this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a Maven container. + +To run the static code tests for all components: + +```sh +$ make test-all +``` + +To run the static code tests for a specific component, replace `all` with the folder name (`sda`, `sda-admin`, `sda-download`, `sda-sftp-inbox`), for example: + +```sh +$ make test-sda +``` \ No newline at end of file From 5af56ec7ca7988fe5ced277d2dd42315bec91190 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Thu, 5 Dec 2024 12:30:24 +0000 Subject: [PATCH 04/10] move building of the sda-admin CLI to a separate section --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d3132c46..423fe88dc 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,18 @@ Build the required Docker images for all SDA services: $ make build-all ``` -You can also build images for individual services by replacing `all` with the folder name (`postgresql`, `rabbitmq`, `sda`, `sda-download`, `sda-sftp-inbox`, `sda-admin`), for example +You can also build images for individual services by replacing `all` with the folder name (`postgresql`, `rabbitmq`, `sda`, `sda-download`, `sda-sftp-inbox`), for example ```sh $ make build-sda ``` +To build the CLI for `sda-admin`: + +```sh +$ make build-sda-admin +``` + ### Running the services #### Start services with Docker Compose From 9fc89efb07b42e63a042fa4452c79d1221d9a6ea Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Fri, 6 Dec 2024 23:36:27 +0100 Subject: [PATCH 05/10] add commands that bring up and bring down services --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index 9228edbb4..728ad5894 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,21 @@ docker-version-check: echo "Docker buildx does not exist can't continue"; \ fi; +# bring up the services +sda-s3-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml up -d +sda-posix-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml up -d +sda-sync-up: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml up -d + +# bring down the services +sda-s3-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans +sda-posix-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans +sda-sync-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans # run intrgration tests, same as being run in Github Actions during a PR integrationtest-postgres: build-postgresql From 805942d0a01859beec0c05afd00b81a2f367dd7d Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Fri, 6 Dec 2024 23:37:01 +0100 Subject: [PATCH 06/10] update README --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 423fe88dc..14fe43c00 100644 --- a/README.md +++ b/README.md @@ -50,23 +50,25 @@ $ make build-sda-admin ### Running the services #### Start services with Docker Compose -The following command will build all required images, bring up all services using the Docker Compose file [sda-s3-integration.yml](.github/integration/sda-s3-integration.yml) (configured for S3 as the storage method) and run the integration test: +The following command will bring up all services using the Docker Compose file [sda-s3-integration.yml](.github/integration/sda-s3-integration.yml) (configured for S3 as the storage backend): ```sh -$ make integrationtest-sda-s3-run +$ make sda-s3-up ``` #### Shut down all services and clean up resources The following command will shut down all services and clean up all related resources: ```sh -$ make integrationtest-sda-s3-down +$ make sda-s3-down ``` -For the setup with POSIX as the storage method, use -`make integrationtest-sda-posix-run` and `make integrationtest-sda-posix-down` to start and shut down services. For the setup including the [`sync`](https://github.com/neicnordic/sda-sync) service, use `make integrationtest-sda-sync-run` and `make integrationtest-sda-sync-down` to start and shut down services. +For the setup with POSIX as the storage backend, use +`make sda-posix-up` and `make sda-posix-down` to start and shut down services. -#### Running the integration tests +For the setup including the [`sync`](https://github.com/neicnordic/sda-sync) service, use `make sda-sync-up` and `make sda-sync-down` to start and shut down services. + +### Running the integration tests This will build all required images, bring up the services, run the integration test, and then shut down services and clean up resources. The same test runs on every pull request (PR) in GitHub. - Integration test for the database: @@ -81,11 +83,11 @@ This will build all required images, bring up the services, run the integration ```sh make integrationtest-sda ``` -- Integration test for SDA using POSIX as the storage method: +- Integration test for SDA using POSIX as the storage backend: ```sh make integrationtest-sda-posix ``` -- Integration test for SDA using S3 as the storage method: +- Integration test for SDA using S3 as the storage backend: ```sh make integrationtest-sda-s3 ``` @@ -93,6 +95,36 @@ This will build all required images, bring up the services, run the integration ```sh make integrationtest-sda-sync ``` +#### Running the integration tests without shutting down the services +This will run the integration tests and keep the services running after the tests are finished. + +- Integration test for SDA using POSIX as the storage backend: + ```sh + make integrationtest-sda-posix-run + ``` +- Integration test for SDA using S3 as the storage backend: + ```sh + make integrationtest-sda-s3-run + ``` +- Integration test for SDA including the sync service: + ```sh + make integrationtest-sda-sync-run + ``` + +After that, you will need to shut down the services manually. + +- Shut down services for SDA using POSIX as the storage backend + ```sh + make integrationtest-sda-posix-down + ``` +- Shut down services for SDA using S3 as the storage backend + ```sh + make integrationtest-sda-s3-down + ``` +- Shut down services for SDA including the sync service: + ```sh + make integrationtest-sda-sync-down + ``` ### Linting the Go code From 6d5d496792ff4d9f4f8c6018dca3315165024c32 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 9 Dec 2024 15:55:16 +0100 Subject: [PATCH 07/10] print message on passed version check --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 728ad5894..78e4c86f6 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,10 @@ go-version-check: ( $${GO_VERSION_ARR[1]} -lt $${GO_VERSION_REQ[1]} ||\ ( $${GO_VERSION_ARR[1]} -eq $${GO_VERSION_REQ[1]} && $${GO_VERSION_ARR[2]} -lt $${GO_VERSION_REQ[2]} )))\ ]]; then\ - echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}.";\ - exit 1;\ + echo "Version check failed. SDA requires Go $${GO_VERSION_MIN} to build."; \ + exit 1; \ + else \ + echo "Version check passed. Installed GO version: $${GO_VERSION}."; \ fi; docker-version-check: @@ -66,7 +68,9 @@ docker-version-check: fi; \ if [ ! $$(docker buildx version | cut -d' ' -f 2) ]; then \ echo "Docker buildx does not exist can't continue"; \ - fi; + exit 1;\ + fi; \ + echo "Version check passed. Installed Docker version: $${DOCKER_VERSION} and Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; # bring up the services sda-s3-up: From 452f3f948e51d4657477893f525024686396cf7c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 9 Dec 2024 17:03:49 +0100 Subject: [PATCH 08/10] Update Makefile Co-authored-by: Joakim Bygdell --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 78e4c86f6..1e9a623da 100644 --- a/Makefile +++ b/Makefile @@ -49,11 +49,10 @@ go-version-check: ( $${GO_VERSION_ARR[1]} -lt $${GO_VERSION_REQ[1]} ||\ ( $${GO_VERSION_ARR[1]} -eq $${GO_VERSION_REQ[1]} && $${GO_VERSION_ARR[2]} -lt $${GO_VERSION_REQ[2]} )))\ ]]; then\ - echo "Version check failed. SDA requires Go $${GO_VERSION_MIN} to build."; \ + echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}."; \ exit 1; \ - else \ - echo "Version check passed. Installed GO version: $${GO_VERSION}."; \ - fi; + fi; \ + echo "GO version: $${GO_VERSION}."; docker-version-check: @DOCKER_VERSION=$$(docker version -f "{{.Server.Version}}" | cut -d'.' -f 1); \ @@ -70,7 +69,8 @@ docker-version-check: echo "Docker buildx does not exist can't continue"; \ exit 1;\ fi; \ - echo "Version check passed. Installed Docker version: $${DOCKER_VERSION} and Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; + echo "Docker version: $${DOCKER_VERSION}."; \ + echo "Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; # bring up the services sda-s3-up: From 06d83ad5dd3499ac8a94a3d46f514ce215f9a501 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 10 Dec 2024 10:47:51 +0100 Subject: [PATCH 09/10] remove unnecessary trailing semicolon --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1e9a623da..b3217c3f7 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ go-version-check: echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}."; \ exit 1; \ fi; \ - echo "GO version: $${GO_VERSION}."; + echo "GO version: $${GO_VERSION}." docker-version-check: @DOCKER_VERSION=$$(docker version -f "{{.Server.Version}}" | cut -d'.' -f 1); \ @@ -70,7 +70,7 @@ docker-version-check: exit 1;\ fi; \ echo "Docker version: $${DOCKER_VERSION}."; \ - echo "Docker Compose version: $${DOCKER_COMPOSE_VERSION}."; + echo "Docker Compose version: $${DOCKER_COMPOSE_VERSION}." # bring up the services sda-s3-up: From 0bac334e3f54af333a992115f8573aff3d3c0c1e Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 10 Dec 2024 12:54:31 +0100 Subject: [PATCH 10/10] refactor: sentence reformatting Co-authored-by: Joakim Bygdell --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 14fe43c00..4538ca0df 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ You can also build images for individual services by replacing `all` with the fo $ make build-sda ``` -To build the CLI for `sda-admin`: +To build the `sda-admin` CLI tool: ```sh $ make build-sda-admin