diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index ae0565c8e..f2db39dca 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -208,49 +208,6 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared - sync: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-sync ] - container_name: sync - depends_on: - credentials: - condition: service_completed_successfully - minio: - condition: service_healthy - postgres: - condition: service_healthy - rabbitmq: - condition: service_healthy - environment: - - BROKER_PASSWORD=sync - - BROKER_USER=sync - - BROKER_QUEUE=mapping_stream - - DB_PASSWORD=sync - - DB_USER=sync - restart: always - volumes: - - ./sda/config.yaml:/config.yaml - - shared:/shared - - sync-api: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-syncapi ] - container_name: sync-api - depends_on: - credentials: - condition: service_completed_successfully - rabbitmq: - condition: service_healthy - environment: - - BROKER_PASSWORD=sync - - BROKER_USER=sync - - BROKER_EXCHANGE=sda.dead - ports: - - "18080:8080" - restart: always - volumes: - - ./sda/config.yaml:/config.yaml - oidc: container_name: oidc command: @@ -398,10 +355,6 @@ services: condition: service_started s3inbox: condition: service_started - sync: - condition: service_started - sync-api: - condition: service_started verify: condition: service_started api: diff --git a/.github/integration/sda-sync-integration.yml b/.github/integration/sda-sync-integration.yml new file mode 100644 index 000000000..5160ab962 --- /dev/null +++ b/.github/integration/sda-sync-integration.yml @@ -0,0 +1,361 @@ +services: + credentials: + container_name: credentials + command: + - "/bin/sh" + - "/scripts/make_sda_credentials.sh" + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - PGPASSWORD=rootpasswd + image: python:3.11-slim + volumes: + - ./scripts:/scripts + - shared:/shared + + postgres: + build: + context: ../../postgresql + container_name: postgres + environment: + - POSTGRES_PASSWORD=rootpasswd + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 2s + retries: 6 + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + ports: + - "15432:5432" + restart: always + volumes: + - postgres_data:/var/lib/postgresql/data + + rabbitmq: + build: + context: ../../rabbitmq + container_name: rabbitmq + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms", + ] + interval: 10s + timeout: 5s + retries: 6 + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + ports: + - "15672:15672" + restart: always + volumes: + - rabbitmq_data:/var/lib/rabbitmq + + minio: + image: minio/minio:RELEASE.2023-05-18T00-05-36Z + command: server /data --console-address ":9001" + container_name: s3 + environment: + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretKey + - MINIO_SERVER_URL=http://127.0.0.1:9000 + healthcheck: + test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 2s + retries: 6 + ports: + - "19000:9000" + - "19001:9001" + volumes: + - minio_data:/data + + s3inbox: + build: + args: + GOLANG_VERSION: ${GOLANG_VERSION:-1.20} + context: ../../sda + target: debug + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-s3inbox ] + container_name: s3inbox + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + oidc: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=inbox + - BROKER_USER=inbox + - BROKER_ROUTINGKEY=inbox + - DB_PASSWORD=inbox + - DB_USER=inbox + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + ports: + - "18000:8000" + - "18001:8001" + + ingest: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-ingest ] + container_name: ingest + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=ingest + - BROKER_USER=ingest + - BROKER_QUEUE=ingest + - BROKER_ROUTINGKEY=archived + - DB_PASSWORD=ingest + - DB_USER=ingest + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + verify: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-verify ] + container_name: verify + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=verify + - BROKER_USER=verify + - BROKER_QUEUE=archived + - BROKER_ROUTINGKEY=verified + - DB_PASSWORD=verify + - DB_USER=verify + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + finalize: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-finalize ] + container_name: finalize + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=finalize + - BROKER_USER=finalize + - BROKER_QUEUE=accession + - BROKER_ROUTINGKEY=completed + - DB_PASSWORD=finalize + - DB_USER=finalize + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + mapper: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-mapper ] + container_name: mapper + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=mapper + - BROKER_USER=mapper + - BROKER_QUEUE=mappings + - DB_PASSWORD=mapper + - DB_USER=mapper + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + sync: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-sync ] + container_name: sync + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=sync + - BROKER_USER=sync + - BROKER_QUEUE=mapping_stream + - DB_PASSWORD=sync + - DB_USER=sync + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + sync-api: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-syncapi ] + container_name: sync-api + depends_on: + credentials: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=sync + - BROKER_USER=sync + - BROKER_EXCHANGE=sda.dead + ports: + - "18080:8080" + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + + oidc: + container_name: oidc + command: + - /bin/sh + - -c + - | + pip install --upgrade pip + pip install aiohttp Authlib joserfc requests + python -u /oidc.py + depends_on: + credentials: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] + interval: 10s + timeout: 2s + retries: 6 + image: python:3.11-slim + ports: + - "8080:8080" + restart: always + volumes: + - ./sda/oidc.py:/oidc.py + - shared:/shared + + api: + command: [ sda-api ] + container_name: api + depends_on: + credentials: + condition: service_completed_successfully + postgres: + condition: service_healthy + oidc: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=api + - BROKER_USER=api + - DB_PASSWORD=api + - DB_USER=api + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + ports: + - "8090:8080" + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - ./sda/rbac.json:/rbac.json + - shared:/shared + + reencrypt: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [ sda-reencrypt ] + container_name: reencrypt + depends_on: + credentials: + condition: service_completed_successfully + ports: + - "50051:50051" + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + integration_test: + container_name: tester + command: + - "/bin/sh" + - "/tests/run_scripts.sh" + - "/tests/sda" + depends_on: + credentials: + condition: service_completed_successfully + finalize: + condition: service_started + ingest: + condition: service_started + mapper: + condition: service_started + s3inbox: + condition: service_started + sync: + condition: service_started + sync-api: + condition: service_started + verify: + condition: service_started + api: + condition: service_started + reencrypt: + condition: service_started + environment: + - PGPASSWORD=rootpasswd + - STORAGETYPE=s3 + - SYNCTEST=true + image: python:3.11-slim-bullseye + profiles: + - tests + volumes: + - shared:/shared + - ./tests:/tests + +volumes: + minio_data: + postgres_data: + rabbitmq_data: + shared: diff --git a/.github/integration/tests/sda/45_sync_test.sh b/.github/integration/tests/sda/45_sync_test.sh index 6ed5543c9..4d44aef99 100644 --- a/.github/integration/tests/sda/45_sync_test.sh +++ b/.github/integration/tests/sda/45_sync_test.sh @@ -3,7 +3,8 @@ set -e cd shared || true -if [ "$STORAGETYPE" = "posix" ]; then +if [ -z "$SYNCTEST" ]; then + echo "sync not tested" exit 0 fi diff --git a/.github/integration/tests/sda/99_auth_test.sh b/.github/integration/tests/sda/99_auth_test.sh index eb63dbba4..433159a76 100644 --- a/.github/integration/tests/sda/99_auth_test.sh +++ b/.github/integration/tests/sda/99_auth_test.sh @@ -1,6 +1,10 @@ #!/bin/sh set -e +if [ -n "$SYNCTEST" ]; then + exit 0 +fi + python -m pip install --upgrade pip pip install tox diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 3c0a0735e..bd7803497 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -229,6 +229,25 @@ jobs: - name: Test sensitive-data-archive run: docker compose -f .github/integration/sda-${{matrix.storage}}-integration.yml run integration_test + sda-sync: + needs: + - build_go_images + - build_server_images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + sync: + - "sda/cmd/sync/*" + sync-api: + - "sda/cmd/syncapi/*" + - name: Test sda-sync + run: docker compose -f .github/integration/sda-sync-integration.yml run integration_test + if: steps.changes.outputs.sync == 'true' || steps.changes.outputs.sync-api == 'true' + chart: needs: - build_go_images diff --git a/Makefile b/Makefile index 9a1194b68..51791cd7e 100644 --- a/Makefile +++ b/Makefile @@ -76,12 +76,39 @@ integrationtest-postgres: build-postgresql integrationtest-rabbitmq: build-rabbitmq build-sda @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml down -v --remove-orphans -integrationtest-sda: build-all + +integrationtest-sda: build_all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans + +integrationtest-sda-posix: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans +integrationtest-sda-posix-run: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test +integrationtest-sda-posix-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans +integrationtest-sda-s3: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans +integrationtest-sda-s3-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test +integrationtest-sda-s3-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans + +integrationtest-sda-sync: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans +integrationtest-sda-sync-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml run integration_test +integrationtest-sda-sync-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans + # lint go code lint-all: lint-sda lint-sda-download lint-sda-admin lint-sda: