Update containers

.env.example

@@ -9,30 +9,14 @@ credentials_PGPASSWORD=rootpass
 
 # auth
 auth_ELIXIR_ID=XC56EL11xx
-auth_ELIXIR_PROVIDER=http://aai-mock:8080/oidc/
 auth_ELIXIR_SECRET=wHPVQaYXmdDHg
-auth_ELIXIR_JWKPATH=jwk
-auth_ELIXIR_REDIRECTURL=https://localhost:8085/elixir/login
-auth_LOG_LEVEL=info
-auth_S3INBOX=localhost:8000
-auth_JWTISSUER=http://auth:8085
-auth_JWTPRIVATEKEY=shared/keys/jwt.key
-auth_JWTSIGNATUREALG=ES256
-auth_SERVER_CERT=shared/cert/server.crt
-auth_SERVER_KEY=shared/cert/auth.key
-auth_RESIGNJWT=False
 
 # rabbitmq
-rabbitmq_MQ_PASSWORD_HASH=C5ufXbYlww6ZBcEqDUB04YdUptO81s+ozI3Ll5GCHTnv8NAm
+rabbitmq_MQ_PASSWORD=test
 rabbitmq_MQ_USER=test
-rabbitmq_MQ_VHOST=gdi
-rabbitmq_NOTLS=true
 
 # postgres
-postgres_DB_LEGA_IN_PASSWORD=lega_in
-postgres_DB_LEGA_OUT_PASSWORD=.
 postgres_POSTGRES_PASSWORD=rootpass
-postgres_NOTLS=true
 
 # s3
 s3_MINIO_ROOT_USER=access
@@ -47,7 +31,7 @@ download_DB_USER=download
 
 # finalize
 finalize_BROKER_PASSWORD=finalize
-finalize_BROKER_QUEUE=accessionIDs
+finalize_BROKER_QUEUE=accession
 finalize_BROKER_ROUTINGKEY=completed
 finalize_BROKER_USER=finalize
 finalize_DB_PASSWORD=finalize
@@ -58,8 +42,8 @@ ingest_BROKER_PASSWORD=ingest
 ingest_BROKER_QUEUE=ingest
 ingest_BROKER_ROUTINGKEY=archived
 ingest_BROKER_USER=ingest
-ingest_DB_PASSWORD=lega_in
-ingest_DB_USER=lega_in
+ingest_DB_PASSWORD=ingest
+ingest_DB_USER=ingest
 ingest_SCHEMA_TYPE=federated
 
 # mapper
@@ -84,5 +68,3 @@ s3inbox_BROKER_ROUTINGKEY=inbox
 s3inbox_BROKER_USER=inbox
 s3inbox_DB_PASSWORD=inbox
 s3inbox_DB_USER=inbox
-s3inbox_SERVER_CONFFILE=/config.yaml
-s3inbox_SERVER_JWTPUBKEYURL=http://aai-mock:8080/oidc/jwk

.github/workflows/test_demo.yaml

@@ -0,0 +1,34 @@
+name: Run Demo
+
+on:
+  pull_request:
+
+jobs:
+  demo:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out code
+        uses : actions/checkout@v4
+
+      - name: Run compose demo
+        run: |
+          cp config/config.yaml.example config/config.yaml
+          cp config/iss.json.example config/iss.json
+          cp .env.example .env
+          docker compose -f docker-compose-demo.yml up -d
+          until [ "$(docker inspect data_loader  --format='{{.State.Status}}')" = "exited" ]; do
+            echo "waithg for data_loader to finish"
+            sleep 5
+          done
+          token=$(curl -s -k https://localhost:8080/tokens | jq -r '.[0]')
+          if [ "$(curl -s -H "Authorization: Bearer $token" http://localhost:8443/metadata/datasets | jq -r '.[]')" != "DATASET0001" ]; then
+            echo "demo failed"
+            exit 1
+          fi
+          if [ "$(curl -s -H "Authorization: Bearer $token" http://localhost:8443/metadata/datasets/DATASET0001/files | jq length)" -ne 4 ]; then
+            echo "demo failed"
+            exit 1
+          fi
+          echo "demo completed sucessfully"
+        shell: bash

README.md

@@ -9,26 +9,27 @@ cp ./config/config.yaml.example ./config/config.yaml
 cp ./config/iss.json.example ./config/iss.json
 cp ./.env.example ./.env
 ```
+
 no further editing to the above files is required for running the stack locally.
 
 ## Starting the full stack with LS-AAI-mock
 
 To bootstrap the *full stack* of `storage-and-interfaces` services use
 the file `docker-compose.yml`. Note that this requires a running [`LS-AAI-mock`](https://github.com/GenomicDataInfrastructure/starter-kit-lsaai-mock) service. To configure the LS-AAI-mock service follow the instructions below.
 
-Add the following line to `/etc/hosts` file:
+Export the variable `DOCKERHOST` with either `$HOSTNAME` or `localhost` unless you want to edit the hosts file as shown below.
 
-```
-127.0.0.1 aai-mock
+```shell
+sudo sh -c "echo '127.0.0.1 dockerhost' >>/etc/hosts"
 ```
 
 First clone the [startet-kit-lsaai-mock](https://github.com/GenomicDataInfrastructure/starter-kit-lsaai-mock) repo.
 
 Under its root folder, change the first two lines of the file `configuration/aai-mock/application.properties` to:
 
-```
-main.oidc.issuer.url=http://aai-mock:8080/oidc/
-web.baseURL=https://aai-mock:8080/oidc
+```conf
+main.oidc.issuer.url=http://${DOCKERHOST}:8080/oidc/
+web.baseURL=https://${DOCKERHOST}:8080/oidc
 ```
 
 and then add the `sda-auth` client by creating a file `configuration/aai-mock/clients/client1.yaml` with the following contents:
@@ -37,32 +38,32 @@ and then add the `sda-auth` client by creating a file `configuration/aai-mock/cl
 client-name: "auth"
 client-id: "XC56EL11xx"
 client-secret: "wHPVQaYXmdDHg"
-redirect-uris: ["https://localhost:8085/elixir/login"]
+redirect-uris: ["http://localhost:8085/elixir/login"]
 token-endpoint-auth-method: "client_secret_basic"
-scope: ["openid", "profile", "email", "ga4gh_passport_v1"]
+scope: ["openid", "profile", "email", "ga4gh_passport_v1", "eduperson_entitlement"]
 grant-types: ["authorization_code"]
-post-logout-redirect-uris: ["https://auth:8085/elixir/login"]
+post-logout-redirect-uris: ["http://localhost:8085/elixir/login"]
 ```
 
-Now that everything should be configured properly, return to the root folder of the `starter-kit-storage-and-interfaces` and run:
+Now that everything should be configured properly, return to the root folder of the `starter-kit-lsaai-mock` and run:
 
 ```shell
 docker compose up -d
 ```
 
-Lastly, *while the `storage-and-interfaces` stack is being deployed*, return to the `starter-kit-lsaai-mock` root folder and run:
+Lastly, return to the root of the `starter-kit-storage-and-interfaces` folder and run:
 
 ```shell
 docker compose up -d
 ```
 
-Note that the above two commands need to be run in that specific order because the `LS-AAI-mock` service expects to connect to an external network `my-app-network` which is created by the `storage-and-interfaces`' compose file.
+Note that the above two commands need to be run in that specific order because the `LS-AAI-mock` compose creates the external network `my-app-network` which is used to communicate with the `aai-mock` service.
 
 ## Starting the stack in standalone demo mode
 
 The file `docker-compose-demo.yml` is used to start the `storage-and-interfaces` services in *demo* mode with an example dataset preloaded and ingested to the sensitive data archive when the deployment is done. This comes with its own python implementation of a mock-oidc in place of LS-AAI and can be run as standalone for demonstration purposes.
 
-The files imported by the data loading script come from here: https://github.com/ga4gh/htsget-refserver/tree/main/data/gcp/gatk-test-data/wgs_bam
+The files imported by the data loading script come from [here:](https://github.com/ga4gh/htsget-refserver/tree/main/data/gcp/gatk-test-data/wgs_bam)
 
 To deploy use the following command:
 

config/TLS-example/README.md

@@ -0,0 +1,116 @@
+# TLS example
+
+This folder contains example files that can be used as a base when the setup requires TLS enabled in the deployment.
+
+**Note** this is not usable as is, manual intervention is required.
+
+## Configuration example
+
+### config.yaml file
+
+In the `config.yaml` the following entries must be set:
+
+```yaml
+archive:
+    url:
+    accesskey:
+    secretkey:
+
+inbox:
+    url:
+    accesskey:
+    secretkey:
+
+c4gh:
+    passphrase:
+
+elixir:
+    id:
+    secret:
+    redirectUrl:
+
+session:
+    domain:
+
+s3inbox:
+```
+
+The `config.yaml` contains comments to help setting the correct value
+
+### env file
+
+The following entries in the env file need to be set:
+
+- `C4GH_KEYPATH`: path to the crypt4gh private key.
+- `rabbitmq_MQ_PASSWORD`: admin password to the RabbitMQ server
+- `rabbitmq_MQ_USER`: username for the RabbitMQ admin
+- `postgres_POSTGRES_PASSWORD`: password for the `postgres` user
+
+After which the file needs to be renamed with a dot prefix (`.env`)
+
+#### service credentials
+
+These credentials can be created manually in Postgres/RabbitMQ, or with the use of the `make_credentials` script once the RabbitMQ and Postgres containers are running. The usernames should be the same as the `container_name` in the docker-compose file
+
+##### RabbitMQ
+
+- finalize_BROKER_PASSWORD
+- ingest_BROKER_PASSWORD
+- mapper_BROKER_PASSWORD
+- verify_BROKER_PASSWORD
+- inbox_BROKER_PASSWORD
+
+##### Postgres
+
+- download_DB_PASSWORD
+- finalize_DB_PASSWORD
+- ingest_DB_PASSWORD
+- mapper_DB_PASSWORD
+- verify_DB_PASSWORD
+- inbox_DB_PASSWORD
+
+### iss.json file
+
+In the `iss.json` file a block for the publicly available endpoint for REMS should to be added.
+
+### certificates
+
+All containers except `auth` needs certificates with DNS entries matching the `container_name` in the docker-compose file.  
+These certificate files should adhere to the [cert-manager](https://cert-manager.io/) naming convention (tls.crt, tls.key, ca.crt).
+
+The following containers also need external certificates generated by a public CA like [Let's Encrypt](https://letsencrypt.org/):
+
+- Auth
+- Download
+- S3inbox
+
+These certificate files should adhere to Let's Encrypts naming schema (fullchain.pem, privkey.pem).
+
+All certificates for a service should be placed in a folder with the same name as the `container_name` and be placed next to the docker-compose file.  
+Permissions for the private key need to be set at `0400` and the owner set as shown below:
+
+- rabbitmq - `100:101`
+- postgres - `70:70`
+- all others - `65534:65534`
+
+If automatic certificate rotation is done through cron it is easier to set the owner to root and group based on the service, with the permissions to `0640` for all files in each folder.
+
+- rabbitmq - `0:101`
+- postgres - `0:70`
+- all others - `0:65534`
+
+## Bootstrapping
+
+First the RabbitMQ and Postgres servers need to be started.
+
+```cmd
+docker compose up -d postgres rabbitmq
+```
+
+When they are running with status `healthy`, the credentials for the services can be created, either manually or by executing the `make_credentials` script.
+
+Once the credentials have been created the rest of the services can be started.
+
+```cmd
+docker compose up -d
+```

config/TLS-example/config.yaml

@@ -0,0 +1,105 @@
+app: # this is for download
+  host: "0.0.0.0"
+  port: "8443"
+  servercert: "/certificates/fullchain.pem"
+  serverkey: "/certificates/privkey.pem"
+
+archive:
+  type: "s3"
+  url: ""
+#  port: only needed if port is not 80 or 443
+  accesskey: ""
+  secretkey: ""
+  bucket: "archive"
+  chunksize: 32
+
+broker:
+  host: "rabbitmq"
+  port: 5671 # This is the default SSL port
+  # user: "" # dont set, this should be unique for each service
+  # password: "" # dont set, this should be unique for each service
+  vhost: "sda"
+  # queue: "" # dont set, this is unique for each service
+  exchange: "sda"
+  # routingKey: "" # dont set, this is unique for each service
+  routingError: "error"
+  ssl: "true"
+  cacert: /certificates/ca.crt # if certificates are signed by an private PKI
+  # verifyPeer: "false" 
+# If verifyPeer is true clientKey and clientCert needs to be set
+  # clientCert: /certificates/tls.crt
+  # clientKey: /certificates/tls.key
+
+c4gh:
+  passphrase: ""
+  filepath: "/c4gh/gdi.sec.pem"
+
+db:
+  host: "postgres"
+  port: 5432
+  # user: "" # dont set, this should be unique for each service
+  # password: "" # dont set, this should be unique for each service
+  database: "sda"
+  sslmode: "verify-ca"
+  cacert: /certificates/ca.crt # if certificates are signed by an private PKI
+  clientCert: /certificates/tls.crt
+  clientKey: /certificates/tls.key
+
+
+elixir: # used by the auth service
+  id: "" # LS AAI client ID
+  secret: "" # LS AAI client secret
+  provider: "https://login.elixir-czech.org/oidc/"
+  jwkpath: "jwk"
+  redirectUrl: "" # public url to the auth endpoint
+
+inbox:
+  type: "s3"
+  url: ""
+#  port: only needed if port is not 80 or 443
+  accesskey: ""
+  secretkey: ""
+  bucket: "inbox"
+  chunksize: 32
+
+log:
+  level: "info"
+  format: "json"
+
+oidc:
+  configuration:
+    url: "https://login.elixir-czech.org/oidc/.well-known/openid-configuration"
+  trusted:
+    iss: "/iss.json"
+
+schema:
+  type: isolated
+
+server:
+  jwtpubkeyurl: "https://login.elixir-czech.org/oidc/jwk"
+  cert: "/certificates/fullchain.pem"
+  key: "/certificates/privkey.pem"
+
+session:
+  # session key expiration time in seconds
+  # default value = -1 for disabled state
+  # a positive integer enables sessions
+  # a negative integer disables sessions
+  expiration: 28800
+  # domain name must be set to the hostname (FQDN) of the service
+  domain: ""
+  # session cookie Secure value, if true, TLS must be active
+  # default value = true
+  secure: true
+  # session cookie HttpOnly value, if true, TLS must be active
+  # default value = true
+  httponly: true
+  # name of session cookie
+  # default value = sda_session_key
+  name: "sda_session_key"
+
+# these are used by the auth container
+s3inbox: "" # public URL to the s3inbox
+resignjwt: "false"
+infoText: "About GDI"
+infoUrl: "https://gdi.onemilliongenomes.eu/" # or this can be a national site where information about the available datasets can be found.