From c30513919010c8ceb3d595b23ce8d2a08bebecd0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 20 Jun 2023 14:31:28 +0200 Subject: [PATCH 01/14] Update sda-db Version bumped to 0.6.0 Uses image from this repository --- charts/sda-db/Chart.yaml | 4 +-- charts/sda-db/README.md | 13 +++----- charts/sda-db/templates/_helpers.tpl | 16 +++++----- charts/sda-db/templates/secrets.yaml | 9 +++--- charts/sda-db/templates/statefulset.yaml | 39 ++++++------------------ charts/sda-db/values.yaml | 18 +++-------- 6 files changed, 34 insertions(+), 65 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index f73dd4842..a751eeb51 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: sda-db -version: "0.5.2" +version: "0.6.0" description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io icon: https://neic.no/assets/images/logo.png sources: -- https://github.com/neicnordic/sda-helm +- https://github.com/neicnordic/sensitive-data-archive diff --git a/charts/sda-db/README.md b/charts/sda-db/README.md index 2024f2a5a..d2856f248 100644 --- a/charts/sda-db/README.md +++ b/charts/sda-db/README.md @@ -1,6 +1,6 @@ # SDA Database -Source repository: [https://github.com/neicnordic/sda-db](https://github.com/neicnordic/sda-db) +Source repository: [https://github.com/neicnordic/sensitive-data-archive](https://github.com/neicnordic/sensitive-data-archive) ## Installing the Chart @@ -8,8 +8,7 @@ Edit the values.yaml file and specify the relevant parts of the `global` section Parameter | Description | Default --------- | ----------- | ------- -`global.pg_in_password` | Password for `lega_in` user, used for `data in` services. |`""` -`global.pg_out_password` | Password for `lega_out` user, used for `data out` services. |`""` +`global.postgresAdminPassword` | PostgreSQL admin password (Random if empty) | `""` `global.tls.enabled` | Enable TLS for all connections. |`true` `global.tls.issuer` | Issuer for TLS certificate creation. |`""` `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` @@ -19,8 +18,8 @@ Parameter | Description | Default `global.tls.CAFile` | CA root certificate. |`ca.crt` `global.tls.verifyPeer` | Require client certificates. |`verify-ca` `externalPkiService.tlsPath` | If an external PKI service is used, this is the path where the certifiates are placed | `""` -`image.repository` | sda-db container image repository | `ghcr.io/neicnordic/sda-db` -`image.tag` | sda-db container image version | `v1.4.0` +`image.repository` | sda-db container image repository | `ghcr.io/neicnordic/sensitive-data-archive` +`image.tag` | sda-db container image version | `` `image.pullPolicy` | sda-db container image pull policy | `IfNotPresent` `networkPolicy.create` | Use network isolation. | `false` `networkPolicy.matchLabels` | App labels that are allowed to connect to the database. | `app: sda-svc` @@ -31,8 +30,6 @@ Parameter | Description | Default `persistence.existingClaim` | Use existing claim. | `null` `persistence.volumePermissions` | Change the owner of the persist volume mountpoint to `RunAsUser:fsGroup`. | `true` `podAnnotations` | `"key": "value"` list of annotations for the pod (optional) | `{}` -`port` | Port the application will listen to (optional) | `5432` -`postgresAdminPassword` | PostgreSQL admin password (optional) | `""` `rbacEnabled` | Use role based access control. |`true` `resources.requests.memory` | Memory request for container. |`128Mi` `resources.requests.cpu` | CPU request for container. |`100m` @@ -40,7 +37,7 @@ Parameter | Description | Default `resources.limits.cpu` | CPU limit for container. |`200m` `revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` `updateStrategyType` | Update strategy type. | `RollingUpdate` -`securityPolicy.create` | Use pod security policy. | `true` +`securityPolicy.create` | Use pod security policy. | `false` `service.type` | Database service type. |`ClusterIP` `service.port` | Database service port. |`5432` diff --git a/charts/sda-db/templates/_helpers.tpl b/charts/sda-db/templates/_helpers.tpl index a17860a6b..ba511299f 100644 --- a/charts/sda-db/templates/_helpers.tpl +++ b/charts/sda-db/templates/_helpers.tpl @@ -30,14 +30,6 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{- define "pgInPassword" -}} - {{- ternary (randAlphaNum 12) .Values.global.pg_in_password (empty .Values.global.pg_in_password) -}} -{{- end -}} - -{{- define "pgOutPassword" -}} - {{- ternary (randAlphaNum 12) .Values.global.pg_out_password (empty .Values.global.pg_out_password) -}} -{{- end -}} - {{- define "pgCert" -}} {{- if .Values.externalPkiService.tlsPath -}} {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/tls.crt" .Values.externalPkiService.tlsPath) "/") -}} @@ -113,3 +105,11 @@ Create chart name and version as used by the chart label. {{- "/var/lib/postgresql/data/pgdata/" }} {{- end -}} {{- end -}} + +{{- define "adminPass" -}} + {{- if .Values.global.postgresAdminPassword }} + {{- printf "%s" (.Values.global.postgresAdminPassword ) | b64enc }} + {{- else }} + {{- randAlphaNum 32 | b64enc }} + {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/sda-db/templates/secrets.yaml b/charts/sda-db/templates/secrets.yaml index 159211c77..8ada8be82 100644 --- a/charts/sda-db/templates/secrets.yaml +++ b/charts/sda-db/templates/secrets.yaml @@ -3,8 +3,7 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }} data: - pgInPasswd: {{ include "pgInPassword" . | b64enc }} - pgOutPasswd: {{ include "pgOutPassword" . | b64enc }} -{{- if .Values.postgresAdminPassword }} - postgresPassword: {{ .Values.postgresAdminPassword | b64enc }} -{{- end }} + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace (include "sda.fullname" .)) | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + {{- $postgresPassword := (get $secretData "postgresPassword") | default (include "adminPass" . ) }} + postgresPassword: {{ $postgresPassword | quote }} diff --git a/charts/sda-db/templates/statefulset.yaml b/charts/sda-db/templates/statefulset.yaml index e3ab1bb79..fd6da055a 100644 --- a/charts/sda-db/templates/statefulset.yaml +++ b/charts/sda-db/templates/statefulset.yaml @@ -77,37 +77,22 @@ spec: resources: {{ toYaml .Values.resources | trim | indent 10 }} env: - - name: DB_LEGA_IN_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }} - key: pgInPasswd - - name: DB_LEGA_OUT_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }} - key: pgOutPasswd - {{- if .Values.postgresAdminPassword }} - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: {{ template "sda.fullname" . }} key: postgresPassword - {{- end }} {{- if .Values.global.tls.enabled }} - - name: PG_SERVER_CERT + - name: POSTGRES_SERVER_CERT value: {{ template "pgCert" . }} - - name: PG_SERVER_KEY + - name: POSTGRES_SERVER_KEY value: {{ template "pgKey" . }} {{- if ne "verify-none" .Values.global.tls.verifyPeer }} - - name: PG_CA + - name: POSTGRES_SERVER_CACERT value: {{ template "caCert" . }} - - name: PG_VERIFY_PEER + - name: POSTGRES_VERIFY_PEER value: {{ .Values.global.tls.verifyPeer }} {{- end }} - {{- else }} - - name: NOTLS - value: "true" {{- end }} - name: PGDATA value: {{ template "pgData" }} @@ -118,21 +103,17 @@ spec: livenessProbe: exec: command: - - pg_isready - - -h - - localhost - - -U - - lega_out + - sh + - -c + - PGPASSWORD=$POSTGRES_PASSWORD psql -U postgres -h localhost -Atq -c "SELECT version();" initialDelaySeconds: 30 timeoutSeconds: 5 readinessProbe: exec: command: - - pg_isready - - -h - - localhost - - -U - - lega_out + - sh + - -c + - PGPASSWORD=$POSTGRES_PASSWORD psql -U postgres -h localhost -Atq -c "SELECT version();" initialDelaySeconds: 5 timeoutSeconds: 1 volumeMounts: diff --git a/charts/sda-db/values.yaml b/charts/sda-db/values.yaml index 749bb0a17..7cb1e03e9 100644 --- a/charts/sda-db/values.yaml +++ b/charts/sda-db/values.yaml @@ -1,6 +1,5 @@ global: - pg_in_password: "" - pg_out_password: "" + postgresAdminPassword: "" tls: enabled: true issuer: "" @@ -25,8 +24,8 @@ externalPkiService: extraSecurityContext: {} image: - repository: ghcr.io/neicnordic/sda-db - tag: v2.1.10 + repository: ghcr.io/neicnordic/sensitive-data-archive + tag: v0.0.44-postgres pullPolicy: IfNotPresent # utilize network isolation @@ -61,13 +60,6 @@ persistence: # podAnnotations: | # "key": "value" -# if a port other than the default 5432 is used in the pod set the value here. -# port: 5432 - -# if an admin user is to be created during the first setup, set the password below -# This can only be done during the first setup -# postgresAdminPassword: - # RBAC is assumed to be enabled in the cluster rbacEnabled: true @@ -84,8 +76,9 @@ resources: ## RevisionHistoryLimit is number of old ReplicaSets to retain to allow rollback. # revisionHistory: 3 +## This is only available on clusters running k8s < v1.25.0 securityPolicy: - create: true + create: false service: type: ClusterIP @@ -93,7 +86,6 @@ service: updateStrategyType: RollingUpdate - # secret containing the TLS certificates for the release tester # if no certificate issuer is used testimage: From dfbc5b8bbb7f9e49e6b6e1b60574631ab0e72133 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 20 Jun 2023 17:31:36 +0200 Subject: [PATCH 02/14] Update sda-mq Version bumped to 0.5.0 Uses image from this repository --- charts/sda-mq/Chart.yaml | 2 +- charts/sda-mq/README.md | 30 +++++++++--------------- charts/sda-mq/templates/_helpers.tpl | 8 +++++++ charts/sda-mq/templates/secrets.yaml | 5 +++- charts/sda-mq/templates/statefulset.yaml | 19 +++++++-------- charts/sda-mq/values.yaml | 18 +++++++------- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index a1b80b3cf..0031e9cf8 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-mq -version: "0.4.6" +version: "0.5.0" description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io icon: https://neic.no/assets/images/logo.png diff --git a/charts/sda-mq/README.md b/charts/sda-mq/README.md index f539e9d83..bc6f0dbbc 100644 --- a/charts/sda-mq/README.md +++ b/charts/sda-mq/README.md @@ -1,6 +1,6 @@ # SDA Message broker -Source repository: [https://github.com/neicnordic/sda-mq](https://github.com/neicnordic/sda-mq) +Source repository: [https://github.com/neicnordic/sensitive-data-archive](https://github.com/neicnordic/sensitive-data-archive) ## Installing the Chart @@ -8,11 +8,11 @@ Edit the values.yaml file and specify the relevant parts of the `config` section Parameter | Description | Default --------- | ----------- | ------- -`image.repository` | sda-mq container image repository | `ghcr.io/neicnordic/sda-mq` -`image.tag` | sda-mq container image version | `v1.3.0` +`image.repository` | sda-mq container image repository | `ghcr.io/neicnordic/sensitive-data-archive` +`image.tag` | sda-mq container image version | `` `image.pullPolicy` | sda-mq container image pull policy | `Always` -`global.adminUser` | Username of admin user |`""` -`global.adminPasswordHash` | Passwordhash for admin user. |`""` +`global.adminUser` | Username of admin user |`admin` +`global.adminPassword` | Password for admin user. |`Random if unset` `global.tls.enabled` | Use TLS for all connections. |`true` `global.tls.issuer` | Issuer for TLS certificate creation. |`""` `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` @@ -21,7 +21,7 @@ Parameter | Description | Default `global.tls.serverCert` | Name of the certificate file. |`""` `global.tls.caCert` | Name of the CA file. |`""` `global.tls.verifyPeer` | Require client certificates. |`true` -`global.vhost` | default vhost is '/' unless specifically named |`""` +`global.vhost` | default vhost is 'sda' unless specifically named |`""` `global.shovel.host` | Hostname of federated server |`""` `global.shovel.pass` | Password to federated server |`""` `global.shovel.port` | Port that federated server listens on |`5671` @@ -33,7 +33,7 @@ Parameter | Description | Default `updateStrategyType` | Update strategy type. | `RollingUpdate` `networkPolicy.create` | Use network isolation. | `false` `networkPolicy.matchLabels` | App labels that are allowed to connect to the Message broker. | `app: sda-svc` -`securityPolicy.create` | Use pod security policy. | `true` +`securityPolicy.create` | Use pod security policy. | `false` `persistence.enabled` | Enable persistence. | `true` `persistence.storageSize` | Volume size. | `8Gi` `persistence.storageClass` | Use specific storage class, by default dynamic provisioning enabled. | `null` @@ -41,10 +41,10 @@ Parameter | Description | Default `persistence.volumePermissions` | Change the owner of the persist volume mountpoint to `RunAsUser:fsGroup`. | `true` `service.type` | Message broker service type. |`ClusterIP` `service.port` | Message broker service port. |`5671` -`resources.requests.memory` | Memory request for container. |`128Mi` -`resources.requests.cpu` | CPU request for container. |`100m` -`resources.limits.memory` | Memory limit for container. |`256Mi` -`resources.limits.cpu` | CPU limit for container. |`200m` +`resources.requests.memory` | Memory request for container. |`1Gi` +`resources.requests.cpu` | CPU request for container. |`1` +`resources.limits.memory` | Memory limit for container. |`2Gi` +`resources.limits.cpu` | CPU limit for container. |`2` `testimage.tls.secretName` | Name of the testers secret that holds the certificates. |`""` `testimage.tls.serverKey` | Name of the testers certificate private key file. |`""` `testimage.tls.serverCert` | Name of testers the certificate file. |`""` @@ -71,11 +71,3 @@ kubectl create secret generic tester-certs \ --from-file=tls.crt\ --from-file=tls.key ``` - -## Password hash - -To create a password hash for the admin user run the followin command: - -```cmd -sh ../dev_tools/scripts/mq-password-generator.sh ADMIN_PASSWORD -``` diff --git a/charts/sda-mq/templates/_helpers.tpl b/charts/sda-mq/templates/_helpers.tpl index dbc94bd05..35b449db1 100644 --- a/charts/sda-mq/templates/_helpers.tpl +++ b/charts/sda-mq/templates/_helpers.tpl @@ -99,3 +99,11 @@ Create chart name and version as used by the chart label. {{- end -}} {{- end -}} {{- end -}} + +{{- define "adminPass" -}} + {{- if .Values.global.adminPassword }} + {{- printf "%s" (.Values.global.adminPassword ) | b64enc }} + {{- else }} + {{- randAlphaNum 32 | b64enc }} + {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/sda-mq/templates/secrets.yaml b/charts/sda-mq/templates/secrets.yaml index a1d8bac23..193d1abab 100644 --- a/charts/sda-mq/templates/secrets.yaml +++ b/charts/sda-mq/templates/secrets.yaml @@ -3,7 +3,10 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }} data: - password_hash: {{ (required "A valid MQ password hash is required" .Values.global.adminPasswordHash) | quote | trimall "\"" | b64enc }} + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace (include "sda.fullname" .)) | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + {{- $adminPassword := (get $secretData "password") | default (include "adminPass" . ) }} + password: {{ $adminPassword | quote }} {{- if and .Values.global.shovel.host (and .Values.global.shovel.user .Values.global.shovel.pass) }} shovel_connection: {{ printf "amqps://%s:%s@%s:%s/%s?server_name_indication=%s" .Values.global.shovel.user .Values.global.shovel.pass (required "A valid MQ shovel host is required" .Values.global.shovel.host) ( .Values.global.shovel.port | quote | trimall "\"" ) .Values.global.shovel.vhost .Values.global.shovel.host | quote | trimall "\"" | b64enc }} {{- end }} diff --git a/charts/sda-mq/templates/statefulset.yaml b/charts/sda-mq/templates/statefulset.yaml index 1a97065f1..b9b1352d5 100644 --- a/charts/sda-mq/templates/statefulset.yaml +++ b/charts/sda-mq/templates/statefulset.yaml @@ -65,13 +65,13 @@ spec: resources: {{ toYaml .Values.resources | trim | indent 10 }} env: - - name: MQ_USER - value: {{ required "The admin username is required" .Values.global.adminUser | quote }} - - name: MQ_PASSWORD_HASH + - name: RABBITMQ_DEFAULT_USER + value: {{ .Values.global.adminUser | default "admin" }} + - name: RABBITMQ_DEFAULT_PASS valueFrom: secretKeyRef: name: {{ template "sda.fullname" . }} - key: password_hash + key: password {{- if and .Values.global.shovel.host (and .Values.global.shovel.user .Values.global.shovel.pass) }} - name: CEGA_CONNECTION valueFrom: @@ -90,16 +90,13 @@ spec: - name: MQ_VERIFY value: {{ template "verifyPeer" . }} {{- end }} - {{- else }} - - name: NOTLS - value: "true" {{- end }} {{- if .Values.global.vhost }} - name: MQ_VHOST value: {{ .Values.global.vhost | quote }} {{- end }} ports: - - containerPort: 15672 + - containerPort: {{ ternary 15671 15672 (.Values.global.tls.enabled )}} name: management protocol: TCP - containerPort: {{ ternary 5671 5672 (.Values.global.tls.enabled )}} @@ -111,8 +108,8 @@ spec: - -ec - rabbitmq-diagnostics -q ping initialDelaySeconds: 20 - periodSeconds: 5 - timeoutSeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 readinessProbe: exec: command: @@ -121,7 +118,7 @@ spec: - rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 10 + timeoutSeconds: 3 volumeMounts: - name: data mountPath: "/var/lib/rabbitmq/" diff --git a/charts/sda-mq/values.yaml b/charts/sda-mq/values.yaml index 305aa62c8..a93af6800 100644 --- a/charts/sda-mq/values.yaml +++ b/charts/sda-mq/values.yaml @@ -1,6 +1,6 @@ global: adminUser: - adminPasswordHash: + adminPassword: tls: enabled: true issuer: "" @@ -10,7 +10,6 @@ global: keyName: tls.key caCert: ca.crt verifyPeer: true -# if a different vhost than the default `/` is to be used vhost: "" # Upstream shovel recipient @@ -34,8 +33,8 @@ externalPkiService: extraSecurityContext: {} image: - repository: ghcr.io/neicnordic/sda-mq - tag: v1.4.38 + repository: ghcr.io/neicnordic/sensitive-data-archive + tag: v0.0.44-rabbitmq pullPolicy: Always # utilize network isolation @@ -56,7 +55,7 @@ networkPolicy: ## If undefined (the default) or set to null, no storageClassName spec is ## set, choosing the default provisioner. persistence: - enabled: false + enabled: true storageSize: 8Gi storageClass: null existingClaim: null @@ -74,19 +73,20 @@ rbacEnabled: true resources: requests: - memory: "128Mi" - cpu: "100m" - limits: memory: "1Gi" cpu: "1" + limits: + memory: "2Gi" + cpu: "2" ## RevisionHistory ## If defined, set the revisionHistoryLimit of the deployment, defaults to 3 ## RevisionHistoryLimit is number of old ReplicaSets to retain to allow rollback. # revisionHistory: 3 +## This is only available on clusters running k8s < v1.25.0 securityPolicy: - create: true + create: false service: type: ClusterIP From b4631c29973b47f337cd57e8a6ddcc099d4d30f5 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 28 Jun 2023 14:51:24 +0200 Subject: [PATCH 03/14] Update sda-svc --- charts/sda-svc/Chart.yaml | 4 +- charts/sda-svc/README.md | 53 +++++----------- charts/sda-svc/templates/_helpers.yaml | 36 +++++------ .../sda-svc/templates/auth-certificate.yaml | 2 + charts/sda-svc/templates/auth-deploy.yaml | 18 +++--- charts/sda-svc/templates/auth-secrets.yaml | 4 +- .../sda-svc/templates/backup-certificate.yaml | 2 + charts/sda-svc/templates/backup-deploy.yaml | 4 +- charts/sda-svc/templates/doa-deploy.yaml | 14 ++--- .../templates/download-certificate.yaml | 2 + charts/sda-svc/templates/download-deploy.yaml | 16 ++++- .../templates/finalize-certificate.yaml | 2 + charts/sda-svc/templates/finalize-deploy.yaml | 6 +- .../sda-svc/templates/inbox-certificate.yaml | 2 + .../sda-svc/templates/ingest-certificate.yaml | 2 + charts/sda-svc/templates/ingest-deploy.yaml | 4 +- .../templates/intercept-certificate.yaml | 2 + .../sda-svc/templates/intercept-deploy.yaml | 4 +- .../sda-svc/templates/mapper-certificate.yaml | 2 + charts/sda-svc/templates/mapper-deploy.yaml | 38 +++++++++++- charts/sda-svc/templates/s3-inbox-deploy.yaml | 7 ++- charts/sda-svc/templates/serviceaccount.yaml | 1 + .../sda-svc/templates/sftp-inbox-deploy.yaml | 4 +- .../sda-svc/templates/verify-certificate.yaml | 2 + charts/sda-svc/templates/verify-deploy.yaml | 4 +- charts/sda-svc/test/release-test.sh | 8 +-- charts/sda-svc/values.yaml | 61 ++++++------------- 27 files changed, 159 insertions(+), 145 deletions(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index ab408cf04..d84ff4220 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 name: sda-svc -version: "0.18.9" +version: "0.20.0" kubeVersion: ">= 1.19.0-0" description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io icon: https://neic.no/assets/images/logo.png sources: -- https://github.com/neicnordic/sda-helm +- https://github.com/neicnordic/sensitive-data-archive diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index ed10fd400..73866f682 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -2,9 +2,8 @@ Source repositories: -- [https://github.com/neicnordic/sda-pipeline](https://github.com/neicnordic/sda-pipeline) +- [https://github.com/neicnordic/sensitive-data-archive](https://github.com/neicnordic/sensitive-data-archive) - [https://github.com/neicnordic/sda-doa](https://github.com/neicnordic/sda-doa) -- [https://github.com/neicnordic/sda-download](https://github.com/neicnordic/sda-download) ## Installing the Chart @@ -17,6 +16,9 @@ The following table lists the configurable parameters of the `sda-svc` chart and Parameter | Description | Default --------- | ----------- | ------- +`global.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` +`global.imageTag` | Tag version to deploy | `` +`global.imagePullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` `global.tlsPath` | This path will be a subpath to the secretsPath | `tls` @@ -120,10 +122,12 @@ Parameter | Description | Default `global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` `global.download.trusted.configFile` | Name of ISS config file | `iss.json` `global.download.trusted.iss` | Array of trusted OIDC endpoints | `` -`global.download.trusted.iss[iss]` | URI to the OIDC service | `https://login.elixir-czech.org/oidc/` -`global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://login.elixir-czech.org/oidc/jwk` -`global.elixir.oidcdHost` | URL to the OIDc service. | `"https://login.elixir-czech.org/oidc/"` -`global.elixir.jwkPath` | Public key path on the OIDC host. | `jwk` +`global.download.trusted.iss[iss]` | URI to the OIDC service | `https://proxy.aai.lifescience-ri.eu` +`global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://proxy.aai.lifescience-ri.eu/OIDC/jwks` +`global.oidc.provider` | URL to the OIDc service. | `"https://proxy.aai.lifescience-ri.eu"` +`global.oidc.jwkPath` | Public key path on the OIDC host. | `/OIDC/jwks` +`global.oidc.id` | User ID to the OIDC host. | `` +`global.oidc.secret` | User credentials to the OIDC host. | `` `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` `global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. |`posix` `global.inbox.path` | Path to the mounted `posix` volume. |`/inbox` @@ -182,25 +186,19 @@ Parameter | Description | Default Parameter | Description | Default --------- | ----------- | ------- -`auth.replicaCount` | desired number of replicas | `1` -`auth.repository` | auth container image repository | `neicnordic/sda-auth` -`auth.imageTag` | auth container image version | `"latest"` -`auth.imagePullPolicy` | auth container image pull policy | `Always` +`auth.replicaCount` | desired number of replicas | `2` `auth.annotations` | Specific annotation for the auth pod | `{}` `auth.resources.requests.memory` | Memory request for container. |`128Mi` `auth.resources.requests.cpu` | CPU request for container. |`100m` `auth.resources.limits.memory` | Memory limit for container. |`256Mi` `auth.resources.limits.cpu` | CPU limit for container. |`250m` -`backup.repository` | inbox container image repository | `neicnordic/sda-pipeline` -`backup.imageTag` | inbox container image version | `latest` -`backup.imagePullPolicy` | inbox container image pull policy | `Always` `backup.annotations` | Specific annotation for the backup pod | `{}` `backup.resources.requests.memory` | Memory request for backup container. |`128Mi` `backup.resources.requests.cpu` | CPU request for backup container. |`100m` `backup.resources.limits.memory` | Memory limit for backup container. |`256Mi` `backup.resources.limits.cpu` | CPU limit for backup container. |`250m` `backup.deploy` | Set to true if the backup service should be active | `false` -`doa.replicaCount` | desired number of replicas | `1` +`doa.replicaCount` | desired number of replicas | `2` `doa.repository` | dataedge container image repository | `neicnordic/sda-doa` `doa.imageTag` | dataedge container image version | `"latest"` `doa.imagePullPolicy` | dataedge container image pull policy | `Always` @@ -210,36 +208,24 @@ Parameter | Description | Default `doa.resources.requests.cpu` | CPU request for dataedge container. |`100m` `doa.resources.limits.memory` | Memory limit for dataedge container. |`1024Mi` `doa.resources.limits.cpu` | CPU limit for dataedge container. |`2000m` -`download.replicaCount` | desired number of replicas | `1` -`download.repository` | dataedge container image repository | `neicnordic/sda-doa` -`download.imageTag` | dataedge container image version | `"latest"` -`download.imagePullPolicy` | dataedge container image pull policy | `Always` +`download.replicaCount` | desired number of replicas | `2` `download.keystorePass` | keystore password | `changeit` `download.annotations` | Specific annotation for the dataedge pod | `{}` `download.resources.requests.memory` | Memory request for dataedge container. |`256Mi` `download.resources.requests.cpu` | CPU request for dataedge container. |`100m` `download.resources.limits.memory` | Memory limit for dataedge container. |`512Mi` `download.resources.limits.cpu` | CPU limit for dataedge container. |`1000m` -`finalize.repository` | inbox container image repository | `neicnordic/sda-pipeline` -`finalize.imageTag` | inbox container image version | `latest` -`finalize.imagePullPolicy` | inbox container image pull policy | `Always` `finalize.annotations` | Specific annotation for the finalize pod | `{}` `finalize.resources.requests.memory` | Memory request for finalize container. |`128Mi` `finalize.resources.requests.cpu` | CPU request for finalize container. |`100m` `finalize.resources.limits.memory` | Memory limit for finalize container. |`256Mi` `finalize.resources.limits.cpu` | CPU limit for finalize container. |`250m` -`ingest.repository` | inbox container image repository | `neicnordic/sda-pipeline` -`ingest.imageTag` | inbox container image version | `latest` -`ingest.imagePullPolicy` | inbox container image pull policy | `Always` `ingest.replicaCount` | desired number of ingest workers | `1` `ingest.annotations` | Specific annotation for the ingest pod | `{}` `ingest.resources.requests.memory` | Memory request for ingest container. |`128Mi` `ingest.resources.requests.cpu` | CPU request for ingest container. |`100m` `ingest.resources.limits.memory` | Memory limit for ingest container. |`512Mi` `ingest.resources.limits.cpu` | CPU limit for ingest container. |`2000m` -`intercept.repository` | intercept container image repository | `neicnordic/sda-pipeline` -`intercept.imageTag` | intercept container image version | `latest` -`intercept.imagePullPolicy` | intercept container image pull policy | `Always` `intercept.replicaCount` | desired number of intercept workers | `1` `intercept.annotations` | Specific annotation for the intercept pod | `{}` `intercept.deploy` | Set to false in a non federated deployment | `true` @@ -247,19 +233,13 @@ Parameter | Description | Default `intercept.resources.requests.cpu` | CPU request for intercept container. |`100m` `intercept.resources.limits.memory` | Memory limit for intercept container. |`128Mi` `intercept.resources.limits.cpu` | CPU limit for intercept container. |`2000m` -`s3Inbox.repository` | S3inbox container image repository | `neicnordic/sda-s3proxy` -`s3Inbox.imageTag` | S3inbox container image version | `latest` -`s3Inbox.imagePullPolicy` | S3inbox container image pull policy | `Always` -`s3Inbox.replicaCount`| desired number of S3inbox containers | `1` +`s3Inbox.replicaCount`| desired number of S3inbox containers | `2` `s3Inbox.annotations` | Specific annotation for the S3inbox pod | `{}` `s3Inbox.resources.requests.memory` | Memory request for s3Inbox container. |`128Mi` `s3Inbox.resources.requests.cpu` | CPU request for s3Inbox container. |`100m` `s3Inbox.resources.limits.memory` | Memory limit for s3Inbox container. |`1024Mi` `s3Inbox.resources.limits.cpu` | CPU limit for s3Inbox container. |`1000m` -`sftpInbox.repository` | sftp inbox container image repository | `neicnordic/sda-inbox-sftp` -`sftpInbox.imageTag` | sftp inbox container image version | `latest` -`sftpInbox.imagePullPolicy` | sftp inbox container image pull policy | `Always` -`sftpInbox.replicaCount`| desired number of sftp inbox containers | `1` +`sftpInbox.replicaCount`| desired number of sftp inbox containers | `2` `sftpInbox.keystorePass` | sftp inbox keystore password | `changeit` `sftpInbox.nodeHostname` | Node name if the sftp inbox needs to be deployed on a specific node | `""` `sftpInbox.annotations` | Specific annotation for the sftp inbox pod | `{}` @@ -267,9 +247,6 @@ Parameter | Description | Default `sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. |`100m` `sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. |`256Mi` `sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. |`250m` -`verify.repository` | inbox container image repository | `neicnordic/sda-pipeline` -`verify.imageTag` | inbox container image version | `latest` -`verify.imagePullPolicy` | inbox container image pull policy | `Always` `verify.replicaCount`| desired number of verify containers | `1` `verify.annotations` | Specific annotation for the verify pod | `{}` `verify.resources.requests.memory` | Memory request for verify container. |`128Mi` diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 1a1588e83..4e32654f6 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -135,10 +135,10 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserBackup" -}} -{{- ternary "lega_in" .Values.credentials.backup.dbUser (empty .Values.credentials.backup.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.backup.dbUser (empty .Values.credentials.backup.dbUser) -}} {{- end -}} {{- define "dbPassBackup" -}} -{{- ternary .Values.global.db.passIngest .Values.credentials.backup.dbPassword (empty .Values.credentials.backup.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.backup.dbPassword (empty .Values.credentials.backup.dbPassword) -}} {{- end -}} {{- define "mqUserBackup" -}} {{- ternary .Values.global.broker.username .Values.credentials.backup.mqUser (empty .Values.credentials.backup.mqUser) -}} @@ -149,10 +149,10 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserDoa" -}} -{{- ternary "lega_out" .Values.credentials.doa.dbUser (empty .Values.credentials.doa.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.doa.dbUser (empty .Values.credentials.doa.dbUser) -}} {{- end -}} {{- define "dbPassDoa" -}} -{{- ternary .Values.global.db.passOutgest .Values.credentials.doa.dbPassword (empty .Values.credentials.doa.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.doa.dbPassword (empty .Values.credentials.doa.dbPassword) -}} {{- end -}} {{- define "mqUserDoa" -}} {{- ternary .Values.global.broker.username .Values.credentials.doa.mqUser (empty .Values.credentials.doa.mqUser) -}} @@ -163,18 +163,18 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserDownload" -}} -{{- ternary "lega_out" .Values.credentials.download.dbUser (empty .Values.credentials.download.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.download.dbUser (empty .Values.credentials.download.dbUser) -}} {{- end -}} {{- define "dbPassDownload" -}} -{{- ternary .Values.global.db.passOutgest .Values.credentials.download.dbPassword (empty .Values.credentials.download.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.download.dbPassword (empty .Values.credentials.download.dbPassword) -}} {{- end -}} {{/**/}} {{- define "dbUserFinalize" -}} -{{- ternary "lega_in" .Values.credentials.finalize.dbUser (empty .Values.credentials.finalize.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.finalize.dbUser (empty .Values.credentials.finalize.dbUser) -}} {{- end -}} {{- define "dbPassFinalize" -}} -{{- ternary .Values.global.db.passIngest .Values.credentials.finalize.dbPassword (empty .Values.credentials.finalize.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.finalize.dbPassword (empty .Values.credentials.finalize.dbPassword) -}} {{- end -}} {{- define "mqUserFinalize" -}} {{- ternary .Values.global.broker.username .Values.credentials.finalize.mqUser (empty .Values.credentials.finalize.mqUser) -}} @@ -185,10 +185,10 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserIngest" -}} -{{- ternary "lega_in" .Values.credentials.ingest.dbUser (empty .Values.credentials.ingest.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.ingest.dbUser (empty .Values.credentials.ingest.dbUser) -}} {{- end -}} {{- define "dbPassIngest" -}} -{{- ternary .Values.global.db.passIngest .Values.credentials.ingest.dbPassword (empty .Values.credentials.ingest.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.ingest.dbPassword (empty .Values.credentials.ingest.dbPassword) -}} {{- end -}} {{- define "mqUserIngest" -}} {{- ternary .Values.global.broker.username .Values.credentials.ingest.mqUser (empty .Values.credentials.ingest.mqUser) -}} @@ -199,10 +199,10 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserInbox" -}} -{{- ternary "lega_in" .Values.credentials.inbox.dbUser (empty .Values.credentials.inbox.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.inbox.dbUser (empty .Values.credentials.inbox.dbUser) -}} {{- end -}} {{- define "dbPassInbox" -}} -{{- ternary .Values.global.db.passIngest .Values.credentials.inbox.dbPassword (empty .Values.credentials.inbox.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.inbox.dbPassword (empty .Values.credentials.inbox.dbPassword) -}} {{- end -}} {{- define "mqUserInbox" -}} {{- ternary .Values.global.broker.username .Values.credentials.inbox.mqUser (empty .Values.credentials.inbox.mqUser) -}} @@ -221,10 +221,10 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserMapper" -}} -{{- ternary "lega_out" .Values.credentials.mapper.dbUser (empty .Values.credentials.mapper.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.mapper.dbUser (empty .Values.credentials.mapper.dbUser) -}} {{- end -}} {{- define "dbPassMapper" -}} -{{- ternary .Values.global.db.passOutgest .Values.credentials.mapper.dbPassword (empty .Values.credentials.mapper.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.mapper.dbPassword (empty .Values.credentials.mapper.dbPassword) -}} {{- end -}} {{- define "mqUserMapper" -}} {{- ternary .Values.global.broker.username .Values.credentials.mapper.mqUser (empty .Values.credentials.mapper.mqUser) -}} @@ -236,10 +236,10 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserReleaseTest" -}} -{{- ternary "lega_in" .Values.credentials.releasetest.dbUser (empty .Values.credentials.releasetest.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.releasetest.dbUser (empty .Values.credentials.releasetest.dbUser) -}} {{- end -}} {{- define "dbPassReleaseTest" -}} -{{- ternary .Values.global.db.passIngest .Values.credentials.releasetest.dbPassword (empty .Values.credentials.releasetest.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.releasetest.dbPassword (empty .Values.credentials.releasetest.dbPassword) -}} {{- end -}} {{- define "mqUserReleaseTest" -}} {{- ternary .Values.global.broker.username .Values.credentials.releasetest.mqUser (empty .Values.credentials.releasetest.mqUser) -}} @@ -252,10 +252,10 @@ Create chart name and version as used by the chart label. {{/**/}} {{- define "dbUserVerify" -}} -{{- ternary "lega_in" .Values.credentials.verify.dbUser (empty .Values.credentials.verify.dbUser) -}} +{{- ternary .Values.global.db.user .Values.credentials.verify.dbUser (empty .Values.credentials.verify.dbUser) -}} {{- end -}} {{- define "dbPassVerify" -}} -{{- ternary .Values.global.db.passIngest .Values.credentials.verify.dbPassword (empty .Values.credentials.verify.dbPassword) -}} +{{- ternary .Values.global.db.password .Values.credentials.verify.dbPassword (empty .Values.credentials.verify.dbPassword) -}} {{- end -}} {{- define "mqUserVerify" -}} {{- ternary .Values.global.broker.username .Values.credentials.verify.mqUser (empty .Values.credentials.verify.mqUser) -}} diff --git a/charts/sda-svc/templates/auth-certificate.yaml b/charts/sda-svc/templates/auth-certificate.yaml index f9693d639..7d7a2b743 100644 --- a/charts/sda-svc/templates/auth-certificate.yaml +++ b/charts/sda-svc/templates/auth-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} {{- if eq "s3" .Values.global.inbox.storageType }} apiVersion: cert-manager.io/v1 @@ -36,3 +37,4 @@ spec: group: cert-manager.io {{- end -}} {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index 48030c25f..cef75c1dd 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -53,13 +53,13 @@ spec: serviceAccountName: {{ .Release.Name }} {{- end }} securityContext: - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 containers: - name: auth - image: "{{ .Values.auth.repository }}:{{ .Values.auth.imageTag }}" - imagePullPolicy: {{ .Values.auth.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-auth" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: allowPrivilegeEscalation: false {{- if .Values.global.extraSecurityContext }} @@ -79,12 +79,12 @@ spec: valueFrom: secretKeyRef: name: {{ template "sda.fullname" . }}-auth - key: elixirID + key: oidcID - name: ELIXIR_SECRET valueFrom: secretKeyRef: name: {{ template "sda.fullname" . }}-auth - key: elixirSecret + key: oidcSecret {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - name: CEGA_ID valueFrom: @@ -104,11 +104,11 @@ spec: - name: ELIXIR_REDIRECTURL value: {{ template "authRedirect" .}} - name: ELIXIR_PROVIDER - value: "{{ .Values.global.elixir.provider }}" + value: "{{ .Values.global.oidc.provider }}" - name: ELIXIR_SCOPE value: "ga4gh_passport_v1" - name: ELIXIR_JWKPATH - value: {{ .Values.global.elixir.jwkPath | quote }} + value: {{ .Values.global.oidc.jwkPath | quote }} {{- if .Values.global.auth.corsOrigins }} - name: CORS_ORIGINS value: {{ .Values.global.auth.corsOrigins | quote }} diff --git a/charts/sda-svc/templates/auth-secrets.yaml b/charts/sda-svc/templates/auth-secrets.yaml index 647f0180f..7127bd259 100644 --- a/charts/sda-svc/templates/auth-secrets.yaml +++ b/charts/sda-svc/templates/auth-secrets.yaml @@ -7,8 +7,8 @@ metadata: name: {{ template "sda.fullname" . }}-auth type: Opaque data: - elixirID: {{ .Values.global.auth.elixirID | quote | trimall "\"" | b64enc }} - elixirSecret: {{ .Values.global.auth.elixirSecret | quote | trimall "\"" | b64enc }} + oidcID: {{ .Values.global.oidc.id | quote | trimall "\"" | b64enc }} + oidcSecret: {{ .Values.global.oidc.secret | quote | trimall "\"" | b64enc }} {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} cegaID: {{ .Values.global.cega.user | quote | trimall "\"" | b64enc }} cegaSecret: {{ .Values.global.cega.password | quote | trimall "\"" | b64enc }} diff --git a/charts/sda-svc/templates/backup-certificate.yaml b/charts/sda-svc/templates/backup-certificate.yaml index 44753ace8..cbde0c6d2 100644 --- a/charts/sda-svc/templates/backup-certificate.yaml +++ b/charts/sda-svc/templates/backup-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} {{- if .Values.backup.deploy}} apiVersion: cert-manager.io/v1 @@ -36,3 +37,4 @@ spec: group: cert-manager.io {{- end -}} {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/sda-svc/templates/backup-deploy.yaml b/charts/sda-svc/templates/backup-deploy.yaml index e733921a2..3593da14a 100644 --- a/charts/sda-svc/templates/backup-deploy.yaml +++ b/charts/sda-svc/templates/backup-deploy.yaml @@ -61,8 +61,8 @@ spec: {{- end }} containers: - name: backup - image: "{{ .Values.backup.repository }}:{{ .Values.backup.imageTag }}" - imagePullPolicy: {{ .Values.backup.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-backup"] securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-svc/templates/doa-deploy.yaml b/charts/sda-svc/templates/doa-deploy.yaml index e38804476..31e1d2486 100644 --- a/charts/sda-svc/templates/doa-deploy.yaml +++ b/charts/sda-svc/templates/doa-deploy.yaml @@ -181,9 +181,9 @@ spec: - name: ARCHIVE_PATH value: {{ .Values.global.archive.volumePath | quote }} {{- end }} - {{- if .Values.global.elixir.pubKey }} + {{- if .Values.global.oidc.pubKey }} - name: PASSPORT_PUBLIC_KEY_PATH - value: "{{ include "secretsPath" . }}/{{ .Values.global.elixir.pubKey }}" + value: "{{ include "secretsPath" . }}/{{ .Values.global.oidc.pubKey }}" {{- end }} - name: KEYSTORE_PATH value: {{ ternary (print "/etc/ssl/certs/java/doa.p12") (printf "%s/%s" .Values.global.secretsPath .Values.doa.tls.keyStore) (empty .Values.global.pkiService) }} @@ -200,7 +200,7 @@ spec: - name: CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH value: "{{ template "c4ghPath" . }}/passphrase" - name: OPENID_CONFIGURATION_URL - value: "{{ .Values.global.elixir.oidcdHost }}.well-known/openid-configuration" + value: "{{ .Values.global.oidc.provider }}/.well-known/openid-configuration" - name: OUTBOX_ENABLED value: {{ .Values.global.doa.outbox.enabled | quote }} {{- if .Values.global.doa.outbox.enabled }} @@ -266,7 +266,7 @@ spec: mountPath: "/etc/ssl/certs/java" {{- end }} {{- if not .Values.global.vaultSecrets }} - {{- if .Values.global.elixir.pubKey }} + {{- if .Values.global.oidc.pubKey }} - name: jwt-token mountPath: {{ include "secretsPath" . }} {{- end }} @@ -293,7 +293,7 @@ spec: {{- end }} {{- end }} {{- if not .Values.global.vaultSecrets }} - {{- if .Values.global.elixir.pubKey }} + {{- if .Values.global.oidc.pubKey }} - name: jwt-token projected: defaultMode: 0440 @@ -301,8 +301,8 @@ spec: - secret: name: {{ template "sda.fullname" . }}-doa items: - - key: {{ .Values.global.elixir.pubKey }} - path: {{ .Values.global.elixir.pubKey }} + - key: {{ .Values.global.oidc.pubKey }} + path: {{ .Values.global.oidc.pubKey }} {{- end }} - name: c4gh-key secret: diff --git a/charts/sda-svc/templates/download-certificate.yaml b/charts/sda-svc/templates/download-certificate.yaml index 2455fcc58..8e1b0bfce 100644 --- a/charts/sda-svc/templates/download-certificate.yaml +++ b/charts/sda-svc/templates/download-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if .Values.global.download.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 @@ -37,3 +38,4 @@ spec: group: cert-manager.io {{- end -}} {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index d8f1a28c6..9429b8319 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -75,8 +75,8 @@ spec: {{- end }} containers: - name: download - image: "{{ .Values.download.repository }}:{{ .Values.download.imageTag }}" - imagePullPolicy: {{ .Values.download.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-download" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: allowPrivilegeEscalation: false command: ["sda-download"] @@ -106,7 +106,7 @@ spec: value: "{{ .Values.global.archive.volumePath }}" {{- end }} - name: OIDC_CONFIGURATION_URL - value: "{{ .Values.global.elixir.oidcConfig }}/.well-known/openid-configuration" + value: "{{ .Values.global.oidc.provider }}/.well-known/openid-configuration" {{- if .Values.global.download.trusted.iss }} - name: OIDC_TRUSTED_ISS value: {{ include "trustedIssPath" . }}/{{ default "iss.json" .Values.global.download.trusted.configFile }} @@ -213,6 +213,16 @@ spec: value: {{ .Values.global.ingress.hostName.download }} initialDelaySeconds: 20 periodSeconds: 10 + startupProbe: + httpGet: + path: /health + port: download + scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} + httpHeaders: + - name: Host + value: {{ .Values.global.ingress.hostName.download }} + failureThreshold: 30 + periodSeconds: 10 resources: {{ toYaml .Values.download.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/templates/finalize-certificate.yaml b/charts/sda-svc/templates/finalize-certificate.yaml index 2eeaefcbe..93dcbfaef 100644 --- a/charts/sda-svc/templates/finalize-certificate.yaml +++ b/charts/sda-svc/templates/finalize-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 kind: Certificate @@ -34,3 +35,4 @@ spec: # if you are using an external issuer, change this to that issuer group. group: cert-manager.io {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml index 227a91665..d85a3ce79 100644 --- a/charts/sda-svc/templates/finalize-deploy.yaml +++ b/charts/sda-svc/templates/finalize-deploy.yaml @@ -60,8 +60,8 @@ spec: {{- end }} containers: - name: finalize - image: "{{ .Values.finalize.repository }}:{{ .Values.finalize.imageTag }}" - imagePullPolicy: {{ .Values.finalize.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-finalize"] securityContext: allowPrivilegeEscalation: false @@ -71,7 +71,7 @@ spec: - name: BROKER_EXCHANGE value: {{ default "sda" .Values.global.broker.exchange }} - name: BROKER_QUEUE - value: "accessionIDs" + value: "accession" - name: BROKER_HOST value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - name: BROKER_PORT diff --git a/charts/sda-svc/templates/inbox-certificate.yaml b/charts/sda-svc/templates/inbox-certificate.yaml index 935f3deb4..9692c5519 100644 --- a/charts/sda-svc/templates/inbox-certificate.yaml +++ b/charts/sda-svc/templates/inbox-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 kind: Certificate @@ -35,3 +36,4 @@ spec: # if you are using an external issuer, change this to that issuer group. group: cert-manager.io {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/ingest-certificate.yaml b/charts/sda-svc/templates/ingest-certificate.yaml index 2a9561675..1d6ef9ca5 100644 --- a/charts/sda-svc/templates/ingest-certificate.yaml +++ b/charts/sda-svc/templates/ingest-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 kind: Certificate @@ -34,3 +35,4 @@ spec: # if you are using an external issuer, change this to that issuer group. group: cert-manager.io {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index 411f7fe74..0b8533ade 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -61,8 +61,8 @@ spec: {{- end }} containers: - name: ingest - image: "{{ .Values.ingest.repository }}:{{ .Values.ingest.imageTag }}" - imagePullPolicy: {{ .Values.ingest.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-ingest"] securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-svc/templates/intercept-certificate.yaml b/charts/sda-svc/templates/intercept-certificate.yaml index 364344f38..9f8451d8b 100644 --- a/charts/sda-svc/templates/intercept-certificate.yaml +++ b/charts/sda-svc/templates/intercept-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 kind: Certificate @@ -34,3 +35,4 @@ spec: # if you are using an external issuer, change this to that issuer group. group: cert-manager.io {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/intercept-deploy.yaml b/charts/sda-svc/templates/intercept-deploy.yaml index 72eedc09c..66c2e2615 100644 --- a/charts/sda-svc/templates/intercept-deploy.yaml +++ b/charts/sda-svc/templates/intercept-deploy.yaml @@ -44,8 +44,8 @@ spec: fsGroup: 65534 containers: - name: intercept - image: "{{ .Values.intercept.repository }}:{{ .Values.intercept.imageTag }}" - imagePullPolicy: {{ .Values.intercept.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-intercept"] securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-svc/templates/mapper-certificate.yaml b/charts/sda-svc/templates/mapper-certificate.yaml index e83ac5c55..57f1c32f4 100644 --- a/charts/sda-svc/templates/mapper-certificate.yaml +++ b/charts/sda-svc/templates/mapper-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 kind: Certificate @@ -34,3 +35,4 @@ spec: # if you are using an external issuer, change this to that issuer group. group: cert-manager.io {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml index 64bfcefe2..e1528a882 100644 --- a/charts/sda-svc/templates/mapper-deploy.yaml +++ b/charts/sda-svc/templates/mapper-deploy.yaml @@ -60,8 +60,8 @@ spec: {{- end }} containers: - name: mapper - image: "{{ .Values.mapper.repository }}:{{ .Values.mapper.imageTag }}" - imagePullPolicy: {{ .Values.mapper.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-mapper"] securityContext: allowPrivilegeEscalation: false @@ -116,6 +116,28 @@ spec: value: {{ .Values.global.db.port | quote }} - name: DB_SSLMODE value: {{ template "dbSSLmode" . }} + - name: INBOX_TYPE + {{- if eq "s3" .Values.global.inbox.storageType }} + value: "s3" + - name: INBOX_BUCKET + value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + - name: INBOX_CACERT + value: {{ template "tlsPath" . }}/ca.crt + {{- end }} + - name: INBOX_REGION + value: {{ default "us-east-1" .Values.global.inbox.s3Region }} + - name: INBOX_URL + value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- if .Values.global.inbox.s3Port }} + - name: INBOX_PORT + value: {{ .Values.global.inbox.s3Port | quote }} + {{- end }} + {{- else }} + value: "posix" + - name: INBOX_LOCATION + value: "{{ .Values.global.inbox.path }}/" + {{- end }} {{- if .Values.global.log.format }} - name: LOG_FORMAT value: {{ .Values.global.log.format | quote }} @@ -147,6 +169,18 @@ spec: secretKeyRef: name: {{ template "sda.fullname" . }}-mapper key: dbUser + {{- if eq "s3" .Values.global.inbox.storageType }} + - name: INBOX_ACCESSKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3inbox-keys + key: s3InboxAccessKey + - name: INBOX_SECRETKEY + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-s3inbox-keys + key: s3InboxSecretKey + {{- end }} {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index 1b5b96abb..ebd4f4d06 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -76,8 +76,9 @@ spec: {{- end }} containers: - name: s3inbox - image: "{{ .Values.s3Inbox.repository }}:{{ .Values.s3Inbox.imageTag }}" - imagePullPolicy: {{ .Values.s3Inbox.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + command: ["sda-s3inbox"] securityContext: allowPrivilegeEscalation: false {{- if .Values.global.extraSecurityContext }} @@ -139,7 +140,7 @@ spec: - name: BROKER_VHOST value: {{ include "brokerVhost" . | quote }} - name: BROKER_EXCHANGE - value: {{ .Values.global.broker.exchange | quote }} + value: {{ default "sda" .Values.global.broker.exchange }} - name: BROKER_ROUTINGKEY value: "inbox" - name: BROKER_SSL diff --git a/charts/sda-svc/templates/serviceaccount.yaml b/charts/sda-svc/templates/serviceaccount.yaml index 73d5b9612..1cd9efdc7 100644 --- a/charts/sda-svc/templates/serviceaccount.yaml +++ b/charts/sda-svc/templates/serviceaccount.yaml @@ -9,4 +9,5 @@ metadata: release: {{ .Release.Name }} name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} +automountServiceAccountToken: false {{- end }} diff --git a/charts/sda-svc/templates/sftp-inbox-deploy.yaml b/charts/sda-svc/templates/sftp-inbox-deploy.yaml index 0c2116b59..75e42ef45 100644 --- a/charts/sda-svc/templates/sftp-inbox-deploy.yaml +++ b/charts/sda-svc/templates/sftp-inbox-deploy.yaml @@ -94,8 +94,8 @@ spec: {{- end }} containers: - name: inbox - image: "{{ .Values.sftpInbox.repository }}:{{ .Values.sftpInbox.imageTag }}" - imagePullPolicy: {{ .Values.sftpInbox.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-sftp-inbox" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["java", "-jar", "inbox-0.0.3-SNAPSHOT.jar"] securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-svc/templates/verify-certificate.yaml b/charts/sda-svc/templates/verify-certificate.yaml index d3e5be52e..177f71398 100644 --- a/charts/sda-svc/templates/verify-certificate.yaml +++ b/charts/sda-svc/templates/verify-certificate.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.tls.enabled }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 kind: Certificate @@ -34,3 +35,4 @@ spec: # if you are using an external issuer, change this to that issuer group. group: cert-manager.io {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index b371d166d..54ab72d78 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -61,8 +61,8 @@ spec: {{- end }} containers: - name: verify - image: "{{ .Values.verify.repository }}:{{ .Values.verify.imageTag }}" - imagePullPolicy: {{ .Values.verify.imagePullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}-pipeline" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} command: ["sda-verify"] securityContext: allowPrivilegeEscalation: false diff --git a/charts/sda-svc/test/release-test.sh b/charts/sda-svc/test/release-test.sh index c1f3b2afd..6807930b8 100644 --- a/charts/sda-svc/test/release-test.sh +++ b/charts/sda-svc/test/release-test.sh @@ -5,7 +5,7 @@ if [ "$INBOX_STORAGE_TYPE" == "s3" ]; then cat >> "/tmp/s3cfg" <<-EOF host_base = $INBOX_SERVICE_NAME host_bucket = $INBOX_SERVICE_NAME - access_key = dummy + access_key = test_dummy.org access_token = $INBOX_ACCESS_TOKEN use_https = True ca_certs_file = /tls/ca.crt @@ -14,7 +14,7 @@ if [ "$INBOX_STORAGE_TYPE" == "s3" ]; then cat >> "/tmp/s3cfg" <<-EOF host_base = $INBOX_SERVICE_NAME host_bucket = $INBOX_SERVICE_NAME - access_key = dummy + access_key = test_dummy.org access_token = $INBOX_ACCESS_TOKEN use_https = False EOF @@ -34,7 +34,7 @@ if [ "${DEPLOYMENT_TYPE}" = all ] || [ "${DEPLOYMENT_TYPE}" = external ]; then elif [ "$INBOX_STORAGE_TYPE" == "s3" ]; then if [ "$TLS" == true ]; then echo "Will try connecting to https://$INBOX_SERVICE_NAME/" - if ! s3cmd -c "/tmp/s3cfg" ls s3://dummy ; then + if ! s3cmd -c "/tmp/s3cfg" ls s3://test_dummy.org ; then echo "expected 403 got: $responsecode" echo "Failed inbox verification, bailing out" exit 1 @@ -49,7 +49,7 @@ if [ "${DEPLOYMENT_TYPE}" = all ] || [ "${DEPLOYMENT_TYPE}" = external ]; then fi else echo "Will try connecting to http://$INBOX_SERVICE_NAME/" - if ! s3cmd -c "/tmp/s3cfg" ls s3://dummy ; then + if ! s3cmd -c "/tmp/s3cfg" ls s3://test_dummy.org ; then echo "Failed inbox verification, bailing out" exit 1 fi diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 44a1919d5..8cbd726f6 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -1,8 +1,11 @@ # Default values for SDA services. -# This is a YAML-formatted file. # Declare variables to be passed into your templates. -global: +image: + repository: "ghcr.io/neicnordic/sensitive-data-archive" + tag: "v0.0.44" + pullPolicy: "Always" +global: # Path where the sensitive files can be found, default is "/.secrets". # TLS certificates or C4GH key locations can be set using global.tlsPath or global.c4ghPath respectively, # this path will be a subpath to the secretsPath. @@ -85,7 +88,7 @@ global: rbacEnabled: true podSecurityPolicy: - create: true + create: false # Extra security context to apply to all pods # This should be a multi-line string mapping directly to the a map of @@ -137,10 +140,6 @@ global: copyHeader: false auth: - # @param elixirID, client ID to the Elixir OIDC for the service endpoint - elixirID: - # @param elixirSecret, client secret to the Elixir OIDC for the service endpoint - elixirSecret: # @param jwtSecret, name of the secret holding the jwt signing key jwtSecret: # @param jwtAlg, cipher type of the signing key @@ -169,7 +168,7 @@ global: ssl: true username: "" verifyPeer: true - vhost: "/" + vhost: "sda" prefetchCount: 2 cega: @@ -191,9 +190,9 @@ global: db: host: "" - name: "lega" - passIngest: "" - passOutgest: "" + name: "sda" + user: "" + password: "" port: 5432 sslMode: "verify-full" @@ -230,13 +229,14 @@ global: configPath: "iss" configFile: "iss.json" iss: - - iss: " https://profile.aai.lifescience-ri.eu/" - jku: " https://profile.aai.lifescience-ri.eu/jwk" + - iss: "https://proxy.aai.lifescience-ri.eu" + jku: "https://proxy.aai.lifescience-ri.eu/OIDC/jwks" - elixir: - oidcConfig: "https://proxy.aai.lifescience-ri.eu" - provider: "https://profile.aai.lifescience-ri.eu/" - jwkPath: "jwk" + oidc: + provider: "https://proxy.aai.lifescience-ri.eu" + jwkPath: "/OIDC/jwks" + id: "" + secret: "" inbox: servicePort: 2222 @@ -318,9 +318,6 @@ credentials: auth: name: auth replicaCount: 2 - repository: ghcr.io/neicnordic/sda-auth - imageTag: v0.7.6 - imagePullPolicy: IfNotPresent resources: requests: memory: "128Mi" @@ -339,9 +336,6 @@ backup: name: backup deploy: false replicaCount: 1 - repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.4.27 - imagePullPolicy: IfNotPresent resources: requests: memory: "128Mi" @@ -382,9 +376,6 @@ doa: download: name: download replicaCount: 2 - repository: ghcr.io/neicnordic/sda-download - imageTag: v1.9.14 - imagePullPolicy: IfNotPresent resources: requests: memory: "256Mi" @@ -402,9 +393,6 @@ download: finalize: name: finalize replicaCount: 1 - repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.4.27 - imagePullPolicy: IfNotPresent resources: requests: memory: "128Mi" @@ -422,9 +410,6 @@ finalize: ingest: name: ingest replicaCount: 1 - repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.4.27 - imagePullPolicy: IfNotPresent resources: requests: memory: "128Mi" @@ -443,9 +428,6 @@ intercept: deploy: true name: ingest replicaCount: 1 - repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.4.27 - imagePullPolicy: IfNotPresent resources: requests: memory: "128Mi" @@ -462,9 +444,6 @@ intercept: mapper: replicaCount: 1 - repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.4.27 - imagePullPolicy: IfNotPresent resources: requests: memory: "128Mi" @@ -481,9 +460,6 @@ mapper: s3Inbox: name: s3Inbox - repository: ghcr.io/neicnordic/sda-s3proxy - imageTag: v0.2.38 - imagePullPolicy: IfNotPresent replicaCount: 2 resources: requests: @@ -502,9 +478,6 @@ s3Inbox: sftpInbox: name: sftpInbox - repository: ghcr.io/neicnordic/sda-inbox-sftp - imageTag: v1.12.16 - imagePullPolicy: IfNotPresent replicaCount: 2 resources: requests: From 61d6e5f1854591724f9279573cedb39b1a38eabc Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 7 Aug 2023 16:25:41 +0200 Subject: [PATCH 04/14] Test deployment --- .../scripts/charts/dependencies.sh | 76 +++++++++++ .../scripts/charts/dependencies.yaml | 118 ++++++++++++++++++ .../scripts/charts/deploy_charts.sh | 35 ++++++ .github/integration/scripts/charts/k3d.sh | 14 +++ .../integration/scripts/charts/values.yaml | 92 ++++++++++++++ .github/workflows/chart_test.yml | 87 +++++++++++++ 6 files changed, 422 insertions(+) create mode 100644 .github/integration/scripts/charts/dependencies.sh create mode 100644 .github/integration/scripts/charts/dependencies.yaml create mode 100644 .github/integration/scripts/charts/deploy_charts.sh create mode 100755 .github/integration/scripts/charts/k3d.sh create mode 100644 .github/integration/scripts/charts/values.yaml create mode 100644 .github/workflows/chart_test.yml diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh new file mode 100644 index 000000000..d734a2813 --- /dev/null +++ b/.github/integration/scripts/charts/dependencies.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -ex + +YQ_VERSION="v4.20.1" +C4GH_VERSION="1.7.5" + +random-string() { + head -c 32 /dev/urandom | base64 -w0 | tr -d '/+' | fold -w 32 | head -n 1 +} + +sudo curl -sLO "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -O /usr/bin/yq && + sudo chmod +x /usr/bin/yq + +curl -sL https://github.com/neicnordic/crypt4gh/releases/download/v"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ && + sudo chmod +x /usr/bin/crypt4gh + +# secret for the crypt4gh keypair +C4GHPASSPHRASE="$(random-string)" +export C4GHPASSPHRASE +crypt4gh generate -n c4gh -p "$C4GHPASSPHRASE" +kubectl create secret generic c4gh --from-file="c4gh.sec.pem" --from-file="c4gh.pub.pem" --from-literal=passphrase="${C4GHPASSPHRASE}" + +# secret for the OIDC keypair +openssl ecparam -name prime256v1 -genkey -noout -out "jwt.key" +openssl ec -in "jwt.key" -pubout -out "jwt.pub" +kubectl create secret generic jwk --from-file="jwt.key" --from-file="jwt.pub" + +## OIDC +SELF=$(dirname "$0") +kubectl create configmap oidc --from-file="$SELF/../../sda/oidc.py" + +helm repo add jetstack https://charts.jetstack.io +helm repo add minio https://charts.min.io/ + +helm repo update + +helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --set installCRDs=true + +kubectl create namespace minio +kubectl apply -f .github/integration/scripts/charts/dependencies.yaml + +## S3 storage backend +MINIO_ACCESS="$(random-string)" +export MINIO_ACCESS +MINIO_SECRET="$(random-string)" +export MINIO_SECRET +helm install minio minio/minio \ + --namespace minio \ + --set rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi + +PGPASSWORD="$(random-string)" +export PGPASSWORD + +MQPASSWORD="$(random-string)" +export MQPASSWORD + +TEST_TOKEN="$(bash .github/integration/scripts/sign_jwt.sh ES256 jwt.key)" +export TEST_TOKEN + +## update values file with all credentials +yq -i ' +.global.archive.s3AccessKey = strenv(MINIO_ACCESS) | +.global.archive.s3SecretKey = strenv(MINIO_SECRET) | +.global.backupArchive.s3AccessKey = strenv(MINIO_ACCESS) | +.global.backupArchive.s3SecretKey = strenv(MINIO_SECRET) | +.global.broker.password = strenv(MQPASSWORD) | +.global.c4gh.passphrase = strenv(C4GHPASSPHRASE) | +.global.db.password = strenv(PGPASSWORD) | +.global.inbox.s3AccessKey = strenv(MINIO_ACCESS) | +.global.inbox.s3SecretKey = strenv(MINIO_SECRET) | +.releasetest.secrets.accessToken = strenv(TEST_TOKEN) +' .github/integration/scripts/charts/values.yaml diff --git a/.github/integration/scripts/charts/dependencies.yaml b/.github/integration/scripts/charts/dependencies.yaml new file mode 100644 index 000000000..13d6c2fa5 --- /dev/null +++ b/.github/integration/scripts/charts/dependencies.yaml @@ -0,0 +1,118 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: cert-manager +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: selfsigned-ca + namespace: cert-manager +spec: + isCA: true + commonName: selfsigned-ca + secretName: root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: cert-issuer + namespace: cert-manager +spec: + ca: + secretName: root-secret +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: minio-cert + namespace: minio +spec: + secretName: minio-cert + duration: 24h + commonName: minio + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + dnsNames: + - localhost + - minio + - minio.minio.svc + - minio.minio.svc.cluster.local + ipAddresses: + - 127.0.0.1 + issuerRef: + name: cert-issuer + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: ClusterIssuer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oidc-server +spec: + selector: + matchLabels: + app: oidc-server + replicas: 1 + template: + metadata: + labels: + app: oidc-server + spec: + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: oidc-server + image: neicnordic/mock-oidc:latest + ports: + - containerPort: 8080 + env: + - name: PORT + value: "8080" + - name: HOST + value: oidc-server + - name: CLIENT_ID + value: DfCieZLuBU + - name: CLIENT_SECRET + value: DfCieZLuBU + - name: CLIENT_REDIRECT_URI + value: https://sda-auth/elixir/login + resources: + limits: + cpu: 250m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: oidc-server + labels: + app: oidc-server +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + selector: + app: oidc-server diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh new file mode 100644 index 000000000..7e999dffc --- /dev/null +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -ex + +if [ "$1" == "sda-db" ]; then + ROOTPASS=$(yq e '.global.db.password' .github/integration/scripts/charts/values.yaml) + helm install postgres charts/sda-db \ + --set image.tag=test-postgres \ + --set image.pullPolicy=Never \ + --set global.postgresAdminPassword="$ROOTPASS" \ + --set global.tls.enabled=false \ + --set persistence.enabled=false \ + --set resources=null \ + --wait +fi + +if [ "$1" == "sda-mq" ]; then + ADMINPASS=$(yq e '.global.broker.password' .github/integration/scripts/charts/values.yaml) + helm install broker charts/sda-mq \ + --set image.tag=test-rabbitmq \ + --set image.pullPolicy=Never \ + --set global.adminPassword="$ADMINPASS" \ + --set global.adminUser=admin \ + --set global.tls.enabled=false \ + --set persistence.enabled=false \ + --set resources=null \ + --wait +fi + +if [ "$1" == "sda-svc" ]; then + helm install pipeline charts/sda-svc \ + --set image.tag=test \ + --set image.pullPolicy=Never \ + -f .github/integration/scripts/charts/values.yaml \ + --wait +fi diff --git a/.github/integration/scripts/charts/k3d.sh b/.github/integration/scripts/charts/k3d.sh new file mode 100755 index 000000000..a06206435 --- /dev/null +++ b/.github/integration/scripts/charts/k3d.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +k8s="$(curl -L -s https://dl.k8s.io/release/stable.txt)" + +curl -s -L https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | sudo bash +curl -sLO https://storage.googleapis.com/kubernetes-release/release/"$k8s"/bin/linux/amd64/kubectl +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl + +sudo k3d cluster create sda --image=rancher/k3s:"$k8s"-k3s1 --wait --timeout 10m +sudo k3d kubeconfig merge sda --kubeconfig-switch-context +mkdir -p ~/.kube/ && sudo cp /root/.k3d/kubeconfig-sda.yaml ~/.kube/config +sudo chown $UID:$UID ~/.kube/config && chmod 600 ~/.kube/config diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml new file mode 100644 index 000000000..a2c5c58a5 --- /dev/null +++ b/.github/integration/scripts/charts/values.yaml @@ -0,0 +1,92 @@ +global: + schemaType: "isolated" + ingress: + deploy: false + hostName: + auth: pipeline-sda-svc-auth + download: pipeline-sda-svc-download + s3Inbox: pipeline-sda-svc-inbox + log: + level: "debug" + tls: + enabled: false + issuer: "cert-issuer" + clusterIssuer: "" + archive: + s3Url: "http://minio.minio" + s3Bucket: "archive" + s3Port: 9000 + backupArchive: + storageType: "s3" + s3Url: "http://minio.minio" + s3Bucket: "backup" + s3Port: 9000 + auth: + jwtSecret: jwk + jwtAlg: ES256 + jwtKey: jwt.key + jwtPub: jwt.pub + resignJwt: true + broker: + durable: true + host: "broker-sda-mq" + port: 5672 + routingError: "error" + backupRoutingKey: "backup" + ssl: false + username: "admin" + c4gh: + secretName: c4gh + keyFile: c4gh.sec.pem + publicFile: c4gh.pub.pem + db: + host: "postgres-sda-db" + user: "postgres" + doa: + enabled: false + download: + enabled: true + trusted: + configPath: "iss" + configFile: "iss.json" + iss: + - iss: "http://oidc-server:8080" + jku: "http://oidc-server:8080/jwks" + oidc: + provider: "http://oidc-server:8080" + jwkPath: "/jwks" + id: DfCieZLuBU + secret: DfCieZLuBU + inbox: + storageType: s3 + s3Url: http://minio.minio + s3Port: 9000 + s3Bucket: "inbox" + s3ReadyPath: "/minio/health/ready" + +auth: + replicaCount: 1 + resources: null +backup: + deploy: true + resources: null +doa: + deploy: false +download: + resources: null + replicaCount: 1 +finalize: + resources: null +ingest: + resources: null +intercept: + deploy: false +mapper: + resources: null +releasetest: + run: false +s3Inbox: + resources: null + replicaCount: 1 +verify: + resources: null \ No newline at end of file diff --git a/.github/workflows/chart_test.yml b/.github/workflows/chart_test.yml new file mode 100644 index 000000000..796baec02 --- /dev/null +++ b/.github/workflows/chart_test.yml @@ -0,0 +1,87 @@ +name: Chart deployment test + +on: + pull_request: + paths: + - "charts/**" + +env: + DOCKER_BUILDKIT: 1 + +jobs: + deploy: + name: deployment test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Helm + uses: azure/setup-helm@v3.5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Initialise k3d + run: bash .github/integration/scripts/charts/k3d.sh + shell: bash + + - name: build postgres + shell: bash + run: | + cd postgresql + docker build -t ghcr.io/${{ github.repository }}:test-postgres . + - name: build rabbitmq + shell: bash + run: | + cd rabbitmq + docker build -t ghcr.io/${{ github.repository }}:test-rabbitmq . + - name: build sda + shell: bash + run: | + cd sda + docker build -t ghcr.io/${{ github.repository }}:test . + - name: build sda-auth + shell: bash + run: | + cd sda-auth + docker build -t ghcr.io/${{ github.repository }}:test-auth . + - name: build sda-download + shell: bash + run: | + cd sda-download + docker build -t ghcr.io/${{ github.repository }}:test-download . + - name: build sda-pipeline + shell: bash + run: | + cd sda-pipeline + docker build -t ghcr.io/${{ github.repository }}:test-pipeline . + - name: Import images to cluster + run: k3d image import ghcr.io/${{ github.repository }}:test-postgres ghcr.io/${{ github.repository }}:test-rabbitmq ghcr.io/${{ github.repository }}:test ghcr.io/${{ github.repository }}:test-auth ghcr.io/${{ github.repository }}:test-download ghcr.io/${{ github.repository }}:test-pipeline -c sda + + - name: Deploy external services + run: bash .github/integration/scripts/charts/dependencies.sh + shell: bash + + - name: Deploy DB + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db + + - name: Deploy MQ + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq + shell: bash + + - name: Deploy pipeline + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc + shell: bash + + - name: test + if: always() + run: | + kubectl get secret broker-sda-mq -o json + kubectl get secret pipeline-sda-svc-mapper -o json + kubectl get pods + echo "describe mapper" && kubectl describe pod -l role=mapper + sleep 1 + echo "logs mapper" && kubectl logs -l role=mapper + sleep 1 + echo "describe broker" && kubectl logs -l role=broker + shell: bash \ No newline at end of file From b9887a6b1a8add620b82ef31e8ec76a5031abc04 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 7 Aug 2023 15:32:25 +0200 Subject: [PATCH 05/14] Update RabbitMQ image --- rabbitmq/Dockerfile | 1 + rabbitmq/definitions.json | 85 ++++++++++++++++++++++++++++++----- rabbitmq/docker-entrypoint.sh | 17 +++++-- rabbitmq/federation.json | 4 +- 4 files changed, 89 insertions(+), 18 deletions(-) diff --git a/rabbitmq/Dockerfile b/rabbitmq/Dockerfile index 17ce7d589..958264127 100644 --- a/rabbitmq/Dockerfile +++ b/rabbitmq/Dockerfile @@ -10,6 +10,7 @@ LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sda" LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT ENV RABBITMQ_CONFIG_FILE=/var/lib/rabbitmq/rabbitmq.conf +ENV RABBITMQ_ADVANCED_CONFIG_FILE=/var/lib/rabbitmq/advanced.config COPY *.json /etc/rabbitmq/ diff --git a/rabbitmq/definitions.json b/rabbitmq/definitions.json index 09158f039..9be22ac25 100644 --- a/rabbitmq/definitions.json +++ b/rabbitmq/definitions.json @@ -24,9 +24,51 @@ "read": ".*" } ], - "parameters": [], + "parameters": [ + { + "component": "shovel", + "name": "completed", + "value": { + "ack-mode": "on-confirm", + "dest-queue": "completed", + "dest-protocol": "amqp091", + "dest-uri": "amqp:///sda", + "src-delete-after": "never", + "src-protocol": "amqp091", + "src-queue": "completed_stream", + "src-uri": "amqp:///sda" + }, + "vhost": "sda" + }, + { + "component": "shovel", + "name": "mappings", + "value": { + "ack-mode": "on-confirm", + "dest-queue": "mappings", + "dest-protocol": "amqp091", + "dest-uri": "amqp:///sda", + "src-delete-after": "never", + "src-protocol": "amqp091", + "src-queue": "mapping_stream", + "src-uri": "amqp:///sda" + }, + "vhost": "sda" + } + ], "global_parameters": [], - "policies": [], + "policies": [ + { + "vhost": "sda", + "name": "AE", + "pattern": "^sda$", + "apply-to": "exchanges", + "priority": 0, + "definition": { + "alternate-exchange": "sda.dead" + } + } + ], "queues": [ { "name": "accession", @@ -43,17 +85,24 @@ "arguments": {} }, { - "name": "completed", + "name": "backup", + "vhost": "sda", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "completed_stream", "vhost": "sda", "durable": true, "auto_delete": false, "arguments": { - "x-max-age": "7D", + "x-max-age": "1M", "x-queue-type": "stream" } }, { - "name": "error", + "name": "error_stream", "vhost": "sda", "durable": true, "auto_delete": false, @@ -81,6 +130,13 @@ "vhost": "sda", "durable": true, "auto_delete": false, + "arguments": {} + }, + { + "name": "mapping_stream", + "vhost": "sda", + "durable": true, + "auto_delete": false, "arguments": { "x-max-age": "1M", "x-queue-type": "stream" @@ -108,10 +164,7 @@ "type": "topic", "durable": true, "auto_delete": false, - "internal": false, - "arguments": { - "alternate-exchange": "sda.dead" - } + "internal": false }, { "name": "sda.dead", @@ -145,7 +198,15 @@ "vhost": "sda", "destination_type": "queue", "arguments": {}, - "destination": "completed", + "destination": "backup", + "routing_key": "backup" + }, + { + "source": "sda", + "vhost": "sda", + "destination_type": "queue", + "arguments": {}, + "destination": "completed_stream", "routing_key": "completed" }, { @@ -153,7 +214,7 @@ "vhost": "sda", "destination_type": "queue", "arguments": {}, - "destination": "error", + "destination": "error_stream", "routing_key": "error" }, { @@ -185,7 +246,7 @@ "vhost": "sda", "destination_type": "queue", "arguments": {}, - "destination": "mappings", + "destination": "mapping_stream", "routing_key": "mappings" }, { diff --git a/rabbitmq/docker-entrypoint.sh b/rabbitmq/docker-entrypoint.sh index b39b99ad1..53da5d9ab 100644 --- a/rabbitmq/docker-entrypoint.sh +++ b/rabbitmq/docker-entrypoint.sh @@ -10,10 +10,8 @@ if [[ "$1" == rabbitmq* ]] && [ "$(id -u)" = '0' ]; then exec su-exec rabbitmq "${BASH_SOURCE[0]}" "$@" fi -if [ -z "$RABBITMQ_DEFAULT_USER" ] || [ -z "$RABBITMQ_DEFAULT_PASS" ]; then - RABBITMQ_DEFAULT_USER="guest" - RABBITMQ_DEFAULT_PASS="guest" -fi +RABBITMQ_DEFAULT_USER="${RABBITMQ_DEFAULT_USER:-guest}" +RABBITMQ_DEFAULT_PASS="${RABBITMQ_DEFAULT_PASS:-guest}" sed -e "s/RABBITMQ_DEFAULT_USER/$RABBITMQ_DEFAULT_USER/" -e "s/RABBITMQ_DEFAULT_PASS/$RABBITMQ_DEFAULT_PASS/" \ /etc/rabbitmq/definitions.json >/var/lib/rabbitmq/definitions.json @@ -50,6 +48,17 @@ if [ -n "$CEGA_CONNECTION" ]; then chmod 600 "/var/lib/rabbitmq/federation.json" fi +# This is needed for the streams to work properly +cat >/var/lib/rabbitmq/advanced.config<<-EOF +[ + {rabbit, [ + {default_consumer_prefetch, {false,100}} + ] + } +]. +EOF + +chmod 600 "/var/lib/rabbitmq/advanced.config" chmod 600 "/var/lib/rabbitmq/rabbitmq.conf" chmod 600 "/var/lib/rabbitmq/definitions.json" diff --git a/rabbitmq/federation.json b/rabbitmq/federation.json index d66a78efe..8341a70ba 100644 --- a/rabbitmq/federation.json +++ b/rabbitmq/federation.json @@ -51,7 +51,7 @@ "dest-uri": "amqp:///sda", "src-delete-after": "never", "src-protocol": "amqp091", - "src-queue": "completed", + "src-queue": "completed_stream", "src-uri": "amqp:///sda" }, "vhost": "sda" @@ -67,7 +67,7 @@ "dest-uri": "amqp:///sda", "src-delete-after": "never", "src-protocol": "amqp091", - "src-queue": "error", + "src-queue": "error_stream", "src-uri": "amqp:///sda" }, "vhost": "sda" From 17c154f783ed9e2b3d5c71b22f9e8a074ff3a13f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 7 Aug 2023 15:32:58 +0200 Subject: [PATCH 06/14] Update sda Dockerfile --- sda/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/Dockerfile b/sda/Dockerfile index 3f0854093..ea2edcd89 100644 --- a/sda/Dockerfile +++ b/sda/Dockerfile @@ -8,7 +8,7 @@ COPY . . SHELL ["bash", "-c"] RUN set -ex; for p in cmd/*; do go build -buildvcs=false -o "${p/cmd\//sda-}" "./$p"; done -FROM debian:11-slim AS Debug +FROM debian:bullseye-slim AS Debug ARG SOURCE_COMMIT From fcab16654e84e796b0093cb246bf24518b3ed6e0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 8 Aug 2023 07:33:06 +0200 Subject: [PATCH 07/14] Publish chart on PR close --- .github/workflows/publish_charts.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_charts.yml b/.github/workflows/publish_charts.yml index c931cb7e3..886953b72 100644 --- a/.github/workflows/publish_charts.yml +++ b/.github/workflows/publish_charts.yml @@ -1,14 +1,16 @@ name: Publish charts on: - push: + pull_request: branches: - main paths: - "charts/**" + types: [ closed ] jobs: - release: + release_chart: + if: github.event.pull_request.merged == true runs-on: ubuntu-latest continue-on-error: true steps: From 8286d6d70d6bda45dc60e90780810dd88e2d6482 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 8 Aug 2023 11:22:34 +0200 Subject: [PATCH 08/14] Programatically get latest github release for crypt4gh --- .github/integration/scripts/charts/dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index d734a2813..d0d752e35 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -2,7 +2,7 @@ set -ex YQ_VERSION="v4.20.1" -C4GH_VERSION="1.7.5" +C4GH_VERSION="$(curl -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')" random-string() { head -c 32 /dev/urandom | base64 -w0 | tr -d '/+' | fold -w 32 | head -n 1 @@ -11,7 +11,7 @@ random-string() { sudo curl -sLO "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -O /usr/bin/yq && sudo chmod +x /usr/bin/yq -curl -sL https://github.com/neicnordic/crypt4gh/releases/download/v"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ && +curl -sL https://github.com/neicnordic/crypt4gh/releases/download/"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ && sudo chmod +x /usr/bin/crypt4gh # secret for the crypt4gh keypair From e549c0d8257fee88f790958452aabd09d9c0c00d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Aug 2023 10:52:25 +0200 Subject: [PATCH 09/14] Use matrix to test multiple k8s versions --- .github/integration/scripts/charts/k3d.sh | 14 +++++++++----- .github/workflows/chart_test.yml | 10 +++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/integration/scripts/charts/k3d.sh b/.github/integration/scripts/charts/k3d.sh index a06206435..4afe4425e 100755 --- a/.github/integration/scripts/charts/k3d.sh +++ b/.github/integration/scripts/charts/k3d.sh @@ -4,11 +4,15 @@ set -ex k8s="$(curl -L -s https://dl.k8s.io/release/stable.txt)" curl -s -L https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | sudo bash -curl -sLO https://storage.googleapis.com/kubernetes-release/release/"$k8s"/bin/linux/amd64/kubectl + +if [ -n "$1" ]; then + k8s=$(k3d version list k3s | grep "$1" | head -n 1 | cut -d '-' -f 1) +fi + +curl -sLO https://dl.k8s.io/release/"$k8s"/bin/linux/amd64/kubectl chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl -sudo k3d cluster create sda --image=rancher/k3s:"$k8s"-k3s1 --wait --timeout 10m -sudo k3d kubeconfig merge sda --kubeconfig-switch-context -mkdir -p ~/.kube/ && sudo cp /root/.k3d/kubeconfig-sda.yaml ~/.kube/config -sudo chown $UID:$UID ~/.kube/config && chmod 600 ~/.kube/config +k3d cluster create sda --image=rancher/k3s:"$k8s"-k3s1 --wait --timeout 10m +k3d kubeconfig merge sda --kubeconfig-switch-context +mkdir -p ~/.kube/ && cp ~/.config/kubeconfig-sda.yaml ~/.kube/config diff --git a/.github/workflows/chart_test.yml b/.github/workflows/chart_test.yml index 796baec02..241fee3d4 100644 --- a/.github/workflows/chart_test.yml +++ b/.github/workflows/chart_test.yml @@ -12,6 +12,9 @@ jobs: deploy: name: deployment test runs-on: ubuntu-latest + strategy: + matrix: + version: ["1.26", "1.27"] steps: - name: Checkout uses: actions/checkout@v3 @@ -22,7 +25,12 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Initialise k3d - run: bash .github/integration/scripts/charts/k3d.sh + run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} + shell: bash + + - name: debug + if: failure() + run: k3d version list k3s | grep ${{matrix.version}} shell: bash - name: build postgres From 83bcb9a0ca929c6576a7aea4af2066b6a51a674a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Aug 2023 12:20:48 +0200 Subject: [PATCH 10/14] Move chart tests into `PR container build` actiion --- .../scripts/charts/deploy_charts.sh | 17 ++-- .github/workflows/build_pr_container.yaml | 57 ++++++++++- .github/workflows/chart_test.yml | 95 ------------------- 3 files changed, 67 insertions(+), 102 deletions(-) delete mode 100644 .github/workflows/chart_test.yml diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 7e999dffc..b4b69705a 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -1,11 +1,16 @@ #!/bin/bash set -ex +if [ -z "$2" ];then + echo "PR number missing" + exit 1 +fi + if [ "$1" == "sda-db" ]; then ROOTPASS=$(yq e '.global.db.password' .github/integration/scripts/charts/values.yaml) helm install postgres charts/sda-db \ - --set image.tag=test-postgres \ - --set image.pullPolicy=Never \ + --set image.tag="PR$2-postgres" \ + --set image.pullPolicy=IfNotPresent \ --set global.postgresAdminPassword="$ROOTPASS" \ --set global.tls.enabled=false \ --set persistence.enabled=false \ @@ -16,8 +21,8 @@ fi if [ "$1" == "sda-mq" ]; then ADMINPASS=$(yq e '.global.broker.password' .github/integration/scripts/charts/values.yaml) helm install broker charts/sda-mq \ - --set image.tag=test-rabbitmq \ - --set image.pullPolicy=Never \ + --set image.tag="PR$2-rabbitmq" \ + --set image.pullPolicy=IfNotPresent \ --set global.adminPassword="$ADMINPASS" \ --set global.adminUser=admin \ --set global.tls.enabled=false \ @@ -28,8 +33,8 @@ fi if [ "$1" == "sda-svc" ]; then helm install pipeline charts/sda-svc \ - --set image.tag=test \ - --set image.pullPolicy=Never \ + --set image.tag="PR$2" \ + --set image.pullPolicy=IfNotPresent \ -f .github/integration/scripts/charts/values.yaml \ --wait fi diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 9be96eb9d..f1e565f0f 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -236,4 +236,59 @@ jobs: uses: actions/checkout@v3 - name: Test sensitive-data-archive - run: docker compose -f .github/integration/sda-integration.yml run integration_test \ No newline at end of file + run: docker compose -f .github/integration/sda-integration.yml run integration_test + + chart: + needs: + - build_go_images + - build_server_images + - build_java_images + runs-on: ubuntu-latest + strategy: + matrix: + version: ["1.26", "1.27"] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Helm + uses: azure/setup-helm@v3.5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Initialise k3d + run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} + shell: bash + + - name: debug + if: failure() + run: k3d version list k3s | grep ${{matrix.version}} + shell: bash + + - name: Deploy external services + run: bash .github/integration/scripts/charts/dependencies.sh + shell: bash + + - name: Deploy DB + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db ${{ github.event.number }} + + - name: Deploy MQ + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} + shell: bash + + - name: Deploy pipeline + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} + shell: bash + + - name: test + if: always() + run: | + kubectl get secret broker-sda-mq -o json + kubectl get secret pipeline-sda-svc-mapper -o json + kubectl get pods + echo "describe mapper" && kubectl describe pod -l role=mapper + sleep 1 + echo "logs mapper" && kubectl logs -l role=mapper + sleep 1 + echo "describe broker" && kubectl logs -l role=broker + shell: bash \ No newline at end of file diff --git a/.github/workflows/chart_test.yml b/.github/workflows/chart_test.yml deleted file mode 100644 index 241fee3d4..000000000 --- a/.github/workflows/chart_test.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Chart deployment test - -on: - pull_request: - paths: - - "charts/**" - -env: - DOCKER_BUILDKIT: 1 - -jobs: - deploy: - name: deployment test - runs-on: ubuntu-latest - strategy: - matrix: - version: ["1.26", "1.27"] - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Helm - uses: azure/setup-helm@v3.5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Initialise k3d - run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} - shell: bash - - - name: debug - if: failure() - run: k3d version list k3s | grep ${{matrix.version}} - shell: bash - - - name: build postgres - shell: bash - run: | - cd postgresql - docker build -t ghcr.io/${{ github.repository }}:test-postgres . - - name: build rabbitmq - shell: bash - run: | - cd rabbitmq - docker build -t ghcr.io/${{ github.repository }}:test-rabbitmq . - - name: build sda - shell: bash - run: | - cd sda - docker build -t ghcr.io/${{ github.repository }}:test . - - name: build sda-auth - shell: bash - run: | - cd sda-auth - docker build -t ghcr.io/${{ github.repository }}:test-auth . - - name: build sda-download - shell: bash - run: | - cd sda-download - docker build -t ghcr.io/${{ github.repository }}:test-download . - - name: build sda-pipeline - shell: bash - run: | - cd sda-pipeline - docker build -t ghcr.io/${{ github.repository }}:test-pipeline . - - name: Import images to cluster - run: k3d image import ghcr.io/${{ github.repository }}:test-postgres ghcr.io/${{ github.repository }}:test-rabbitmq ghcr.io/${{ github.repository }}:test ghcr.io/${{ github.repository }}:test-auth ghcr.io/${{ github.repository }}:test-download ghcr.io/${{ github.repository }}:test-pipeline -c sda - - - name: Deploy external services - run: bash .github/integration/scripts/charts/dependencies.sh - shell: bash - - - name: Deploy DB - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db - - - name: Deploy MQ - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq - shell: bash - - - name: Deploy pipeline - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc - shell: bash - - - name: test - if: always() - run: | - kubectl get secret broker-sda-mq -o json - kubectl get secret pipeline-sda-svc-mapper -o json - kubectl get pods - echo "describe mapper" && kubectl describe pod -l role=mapper - sleep 1 - echo "logs mapper" && kubectl logs -l role=mapper - sleep 1 - echo "describe broker" && kubectl logs -l role=broker - shell: bash \ No newline at end of file From 4a8984f0366ac10664ea6a4bb3e073896efd1451 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 24 Aug 2023 12:46:48 +0200 Subject: [PATCH 11/14] [sda-mq] Add management port to service --- charts/sda-mq/templates/service.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/sda-mq/templates/service.yaml b/charts/sda-mq/templates/service.yaml index fcda5f364..b7df680e1 100644 --- a/charts/sda-mq/templates/service.yaml +++ b/charts/sda-mq/templates/service.yaml @@ -14,6 +14,8 @@ spec: port: 4369 - name: rabbitmq-dist port: 25672 + - name: management + port: {{ ternary 15671 15672 (.Values.global.tls.enabled )}} selector: app: {{ template "sda.fullname" . }} From d3c115e43c9ae8d98c1e888fd10ccce9c262a113 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 24 Aug 2023 13:00:49 +0200 Subject: [PATCH 12/14] Add ingress for management access --- charts/sda-mq/README.md | 6 ++++ charts/sda-mq/templates/ingress.yaml | 45 ++++++++++++++++++++++++++++ charts/sda-mq/values.yaml | 10 +++++++ 3 files changed, 61 insertions(+) create mode 100644 charts/sda-mq/templates/ingress.yaml diff --git a/charts/sda-mq/README.md b/charts/sda-mq/README.md index bc6f0dbbc..c7e591d0e 100644 --- a/charts/sda-mq/README.md +++ b/charts/sda-mq/README.md @@ -13,6 +13,12 @@ Parameter | Description | Default `image.pullPolicy` | sda-mq container image pull policy | `Always` `global.adminUser` | Username of admin user |`admin` `global.adminPassword` | Password for admin user. |`Random if unset` +`global.ingress.annotations` | extra annotations for the ingress objects | `""` +`global.ingress.ingressClassName` | class of the ingress controller | `"nginx"` +`global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` +`global.ingress.hostName` | hostname for the ingress endpoint | `""` +`global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` +`global.ingress.secretName` | The name of a manually created secret holding the certificates for the ingress enpoint. | `""` `global.tls.enabled` | Use TLS for all connections. |`true` `global.tls.issuer` | Issuer for TLS certificate creation. |`""` `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` diff --git a/charts/sda-mq/templates/ingress.yaml b/charts/sda-mq/templates/ingress.yaml new file mode 100644 index 000000000..fd0f30d58 --- /dev/null +++ b/charts/sda-mq/templates/ingress.yaml @@ -0,0 +1,45 @@ +{{- if .Values.global.ingress.hostname }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ template "sda.fullname" . }}-ingress + labels: + app: {{ template "sda.fullname" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + {{- if eq "nginx" .Values.global.ingress.ingressClassName }} + nginx.ingress.kubernetes.io/rewrite-target: "/" + nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}" + {{- end }} + {{- if .Values.global.ingress.clusterIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }} + {{- else if .Values.global.ingress.issuer }} + cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }} + {{- end }} +{{- if .Values.global.ingress.annotations }} +{{ toYaml .Values.global.ingress.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.global.ingress.ingressClassName }} + ingressClassName: {{ .Values.global.ingress.ingressClassName }} +{{- end }} + rules: + - host: {{ required "An ingress hostname is required!" .Values.global.ingress.hostName }} + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: {{ template "sda.fullname" . }} + port: + number: {{ ternary 15671 15672 .Values.global.tls.enabled }} +{{- if .Values.global.tls.enabled }} + tls: + - hosts: + - {{ required "An ingress hostname is required!" .Values.global.ingress.hostName }} + secretName: {{ if .Values.global.ingress.secretName }}{{ .Values.global.ingress.secretName }}{{- else }}"{{ template "sda.fullname" . }}-ingress"{{- end }} +{{- end }} +{{- end }} diff --git a/charts/sda-mq/values.yaml b/charts/sda-mq/values.yaml index a93af6800..085679e0e 100644 --- a/charts/sda-mq/values.yaml +++ b/charts/sda-mq/values.yaml @@ -1,6 +1,16 @@ global: adminUser: adminPassword: + ingress: + # extra annotations for the ingress + annotations: {} + hostname: "" + ingressClassName: "nginx" + issuer: "" + clusterIssuer: "" + # If the certificates is generated by external providers + # the secrets containing them needs to be created manually. + secretName: "" tls: enabled: true issuer: "" From e6380b61a97dce25722789cc4f3b84921dd99c17 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 24 Aug 2023 15:16:06 +0200 Subject: [PATCH 13/14] Delete `sda-helm` folder --- sda-helm/.github/dependabot.yml | 11 - sda-helm/.github/workflows/lint.yml | 44 -- .../.github/workflows/manual_release.yaml | 27 - .../.github/workflows/sda-pipeline-notls.yml | 39 -- sda-helm/.github/workflows/sda-pipeline.yml | 94 --- sda-helm/.github/workflows/shellcheck.yml | 12 - sda-helm/.gitignore | 10 - sda-helm/LICENSE | 661 ------------------ sda-helm/README.md | 27 - sda-helm/dev_tools/cega/cega-issuer.yaml | 34 - sda-helm/dev_tools/cega/cega.conf | 15 - sda-helm/dev_tools/cega/cega.json | 23 - sda-helm/dev_tools/cega/cega.plugins | 1 - sda-helm/dev_tools/cega/deploy.yaml | 172 ----- sda-helm/dev_tools/cega/users.json | 7 - sda-helm/dev_tools/cega/users.py | 103 --- sda-helm/dev_tools/config/cert-issuer.yaml | 30 - sda-helm/dev_tools/config/minio-issuer.yaml | 33 - sda-helm/dev_tools/config/no-tls.yaml | 72 -- sda-helm/dev_tools/config/oidc.yaml | 56 -- sda-helm/dev_tools/config/orch.yaml | 18 - sda-helm/dev_tools/config/posix-volumes.yaml | 35 - sda-helm/dev_tools/config/posix.yaml | 76 -- sda-helm/dev_tools/config/s3.yaml | 106 --- sda-helm/dev_tools/scripts/cleanup.sh | 16 - .../scripts/create-s3-buckets-no-tls.sh | 27 - .../dev_tools/scripts/create-s3-buckets.sh | 28 - sda-helm/dev_tools/scripts/create-secrets.sh | 37 - sda-helm/dev_tools/scripts/deploy-cega.sh | 19 - .../dev_tools/scripts/deploy-cert-manager.sh | 12 - sda-helm/dev_tools/scripts/deploy-db.sh | 25 - sda-helm/dev_tools/scripts/deploy-minio.sh | 20 - sda-helm/dev_tools/scripts/deploy-mq.sh | 28 - sda-helm/dev_tools/scripts/deploy-oidc.sh | 16 - sda-helm/dev_tools/scripts/init-k3d.sh | 18 - .../dev_tools/scripts/install-kube-deps.sh | 17 - .../dev_tools/scripts/install-sda-deps.sh | 24 - sda-helm/dev_tools/scripts/java-certs.sh | 60 -- sda-helm/dev_tools/scripts/make-certs.sh | 83 --- .../scripts/mq-password-generator.sh | 7 - sda-helm/dev_tools/scripts/run-helm-test.sh | 17 - .../dev_tools/scripts/sda/deploy-no-tls.sh | 96 --- sda-helm/dev_tools/scripts/sda/deploy-orch.sh | 10 - .../dev_tools/scripts/sda/deploy-posix.sh | 23 - sda-helm/dev_tools/scripts/sda/deploy-s3.sh | 27 - sda-helm/dev_tools/scripts/sign_jwt.sh | 68 -- sda-helm/dev_tools/scripts/ssl.cnf | 236 ------- sda-helm/dev_tools/scripts/svc-setup.sh | 21 - sda-helm/dev_tools/scripts/wait-for-pods.sh | 47 -- 49 files changed, 2688 deletions(-) delete mode 100644 sda-helm/.github/dependabot.yml delete mode 100644 sda-helm/.github/workflows/lint.yml delete mode 100644 sda-helm/.github/workflows/manual_release.yaml delete mode 100644 sda-helm/.github/workflows/sda-pipeline-notls.yml delete mode 100644 sda-helm/.github/workflows/sda-pipeline.yml delete mode 100644 sda-helm/.github/workflows/shellcheck.yml delete mode 100644 sda-helm/.gitignore delete mode 100644 sda-helm/LICENSE delete mode 100644 sda-helm/README.md delete mode 100644 sda-helm/dev_tools/cega/cega-issuer.yaml delete mode 100644 sda-helm/dev_tools/cega/cega.conf delete mode 100644 sda-helm/dev_tools/cega/cega.json delete mode 100644 sda-helm/dev_tools/cega/cega.plugins delete mode 100644 sda-helm/dev_tools/cega/deploy.yaml delete mode 100644 sda-helm/dev_tools/cega/users.json delete mode 100644 sda-helm/dev_tools/cega/users.py delete mode 100644 sda-helm/dev_tools/config/cert-issuer.yaml delete mode 100644 sda-helm/dev_tools/config/minio-issuer.yaml delete mode 100644 sda-helm/dev_tools/config/no-tls.yaml delete mode 100644 sda-helm/dev_tools/config/oidc.yaml delete mode 100644 sda-helm/dev_tools/config/orch.yaml delete mode 100644 sda-helm/dev_tools/config/posix-volumes.yaml delete mode 100644 sda-helm/dev_tools/config/posix.yaml delete mode 100644 sda-helm/dev_tools/config/s3.yaml delete mode 100644 sda-helm/dev_tools/scripts/cleanup.sh delete mode 100644 sda-helm/dev_tools/scripts/create-s3-buckets-no-tls.sh delete mode 100755 sda-helm/dev_tools/scripts/create-s3-buckets.sh delete mode 100755 sda-helm/dev_tools/scripts/create-secrets.sh delete mode 100755 sda-helm/dev_tools/scripts/deploy-cega.sh delete mode 100644 sda-helm/dev_tools/scripts/deploy-cert-manager.sh delete mode 100755 sda-helm/dev_tools/scripts/deploy-db.sh delete mode 100755 sda-helm/dev_tools/scripts/deploy-minio.sh delete mode 100755 sda-helm/dev_tools/scripts/deploy-mq.sh delete mode 100755 sda-helm/dev_tools/scripts/deploy-oidc.sh delete mode 100755 sda-helm/dev_tools/scripts/init-k3d.sh delete mode 100755 sda-helm/dev_tools/scripts/install-kube-deps.sh delete mode 100755 sda-helm/dev_tools/scripts/install-sda-deps.sh delete mode 100755 sda-helm/dev_tools/scripts/java-certs.sh delete mode 100755 sda-helm/dev_tools/scripts/make-certs.sh delete mode 100644 sda-helm/dev_tools/scripts/mq-password-generator.sh delete mode 100755 sda-helm/dev_tools/scripts/run-helm-test.sh delete mode 100644 sda-helm/dev_tools/scripts/sda/deploy-no-tls.sh delete mode 100755 sda-helm/dev_tools/scripts/sda/deploy-orch.sh delete mode 100755 sda-helm/dev_tools/scripts/sda/deploy-posix.sh delete mode 100755 sda-helm/dev_tools/scripts/sda/deploy-s3.sh delete mode 100644 sda-helm/dev_tools/scripts/sign_jwt.sh delete mode 100644 sda-helm/dev_tools/scripts/ssl.cnf delete mode 100755 sda-helm/dev_tools/scripts/svc-setup.sh delete mode 100755 sda-helm/dev_tools/scripts/wait-for-pods.sh diff --git a/sda-helm/.github/dependabot.yml b/sda-helm/.github/dependabot.yml deleted file mode 100644 index f21a6b813..000000000 --- a/sda-helm/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - open-pull-requests-limit: 10 - reviewers: - - "dbampalikis" - - "jbygdell" - - "blankdots" diff --git a/sda-helm/.github/workflows/lint.yml b/sda-helm/.github/workflows/lint.yml deleted file mode 100644 index 2e9b2a080..000000000 --- a/sda-helm/.github/workflows/lint.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Helm linter - -on: [push] - -jobs: - lint_sda-db: - name: Lint sda-db - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - name: Install helm3 - run: bash ./dev_tools/scripts/install-kube-deps.sh - - name: Lint sda-db - run: helm lint charts/sda-db - - lint_sda-mq: - name: Lint sda-mq - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - name: Install helm3 - run: bash ./dev_tools/scripts/install-kube-deps.sh - - name: Lint sda-mq - run: helm lint charts/sda-mq - - lint_sda-svc: - name: Lint sda-svc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - name: Install helm3 - run: bash ./dev_tools/scripts/install-kube-deps.sh - - name: Lint sda-svc - run: helm lint charts/sda-svc - - lint_sda-orch: - name: Lint sda-orch - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - name: Install helm3 - run: bash ./dev_tools/scripts/install-kube-deps.sh - - name: Lint sda-orch - run: helm lint charts/sda-orch diff --git a/sda-helm/.github/workflows/manual_release.yaml b/sda-helm/.github/workflows/manual_release.yaml deleted file mode 100644 index 598433fee..000000000 --- a/sda-helm/.github/workflows/manual_release.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Manually Release Charts - -on: [workflow_dispatch] - -jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3.3.0 - with: - fetch-depth: 0 - - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - - name: Install Helm - uses: azure/setup-helm@v3.5 - with: - version: v3.4.0 - - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.5.0 - env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/sda-helm/.github/workflows/sda-pipeline-notls.yml b/sda-helm/.github/workflows/sda-pipeline-notls.yml deleted file mode 100644 index ce1bf1583..000000000 --- a/sda-helm/.github/workflows/sda-pipeline-notls.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: standalone sda deployment without TLS - -on: [push,pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - name: Install kube dependencies - run: bash ./dev_tools/scripts/install-kube-deps.sh - - name: Initialise k3d - run: bash ./dev_tools/scripts/init-k3d.sh - - name: Wait for k3d to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh metrics-server k8s-app kube-system - - name: Install sda dependencies - run: bash ./dev_tools/scripts/install-sda-deps.sh - - name: Create certificates - run: bash ./dev_tools/scripts/make-certs.sh - - name: Create secrets - run: bash ./dev_tools/scripts/create-secrets.sh - - name: Deploy SDA database - run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh database - - name: Deploy SDA message broker - run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh broker - - name: Deploy SDA orchestrator - run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh orchestrate - - name: Deploy mock oidc server - run: bash ./dev_tools/scripts/deploy-oidc.sh - - name: Deploy minio - run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh minio - - name: Create s3 buckets - run: bash ./dev_tools/scripts/create-s3-buckets-no-tls.sh - - name: Deploy the SDA stack - run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh pipeline - - name: Wait for sda to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh standalone_s3_svc_list - - name: Run helm test - run: bash ./dev_tools/scripts/run-helm-test.sh diff --git a/sda-helm/.github/workflows/sda-pipeline.yml b/sda-helm/.github/workflows/sda-pipeline.yml deleted file mode 100644 index 745f8c203..000000000 --- a/sda-helm/.github/workflows/sda-pipeline.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: sda-pipeline deployment - -on: [push, pull_request] - -jobs: - build: - strategy: - fail-fast: false - matrix: - inbox: [posix, s3] - deployment: [federated, standalone] - cert: [issuer, manual] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - - name: Install kube dependencies - run: bash ./dev_tools/scripts/install-kube-deps.sh - - - name: Initialise k3d - run: bash ./dev_tools/scripts/init-k3d.sh - - - name: Wait for k3d to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh metrics-server k8s-app kube-system - - - name: Install sda dependencies - run: bash ./dev_tools/scripts/install-sda-deps.sh - - - name: Create certificates - if: matrix.cert == 'manual' - run: bash ./dev_tools/scripts/make-certs.sh - - - name: Create certificate issuer - if: matrix.cert == 'issuer' - run: bash ./dev_tools/scripts/deploy-cert-manager.sh - - - name: Create secrets - run: bash ./dev_tools/scripts/create-secrets.sh - - - name: Set up services configuration - if: matrix.cert == 'manual' - run: bash ./dev_tools/scripts/svc-setup.sh - - - name: Deploy SDA database - run: bash ./dev_tools/scripts/deploy-db.sh "${{ matrix.cert }}" - - - name: Wait for database to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh database - - - name: Deploy mock oidc server - if: matrix.inbox == 's3' - run: bash ./dev_tools/scripts/deploy-oidc.sh - - - name: Deploy minio - if: matrix.inbox == 's3' - run: bash ./dev_tools/scripts/deploy-minio.sh "${{ matrix.cert }}" - - - name: Wait for minio to become ready - if: matrix.inbox == 's3' - run: bash ./dev_tools/scripts/wait-for-pods.sh minio app - - - name: Create s3 buckets - if: matrix.inbox == 's3' - run: bash ./dev_tools/scripts/create-s3-buckets.sh - - - name: Start CEGA services - if: matrix.deployment == 'federated' - run: bash ./dev_tools/scripts/deploy-cega.sh "${{ matrix.cert }}" - - - name: Wait for CEGA to become ready - if: matrix.deployment == 'federated' - run: bash ./dev_tools/scripts/wait-for-pods.sh cega-mq app - - - name: Deploy SDA message broker - run: bash ./dev_tools/scripts/deploy-mq.sh "${{ matrix.deployment }}" "${{ matrix.cert }}" - - - name: Wait for broker to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh broker - - - name: Deploy the SDA stack for posix - if: matrix.inbox == 'posix' - run: | - kubectl apply -f dev_tools/config/posix-volumes.yaml; - bash ./dev_tools/scripts/sda/deploy-posix.sh "${{ matrix.deployment }}" "${{ matrix.cert }}" - - - name: Deploy the SDA stack for s3 - if: matrix.inbox == 's3' - run: bash ./dev_tools/scripts/sda/deploy-s3.sh "${{ matrix.deployment }}" "${{ matrix.cert }}" - - - name: Wait for sda to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ format('{0}_{1}_svc_list', matrix.deployment, matrix.inbox) }}" - - - name: Run helm test - run: bash ./dev_tools/scripts/run-helm-test.sh diff --git a/sda-helm/.github/workflows/shellcheck.yml b/sda-helm/.github/workflows/shellcheck.yml deleted file mode 100644 index 4bd810355..000000000 --- a/sda-helm/.github/workflows/shellcheck.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Scripts linter - -on: [push] - -jobs: - shellcheck: - name: Shellcheck - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master diff --git a/sda-helm/.gitignore b/sda-helm/.gitignore deleted file mode 100644 index 366f3999f..000000000 --- a/sda-helm/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -*.pem -*.csr -*.crt -*.key -*.p12 -cacerts -*.pub -*.sec -sda-deploy-init/* -LocalEGA-helm/* \ No newline at end of file diff --git a/sda-helm/LICENSE b/sda-helm/LICENSE deleted file mode 100644 index 0ad25db4b..000000000 --- a/sda-helm/LICENSE +++ /dev/null @@ -1,661 +0,0 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. diff --git a/sda-helm/README.md b/sda-helm/README.md deleted file mode 100644 index 4b5d5ca1c..000000000 --- a/sda-helm/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# SDA-helm - -[![GitHub](https://img.shields.io/github/license/neicnordic/sda-helm?style=plastic)](https://www.gnu.org/licenses/agpl-3.0) -![GitHub Actions linter](https://github.com/neicnordic/sda-helm/workflows/Helm%20linter/badge.svg) -![GitHub tag (latest SemVer)](https://img.shields.io/github/v/tag/neicnordic/sda-helm?sort=semver&style=plastic) - -## Info - -This repositroy contains helmcharts for deploying a Sensitive Data Archive solution that is compatible with the European Genome Archives federated archiving model. - -The charts are compatible with kubernetes versions >= 1.19.0 and are tested against kubernetes version 1.25.6 - -## sda-db - -This chart deploys a pre-configured database instance for Sensitive Data Archive, the schemas match European Genome Archives federated archiving model. - -## sda-mq - -This chart deploys a pre-configured message broker designed to work European Genome Archives federated archive setup. - -## sda-svc - -This chart deploys the service components needed for the Sensitive Data Archive solution. - -## sda-orch - -This chart deploys the orchestrate service needed for the Sensitive Data Archive standalone solution (No European Genome Archive connection). diff --git a/sda-helm/dev_tools/cega/cega-issuer.yaml b/sda-helm/dev_tools/cega/cega-issuer.yaml deleted file mode 100644 index 85d775ae0..000000000 --- a/sda-helm/dev_tools/cega/cega-issuer.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: cega-certs -spec: - # Secret names are always required. - secretName: cega-certs - - duration: 2160h # 90d - - # The use of the common name field has been deprecated since 2000 and is - # discouraged from being used. - commonName: cega - isCA: false - privateKey: - algorithm: ECDSA - size: 256 - usages: - - server auth - # At least one of a DNS Name, URI, or IP address is required. - dnsNames: - - cega-mq - - cega-users - ipAddresses: - - 127.0.0.1 - # Issuer references are always required. - issuerRef: - name: ca-issuer - # We can reference ClusterIssuers by changing the kind here. - # The default value is Issuer (i.e. a locally namespaced Issuer) - kind: Issuer - # This is optional since cert-manager will default to this value however - # if you are using an external issuer, change this to that issuer group. - group: cert-manager.io diff --git a/sda-helm/dev_tools/cega/cega.conf b/sda-helm/dev_tools/cega/cega.conf deleted file mode 100644 index acaeb0eb6..000000000 --- a/sda-helm/dev_tools/cega/cega.conf +++ /dev/null @@ -1,15 +0,0 @@ -listeners.ssl.default = 5671 -ssl_options.cacertfile = /etc/rabbitmq/ssl/ca.crt -ssl_options.certfile = /etc/rabbitmq/ssl/tls.crt -ssl_options.keyfile = /etc/rabbitmq/ssl/tls.key -ssl_options.verify = verify_none -ssl_options.fail_if_no_peer_cert = true -ssl_options.versions.1 = tlsv1.2 -management.load_definitions = /etc/rabbitmq/conf/cega.json -management.listener.port = 15671 -management.listener.ssl = true -management.listener.ssl_opts.cacertfile = /etc/rabbitmq/ssl/ca.crt -management.listener.ssl_opts.certfile = /etc/rabbitmq/ssl/tls.crt -management.listener.ssl_opts.keyfile = /etc/rabbitmq/ssl/tls.key -default_vhost = lega -disk_free_limit.absolute = 1GB diff --git a/sda-helm/dev_tools/cega/cega.json b/sda-helm/dev_tools/cega/cega.json deleted file mode 100644 index 57991b559..000000000 --- a/sda-helm/dev_tools/cega/cega.json +++ /dev/null @@ -1,23 +0,0 @@ -{"rabbit_version":"3.7", - "users":[{"name":"lega", - "password_hash":"tBwQTdorHZnIdJI7AUK71L56JVbYhjfhNoVO2y1nWmt2Cgdm","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"}], "vhosts":[{"name":"lega"}], - "permissions":[{"user":"lega", "vhost":"lega", "configure":".*", "write":".*", "read":".*"}], - - "parameters":[], "global_parameters":[{"name":"cluster_name", "value":"rabbit@localhost"}], - "policies":[], - "queues":[{"name":"v1.files.inbox", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}}, - {"name":"v1.stableIDs", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}}, - {"name":"v1.files", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}}, - {"name":"v1.files.completed", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}}, - {"name":"v1.files.verified", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}}, - {"name":"v1.files.error", "vhost":"lega", "durable":true, "auto_delete":false, "arguments":{}}], - "exchanges":[{"name":"localega.v1", "vhost":"lega", "type":"topic", "durable":true, "auto_delete":false, "internal":false, "arguments":{}}], - "bindings":[ - {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.stableIDs","routing_key":"stableIDs"}, - {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files","routing_key":"files"}, - {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.inbox","routing_key":"files.inbox"}, - {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.error","routing_key":"files.error"}, - {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.verified","routing_key":"files.verified"}, - {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.completed","routing_key":"files.completed"}] - -} diff --git a/sda-helm/dev_tools/cega/cega.plugins b/sda-helm/dev_tools/cega/cega.plugins deleted file mode 100644 index 7e8d295ec..000000000 --- a/sda-helm/dev_tools/cega/cega.plugins +++ /dev/null @@ -1 +0,0 @@ -[rabbitmq_federation,rabbitmq_federation_management,rabbitmq_management,rabbitmq_shovel,rabbitmq_shovel_management]. \ No newline at end of file diff --git a/sda-helm/dev_tools/cega/deploy.yaml b/sda-helm/dev_tools/cega/deploy.yaml deleted file mode 100644 index 524dd23cd..000000000 --- a/sda-helm/dev_tools/cega/deploy.yaml +++ /dev/null @@ -1,172 +0,0 @@ ---- -# Source: cega/templates/cega-deploy.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cega-users - labels: - role: fake-users -spec: - replicas: 1 - selector: - matchLabels: - app: cega-users - template: - metadata: - labels: - app: cega-users - role: fake-users - spec: - serviceAccountName: cega - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: cega-users - image: "egarchive/lega-base:release.v0.2.0" - imagePullPolicy: "Always" - command: ["python", "/cega/users.py", "0.0.0.0", "8443", "/cega/users.json"] - securityContext: - allowPrivilegeEscalation: false - env: - - name: LEGA_INSTANCES - value: legatest - - name: CEGA_USERS_PASSWORD - value: OfEoDPVadIfd4CZUWkisrrVQbJ2yQPIH - - name: CEGA_USERS_USER - value: legatest - ports: - - name: cega-users - containerPort: 443 - protocol: TCP - volumeMounts: - - name: cega-config - mountPath: /cega - - name: cega-certs - mountPath: /tls/ - volumes: - - name: cega-config - secret: - secretName: cega-users-config - defaultMode: 0440 - - name: cega-certs - secret: - secretName: cega-certs - defaultMode: 0440 ---- -# Source: cega/templates/cegamq-deploy.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cega-mq - labels: - role: cega-broker - app: cega-mq -spec: - replicas: 1 - selector: - matchLabels: - app: cega-mq - template: - metadata: - labels: - app: cega-mq - spec: - serviceAccountName: cega - securityContext: - runAsUser: 100 - runAsGroup: 101 - fsGroup: 101 - containers: - - name: cega-mq - image: "rabbitmq:3.7.8-management-alpine" - imagePullPolicy: "IfNotPresent" - securityContext: - allowPrivilegeEscalation: false - env: - - name: RABBITMQ_CONFIG_FILE - value: /etc/rabbitmq/conf/cega - - name: RABBITMQ_ENABLED_PLUGINS_FILE - value: /etc/rabbitmq/conf/cega.plugins - ports: - - containerPort: 15671 - protocol: TCP - name: https - - containerPort: 15672 - protocol: TCP - name: http - - containerPort: 5672 - name: amqp - - containerPort: 5671 - name: amqps - volumeMounts: - - name: conf - mountPath: /etc/rabbitmq/conf - - name: rabbitmq - mountPath: /var/lib/rabbitmq - - name: ssl-certs - mountPath: /etc/rabbitmq/ssl - volumes: - - name: ssl-certs - secret: - secretName: cega-certs - defaultMode: 0440 - - name: conf - secret: - secretName: cega-mq-config - defaultMode: 0440 - - name: rabbitmq - emptyDir: {} ---- -# Source: cega/templates/cega-svc.yaml -apiVersion: v1 -kind: Service -metadata: - name: cega-users - labels: - app: cega-users -spec: - ports: - - port: 443 - targetPort: 8443 - protocol: TCP - selector: - app: cega-users ---- -# Source: cega/templates/cegamq-svc.yaml -apiVersion: v1 -kind: Service -metadata: - name: cega-mq - labels: - app: cega-mq -spec: - ports: - - port: 5672 - targetPort: 5672 - protocol: TCP - name: amqp - - port: 5671 - targetPort: 5671 - protocol: TCP - name: amqps - - port: 4369 - name: epmd - - port: 25672 - name: rabbitmq-dist - selector: - app: cega-mq ---- -# Source: cega/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: cega-serviceAccount - chart: cega-0.3.3 - heritage: Helm - release: cega - name: cega - namespace: default - diff --git a/sda-helm/dev_tools/cega/users.json b/sda-helm/dev_tools/cega/users.json deleted file mode 100644 index 953704079..000000000 --- a/sda-helm/dev_tools/cega/users.json +++ /dev/null @@ -1,7 +0,0 @@ -[{"username": "dummy", - "uid": 1, - "passwordHash": "wW94fVzPmrB2KiDuhBq2oVx416YMBIC8Q0HGCeTBjPmOo/0R", - "gecos": "dummy user", - "sshPublicKey": ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDI7vOtisP6sEfK25RqVJKJ+xuLn42eDUBGYJsKbo638s+Ft3LbriL00CwGAElu8bB2LEIO/lMPphqrci039eysLXeS9y0ZhNMyqBc6Z5FH7Bp6RZquEikXs6ALuLKd30u8CyRZ3JuH/C3W4ptbfaeBAwnhUXCaxttgF5rdWZbkDP/Lyc2n3PdcHvEWNCo2fvs3QgxCgoYMsRhUAOv+n58wrYoY4sMBXmeWauxfOEKhGQqPoz8c+l68oh5oKcXtPW8ETc2id3kiwEfZMkB/p/MqFbFNqcHhGmmZAgzP0YbH2KUGYp8WiIEZpy6dTbv2eQfkoqFcJilm0iOJ40aDiJrv6tFSx/RPc6togj+kxPnXK5daYmhUkzm0AS7qUWL9QSd9S0nViav+NJzhhchpcWjU7fSnck7dRj+pyr/PLNUW6Dc8LETPnWLkLaIR3AgLPp8lq9JPbfHrfSNNleElRTpjweqAY/6ViRo690xjupZIPWiYe56vx3H13FbkngSuS7YEFGJeg3leHFql8BfmW9GMm3b8H56wWsE9ewBSVOGNNeqFR+3TBCL4wk5IMo5nK2yp9eQy6shgN4fbES93KQr7VkIx2rQna8JJJLnhZoU2iHtG57EIAKjtF69us4aPWnY1NCr19xGzTqN8HUpaaGYH4EMLuukRcPm0ebCjeDUyxQ=="], - "enabled": null -}] diff --git a/sda-helm/dev_tools/cega/users.py b/sda-helm/dev_tools/cega/users.py deleted file mode 100644 index 80bdc8421..000000000 --- a/sda-helm/dev_tools/cega/users.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3.6 -# -*- coding: utf-8 -*- - -''' -Test server to act as CentralEGA endpoint for users - -:author: Frédéric Haziza -:copyright: (c) 2018, EGA System Developers. -''' - -import sys -import os -import logging -import asyncio -import json -from base64 import b64decode -import ssl - -from aiohttp import web - -#logging.basicConfig(format='[%(asctime)s][%(levelname)-8s] (L:%(lineno)s) %(message)s', datefmt='%Y-%m-%d %H:%M:%S') -logging.basicConfig(format='[%(levelname)-8s] (L:%(lineno)s) %(message)s') -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.INFO) - -filepath = None -instances = {} -store = None -usernames = {} - -def fetch_user_info(identifier, query): - LOG.info(f'Requesting User {identifier} [type {id_type}]') - try: - pos = usernames.get(identifier, None) - return store[pos] if pos is not None else None - except: - raise web.HTTPBadRequest(text="Missing or wrong idType") - -async def user(request): - # Authenticate - auth_header = request.headers.get('AUTHORIZATION') - if not auth_header: - raise web.HTTPUnauthorized(text=f'Protected access\n') - _, token = auth_header.split(None, 1) # Skipping the Basic keyword - instance, passwd = b64decode(token).decode().split(':', 1) - info = instances.get(instance) - if info is None or info != passwd: - raise web.HTTPUnauthorized(text=f'Protected access\n') - - # Reload users list - load_users() - - # Find user - user_info = fetch_user_info(request.match_info['identifier'], request.rel_url.query) - if user_info is None: - raise web.HTTPBadRequest(text=f'No info for that user\n') - return web.json_response(user_info) - -def main(): - - if len(sys.argv) < 3: - print('Usage: {sys.argv[0] }', file=sys.stderr) - sys.exit(2) - - host = sys.argv[1] - port = sys.argv[2] - - global filepath - filepath = sys.argv[3] - - server = web.Application() - load_users() - - # Registering the routes - server.router.add_get('/username/{identifier}', user, name='user') - - # SSL settings - cacertfile = '/tls/ca.crt' - certfile = '/tls/tls.crt' - keyfile = '/tls/tls.key' - - ssl_ctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=cacertfile) - ssl_ctx.check_hostname = False - ssl_ctx.verify_mode = ssl.CERT_NONE - - ssl_ctx.load_cert_chain(certfile, keyfile=keyfile) - - # aaaand... cue music - web.run_app(server, host=host, port=port, shutdown_timeout=0, ssl_context=ssl_ctx) - - -def load_users(): - # Initialization - global filepath, instances, store, usernames, uids - instances[os.environ[f'CEGA_USERS_USER']] = os.environ[f'CEGA_USERS_PASSWORD'] #'legatest' # Hard-coding legatest:legatest - with open(filepath, 'rt') as f: - store = json.load(f) - for i, d in enumerate(store): - usernames[d['username']] = i # No KeyError, should be there - - -if __name__ == '__main__': - main() diff --git a/sda-helm/dev_tools/config/cert-issuer.yaml b/sda-helm/dev_tools/config/cert-issuer.yaml deleted file mode 100644 index 1c2440aac..000000000 --- a/sda-helm/dev_tools/config/cert-issuer.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: selfsigned-ca -spec: - isCA: true - commonName: selfsigned-ca - secretName: root-secret - privateKey: - algorithm: ECDSA - size: 256 - issuerRef: - name: selfsigned-issuer - kind: Issuer - group: cert-manager.io ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: ca-issuer -spec: - ca: - secretName: root-secret diff --git a/sda-helm/dev_tools/config/minio-issuer.yaml b/sda-helm/dev_tools/config/minio-issuer.yaml deleted file mode 100644 index e33f40b57..000000000 --- a/sda-helm/dev_tools/config/minio-issuer.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: minio-certs -spec: - # Secret names are always required. - secretName: minio-certs - - duration: 2160h # 90d - - # The use of the common name field has been deprecated since 2000 and is - # discouraged from being used. - commonName: cega - isCA: false - privateKey: - algorithm: ECDSA - size: 256 - usages: - - server auth - # At least one of a DNS Name, URI, or IP address is required. - dnsNames: - - minio - ipAddresses: - - 127.0.0.1 - # Issuer references are always required. - issuerRef: - name: ca-issuer - # We can reference ClusterIssuers by changing the kind here. - # The default value is Issuer (i.e. a locally namespaced Issuer) - kind: Issuer - # This is optional since cert-manager will default to this value however - # if you are using an external issuer, change this to that issuer group. - group: cert-manager.io diff --git a/sda-helm/dev_tools/config/no-tls.yaml b/sda-helm/dev_tools/config/no-tls.yaml deleted file mode 100644 index 1cffc06a0..000000000 --- a/sda-helm/dev_tools/config/no-tls.yaml +++ /dev/null @@ -1,72 +0,0 @@ -global: - c4ghPath: c4gh - jwtPath: jwt - tlsPath: tls - podSecurityPolicy: - create: false - archive: - storageType: s3 - s3Url: "http://minio" - s3Port: 9000 - s3Bucket: "archive" - s3ReadyPath: "/minio/health/ready" - auth: - elixirID: DfCieZLuBU - elixirSecret: DfCieZLuBU - jwtSecret: oidc - jwtAlg: ES256 - jwtKey: jwt.key - jwtPub: jwt.pub - backupArchive: - storageType: s3 - s3Url: "http://minio" - s3Port: 9000 - s3Bucket: "backup" - s3ReadyPath: "/minio/health/ready" - broker: - host: "broker-sda-mq" - exchange: "sda" - port: 5672 - password: "admin" - username: "admin" - backupRoutingKey: "backup" - db: - host: "postgres-sda-db" - c4gh: - secretName: c4gh - keyFile: c4gh.sec.pem - publicFile: c4gh.pub.pem - elixir: - provider: "http://oidc-server:8080" - jwkPath: "" - inbox: - storageType: s3 - # existingClaim: test - s3Url: http://minio - s3Port: 9000 - s3Bucket: "inbox" - s3ReadyPath: "/minio/health/ready" - ingress: - deploy: false - hostName: - auth: sda-sda-svc-auth - download: sda-sda-svc-download - s3Inbox: sda-sda-svc-inbox - logLevel: debug - schemaType: isolated - tls: - enabled: false -auth: - replicaCount: 1 -backup: - deploy: true -doa: - replicaCount: 1 -download: - replicaCount: 1 -intercept: - deploy: false -releasetest: - run: true -s3Inbox: - replicaCount: 1 diff --git a/sda-helm/dev_tools/config/oidc.yaml b/sda-helm/dev_tools/config/oidc.yaml deleted file mode 100644 index d8904049c..000000000 --- a/sda-helm/dev_tools/config/oidc.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: oidc-server -spec: - selector: - matchLabels: - app: oidc-server - replicas: 1 - template: - metadata: - labels: - app: oidc-server - spec: - securityContext: - runAsNonRoot: true - runAsGroup: 1000 - runAsUser: 1000 - fsGroup: 1000 - containers: - - name: oidc-server - image: "neicnordic/mock-oidc:latest" - ports: - - containerPort: 8080 - env: - - name: PORT - value: "8080" - - name: HOST - value: oidc-server - - name: CLIENT_ID - value: DfCieZLuBU - - name: CLIENT_SECRET - value: DfCieZLuBU - - name: CLIENT_REDIRECT_URI - value: http://sda-auth/elixir/login - resources: - limits: - cpu: 250m - memory: 256Mi - requests: - cpu: 100m - memory: 128Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: oidc-server - labels: - app: oidc-server -spec: - type: ClusterIP - ports: - - port: 8080 - targetPort: 8080 - selector: - app: oidc-server diff --git a/sda-helm/dev_tools/config/orch.yaml b/sda-helm/dev_tools/config/orch.yaml deleted file mode 100644 index 451f2c153..000000000 --- a/sda-helm/dev_tools/config/orch.yaml +++ /dev/null @@ -1,18 +0,0 @@ -broker: - vhost: "sda" - host: "broker-sda-mq" - exchange: "sda" - password: "admin" - username: "admin" - queue: - inbox: inbox - completed: completed - verify: archived - files: files - stableid: accessionIDs -logLevel: debug -tls: - secretName: orch-certs - cert: orch.crt - key: orch.key - caCert: ca.crt diff --git a/sda-helm/dev_tools/config/posix-volumes.yaml b/sda-helm/dev_tools/config/posix-volumes.yaml deleted file mode 100644 index e16f58a8a..000000000 --- a/sda-helm/dev_tools/config/posix-volumes.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: inbox-volume -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: local-path ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: archive-volume -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: local-path ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: backup-volume -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: local-path diff --git a/sda-helm/dev_tools/config/posix.yaml b/sda-helm/dev_tools/config/posix.yaml deleted file mode 100644 index b7cd460fe..000000000 --- a/sda-helm/dev_tools/config/posix.yaml +++ /dev/null @@ -1,76 +0,0 @@ -global: - c4ghPath: c4gh - jwtPath: jwt - tlsPath: tls - podSecurityPolicy: - create: false - archive: - storageType: posix - existingClaim: archive-volume - volumePath: "/archive" - backupArchive: - storageType: posix - existingClaim: backup-volume - volumePath: "/backup" - broker: - host: "broker-sda-mq" - exchange: "sda" - port: 5671 - verifyPeer: true - password: "admin" - username: "admin" - backupRoutingKey: "backup" - c4gh: - secretName: c4gh - keyFile: c4gh.sec.pem - publicFile: c4gh.pub.pem - cega: - host: "cega-users" - user: "legatest" - db: - host: "postgres-sda-db" - sslMode: "verify-ca" - inbox: - storageType: posix - path: "/inbox" - user: "lega" - existingClaim: inbox-volume - ingress: - deploy: false - hostName: - download: sda-sda-svc-download - logLevel: debug - tls: - enabled: true -backup: - deploy: true - tls: - secretName: "backup-certs" -doa: - tls: - secretName: "doa-certs" -finalize: - tls: - secretName: "finalize-certs" -ingest: - tls: - secretName: "ingest-certs" -intercept: - tls: - secretName: "intercept-certs" -mapper: - tls: - secretName: "mapper-certs" -sftpInbox: - tls: - secretName: "inbox-certs" -verify: - tls: - secretName: "verify-certs" -download: - tls: - secretName: "download-certs" -releasetest: - run: true - tls: - secretName: "tester-certs" diff --git a/sda-helm/dev_tools/config/s3.yaml b/sda-helm/dev_tools/config/s3.yaml deleted file mode 100644 index ef4088f52..000000000 --- a/sda-helm/dev_tools/config/s3.yaml +++ /dev/null @@ -1,106 +0,0 @@ -global: - c4ghPath: c4gh - jwtPath: jwt - tlsPath: tls - podSecurityPolicy: - create: false - archive: - storageType: s3 - s3Url: "https://minio" - s3Bucket: "archive" - s3CaFile: "ca.crt" - s3AccessKey: idDQBxLpXoM8l88l - s3SecretKey: ABd6XCIvNWj7JULbrqBf8tB7q9BoHJSc - auth: - elixirID: DfCieZLuBU - elixirSecret: DfCieZLuBU - jwtSecret: oidc - jwtAlg: ES256 - jwtKey: jwt.key - jwtPub: jwt.pub - backupArchive: - storageType: s3 - s3Url: "https://minio" - s3Bucket: "backup" - s3CaFile: "ca.crt" - s3AccessKey: idDQBxLpXoM8l88l - s3SecretKey: ABd6XCIvNWj7JULbrqBf8tB7q9BoHJSc - broker: - host: "broker-sda-mq" - exchange: "sda" - port: 5671 - verifyPeer: true - password: "admin" - username: "admin" - backupRoutingKey: "backup" - cega: - host: "cega-users" - user: "legatest" - password: 3iSGc5loLN7hu2EwT2Z2CzCT26iUrk3t - db: - host: "postgres-sda-db" - sslMode: "verify-full" - c4gh: - secretName: c4gh - keyFile: c4gh.sec.pem - publicFile: c4gh.sec.pub - elixir: - provider: "http://oidc-server:8080" - jwkPath: "" - inbox: - storageType: s3 - # existingClaim: test - s3Url: "https://minio" - s3Bucket: "inbox" - s3AccessKey: idDQBxLpXoM8l88l - s3SecretKey: ABd6XCIvNWj7JULbrqBf8tB7q9BoHJSc - s3CaFile: ca.crt - s3ReadyPath: "/minio/health/ready" - ingress: - deploy: false - hostName: - auth: sda-sda-svc-auth - download: sda-sda-svc-download - s3Inbox: sda-sda-svc-inbox - logLevel: debug - tls: - enabled: true -auth: - replicaCount: 1 - tls: - secretName: "auth-certs" -backup: - deploy: true - tls: - secretName: "backup-certs" -doa: - replicaCount: 1 - tls: - secretName: "doa-certs" -finalize: - tls: - secretName: "finalize-certs" -ingest: - tls: - secretName: "ingest-certs" -intercept: - tls: - secretName: "intercept-certs" -mapper: - tls: - secretName: "mapper-certs" -s3Inbox: - replicaCount: 1 - tls: - secretName: "inbox-certs" -verify: - tls: - secretName: "verify-certs" -download: - replicaCount: 1 - tls: - secretName: "download-certs" -releasetest: - run: true - tls: - secretName: "tester-certs" diff --git a/sda-helm/dev_tools/scripts/cleanup.sh b/sda-helm/dev_tools/scripts/cleanup.sh deleted file mode 100644 index 1ead16b46..000000000 --- a/sda-helm/dev_tools/scripts/cleanup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -if [ "$1" = "db" ] || [ "$1" = "all" ];then - helm uninstall postgres - kubectl delete secret db-certs -fi - -if [ "$1" = "mq" ] || [ "$1" = "all" ];then - helm uninstall broker - kubectl delete secret mq-certs -fi - -if [ "$1" = "sda" ] || [ "$1" = "all" ];then - helm uninstall sda - kubectl delete secret auth-certs backup-certs doa-certs finalize-certs inbox-certs ingest-certs intercept-certs mapper-certs tester-certs verify-certs ca-root download-certs -fi diff --git a/sda-helm/dev_tools/scripts/create-s3-buckets-no-tls.sh b/sda-helm/dev_tools/scripts/create-s3-buckets-no-tls.sh deleted file mode 100644 index 634aec22f..000000000 --- a/sda-helm/dev_tools/scripts/create-s3-buckets-no-tls.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e - -if [ ! -f s3cmd.conf ]; then - cat >> "s3cmd.conf" <>"s3cmd.conf" < /dev/null -then - wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v5.4.6 bash -fi - -k3d cluster list | grep "sda" -cluster_exists=$? - -if [ $cluster_exists -ne 0 ]; then - sudo k3d cluster create sda --image=rancher/k3s:v1.25.6-rc1-k3s1-amd64 - sudo k3d kubeconfig merge sda --kubeconfig-switch-context - sudo mkdir -p ~/.kube/ && sudo cp /root/.k3d/kubeconfig-sda.yaml ~/.kube/config - sudo chmod 666 ~/.kube/config -else - echo "Cluster sda already exists!" -fi diff --git a/sda-helm/dev_tools/scripts/install-kube-deps.sh b/sda-helm/dev_tools/scripts/install-kube-deps.sh deleted file mode 100755 index cbc566fbb..000000000 --- a/sda-helm/dev_tools/scripts/install-kube-deps.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e - -if [ "$OSTYPE" == "linux-gnu" ]; then - BTYPE="linux" -elif [ "$OSTYPE" == "darwin" ]; then - BTYPE="darwin" -fi - -curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/"$BTYPE"/amd64/kubectl -chmod +x ./kubectl -sudo mv ./kubectl /usr/local/bin/kubectl - -wget https://get.helm.sh/helm-v3.11.0-"$BTYPE"-amd64.tar.gz -O - | tar -xz -sudo cp "$BTYPE"-amd64/helm /usr/local/bin/helm - -rm -r ./*-amd64/ diff --git a/sda-helm/dev_tools/scripts/install-sda-deps.sh b/sda-helm/dev_tools/scripts/install-sda-deps.sh deleted file mode 100755 index deb869393..000000000 --- a/sda-helm/dev_tools/scripts/install-sda-deps.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -e - -YQ_VERSION=v4.20.1 -YQ_BINARY=yq_linux_amd64 -C4GH_VERSION=1.4.0 -# Workaround for some MacOS installations -#export PATH=$PATH:/home/ubuntu/.local/bin - -if [ ! -d LocalEGA-helm ]; then - git clone https://github.com/nbisweden/LocalEGA-helm -fi - -# install s3cmd -pip install s3cmd - - -# install yq for creating secrets -sudo wget "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY}" -O /usr/bin/yq &&\ - sudo chmod +x /usr/bin/yq - -# install crypt4gh -curl -L https://github.com/elixir-oslo/crypt4gh/releases/download/v"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ &&\ - sudo chmod +x /usr/bin/crypt4gh diff --git a/sda-helm/dev_tools/scripts/java-certs.sh b/sda-helm/dev_tools/scripts/java-certs.sh deleted file mode 100755 index 027f3e2a1..000000000 --- a/sda-helm/dev_tools/scripts/java-certs.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -[ "${BASH_VERSINFO[0]}" -lt 4 ] && echo 'Bash 4 (or higher) is required' 1>&2 && exit 1 - -if ! [ -x "$(command -v keytool)" ]; then - echo 'Error: Keytool is not installed.' >&2 - exit 1 -fi - -if ! [ -x "$(command -v openssl)" ]; then - echo 'Error: Openssl is not installed.' >&2 - exit 1 -fi - -CONFPATH="sda-deploy-init/config/certs" -STORETYPE=PKCS12 -STOREPASS=changeit -services_input="doa,inbox" - -IFS=',' read -r -a services <<< "$services_input" - -# remove previous alias if keystore exists -# becomes problemantic if password changed -if [[ -f "${CONFPATH}"/cacerts ]]; then - keytool -delete -alias legaCA \ - -keystore "${CONFPATH}"/cacerts \ - -storepass "${STOREPASS}" -noprompt -fi - -# create java keystore for each service -for service in "${services[@]}"; do - if [[ "${STORETYPE}" == "JKS" ]]; then - keytool -import -alias "${service}" \ - -keystore "${CONFPATH}/${service}.jks" \ - -file "${CONFPATH}/${service}".ca.crt.der \ - -storepass "${STOREPASS}" -noprompt - else - openssl pkcs12 -export -out "${CONFPATH}/${service}".p12 \ - -inkey "${CONFPATH}/${service}".key \ - -in "${CONFPATH}/${service}".crt \ - -passout pass:"${STOREPASS}" - openssl pkcs8 -topk8 \ - -inform pem \ - -outform der \ - -in "${CONFPATH}/${service}".key \ - -out "${CONFPATH}/${service}".key.der \ - -nocrypt - fi -done - -# create java CAroot truststore -keytool -import -trustcacerts -file "${CONFPATH}"/ca.crt \ - -alias legaCA -storetype JKS \ - -keystore "${CONFPATH}"/cacerts \ - -storepass "${STOREPASS}" -noprompt - -# create DER format key - diff --git a/sda-helm/dev_tools/scripts/make-certs.sh b/sda-helm/dev_tools/scripts/make-certs.sh deleted file mode 100755 index 0f4b0c1c9..000000000 --- a/sda-helm/dev_tools/scripts/make-certs.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/sh - -basedir="sda-deploy-init/config/certs" -days=1200 - -mkdir -p "${basedir}" - -# create CA certificate -openssl req -config "$(dirname "$0")"/ssl.cnf -new -sha256 -nodes -extensions v3_ca -out "./${basedir}/ca.csr" -keyout "./${basedir}/ca.key" -openssl req -config "$(dirname "$0")"/ssl.cnf -key "./${basedir}/ca.key" -x509 -new -days 7300 -sha256 -nodes -extensions v3_ca -out "./${basedir}/ca.crt" - -# Create certificate for MQ -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/server.key" -out "./${basedir}/mq.csr" -extensions mq_cert -openssl x509 -req -in "./${basedir}/mq.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/server.crt" -extensions mq_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for DB -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/pg.key" -out "./${basedir}/pg.csr" -extensions db_cert -openssl x509 -req -in "./${basedir}/pg.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/pg.crt" -extensions db_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for minio -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/private.key" -out "./${basedir}/s3.csr" -extensions minio_cert -openssl x509 -req -in "./${basedir}/s3.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/public.crt" -extensions minio_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create client certificate -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/client.key" -out "./${basedir}/client.csr" -extensions client_cert -subj "/CN=lega_in/CN=admin/" -openssl x509 -req -in "./${basedir}/client.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/client.crt" -extensions client_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for inbox -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/inbox.key" -out "./${basedir}/inbox.csr" -extensions inbox_cert -subj "/CN=lega_in/CN=admin/" -openssl x509 -req -in "./${basedir}/inbox.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/inbox.crt" -extensions inbox_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for ingest -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/ingest.key" -out "./${basedir}/ingest.csr" -extensions ingest_cert -subj "/CN=lega_in/CN=admin/" -openssl x509 -req -in "./${basedir}/ingest.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/ingest.crt" -extensions ingest_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for intercept -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/intercept.key" -out "./${basedir}/intercept.csr" -extensions intercept_cert -subj "/CN=admin" -openssl x509 -req -in "./${basedir}/intercept.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/intercept.crt" -extensions intercept_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for finalize -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/finalize.key" -out "./${basedir}/finalize.csr" -extensions finalize_cert -subj "/CN=lega_in/CN=admin" -openssl x509 -req -in "./${basedir}/finalize.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/finalize.crt" -extensions finalize_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for verify -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/verify.key" -out "./${basedir}/verify.csr" -extensions verify_cert -subj "/CN=lega_in/CN=admin/" -openssl x509 -req -in "./${basedir}/verify.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/verify.crt" -extensions verify_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for doa -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/doa.key" -out "./${basedir}/doa.csr" -extensions doa_cert -subj "/CN=lega_out/CN=admin/" -openssl x509 -req -in "./${basedir}/doa.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/doa.crt" -extensions doa_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for download -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/download.key" -out "./${basedir}/download.csr" -extensions download_cert -subj "/CN=lega_out/CN=admin/" -openssl x509 -req -in "./${basedir}/download.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/download.crt" -extensions download_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for orch -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/orch.key" -out "./${basedir}/orch.csr" -extensions orch_cert -subj "/CN=admin" -openssl x509 -req -in "./${basedir}/orch.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/orch.crt" -extensions orch_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for mapper -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/mapper.key" -out "./${basedir}/mapper.csr" -extensions mapper_cert -subj "/CN=lega_out/CN=admin" -openssl x509 -req -in "./${basedir}/mapper.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/mapper.crt" -extensions mapper_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for backup -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/backup.key" -out "./${basedir}/backup.csr" -extensions backup_cert -subj "/CN=admin/CN=lega_in" -openssl x509 -req -in "./${basedir}/backup.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/backup.crt" -extensions backup_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for auth -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/auth.key" -out "./${basedir}/auth.csr" -extensions auth_cert -openssl x509 -req -in "./${basedir}/auth.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/auth.crt" -extensions auth_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for tester -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/tester.key" -out "./${basedir}/tester.csr" -extensions tester_cert -subj "/CN=lega_in/CN=admin/" -openssl x509 -req -in "./${basedir}/tester.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/tester.crt" -extensions tester_cert -extfile "$(dirname "$0")"/ssl.cnf - -# Create certificate for cega -openssl req -config "$(dirname "$0")"/ssl.cnf -new -nodes -newkey rsa:4096 -keyout "./${basedir}/cega.key" -out "./${basedir}/cega.csr" -extensions cega -openssl x509 -req -in "./${basedir}/cega.csr" -days "${days}" -CA "./${basedir}/ca.crt" -CAkey "./${basedir}/ca.key" -set_serial 01 -out "./${basedir}/cega.crt" -extensions cega -extfile "$(dirname "$0")"/ssl.cnf - -# Create token -openssl req -nodes -new -x509 -keyout "./${basedir}/token.key" -out "./${basedir}/token.pub" -config "$(dirname "$0")"/ssl.cnf - -chmod 644 "./${basedir}/"* diff --git a/sda-helm/dev_tools/scripts/mq-password-generator.sh b/sda-helm/dev_tools/scripts/mq-password-generator.sh deleted file mode 100644 index a0325a1c6..000000000 --- a/sda-helm/dev_tools/scripts/mq-password-generator.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -SALT=$(od -A n -t x -N 4 /dev/urandom) -PASS=$SALT$(echo -n "$1" | xxd -ps | tr -d '\n' | tr -d ' ') -PASS=$(echo -n "$PASS" | xxd -r -p | sha256sum | head -c 128) -PASS=$(echo -n "$SALT$PASS" | xxd -r -p | base64 -w0) -echo -n "$PASS" diff --git a/sda-helm/dev_tools/scripts/run-helm-test.sh b/sda-helm/dev_tools/scripts/run-helm-test.sh deleted file mode 100755 index 4c2458e6d..000000000 --- a/sda-helm/dev_tools/scripts/run-helm-test.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -if [ -n "$1" ]; then RELEASE_LIST=$1; else RELEASE_LIST="broker postgres sda"; fi - -for release in $RELEASE_LIST; do - echo "Testing $release" - helm test "$release" - r=$? - - if [ "$r" -ne 0 ]; then - kubectl get pod -o name | while read -r pod; do - echo "All logs for $pod" - kubectl logs --all-containers "$pod" - done - exit "$r" - fi -done diff --git a/sda-helm/dev_tools/scripts/sda/deploy-no-tls.sh b/sda-helm/dev_tools/scripts/sda/deploy-no-tls.sh deleted file mode 100644 index 665f37a44..000000000 --- a/sda-helm/dev_tools/scripts/sda/deploy-no-tls.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/sh -set -e - -if [ "$1" = "database" ]; then - DB_IN_PASS=$(grep pg_in_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - DB_OUT_PASS=$(grep pg_out_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - - helm install postgres charts/sda-db \ - --set global.pg_in_password="$DB_IN_PASS",global.pg_out_password="$DB_OUT_PASS",securityPolicy.create=false,global.tls.enabled=false,persistence.enabled=false - - RETRY_TIMES=0 - until kubectl get pods -l role=database -o jsonpath='{..status.containerStatuses[*].ready}' | grep "true"; do - echo "waiting for database to become ready" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - kubectl describe pod -l role=datbase - kubectl logs -l role=datbase - exit 1 - fi - sleep 10 - done - exit 0 -fi - -if [ "$1" = "broker" ]; then - HASH="$(/bin/sh dev_tools/scripts/mq-password-generator.sh admin)" - helm install broker charts/sda-mq \ - --set securityPolicy.create=false,global.adminUser=admin,global.adminPasswordHash="$HASH",global.tls.enabled=false,global.vhost=sda,persistence.enabled=false - - RETRY_TIMES=0 - until kubectl get pods -l role=broker -o jsonpath='{..status.containerStatuses[*].ready}' | grep "true"; do - echo "waiting for broker to become ready" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - kubectl describe pod -l role=broker - kubectl logs -l role=broker - exit 1 - fi - sleep 10 - done - exit 0 -fi - -if [ "$1" = "orchestrate" ]; then - helm install orch charts/sda-orch -f dev_tools/config/orch.yaml \ - --set tls.enabled=false,broker.port=5672,broker.queue.verify=verified,broker.queue.inbox=files - - RETRY_TIMES=0 - until kubectl get pods -l role=orchestrate -o jsonpath='{..status.containerStatuses[*].ready}' | grep "true"; do - echo "waiting for orch to become ready" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - kubectl describe pod -l role=orchestrate - kubectl logs -l role=orchestrate - exit 1 - fi - sleep 10 - done - exit 0 -fi - -if [ "$1" = "minio" ]; then - helm repo add minio https://helm.min.io/ - helm repo update - - MINIO_ACCESS=$(grep s3_access_key sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - MINIO_SECRET=$(grep s3_secret_key sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - - helm install minio minio/minio \ - --set accessKey="$MINIO_ACCESS",secretKey="$MINIO_SECRET",tls.enabled=false,persistence.enabled=false,service.port=9000 \ - --version 8.0.8 - - RETRY_TIMES=0 - until kubectl get pods -l app=minio -o jsonpath='{..status.containerStatuses[*].ready}' | grep "true"; do - echo "waiting for minio to become ready" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - kubectl describe pod -l app=minio - kubectl logs -l app=minio - exit 1 - fi - sleep 10 - done - exit 0 -fi - -if [ "$1" = "pipeline" ]; then - DB_IN_PASS=$(grep pg_in_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - DB_OUT_PASS=$(grep pg_out_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - S3_ACCESS_KEY=$(grep s3_access_key sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - S3_SECRET_KEY=$(grep s3_secret_key sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - C4GH_PASSPHRASE=$(grep c4gh_passphrase sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - token="$(bash dev_tools/scripts/sign_jwt.sh ES256 sda-deploy-init/config/jwt.key)" - helm install sda charts/sda-svc -f dev_tools/config/no-tls.yaml \ - --set global.archive.s3AccessKey="$S3_ACCESS_KEY",global.archive.s3SecretKey="$S3_SECRET_KEY",global.backupArchive.s3AccessKey="$S3_ACCESS_KEY",global.backupArchive.s3SecretKey="$S3_SECRET_KEY",global.broker.vhost=sda,global.c4gh.passphrase="$C4GH_PASSPHRASE",global.db.passIngest="$DB_IN_PASS",global.db.passOutgest="$DB_OUT_PASS",global.inbox.s3AccessKey="$S3_ACCESS_KEY",global.inbox.s3SecretKey="$S3_SECRET_KEY",releasetest.secrets.accessToken="$token" -fi diff --git a/sda-helm/dev_tools/scripts/sda/deploy-orch.sh b/sda-helm/dev_tools/scripts/sda/deploy-orch.sh deleted file mode 100755 index 7f9b5f3fd..000000000 --- a/sda-helm/dev_tools/scripts/sda/deploy-orch.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -## sda-orch certs -kubectl create secret generic orch-certs \ ---from-file=sda-deploy-init/config/certs/ca.crt \ ---from-file=sda-deploy-init/config/certs/orch.crt \ ---from-file=sda-deploy-init/config/certs/orch.key - -helm install orch charts/sda-orch -f dev_tools/config/orch.yaml diff --git a/sda-helm/dev_tools/scripts/sda/deploy-posix.sh b/sda-helm/dev_tools/scripts/sda/deploy-posix.sh deleted file mode 100755 index 891b06cc7..000000000 --- a/sda-helm/dev_tools/scripts/sda/deploy-posix.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -e - -if [ "$1" = "standalone" ]; then - INTERCEPT=false - SCHEMA=isolated -else - INTERCEPT=true - CEGA_USERS_PASS=$(grep cega_users_pass dev_tools/config/cega.yaml | awk '{print $2}' | sed -e 's/\"//g') - SCHEMA=federated -fi - -DB_IN_PASS=$(grep pg_in_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') -DB_OUT_PASS=$(grep pg_out_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') -C4GH_PASSPHRASE=$(grep c4gh_passphrase sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') - -if [ "$2" = "issuer" ]; then - helm install sda charts/sda-svc -f dev_tools/config/posix.yaml \ - --set global.broker.vhost=/sda,global.c4gh.passphrase="$C4GH_PASSPHRASE",global.cega.password="$CEGA_USERS_PASS",global.db.passIngest="$DB_IN_PASS",global.db.passOutgest="$DB_OUT_PASS",global.schemaType="$SCHEMA",intercept.deploy="$INTERCEPT",global.tls.issuer=ca-issuer -else - helm install sda charts/sda-svc -f dev_tools/config/posix.yaml \ - --set global.broker.vhost=/sda,global.c4gh.passphrase="$C4GH_PASSPHRASE",global.cega.password="$CEGA_USERS_PASS",global.db.passIngest="$DB_IN_PASS",global.db.passOutgest="$DB_OUT_PASS",global.schemaType="$SCHEMA",intercept.deploy="$INTERCEPT" -fi diff --git a/sda-helm/dev_tools/scripts/sda/deploy-s3.sh b/sda-helm/dev_tools/scripts/sda/deploy-s3.sh deleted file mode 100755 index e1423a4d1..000000000 --- a/sda-helm/dev_tools/scripts/sda/deploy-s3.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e - -if [ "$1" = "federated" ]; then - INTERCEPT=true - SCHEMA=federated -else - INTERCEPT=false - SCHEMA=isolated -fi -DB_IN_PASS=$(grep pg_in_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') -DB_OUT_PASS=$(grep pg_out_password sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') -S3_ACCESS_KEY=$(grep s3_access_key sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') -S3_SECRET_KEY=$(grep s3_secret_key sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') -C4GH_PASSPHRASE=$(grep c4gh_passphrase sda-deploy-init/config/trace.yml | awk '{print $2}' | sed -e 's/\"//g') -CEGA_USERS_PASS="OfEoDPVadIfd4CZUWkisrrVQbJ2yQPIH" - -token="$(bash dev_tools/scripts/sign_jwt.sh ES256 sda-deploy-init/config/jwt.key)" - -if [ "$2" = "issuer" ]; then - helm install sda charts/sda-svc -f dev_tools/config/s3.yaml \ - --set global.broker.vhost=sda,global.archive.s3AccessKey="$S3_ACCESS_KEY",global.archive.s3SecretKey="$S3_SECRET_KEY",global.backupArchive.s3AccessKey="$S3_ACCESS_KEY",global.backupArchive.s3SecretKey="$S3_SECRET_KEY",global.c4gh.passphrase="$C4GH_PASSPHRASE",global.cega.password="$CEGA_USERS_PASS",global.db.passIngest="$DB_IN_PASS",global.db.passOutgest="$DB_OUT_PASS",global.doa.enabled=true,global.inbox.s3AccessKey="$S3_ACCESS_KEY",global.inbox.s3SecretKey="$S3_SECRET_KEY",global.schemaType="$SCHEMA",intercept.deploy="$INTERCEPT",releasetest.secrets.accessToken="$token",global.tls.issuer=ca-issuer - -else - helm install sda charts/sda-svc -f dev_tools/config/s3.yaml \ - --set global.broker.vhost=sda,global.archive.s3AccessKey="$S3_ACCESS_KEY",global.archive.s3SecretKey="$S3_SECRET_KEY",global.backupArchive.s3AccessKey="$S3_ACCESS_KEY",global.backupArchive.s3SecretKey="$S3_SECRET_KEY",global.c4gh.passphrase="$C4GH_PASSPHRASE",global.cega.password="$CEGA_USERS_PASS",global.db.passIngest="$DB_IN_PASS",global.db.passOutgest="$DB_OUT_PASS",global.doa.enabled=true,global.inbox.s3AccessKey="$S3_ACCESS_KEY",global.inbox.s3SecretKey="$S3_SECRET_KEY",global.schemaType="$SCHEMA",intercept.deploy="$INTERCEPT",releasetest.secrets.accessToken="$token" -fi diff --git a/sda-helm/dev_tools/scripts/sign_jwt.sh b/sda-helm/dev_tools/scripts/sign_jwt.sh deleted file mode 100644 index 90da75478..000000000 --- a/sda-helm/dev_tools/scripts/sign_jwt.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# Inspired by implementation by Will Haley at: -# http://willhaley.com/blog/generate-jwt-with-bash/ - -set -o pipefail - -# Shared content to use as template -header_template='{ - "typ": "JWT", - "kid": "0001" -}' - -build_header() { - jq -c \ - --arg iat_str "$(date +%s)" \ - --arg alg "${1}" \ - ' - ($iat_str | tonumber) as $iat - | .alg = $alg - | .iat = $iat - | .exp = ($iat + 86400) - ' <<<"$header_template" | tr -d '\n' -} - -b64enc() { openssl enc -base64 -A | tr '+/' '-_' | tr -d '='; } -json() { jq -c . | LC_CTYPE=C tr -d '\n'; } -rs_sign() { openssl dgst -binary -sha"${1}" -sign <(printf '%s\n' "$2"); } -es_sign() { openssl dgst -binary -sha"${1}" -sign <(printf '%s\n' "$2") | openssl asn1parse -inform DER | grep INTEGER | cut -d ':' -f 4 | xxd -p -r ; } - -sign() { - if [ -n "$2" ];then - rsa_secret=$(<"$2") - else - echo "no signing key supplied" - exit 1 - fi - local algo payload header sig secret=$rsa_secret - algo=${1:-RS256}; algo=${algo^^} - header=$(build_header "$algo") || return - payload=${3:-$test_payload} - signed_content="$(json <<<"$header" | b64enc).$(json <<<"$payload" | b64enc)" - case $algo in - RS*) sig=$(printf %s "$signed_content" | rs_sign "${algo#RS}" "$secret" | b64enc) ;; - ES*) sig=$(printf %s "$signed_content" | es_sign "${algo#ES}" "$secret" | b64enc) ;; - *) echo "Unknown algorithm" >&2; return 1 ;; - esac - printf '%s.%s\n' "${signed_content}" "${sig}" -} - -iat=$(date +%s) -exp=$(date --date=tomorrow +%s) - -test_payload='{ - "at_hash": "J_fA458SPsXFV6lJQL1l-w", - "aud": "XC56EL11xx", - "email": "dummy.tester@example.org", - "exp": '"$exp"', - "iat": '"$iat"', - "iss": "http://sda-sda-svc-auth", - "kid": "d87f2d01d1a4abb16e1eb88f6561e5067f3a6430174b8fcd0b6bf61434d6c5c8", - "name": "Dummy Tester", - "sid": "1ad14eb5-9b51-40c0-a52a-154a5a3792d5", - "sub": "dummy" -}' - - -sign "$@" diff --git a/sda-helm/dev_tools/scripts/ssl.cnf b/sda-helm/dev_tools/scripts/ssl.cnf deleted file mode 100644 index d6804a4c2..000000000 --- a/sda-helm/dev_tools/scripts/ssl.cnf +++ /dev/null @@ -1,236 +0,0 @@ -[ ca ] -# `man ca` -default_ca = CA_default - -[ CA_default ] -# Directory and file locations. -dir = ./certs -certs = $dir -crl_dir = $dir -new_certs_dir = $dir -database = $dir/index.txt -serial = $dir/serial -RANDFILE = $dir/rand - -# The root key and root certificate. -private_key = $dir/ca-key.pem -certificate = $dir/ca.pem - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -name_opt = ca_default -cert_opt = ca_default -default_days = 7300 -preserve = no -policy = policy_strict - -[ policy_strict ] -# The root CA should only sign intermediate certificates that match. -# See the POLICY FORMAT section of `man ca`. -countryName = match -stateOrProvinceName = match -organizationName = match -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -[ req ] -# Options for the `req` tool (`man req`). -default_bits = 4096 -distinguished_name = req_distinguished_name -string_mask = utf8only -prompt = no - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -# Extension to add when the -x509 option is used. -x509_extensions = v3_ca - -[ req_distinguished_name ] -countryName = SE -stateOrProvinceName = Sweden -localityName = Uppsala -0.organizationName = NBIS -organizationalUnitName = SysDev -commonName = SysDev root CA - -[ v3_ca ] -# Extensions for a typical CA (`man x509v3_config`). -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer -basicConstraints = critical, CA:true -keyUsage = critical, digitalSignature, cRLSign, keyCertSign -#nsCertType = sslCA -nsComment = "NeIC SDA Root CA" - -[ server_client_cert ] -# Extensions for server+client certificates (`man x509v3_config`). -basicConstraints = CA:FALSE -nsCertType = server,client -nsComment = "NeIC SDA Server+Client Certificate" -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = critical, clientAuth, serverAuth - -[ client_cert ] -# Extensions for client certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:localhost,DNS:client,IP:127.0.0.1 - -[ mq_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:broker-sda-mq,DNS:broker-sda-mq.default.svc.cluster.local,DNS:broker-sda-mq.default - -[ db_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:postgres-sda-db,DNS:postgres-sda-db.default.svc.cluster.local,DNS:postgres-sda-db.default - -[ minio_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:minio,DNS:minio.default.svc.cluster.local,DNS:minio.default - - -[ intercept_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:intercept,DNS:intercept.default.svc.cluster.local,DNS:intercept.default - -[ ingest_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:ingest,DNS:ingest.default.svc.cluster.local,DNS:ingest.default - -[ verify_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:verify,DNS:verify.default.svc.cluster.local,DNS:verify.default - -[ finalize_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:finalize,DNS:finalize.default.svc.cluster.local,DNS:finalize.default - -[ mapper_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:mapper,DNS:mapper.default.svc.cluster.local,DNS:mapper.default - -[ backup_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:backup,DNS:backup.default.svc.cluster.local,DNS:backup.default - -[ orch_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:orchestrate,DNS:orchestrate.default.svc.cluster.local,DNS:orchestrate.default - -[ inbox_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:sda-sda-svc-inbox,DNS:inbox.default.svc.cluster.local,DNS:inbox.default,DNS:localhost - -[ doa_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:sda-sda-svc-doa,DNS:doa.default.svc.cluster.local,DNS:doa.default,DNS:localhost - -[ download_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:sda-sda-svc-download,DNS:download.default.svc.cluster.local,DNS:download.default,DNS:localhost - -[ auth_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:sda-sda-svc-auth,DNS:auth.default.svc.cluster.local,DNS:auth.default,DNS:localhost - -[ tester_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:tester,DNS:tester.default.svc.cluster.local,DNS:tester.default - -# CEGA certificates - -[ cega ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:cega-mq,DNS:cega-users - -[ crl_ext ] -# Extension for CRLs (`man x509v3_config`). -authorityKeyIdentifier=keyid:always diff --git a/sda-helm/dev_tools/scripts/svc-setup.sh b/sda-helm/dev_tools/scripts/svc-setup.sh deleted file mode 100755 index fce2f7663..000000000 --- a/sda-helm/dev_tools/scripts/svc-setup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e - -basedir="sda-deploy-init/config" - -## cega config and certs -mkdir -p LocalEGA-helm/ega-charts/cega/config/certs -cp -r dev_tools/cega/* LocalEGA-helm/ega-charts/cega/config/ -cp "${basedir}"/certs/ca.crt LocalEGA-helm/ega-charts/cega/config/certs/ca.crt -cp "${basedir}"/certs/cega.crt LocalEGA-helm/ega-charts/cega/config/certs/tls.crt -cp "${basedir}"/certs/cega.key LocalEGA-helm/ega-charts/cega/config/certs/tls.key - -## sda-svc certs - -for n in backup doa finalize inbox ingest intercept verify mapper auth tester download - do - kubectl create secret generic $n-certs \ - --from-file="${basedir}"/certs/ca.crt \ - --from-file=tls.crt="${basedir}"/certs/$n.crt \ - --from-file=tls.key="${basedir}"/certs/$n.key -done diff --git a/sda-helm/dev_tools/scripts/wait-for-pods.sh b/sda-helm/dev_tools/scripts/wait-for-pods.sh deleted file mode 100755 index cbb89a13b..000000000 --- a/sda-helm/dev_tools/scripts/wait-for-pods.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e - -base_list="backup download finalize inbox ingest mapper verify" - -if [ -n "$1" ]; then - case "$1" in - federated_s3_svc_list) - SVCNAME="$base_list auth intercept" - ;; - - federated_posix_svc_list) - SVCNAME="$base_list intercept" - ;; - - standalone_s3_svc_list) - SVCNAME="$base_list auth" - ;; - - standalone_posix_svc_list) - SVCNAME="$base_list" - ;; - esac -fi - -if [ -n "$2" ]; then - LABEL=$2 -else - LABEL="role" -fi -if [ -n "$3" ]; then - NAMESPACE=${3:-default} -fi - -for p in $SVCNAME; do - RETRY_TIMES=0 - until kubectl get pods -n "$NAMESPACE" -l "$LABEL=$p" -o jsonpath='{..status.containerStatuses[*].ready}' | grep "true"; do - echo "waiting for $p to become ready" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - kubectl describe pod -n "$NAMESPACE" -l "$LABEL"="$p" - kubectl logs -n "$NAMESPACE" -l "$LABEL=$p" - exit 1 - fi - sleep 10 - done -done From 860f90c6f4b77a8dc1961a4ea6390fb5975824c3 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 29 Aug 2023 10:36:12 +0200 Subject: [PATCH 14/14] Bump image versions --- charts/sda-db/values.yaml | 2 +- charts/sda-mq/values.yaml | 2 +- charts/sda-svc/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/sda-db/values.yaml b/charts/sda-db/values.yaml index 7cb1e03e9..eb26e5722 100644 --- a/charts/sda-db/values.yaml +++ b/charts/sda-db/values.yaml @@ -25,7 +25,7 @@ extraSecurityContext: {} image: repository: ghcr.io/neicnordic/sensitive-data-archive - tag: v0.0.44-postgres + tag: v0.0.65-postgres pullPolicy: IfNotPresent # utilize network isolation diff --git a/charts/sda-mq/values.yaml b/charts/sda-mq/values.yaml index 085679e0e..8fab60cbc 100644 --- a/charts/sda-mq/values.yaml +++ b/charts/sda-mq/values.yaml @@ -44,7 +44,7 @@ extraSecurityContext: {} image: repository: ghcr.io/neicnordic/sensitive-data-archive - tag: v0.0.44-rabbitmq + tag: v0.0.65-rabbitmq pullPolicy: Always # utilize network isolation diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 8cbd726f6..85993a1fa 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -2,7 +2,7 @@ # Declare variables to be passed into your templates. image: repository: "ghcr.io/neicnordic/sensitive-data-archive" - tag: "v0.0.44" + tag: "v0.0.65" pullPolicy: "Always" global: