Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FEGA usecase] Merge internal components from sda-pipeline into sda #302

Merged
merged 16 commits into from
Sep 25, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions .github/integration/sda-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ services:
- POSTGRES_PASSWORD=rootpasswd
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s
timeout: 20s
retries: 3
interval: 10s
timeout: 2s
retries: 6
image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres
ports:
- "15432:5432"
Expand All @@ -46,9 +46,9 @@ services:
"-c",
"rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms",
]
interval: 5s
timeout: 20s
retries: 3
interval: 10s
timeout: 2s
retries: 6
image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
ports:
- "15672:15672"
Expand All @@ -66,9 +66,9 @@ services:
- MINIO_SERVER_URL=http://127.0.0.1:9000
healthcheck:
test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"]
interval: 5s
timeout: 20s
retries: 3
interval: 10s
timeout: 2s
retries: 6
ports:
- "19000:9000"
- "19001:9001"
Expand Down Expand Up @@ -122,9 +122,9 @@ services:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)']
interval: 5s
timeout: 20s
retries: 3
interval: 10s
timeout: 2s
retries: 6
image: python:3.10-slim
ports:
- "8080:8080"
Expand Down
5 changes: 3 additions & 2 deletions .github/integration/sda/config.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
log:
format: "json"
aws:
url: "http://s3:9000"
inbox:
url: "http://s3"
port: 9000
readypath: "/minio/health/ready"
accessKey: "access"
secretKey: "secretKey"
Expand Down
8 changes: 0 additions & 8 deletions charts/sda-svc/templates/_helpers.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -289,14 +289,6 @@ Create chart name and version as used by the chart label.
{{- end -}}
{{- end -}}

{{- define "S3InboxURL" -}}
{{- if .Values.global.inbox.s3Port }}
{{- printf "%s:%v" .Values.global.inbox.s3Url .Values.global.inbox.s3Port }}
{{- else }}
{{- printf "%s" .Values.global.inbox.s3Url }}
{{- end }}
{{- end -}}

{{- define "S3ArchiveURL" -}}
{{- if .Values.global.inbox.s3Port }}
{{- printf "%s:%v" .Values.global.inbox.s3Url .Values.global.inbox.s3Port }}
Expand Down
20 changes: 12 additions & 8 deletions charts/sda-svc/templates/s3-inbox-deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,12 @@ spec:
{{- end }}
env:
{{- if not .Values.global.vaultSecrets }}
- name: AWS_ACCESSKEY
- name: INBOX_ACCESSKEY
valueFrom:
secretKeyRef:
name: {{ template "sda.fullname" . }}-inbox
key: s3InboxAccessKey
- name: AWS_SECRETKEY
- name: INBOX_SECRETKEY
valueFrom:
secretKeyRef:
name: {{ template "sda.fullname" . }}-inbox
Expand All @@ -117,20 +117,24 @@ spec:
- name: SERVER_CONFFILE
value: {{ include "confFile" .}}
{{- end }}
- name: AWS_URL
value: {{ template "S3InboxURL" . }}
- name: INBOX_URL
value: {{ .Values.global.inbox.s3Url | quote }}
{{- if .Values.global.inbox.s3Port }}
- name: INBOX_PORT
value: {{ .Values.global.inbox.s3Port | quote }}
{{- end }}
{{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }}
- name: AWS_CACERT
- name: INBOX_CACERT
value: "{{ include "tlsPath" . }}/ca.crt"
{{- end }}
{{- if .Values.global.inbox.s3Region }}
- name: AWS_REGION
- name: INBOX_REGION
value: {{ .Values.global.inbox.s3Region | quote }}
{{- end }}
- name: AWS_BUCKET
- name: INBOX_BUCKET
value: {{ .Values.global.inbox.s3Bucket | quote }}
{{- if .Values.global.inbox.s3ReadyPath }}
- name: AWS_READYPATH
- name: INBOX_READYPATH
value: {{ .Values.global.inbox.s3ReadyPath }}
{{- end }}
- name: BROKER_HOST
Expand Down
86 changes: 0 additions & 86 deletions sda/cmd/s3inbox/bucket.go

This file was deleted.

88 changes: 0 additions & 88 deletions sda/cmd/s3inbox/bucket_test.go

This file was deleted.

12 changes: 7 additions & 5 deletions sda/cmd/s3inbox/healthchecks.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,8 @@ import (
"strconv"
"time"

"github.com/neicnordic/sensitive-data-archive/internal/config"

"github.com/heptiolabs/healthcheck"
"github.com/neicnordic/sensitive-data-archive/internal/config"
)

// HealthCheck registers and endpoint for healthchecking the service
Expand All @@ -25,9 +24,12 @@ type HealthCheck struct {
// NewHealthCheck creates a new healthchecker. It needs to know where to find
// the backend S3 storage and the Message Broker so it can report readiness.
func NewHealthCheck(port int, db *sql.DB, conf *config.Config, tlsConfig *tls.Config) *HealthCheck {
s3URL := conf.S3.URL
if conf.S3.Readypath != "" {
s3URL = conf.S3.URL + conf.S3.Readypath
s3URL := conf.Inbox.S3.URL
if conf.Inbox.S3.Port != 0 {
s3URL = fmt.Sprintf("%s:%d", s3URL, conf.Inbox.S3.Port)
}
if conf.Inbox.S3.Readypath != "" {
s3URL += conf.Inbox.S3.Readypath
}

brokerURL := fmt.Sprintf("%s:%d", conf.Broker.Host, conf.Broker.Port)
Expand Down
12 changes: 6 additions & 6 deletions sda/cmd/s3inbox/healthchecks_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,16 @@ func (suite *HealthcheckTestSuite) SetupTest() {
viper.Set("broker.routingkey", "ingest")
viper.Set("broker.exchange", "sda")
viper.Set("broker.vhost", "sda")
viper.Set("aws.url", "http://localhost:8080")
viper.Set("aws.accesskey", "testaccess")
viper.Set("aws.secretkey", "testsecret")
viper.Set("aws.bucket", "testbucket")
viper.Set("inbox.url", "http://localhost:8080")
viper.Set("inbox.accesskey", "testaccess")
viper.Set("inbox.secretkey", "testsecret")
viper.Set("inbox.bucket", "testbucket")
viper.Set("server.jwtpubkeypath", "testpath")
}

func (suite *HealthcheckTestSuite) TestHttpsGetCheck() {
db, _, _ := sqlmock.New()
conf, err := config.NewConfig()
conf, err := config.NewConfig("s3inbox")
assert.NoError(suite.T(), err)
assert.NotNil(suite.T(), conf)
h := NewHealthCheck(8888,
Expand All @@ -68,7 +68,7 @@ func (suite *HealthcheckTestSuite) TestHealthchecks() {

db, mock, _ := sqlmock.New(sqlmock.MonitorPingsOption(true))
mock.ExpectPing()
conf, err := config.NewConfig()
conf, err := config.NewConfig("s3inbox")
assert.NoError(suite.T(), err)
assert.NotNil(suite.T(), conf)
h := NewHealthCheck(8888,
Expand Down
Loading
Loading