From 46c1bc7cbabbf9870f1fdc2b120c8fa46a43fcfd Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 22 Oct 2024 11:01:34 +0200 Subject: [PATCH 01/15] [postgres] add table for userinfo --- postgresql/initdb.d/01_main.sql | 8 ++++++ postgresql/initdb.d/04_grants.sql | 1 + postgresql/migratedb.d/14.sql | 43 +++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 postgresql/migratedb.d/14.sql diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index eb3394f9c..df4a8f95a 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -74,6 +74,14 @@ CREATE TABLE files ( CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path) ); +-- The user info is used by auth to be able to link users to their name and email +CREATE TABLE userinfo ( + id TEXT PRIMARY KEY, + name TEXT, + email TEXT, + groups TEXT[] +); + -- To allow for multiple checksums per file, we use a dedicated table for it CREATE TABLE checksums ( id SERIAL PRIMARY KEY, diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index 1eadb112b..acdc80144 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -170,6 +170,7 @@ GRANT SELECT ON sda.datasets TO api; GRANT SELECT ON sda.dataset_event_log TO api; GRANT INSERT ON sda.encryption_keys TO api; GRANT UPDATE ON sda.encryption_keys TO api; +GRANT SELECT, INSERT, UPDATE ON sda.userinfo TO download; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO api; diff --git a/postgresql/migratedb.d/14.sql b/postgresql/migratedb.d/14.sql new file mode 100644 index 000000000..aee3aa072 --- /dev/null +++ b/postgresql/migratedb.d/14.sql @@ -0,0 +1,43 @@ +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 13; + changes VARCHAR := 'Add userinfo and create AUTH user'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + -- Temporary function for creating roles if they do not already exist. + CREATE FUNCTION create_role_if_not_exists(role_name NAME) RETURNS void AS $created$ + BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = role_name) THEN + RAISE NOTICE 'Role "%" already exists. Skipping.', role_name; + ELSE + BEGIN + EXECUTE format('CREATE ROLE %I', role_name); + EXCEPTION + WHEN duplicate_object THEN + RAISE NOTICE 'Role "%" was just created by a concurrent transaction. Skipping.', role_name; + END; + END IF; + END; + $created$ LANGUAGE plpgsql; + + CREATE TABLE IF NOT EXISTS sda.userinfo ( + id TEXT PRIMARY KEY, + name TEXT, + email TEXT, + groups TEXT[] + ); + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ \ No newline at end of file From 65388d65aca1aa5e8ab830c9a4696cc97e864413 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 25 Oct 2024 14:35:50 +0200 Subject: [PATCH 02/15] [postgres] add user auth --- postgresql/initdb.d/04_grants.sql | 8 ++++++-- postgresql/migratedb.d/14.sql | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index acdc80144..5c6169ac3 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -170,7 +170,6 @@ GRANT SELECT ON sda.datasets TO api; GRANT SELECT ON sda.dataset_event_log TO api; GRANT INSERT ON sda.encryption_keys TO api; GRANT UPDATE ON sda.encryption_keys TO api; -GRANT SELECT, INSERT, UPDATE ON sda.userinfo TO download; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO api; @@ -180,10 +179,15 @@ GRANT SELECT ON local_ega_ebi.file TO api; GRANT SELECT ON local_ega_ebi.file_dataset TO api; -------------------------------------------------------------------------------- +CREATE ROLE auth; +GRANT USAGE ON SCHEMA sda TO api; +GRANT SELECT, INSERT, UPDATE ON sda.userinfo TO auth; +-------------------------------------------------------------------------------- + -- lega_in permissions GRANT base, ingest, verify, finalize, sync, api TO lega_in; -- lega_out permissions GRANT mapper, download, api TO lega_out; -GRANT base TO api, download, inbox, ingest, finalize, mapper, verify +GRANT base TO api, download, inbox, ingest, finalize, mapper, verify, auth; diff --git a/postgresql/migratedb.d/14.sql b/postgresql/migratedb.d/14.sql index aee3aa072..795f6b3c1 100644 --- a/postgresql/migratedb.d/14.sql +++ b/postgresql/migratedb.d/14.sql @@ -36,6 +36,11 @@ BEGIN groups TEXT[] ); + PERFORM create_role_if_not_exists('auth'); + GRANT USAGE ON SCHEMA sda TO auth; + GRANT SELECT, INSERT, UPDATE ON sda.userinfo TO auth; + + GRANT base TO api, download, inbox, ingest, finalize, mapper, verify, auth; ELSE RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; END IF; From bb4d85e03fd51e49e9c1ecfd9071edbf12fb1112 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 22 Oct 2024 11:09:26 +0200 Subject: [PATCH 03/15] [db] insert user info into db --- sda/internal/database/db_functions.go | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index b83f98ada..bf8843db7 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/lib/pq" log "github.com/sirupsen/logrus" ) @@ -927,3 +928,35 @@ func (dbs *SDAdb) ListUserDatasets(submissionUser string) ([]DatasetInfo, error) return datasets, nil } + +func (dbs *SDAdb) UpdateUserInfo(userID, name, email string, groups []string) error { + var ( + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + err = dbs.updateUserInfo(userID, name, email, groups) + count++ + } + + return err +} +func (dbs *SDAdb) updateUserInfo(userID, name, email string, groups []string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const query = "INSERT INTO sda.userinfo(id, name, email, groups) VALUES($1, $2, $3, $4)" + + "ON CONFLICT (id)" + + "DO UPDATE SET name = excluded.name, email = excluded.email, groups = excluded.groups;" + + result, err := db.Exec(query, userID, name, email, pq.Array(groups)) + if err != nil { + return err + } + if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { + return errors.New("something went wrong with the query zero rows were changed") + } + + return nil +} From ef5143e12e47812f55f2303d3ec8b7df7d373bad Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 22 Oct 2024 12:54:39 +0200 Subject: [PATCH 04/15] [auth] save userinfo to db --- sda/cmd/auth/main.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 46424e563..8d25c7b83 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -16,6 +16,7 @@ import ( "github.com/kataras/iris/v12/sessions" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" log "github.com/sirupsen/logrus" "golang.org/x/oauth2" ) @@ -263,13 +264,17 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { return nil } + err = auth.Config.DB.UpdateUserInfo(idStruct.User, idStruct.Profile, idStruct.Email, idStruct.EdupersonEntitlement) + if err != nil { + log.Warnf("Could not log user info for %s (%s, %s)", idStruct.User, idStruct.Name, idStruct.Email) + } if auth.Config.ResignJwt { claims := map[string]interface{}{ jwt.ExpirationKey: time.Now().UTC().Add(time.Duration(auth.Config.JwtTTL) * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: auth.Config.JwtIssuer, - jwt.SubjectKey: idStruct.User, + jwt.SubjectKey: idStruct.Profile, } token, expDate, err := generateJwtToken(claims, auth.Config.JwtPrivateKey, auth.Config.JwtSignatureAlg) if err != nil { @@ -419,6 +424,18 @@ func main() { log.Fatalf("Failed to read public key: %s", err.Error()) } + // Connect to DB + config.Auth.DB, err = database.NewSDAdb(config.Database) + if err != nil { + log.Error(err) + panic(err) + } + if config.Auth.DB.Version < 8 { + log.Error("database schema v8 is required") + panic(err) + } + defer config.Auth.DB.Close() + // Endpoint for client login info app.Get("/info", authHandler.getInfo) From 2be9dab6bc46e3d2b6e4ea668be0071cf8715eec Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 25 Oct 2024 14:33:39 +0200 Subject: [PATCH 05/15] [tests] make sure auth can connect to db --- .github/integration/scripts/make_db_credentials.sh | 2 +- .github/integration/scripts/make_sda_credentials.sh | 2 +- .github/integration/sda-s3-integration.yml | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/integration/scripts/make_db_credentials.sh b/.github/integration/scripts/make_db_credentials.sh index bef674cb2..4350f1d17 100644 --- a/.github/integration/scripts/make_db_credentials.sh +++ b/.github/integration/scripts/make_db_credentials.sh @@ -4,7 +4,7 @@ set -e apt-get -o DPkg::Lock::Timeout=60 update > /dev/null apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client >/dev/null -for n in api download finalize inbox ingest mapper sync verify; do +for n in api auth download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" psql -U postgres -h migrate -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 3b4289313..7c573c6c7 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -14,7 +14,7 @@ apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl post pip install --upgrade pip > /dev/null pip install aiohttp Authlib joserfc requests > /dev/null -for n in api download finalize inbox ingest mapper sync verify; do +for n in api auth download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" psql -U postgres -h postgres -d sda -c "GRANT base TO $n;" diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index f2db39dca..7d01b3c42 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -302,10 +302,14 @@ services: depends_on: cega-nss: condition: service_started + postgres: + condition: service_healthy environment: - AUTH_RESIGNJWT=true - AUTH_CEGA_ID=test - AUTH_CEGA_SECRET=test + - DB_PASSWORD=auth + - DB_USER=auth image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8888:8080" @@ -318,12 +322,16 @@ services: command: [ sda-auth ] container_name: auth-oidc depends_on: + postgres: + condition: service_healthy oidc: condition: service_healthy environment: - AUTH_RESIGNJWT=false - OIDC_ID=XC56EL11xx - OIDC_SECRET=wHPVQaYXmdDHg + - DB_PASSWORD=api + - DB_USER=api image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8889:8080" From a65380067eb26b6a9304b76b2e6e1ca55592bd6d Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 24 Oct 2024 14:26:26 +0200 Subject: [PATCH 06/15] [config] auth reads db password --- sda/internal/config/config.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 7c53d4ff7..ae08188da 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -114,6 +114,7 @@ type OrchestratorConf struct { type AuthConf struct { OIDC OIDCConfig + DB *database.SDAdb Cega CegaConfig JwtIssuer string JwtPrivateKey string @@ -218,6 +219,11 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = []string{ "auth.s3Inbox", "auth.publicFile", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", } if viper.GetString("auth.cega.id") != "" && viper.GetString("auth.cega.secret") != "" { @@ -531,6 +537,10 @@ func NewConfig(app string) (*Config, error) { } c.Auth.S3Inbox = viper.GetString("auth.s3Inbox") + err := c.configDatabase() + if err != nil { + return nil, err + } case "finalize": if viper.GetString("archive.type") != "" && viper.GetString("backup.type") != "" { c.configArchive() From d07c6a7c08512e16bcc05e92db1ee2fa578056f9 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 24 Oct 2024 15:36:24 +0200 Subject: [PATCH 07/15] [tests] add db unit tests for updating userinfo --- sda/internal/database/db_functions_test.go | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 3bdb373fe..de99d72d3 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/google/uuid" + "github.com/lib/pq" "github.com/stretchr/testify/assert" ) @@ -930,3 +931,46 @@ func (suite *DatabaseTests) TestListUserDatasets() { assert.Equal(suite.T(), 2, len(datasets)) assert.Equal(suite.T(), "test-user-dataset-01", datasets[0].DatasetID) } + +func (suite *DatabaseTests) TestInsertUserInfo() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // Verify that a user can be inserted + var groups []string + userID, name, email := "12334556testuser@lifescience.ru", "Test User", "test.user@example.org" + err = db.UpdateUserInfo(userID, name, email, groups) + assert.NoError(suite.T(), err, "could not insert user info: %v", err) + var exists bool + err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.userinfo WHERE id=$1)", userID).Scan(&exists) + assert.NoError(suite.T(), err, "failed to verify user info existence") + assert.True(suite.T(), exists, "user info was not added to the database") + + // Insert new information about the user + groups = append(groups, "appleGroup", "bananaGroup") + name = "newName" + err = db.UpdateUserInfo(userID, name, email, groups) + assert.NoError(suite.T(), err, "could not insert updated user info: %v", err) +} + +func (suite *DatabaseTests) TestUpdateUserInfo() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // Insert a userID + var groups []string + userID, name, email := "12334556testuser@lifescience.ru", "Test User", "test.user@example.org" + err = db.UpdateUserInfo(userID, name, email, groups) + assert.NoError(suite.T(), err, "could not insert user info: %v", err) + // Verify that the userID is connected to the new details + var numRows int + err = db.DB.QueryRow("SELECT COUNT(*) FROM sda.userinfo WHERE id=$1", userID).Scan(&numRows) + assert.NoError(suite.T(), err, "could select user info: %v", err) + assert.Equal(suite.T(), 1, numRows, "there should be exactly 1 row about %v in userinfo table", userID) + var name2 string + var groups2 []string + err = db.DB.QueryRow("SELECT name, groups FROM sda.userinfo WHERE id=$1", userID).Scan(&name2, pq.Array(&groups2)) + assert.NoError(suite.T(), err, "could select user info: %v", err) + assert.Equal(suite.T(), name, name2, "user info table did not update correctly") + assert.Equal(suite.T(), groups, groups2, "user info table did not update correctly") +} From 6700f46bc2b6add71101dbdde67aa3e8b79fd6fc Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 4 Nov 2024 11:22:57 +0100 Subject: [PATCH 08/15] fixup: use correct versions and db user --- .github/integration/sda-s3-integration.yml | 4 ++-- postgresql/initdb.d/04_grants.sql | 2 +- postgresql/migratedb.d/14.sql | 2 +- sda/cmd/auth/main.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 7d01b3c42..688ea7dea 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -330,8 +330,8 @@ services: - AUTH_RESIGNJWT=false - OIDC_ID=XC56EL11xx - OIDC_SECRET=wHPVQaYXmdDHg - - DB_PASSWORD=api - - DB_USER=api + - DB_PASSWORD=auth + - DB_USER=auth image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8889:8080" diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index 5c6169ac3..aa8a423f4 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -180,7 +180,7 @@ GRANT SELECT ON local_ega_ebi.file_dataset TO api; -------------------------------------------------------------------------------- CREATE ROLE auth; -GRANT USAGE ON SCHEMA sda TO api; +GRANT USAGE ON SCHEMA sda TO auth; GRANT SELECT, INSERT, UPDATE ON sda.userinfo TO auth; -------------------------------------------------------------------------------- diff --git a/postgresql/migratedb.d/14.sql b/postgresql/migratedb.d/14.sql index 795f6b3c1..24a9cff6e 100644 --- a/postgresql/migratedb.d/14.sql +++ b/postgresql/migratedb.d/14.sql @@ -40,7 +40,7 @@ BEGIN GRANT USAGE ON SCHEMA sda TO auth; GRANT SELECT, INSERT, UPDATE ON sda.userinfo TO auth; - GRANT base TO api, download, inbox, ingest, finalize, mapper, verify, auth; + GRANT base TO auth; ELSE RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; END IF; diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 8d25c7b83..7367a045d 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -430,8 +430,8 @@ func main() { log.Error(err) panic(err) } - if config.Auth.DB.Version < 8 { - log.Error("database schema v8 is required") + if config.Auth.DB.Version < 14 { + log.Error("database schema v14 is required") panic(err) } defer config.Auth.DB.Close() From 80e861a8dd9461360014063bc6efa5894bb2ba35 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 4 Nov 2024 14:59:19 +0100 Subject: [PATCH 09/15] Remove unnecessary import --- sda/internal/database/database.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 2cb11797a..651e097c0 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -8,9 +8,6 @@ import ( "time" log "github.com/sirupsen/logrus" - - // Needed implicitly to enable Postgres driver - _ "github.com/lib/pq" ) // DBConf stores information about how to connect to the database backend From c526a2707ee3870a8468913186779f6528329b49 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 12 Nov 2024 14:42:59 +0100 Subject: [PATCH 10/15] [postgres] update schema version --- postgresql/initdb.d/01_main.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index df4a8f95a..cc27cf3df 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -27,7 +27,8 @@ VALUES (0, now(), 'Created with version'), (10, now(), 'Create Inbox user'), (11, now(), 'Grant select permission to download on dataset_event_log'), (12, now(), 'Add key hash'), - (13, now(), 'Create API user'); + (13, now(), 'Create API user'), + (14, now(), 'Create Auth user'); -- Datasets are used to group files, and permissions are set on the dataset -- level From 1752d3d111319c1836f14f75e05eb1ebc71d411f Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 14 Nov 2024 15:10:18 +0100 Subject: [PATCH 11/15] [charts] add auth db user --- charts/sda-svc/README.md | 2 ++ charts/sda-svc/templates/_helpers.yaml | 8 +++++ .../sda-svc/templates/auth-certificate.yaml | 1 + charts/sda-svc/templates/auth-deploy.yaml | 31 +++++++++++++++++++ charts/sda-svc/templates/auth-secrets.yaml | 2 ++ charts/sda-svc/values.yaml | 4 +++ 6 files changed, 48 insertions(+) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index a1d38b365..a184b3d40 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -177,6 +177,8 @@ Parameter | Description | Default `credentials.api.dbUser` | Database user for api | `""` `credentials.api.dbPassword` | Database password for api | `""` `credentials.api.mqUser` | Broker user for api | `""` +`credentials.auth.dbUser` | Database user for auth | `""` +`credentials.auth.dbPassword` | Database password for auth | `""` `credentials.api.mqPassword` | Broker password for api | `""` `credentials.doa.dbUser` | Database user for doa | `""` `credentials.doa.dbPassword` | Database password for doa| `""` diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index c25468dff..7079c7176 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -147,6 +147,14 @@ Create chart name and version as used by the chart label. {{- ternary .Values.global.broker.password .Values.credentials.api.mqPassword (empty .Values.credentials.api.mqPassword) -}} {{- end -}} +{{/**/}} +{{- define "dbUserAuth" -}} +{{- ternary .Values.global.db.user .Values.credentials.auth.dbUser (empty .Values.credentials.auth.dbUser) -}} +{{- end -}} +{{- define "dbPassAuth" -}} +{{- ternary .Values.global.db.password .Values.credentials.auth.dbPassword (empty .Values.credentials.auth.dbPassword) -}} +{{- end -}} + {{/**/}} {{- define "dbUserSync" -}} {{- ternary .Values.global.db.user .Values.credentials.sync.dbUser (empty .Values.credentials.sync.dbUser) -}} diff --git a/charts/sda-svc/templates/auth-certificate.yaml b/charts/sda-svc/templates/auth-certificate.yaml index 7d7a2b743..8fe4e6aa1 100644 --- a/charts/sda-svc/templates/auth-certificate.yaml +++ b/charts/sda-svc/templates/auth-certificate.yaml @@ -20,6 +20,7 @@ spec: size: 256 usages: - server auth + - client auth # At least one of a DNS Name, URI, or IP address is required. dnsNames: - {{ template "sda.fullname" . }}-auth diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index 1d722b422..b7d2542f0 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -159,6 +159,37 @@ spec: - name: SERVER_KEY value: {{ template "tlsPath" . }}/tls.key {{- end }} + {{- if .Values.global.tls.enabled }} + - name: DB_CACERT + value: {{ include "tlsPath" . }}/ca.crt + {{- if ne "verify-none" .Values.global.db.sslMode }} + - name: DB_CLIENTCERT + value: {{ include "tlsPath" . }}/tls.crt + - name: DB_CLIENTKEY + value: {{ include "tlsPath" . }}/tls.key + {{- end }} + - name: DB_SSLMODE + value: {{ .Values.global.db.sslMode | quote }} + {{- else }} + - name: DB_SSLMODE + value: "disable" + {{- end }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-api + key: dbPassword + - name: DB_USER + valueFrom: + secretKeyRef: + name: {{ template "sda.fullname" . }}-api + key: dbUser + - name: DB_DATABASE + value: {{ default "lega" .Values.global.db.name | quote }} + - name: DB_HOST + value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} + - name: DB_PORT + value: {{ .Values.global.db.port | quote }} ports: - name: auth containerPort: 8080 diff --git a/charts/sda-svc/templates/auth-secrets.yaml b/charts/sda-svc/templates/auth-secrets.yaml index 0f692d054..81de9f9e6 100644 --- a/charts/sda-svc/templates/auth-secrets.yaml +++ b/charts/sda-svc/templates/auth-secrets.yaml @@ -15,6 +15,8 @@ data: cegaID: {{ .Values.global.cega.user | quote | trimall "\"" | b64enc }} cegaSecret: {{ .Values.global.cega.password | quote | trimall "\"" | b64enc }} {{- end }} + dbPassword: {{ required "DB password is required" (include "dbPassAuth" .) | b64enc }} + dbUser: {{ required "DB user is required" (include "dbUserAuth" .) | b64enc }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index f102bd84f..a74bbe93b 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -300,6 +300,10 @@ credentials: mqUser: "" mqPassword: "" + auth: + dbUser: "" + dbPassword: "" + doa: dbUser: "" dbPassword: "" From 4970fab7156d1b80b161e896f965640045c1c02f Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 21 Nov 2024 17:05:38 +0100 Subject: [PATCH 12/15] fix: make sure auth can access db instance --- sda/cmd/auth/main.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 7367a045d..2c878a404 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -402,6 +402,18 @@ func main() { app.Use(sess.Handler()) + // Connect to DB + authHandler.Config.DB, err = database.NewSDAdb(config.Database) + if err != nil { + log.Error(err) + panic(err) + } + if authHandler.Config.DB.Version < 14 { + log.Error("database schema v14 is required") + panic(err) + } + defer authHandler.Config.DB.Close() + app.RegisterView(iris.HTML(authHandler.htmlDir, ".html")) app.HandleDir("/public", iris.Dir(authHandler.staticDir)) @@ -424,18 +436,6 @@ func main() { log.Fatalf("Failed to read public key: %s", err.Error()) } - // Connect to DB - config.Auth.DB, err = database.NewSDAdb(config.Database) - if err != nil { - log.Error(err) - panic(err) - } - if config.Auth.DB.Version < 14 { - log.Error("database schema v14 is required") - panic(err) - } - defer config.Auth.DB.Close() - // Endpoint for client login info app.Get("/info", authHandler.getInfo) From dc8fbf3406e55903e864bd69ba2a1ceb93a81aa1 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 21 Nov 2024 17:18:59 +0100 Subject: [PATCH 13/15] [api] refactor: linter upset about defer before log.fatal --- sda/cmd/auth/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 2c878a404..378fc6a65 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -412,7 +412,6 @@ func main() { log.Error("database schema v14 is required") panic(err) } - defer authHandler.Config.DB.Close() app.RegisterView(iris.HTML(authHandler.htmlDir, ".html")) app.HandleDir("/public", iris.Dir(authHandler.staticDir)) @@ -439,6 +438,8 @@ func main() { // Endpoint for client login info app.Get("/info", authHandler.getInfo) + defer authHandler.Config.DB.Close() // needs to be after Fatalf + app.UseGlobal(globalHeaders) if config.Server.Cert != "" && config.Server.Key != "" { From 404191b9675df8589317b1964497d3053fb5ec21 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 25 Nov 2024 14:44:47 +0100 Subject: [PATCH 14/15] [auth] refactor: use panic to avoid fatalf, clean up logs --- sda/cmd/auth/main.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 378fc6a65..b2b8c17ed 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -266,7 +266,7 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { } err = auth.Config.DB.UpdateUserInfo(idStruct.User, idStruct.Profile, idStruct.Email, idStruct.EdupersonEntitlement) if err != nil { - log.Warnf("Could not log user info for %s (%s, %s)", idStruct.User, idStruct.Name, idStruct.Email) + log.Warn("Could not log user info.") } if auth.Config.ResignJwt { @@ -412,6 +412,7 @@ func main() { log.Error("database schema v14 is required") panic(err) } + defer authHandler.Config.DB.Close() app.RegisterView(iris.HTML(authHandler.htmlDir, ".html")) app.HandleDir("/public", iris.Dir(authHandler.staticDir)) @@ -432,14 +433,12 @@ func main() { authHandler.pubKey, err = readPublicKeyFile(authHandler.Config.PublicFile) if err != nil { - log.Fatalf("Failed to read public key: %s", err.Error()) + log.Panicf("Failed to read public key: %s", err.Error()) } // Endpoint for client login info app.Get("/info", authHandler.getInfo) - defer authHandler.Config.DB.Close() // needs to be after Fatalf - app.UseGlobal(globalHeaders) if config.Server.Cert != "" && config.Server.Key != "" { From 78a54bd86dce0d37f067bb0ffeff77fd1a402f57 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 29 Nov 2024 11:27:02 +0100 Subject: [PATCH 15/15] [tests] rework tests --- sda/internal/database/db_functions_test.go | 53 ++++++++++++---------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index de99d72d3..b71b8e6b8 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -932,45 +932,52 @@ func (suite *DatabaseTests) TestListUserDatasets() { assert.Equal(suite.T(), "test-user-dataset-01", datasets[0].DatasetID) } -func (suite *DatabaseTests) TestInsertUserInfo() { +func (suite *DatabaseTests) TestUpdateUserInfo() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - // Verify that a user can be inserted + // Insert a userID var groups []string userID, name, email := "12334556testuser@lifescience.ru", "Test User", "test.user@example.org" err = db.UpdateUserInfo(userID, name, email, groups) assert.NoError(suite.T(), err, "could not insert user info: %v", err) - var exists bool - err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.userinfo WHERE id=$1)", userID).Scan(&exists) - assert.NoError(suite.T(), err, "failed to verify user info existence") - assert.True(suite.T(), exists, "user info was not added to the database") - - // Insert new information about the user - groups = append(groups, "appleGroup", "bananaGroup") - name = "newName" - err = db.UpdateUserInfo(userID, name, email, groups) - assert.NoError(suite.T(), err, "could not insert updated user info: %v", err) + // Verify that the userID is connected to the details + var numRows int + err = db.DB.QueryRow("SELECT COUNT(*) FROM sda.userinfo WHERE id=$1", userID).Scan(&numRows) + assert.NoError(suite.T(), err, "could select user info: %v", err) + assert.Equal(suite.T(), 1, numRows, "there should be exactly 1 row about %v in userinfo table", userID) + var name2 string + err = db.DB.QueryRow("SELECT name FROM sda.userinfo WHERE id=$1", userID).Scan(&name2) + assert.NoError(suite.T(), err, "could not select user info: %v", err) + assert.Equal(suite.T(), name, name2, "user info table did not update correctly") } -func (suite *DatabaseTests) TestUpdateUserInfo() { +func (suite *DatabaseTests) TestUpdateUserInfo_newInfo() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - // Insert a userID + // Insert a user var groups []string userID, name, email := "12334556testuser@lifescience.ru", "Test User", "test.user@example.org" err = db.UpdateUserInfo(userID, name, email, groups) assert.NoError(suite.T(), err, "could not insert user info: %v", err) - // Verify that the userID is connected to the new details + var exists bool + err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.userinfo WHERE id=$1)", userID).Scan(&exists) + assert.NoError(suite.T(), err, "failed to verify user info existence") + assert.True(suite.T(), exists, "user info was not added to the database") + + // Insert new information about the user and verify that there is still only 1 row, + // and that this row is updated var numRows int err = db.DB.QueryRow("SELECT COUNT(*) FROM sda.userinfo WHERE id=$1", userID).Scan(&numRows) - assert.NoError(suite.T(), err, "could select user info: %v", err) - assert.Equal(suite.T(), 1, numRows, "there should be exactly 1 row about %v in userinfo table", userID) - var name2 string - var groups2 []string - err = db.DB.QueryRow("SELECT name, groups FROM sda.userinfo WHERE id=$1", userID).Scan(&name2, pq.Array(&groups2)) - assert.NoError(suite.T(), err, "could select user info: %v", err) - assert.Equal(suite.T(), name, name2, "user info table did not update correctly") - assert.Equal(suite.T(), groups, groups2, "user info table did not update correctly") + assert.NoError(suite.T(), err, "could not verify db count", err) + assert.Equal(suite.T(), 1, numRows, "there should be exactly one row in userinfo") + var dbgroups []string + groups = append(groups, "appleGroup", "bananaGroup") + name = "newName" + err = db.UpdateUserInfo(userID, name, email, groups) + assert.NoError(suite.T(), err, "could not insert updated user info: %v", err) + err = db.DB.QueryRow("SELECT groups FROM sda.userinfo WHERE id=$1", userID).Scan(pq.Array(&dbgroups)) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), groups, dbgroups) }