From baeb893f2520b45f36cd66de23db0bdc491543de Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Thu, 26 Sep 2024 17:37:41 +0200
Subject: [PATCH 01/17] Add api role in postgres DB

---
 postgresql/initdb.d/04_grants.sql | 25 ++++++++++++--
 postgresql/migratedb.d/13.sql     | 56 +++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 3 deletions(-)
 create mode 100644 postgresql/migratedb.d/13.sql

diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql
index 5cb5c9807..074b639a2 100644
--- a/postgresql/initdb.d/04_grants.sql
+++ b/postgresql/initdb.d/04_grants.sql
@@ -158,10 +158,29 @@ GRANT SELECT ON local_ega_ebi.file_dataset TO download;
 
 --------------------------------------------------------------------------------
 
+CREATE ROLE api;
+
+GRANT USAGE ON SCHEMA sda TO api;
+GRANT SELECT ON sda.files TO api;
+GRANT SELECT ON sda.file_dataset TO api;
+GRANT SELECT ON sda.checksums TO api;
+GRANT SELECT ON sda.file_event_log TO api;
+GRANT SELECT ON sda.encryption_keys TO api;
+GRANT INSERT ON sda.encryption_keys TO api;
+GRANT UPDATE ON sda.encryption_keys TO api;
+
+-- legacy schema
+GRANT USAGE ON SCHEMA local_ega TO api;
+GRANT USAGE ON SCHEMA local_ega_ebi TO api;
+GRANT SELECT ON local_ega.files TO api;
+GRANT SELECT ON local_ega_ebi.file TO api;
+GRANT SELECT ON local_ega_ebi.file_dataset TO api;
+
+--------------------------------------------------------------------------------
 -- lega_in permissions
-GRANT base, ingest, verify, finalize, sync TO lega_in;
+GRANT base, ingest, verify, finalize, sync, api TO lega_in;
 
 -- lega_out permissions
-GRANT mapper, download TO lega_out;
+GRANT mapper, download, api TO lega_out;
 
-GRANT base TO download, inbox, ingest, finalize, mapper, verify
+GRANT base TO api, download, inbox, ingest, finalize, mapper, verify
diff --git a/postgresql/migratedb.d/13.sql b/postgresql/migratedb.d/13.sql
new file mode 100644
index 000000000..859b977a9
--- /dev/null
+++ b/postgresql/migratedb.d/13.sql
@@ -0,0 +1,56 @@
+DO
+$$
+DECLARE
+-- The version we know how to do migration from, at the end of a successful migration
+-- we will no longer be at this version.
+  sourcever INTEGER := 12;
+  changes VARCHAR := 'Create API user';
+BEGIN
+-- No explicit transaction handling here, this all happens in a transaction
+-- automatically
+  IF (select max(version) from sda.dbschema_version) = sourcever then
+    RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1;
+    RAISE NOTICE 'Changes: %', changes;
+    INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes);
+
+    -- Temporary function for creating roles if they do not already exist.
+    CREATE FUNCTION create_role_if_not_exists(role_name NAME) RETURNS void AS $created$
+    BEGIN
+        IF EXISTS (
+            SELECT FROM pg_catalog.pg_roles
+            WHERE  rolname = role_name) THEN
+                RAISE NOTICE 'Role "%" already exists. Skipping.', role_name;
+        ELSE
+            BEGIN
+                EXECUTE format('CREATE ROLE %I', role_name);
+            EXCEPTION
+                WHEN duplicate_object THEN
+                    RAISE NOTICE 'Role "%" was just created by a concurrent transaction. Skipping.', role_name;
+            END;
+        END IF;
+    END;
+    $created$ LANGUAGE plpgsql;
+
+    PERFORM create_role_if_not_exists('api');
+    GRANT USAGE ON SCHEMA sda TO api;
+    GRANT SELECT ON sda.files TO api;
+    GRANT SELECT ON sda.file_event_log TO api;
+    GRANT SELECT ON sda.file_dataset TO api;
+    GRANT SELECT  ON sda.checksums TO api;
+    GRANT SELECT, INSERT, UPDATE ON sda.encryption_keys TO api;
+    GRANT USAGE ON SCHEMA local_ega TO api;
+    GRANT USAGE ON SCHEMA local_ega_ebi TO api;
+    GRANT SELECT ON local_ega.files TO api;
+    GRANT SELECT ON local_ega_ebi.file TO api;
+    GRANT SELECT ON local_ega_ebi.file_dataset TO api;
+
+    GRANT base TO api, download, inbox, ingest, finalize, mapper, verify;
+
+    -- Drop temporary user creation function
+    DROP FUNCTION create_role_if_not_exists;
+
+  ELSE
+    RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1;
+  END IF;
+END
+$$

From 2a6992c577d86431005e7fd531bbda563067058c Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Thu, 26 Sep 2024 17:47:44 +0200
Subject: [PATCH 02/17] Create credentials for api user in scripts

---
 .github/integration/scripts/make_db_credentials.sh  | 2 +-
 .github/integration/scripts/make_sda_credentials.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/integration/scripts/make_db_credentials.sh b/.github/integration/scripts/make_db_credentials.sh
index 5d2e40e7d..bef674cb2 100644
--- a/.github/integration/scripts/make_db_credentials.sh
+++ b/.github/integration/scripts/make_db_credentials.sh
@@ -4,7 +4,7 @@ set -e
 apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
 apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client >/dev/null
 
-for n in download finalize inbox ingest mapper sync verify; do
+for n in api download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h migrate -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh
index 36fa29f8e..3b4289313 100644
--- a/.github/integration/scripts/make_sda_credentials.sh
+++ b/.github/integration/scripts/make_sda_credentials.sh
@@ -14,7 +14,7 @@ apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl post
 pip install --upgrade pip > /dev/null
 pip install aiohttp Authlib joserfc requests > /dev/null
 
-for n in download finalize inbox ingest mapper sync verify; do
+for n in api download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
     psql -U postgres -h postgres -d sda -c "GRANT base TO $n;"
@@ -92,4 +92,4 @@ if [ ! -f "/shared/grpcurl" ]; then
     echo "downloading grpcurl"
     latest_grpculr=$(curl --retry 100 -sL https://api.github.com/repos/fullstorydev/grpcurl/releases/latest | jq -r '.name' | sed -e 's/v//')
     curl --retry 100 -s -L "https://github.com/fullstorydev/grpcurl/releases/download/v${latest_grpculr}/grpcurl_${latest_grpculr}_linux_x86_64.tar.gz" | tar -xz -C /shared/ && chmod +x /shared/grpcurl
-fi
\ No newline at end of file
+fi

From 6b18a7f18649356430c509f7e7037d74924a92ba Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Fri, 27 Sep 2024 16:00:25 +0200
Subject: [PATCH 03/17] Endpoint for adding the key in the database

- Add the structure
- Add the database functions for inserting the key and its description
in the encryption_keys table
- Add the endpoint and the function for calling the db functions
---
 sda/cmd/api/api.go                    | 33 ++++++++++++++++++++++++---
 sda/internal/database/db_functions.go | 33 +++++++++++++++++++++++++++
 sda/internal/schema/schema.go         |  5 ++++
 3 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index 4dde5b1fe..b6323c993 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -23,9 +23,11 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-var Conf *config.Config
-var err error
-var auth *userauth.ValidateFromToken
+var (
+	Conf *config.Config
+	err  error
+	auth *userauth.ValidateFromToken
+)
 
 func main() {
 	Conf, err = config.NewConfig("api")
@@ -80,6 +82,7 @@ func setup(config *config.Config) *http.Server {
 		r.POST("/file/accession", isAdmin(), setAccession)             // assign accession ID to a file
 		r.POST("/dataset/create", isAdmin(), createDataset)            // maps a set of files to a dataset
 		r.POST("/dataset/release/*dataset", isAdmin(), releaseDataset) // Releases a dataset to be accessible
+		r.POST("/key/hashed", isAdmin(), addHashedKey)                 // Adds a hashed key to the database
 		r.GET("/users", isAdmin(), listActiveUsers)                    // Lists all users
 		r.GET("/users/:username/files", isAdmin(), listUserFiles)      // Lists all unmapped files for a user
 	}
@@ -414,3 +417,27 @@ func listUserFiles(c *gin.Context) {
 	c.Writer.Header().Set("Content-Type", "application/json")
 	c.JSON(200, files)
 }
+
+// addHashedKey function adds a hashed public key and its description to the database
+func addHashedKey(c *gin.Context) {
+	var keyhash schema.KeyhashInsertion
+	if err := c.BindJSON(&keyhash); err != nil {
+		c.AbortWithStatusJSON(
+			http.StatusBadRequest,
+			gin.H{
+				"error":  "json decoding : " + err.Error(),
+				"status": http.StatusBadRequest,
+			},
+		)
+
+		return
+	}
+
+	err = Conf.API.DB.AddKeyHash(keyhash.Hash, keyhash.Description)
+    if err != nil {
+        c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+        return
+    }
+
+	c.Status(http.StatusOK)
+}
diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go
index 11abc15e2..3b7975edf 100644
--- a/sda/internal/database/db_functions.go
+++ b/sda/internal/database/db_functions.go
@@ -748,3 +748,36 @@ func (dbs *SDAdb) getDatasetStatus(datasetID string) (string, error) {
 
 	return status, nil
 }
+
+// AddKeyHash adds a key hash and key description in the encryption_keys table
+func (dbs *SDAdb) AddKeyHash(keyHash, keyDescription string) error {
+	var err error
+	// 2, 4, 8, 16, 32 seconds between each retry event.
+	for count := 1; count <= RetryTimes; count++ {
+		err = dbs.addKeyHash(keyHash, keyDescription)
+		if err == nil {
+			break
+		}
+		time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second)
+	}
+
+	return err
+}
+
+func (dbs *SDAdb) addKeyHash(keyHash, keyDescription string) error {
+	dbs.checkAndReconnectIfNeeded()
+	db := dbs.DB
+
+	const query = "INSERT INTO sda.encryption_keys(key_hash, description) VALUES($1, $2) ON CONFLICT DO NOTHING;"
+
+	result, err := db.Exec(query, keyHash, keyDescription)
+	if err != nil {
+		return err
+	}
+
+	if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 {
+		return errors.New("key hash already exists or no rows were updated")
+	}
+
+	return nil
+}
diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go
index c01440469..f8d309422 100644
--- a/sda/internal/schema/schema.go
+++ b/sda/internal/schema/schema.go
@@ -179,3 +179,8 @@ type SyncMetadata struct {
 type Metadata struct {
 	Metadata interface{}
 }
+
+type KeyhashInsertion struct {
+	Hash        string `json:"hash"`
+	Description string `json:"description"`
+}

From 070341014f6ab655ab7710b1cb09ab8fdf59d1da Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Mon, 30 Sep 2024 22:15:59 +0200
Subject: [PATCH 04/17] Unit test for db function

Add unit test for adding key hash in the encryptio_keys table function
---
 sda/internal/database/db_functions_test.go | 29 +++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go
index 5ce59d4b6..ec8a7bd30 100644
--- a/sda/internal/database/db_functions_test.go
+++ b/sda/internal/database/db_functions_test.go
@@ -593,7 +593,13 @@ func (suite *DatabaseTests) TestGetDatasetStatus() {
 		assert.Equal(suite.T(), fileID, corrID)
 
 		checksum := fmt.Sprintf("%x", sha256.New().Sum(nil))
-		fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999}
+		fileInfo := FileInfo{
+			fmt.Sprintf("%x", sha256.New().Sum(nil)),
+			1234,
+			filePath,
+			checksum,
+			999,
+		}
 		err = db.SetArchived(fileInfo, fileID, corrID)
 		if err != nil {
 			suite.FailNow("failed to mark file as Archived")
@@ -634,3 +640,24 @@ func (suite *DatabaseTests) TestGetDatasetStatus() {
 	assert.NoError(suite.T(), err, "got (%v) when no error weas expected")
 	assert.Equal(suite.T(), "deprecated", status)
 }
+
+func (suite *DatabaseTests) TestAddKey() {
+	db, err := NewSDAdb(suite.dbConf)
+	assert.NoError(suite.T(), err, "got (%v) when creating new connection", err)
+
+	// Test registering a new key and its description
+	keyHex := `2d2d2d2d2d424547494e204352595054344748205055424c4943204b4559
+2d2d2d2d2d0a65574d394166785761626d775354627657346e736650646b
+432f6953412b3849712b6e516232555a2b6d6f3d0a2d2d2d2d2d454e4420
+4352595054344748205055424c4943204b45592d2d2d2d2d0a`
+	keyDescription := "this is a test key"
+	err = db.addKeyHash(keyHex, keyDescription)
+	assert.NoError(suite.T(), err, "failed to register key in database")
+
+	// Verify that the key was added
+	var exists bool
+	err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.encryption_keys WHERE key_hash=$1 AND description=$2)", keyHex, keyDescription).
+		Scan(&exists)
+	assert.NoError(suite.T(), err, "failed to verify key hash existence")
+	assert.True(suite.T(), exists, "key hash was not added to the database")
+}

From 2560bf964d1e0cf35eda4caeaaf0621e4c863337 Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Tue, 1 Oct 2024 16:38:34 +0200
Subject: [PATCH 05/17] Unit tests for adding key hash

---
 sda/cmd/api/api_test.go | 60 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index 67eff0872..f17a25bff 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -25,6 +25,7 @@ import (
 	"github.com/neicnordic/sensitive-data-archive/internal/config"
 	"github.com/neicnordic/sensitive-data-archive/internal/database"
 	"github.com/neicnordic/sensitive-data-archive/internal/helper"
+	"github.com/neicnordic/sensitive-data-archive/internal/schema"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
 	log "github.com/sirupsen/logrus"
@@ -1281,3 +1282,62 @@ func (suite *TestSuite) TestListUserFiles() {
 	assert.NoError(suite.T(), err, "failed to list users from DB")
 	assert.Equal(suite.T(), 2, len(files))
 }
+
+func (suite *TestSuite) TestAddHashedKeySuccess() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+
+	r := gin.Default()
+	r.POST("/key/hashed", isAdmin(), addHashedKey)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	client := &http.Client{}
+	assert.NoError(suite.T(), setupJwtAuth())
+
+	// Create a valid request body
+	keyhash := schema.KeyhashInsertion{
+		Hash:        "somehashedkey",
+		Description: "Test key description",
+	}
+	body, err := json.Marshal(keyhash)
+	assert.NoError(suite.T(), err)
+
+	req, err := http.NewRequest("POST", ts.URL+"/key/hashed", bytes.NewBuffer(body))
+	assert.NoError(suite.T(), err)
+	req.Header.Add("Authorization", "Bearer "+suite.Token)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), http.StatusOK, resp.StatusCode)
+	defer resp.Body.Close()
+}
+
+func (suite *TestSuite) TestAddHashedKeyInvalidJSON() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+
+	r := gin.Default()
+	r.POST("/key/hashed", isAdmin(), addHashedKey)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	client := &http.Client{}
+	assert.NoError(suite.T(), setupJwtAuth())
+
+	// Create an invalid request body
+	body := []byte(`{"invalid_json"}`)
+
+	req, err := http.NewRequest("POST", ts.URL+"/key/hashed", bytes.NewBuffer(body))
+	assert.NoError(suite.T(), err)
+	req.Header.Add("Authorization", "Bearer "+suite.Token)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode)
+	defer resp.Body.Close()
+}

From a12800bc593c3797df1bdb11b3473bbffd0ae7df Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Tue, 1 Oct 2024 16:52:51 +0200
Subject: [PATCH 06/17] Replace with api db credentials in compose

---
 .github/integration/sda-s3-integration.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml
index 71f141090..7ff6d2ab6 100644
--- a/.github/integration/sda-s3-integration.yml
+++ b/.github/integration/sda-s3-integration.yml
@@ -292,8 +292,8 @@ services:
       - BROKER_PASSWORD=ingest
       - BROKER_USER=ingest
       - BROKER_ROUTINGKEY=ingest
-      - DB_PASSWORD=download
-      - DB_USER=download
+      - DB_PASSWORD=api
+      - DB_USER=api
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
     ports:
       - "8090:8080"

From 9ed38219672813cc32025af46a0f11e980407e75 Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Mon, 14 Oct 2024 17:40:14 +0200
Subject: [PATCH 07/17] Improve error logging and response to the user

- The write time out of the http server was increased for proper
response in case of duplicate
- Spit the errors to confict and server error
- Add logging
- Update function comments
---
 sda/cmd/api/api.go | 38 ++++++++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index b6323c993..562383e6f 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -96,7 +96,7 @@ func setup(config *config.Config) *http.Server {
 		TLSNextProto:      make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
 		ReadHeaderTimeout: 20 * time.Second,
 		ReadTimeout:       5 * time.Minute,
-		WriteTimeout:      20 * time.Second,
+		WriteTimeout:      2 * time.Minute,
 	}
 
 	return srv
@@ -418,7 +418,12 @@ func listUserFiles(c *gin.Context) {
 	c.JSON(200, files)
 }
 
-// addHashedKey function adds a hashed public key and its description to the database
+// addHashedKey handles the addition of a hashed key to the database.
+// It expects a JSON payload containing the key hash and its description.
+// If the JSON payload is invalid, it responds with a 400 Bad Request status.
+// If there is no row update in the database, it responds with a 409 Conflict status
+// If the database insertion fails, it responds with a 500 Internal Server Error status.
+// On success, it responds with a 200 OK status.
 func addHashedKey(c *gin.Context) {
 	var keyhash schema.KeyhashInsertion
 	if err := c.BindJSON(&keyhash); err != nil {
@@ -430,14 +435,35 @@ func addHashedKey(c *gin.Context) {
 			},
 		)
 
+		log.Errorf("Invalid JSON payload: %v", err)
+
 		return
 	}
 
 	err = Conf.API.DB.AddKeyHash(keyhash.Hash, keyhash.Description)
-    if err != nil {
-        c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
-        return
-    }
+	if err != nil {
+		if strings.Contains(err.Error(), "key hash already exists") {
+			c.AbortWithStatusJSON(
+				http.StatusConflict,
+				gin.H{
+					"error":  err.Error(),
+					"status": http.StatusConflict,
+				},
+			)
+			log.Error("Key hash already exists")
+		} else {
+			c.AbortWithStatusJSON(
+				http.StatusInternalServerError,
+				gin.H{
+					"error":  err.Error(),
+					"status": http.StatusInternalServerError,
+				},
+			)
+			log.Errorf("Database insertion failed: %v", err)
+		}
+
+		return
+	}
 
 	c.Status(http.StatusOK)
 }

From a8374ffbf1d7a1183e0e1a3ee2115dd930e2d01e Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Mon, 14 Oct 2024 17:40:52 +0200
Subject: [PATCH 08/17] Update api readme

---
 sda/cmd/api/api.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 80854d83a..45f7dee6d 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -119,6 +119,24 @@ Admin endpoints are only available to a set of whitelisted users specified in th
     - `401` Token user is not in the list of admins.
     - `500` Internal error due to DB failure.
 
+- `/key/hashed`
+  - accepts `POST` requests with the hex hash of the key and its description
+  - registers the key hash in the database.
+
+  - Error codes
+    - `200` Query execute ok.
+    - `400` Error due to bad payload.
+    - `401` Token user is not in the list of admins.
+    - `409` Key hash already exists in the database.
+    - `500` Internal error due to DB failures.
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"hash": "key-hex-hash", "description": "this is the key description"}' https://HOSTNAME/kes/hashed
+    ```
+
+
 #### Configure Admin users
 
 The users that should have administrative access can be set in two ways:

From d8a838eacb825e2629020639fb2b12bb2e471172 Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Tue, 15 Oct 2024 17:00:09 +0200
Subject: [PATCH 09/17] Grand more privileges in the api role

---
 postgresql/initdb.d/04_grants.sql | 2 ++
 postgresql/migratedb.d/13.sql     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql
index 074b639a2..1eadb112b 100644
--- a/postgresql/initdb.d/04_grants.sql
+++ b/postgresql/initdb.d/04_grants.sql
@@ -166,6 +166,8 @@ GRANT SELECT ON sda.file_dataset TO api;
 GRANT SELECT ON sda.checksums TO api;
 GRANT SELECT ON sda.file_event_log TO api;
 GRANT SELECT ON sda.encryption_keys TO api;
+GRANT SELECT ON sda.datasets TO api;
+GRANT SELECT ON sda.dataset_event_log TO api;
 GRANT INSERT ON sda.encryption_keys TO api;
 GRANT UPDATE ON sda.encryption_keys TO api;
 
diff --git a/postgresql/migratedb.d/13.sql b/postgresql/migratedb.d/13.sql
index 859b977a9..13f07a047 100644
--- a/postgresql/migratedb.d/13.sql
+++ b/postgresql/migratedb.d/13.sql
@@ -37,6 +37,8 @@ BEGIN
     GRANT SELECT ON sda.file_event_log TO api;
     GRANT SELECT ON sda.file_dataset TO api;
     GRANT SELECT  ON sda.checksums TO api;
+    GRANT SELECT ON sda.datasets TO api;
+    GRANT SELECT ON sda.dataset_event_log TO api;
     GRANT SELECT, INSERT, UPDATE ON sda.encryption_keys TO api;
     GRANT USAGE ON SCHEMA local_ega TO api;
     GRANT USAGE ON SCHEMA local_ega_ebi TO api;

From b8bfe9f7ff2822dbe57975fb9b3b18108186daea Mon Sep 17 00:00:00 2001
From: kostas-kou <kkoumpouras@gmail.com>
Date: Tue, 15 Oct 2024 17:13:24 +0200
Subject: [PATCH 10/17] Change endpoint

---
 sda/cmd/api/api.go      | 2 +-
 sda/cmd/api/api.md      | 2 +-
 sda/cmd/api/api_test.go | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index 562383e6f..c7eacc945 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -82,7 +82,7 @@ func setup(config *config.Config) *http.Server {
 		r.POST("/file/accession", isAdmin(), setAccession)             // assign accession ID to a file
 		r.POST("/dataset/create", isAdmin(), createDataset)            // maps a set of files to a dataset
 		r.POST("/dataset/release/*dataset", isAdmin(), releaseDataset) // Releases a dataset to be accessible
-		r.POST("/key/hashed", isAdmin(), addHashedKey)                 // Adds a hashed key to the database
+		r.POST("/c4gh-keys/add", isAdmin(), addHashedKey)              // Adds a key hash to the database
 		r.GET("/users", isAdmin(), listActiveUsers)                    // Lists all users
 		r.GET("/users/:username/files", isAdmin(), listUserFiles)      // Lists all unmapped files for a user
 	}
diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 45f7dee6d..30e0507aa 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -119,7 +119,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
     - `401` Token user is not in the list of admins.
     - `500` Internal error due to DB failure.
 
-- `/key/hashed`
+- `/c4gh-keys/add`
   - accepts `POST` requests with the hex hash of the key and its description
   - registers the key hash in the database.
 
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index f17a25bff..c65b04c0a 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -1289,7 +1289,7 @@ func (suite *TestSuite) TestAddHashedKeySuccess() {
 	Conf.API.Admins = []string{"dummy"}
 
 	r := gin.Default()
-	r.POST("/key/hashed", isAdmin(), addHashedKey)
+	r.POST("/c4gh-keys/add", isAdmin(), addHashedKey)
 	ts := httptest.NewServer(r)
 	defer ts.Close()
 
@@ -1304,7 +1304,7 @@ func (suite *TestSuite) TestAddHashedKeySuccess() {
 	body, err := json.Marshal(keyhash)
 	assert.NoError(suite.T(), err)
 
-	req, err := http.NewRequest("POST", ts.URL+"/key/hashed", bytes.NewBuffer(body))
+	req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body))
 	assert.NoError(suite.T(), err)
 	req.Header.Add("Authorization", "Bearer "+suite.Token)
 	req.Header.Add("Content-Type", "application/json")
@@ -1321,7 +1321,7 @@ func (suite *TestSuite) TestAddHashedKeyInvalidJSON() {
 	Conf.API.Admins = []string{"dummy"}
 
 	r := gin.Default()
-	r.POST("/key/hashed", isAdmin(), addHashedKey)
+	r.POST("/c4gh-keys/add", isAdmin(), addHashedKey)
 	ts := httptest.NewServer(r)
 	defer ts.Close()
 
@@ -1331,7 +1331,7 @@ func (suite *TestSuite) TestAddHashedKeyInvalidJSON() {
 	// Create an invalid request body
 	body := []byte(`{"invalid_json"}`)
 
-	req, err := http.NewRequest("POST", ts.URL+"/key/hashed", bytes.NewBuffer(body))
+	req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body))
 	assert.NoError(suite.T(), err)
 	req.Header.Add("Authorization", "Bearer "+suite.Token)
 	req.Header.Add("Content-Type", "application/json")

From 98f2e2f68b43f699b7b1f7440b954c9e3d181931 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Mon, 21 Oct 2024 12:30:13 +0200
Subject: [PATCH 11/17] Ensure payload is a C4GH public key

---
 sda/cmd/api/api.go            | 44 +++++++++++++++++++++++++++++++----
 sda/internal/schema/schema.go |  4 ++--
 2 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index c7eacc945..b821951a6 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -1,8 +1,11 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
+	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -15,6 +18,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/lestrrat-go/jwx/v2/jwk"
+	"github.com/neicnordic/crypt4gh/keys"
 	"github.com/neicnordic/sensitive-data-archive/internal/broker"
 	"github.com/neicnordic/sensitive-data-archive/internal/config"
 	"github.com/neicnordic/sensitive-data-archive/internal/database"
@@ -82,7 +86,7 @@ func setup(config *config.Config) *http.Server {
 		r.POST("/file/accession", isAdmin(), setAccession)             // assign accession ID to a file
 		r.POST("/dataset/create", isAdmin(), createDataset)            // maps a set of files to a dataset
 		r.POST("/dataset/release/*dataset", isAdmin(), releaseDataset) // Releases a dataset to be accessible
-		r.POST("/c4gh-keys/add", isAdmin(), addHashedKey)              // Adds a key hash to the database
+		r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash)               // Adds a key hash to the database
 		r.GET("/users", isAdmin(), listActiveUsers)                    // Lists all users
 		r.GET("/users/:username/files", isAdmin(), listUserFiles)      // Lists all unmapped files for a user
 	}
@@ -424,9 +428,9 @@ func listUserFiles(c *gin.Context) {
 // If there is no row update in the database, it responds with a 409 Conflict status
 // If the database insertion fails, it responds with a 500 Internal Server Error status.
 // On success, it responds with a 200 OK status.
-func addHashedKey(c *gin.Context) {
-	var keyhash schema.KeyhashInsertion
-	if err := c.BindJSON(&keyhash); err != nil {
+func addC4ghHash(c *gin.Context) {
+	var c4gh schema.C4ghPubKey
+	if err := c.BindJSON(&c4gh); err != nil {
 		c.AbortWithStatusJSON(
 			http.StatusBadRequest,
 			gin.H{
@@ -440,7 +444,37 @@ func addHashedKey(c *gin.Context) {
 		return
 	}
 
-	err = Conf.API.DB.AddKeyHash(keyhash.Hash, keyhash.Description)
+	b64d, err := base64.StdEncoding.DecodeString(c4gh.PubKey)
+	if err != nil {
+		c.AbortWithStatusJSON(
+			http.StatusBadRequest,
+			gin.H{
+				"error":  "base64 decoding : " + err.Error(),
+				"status": http.StatusBadRequest,
+			},
+		)
+
+		log.Errorf("Invalid JSON payload: %v", err)
+
+		return
+	}
+
+	pubKey, err := keys.ReadPublicKey(bytes.NewReader(b64d))
+	if err != nil {
+		c.AbortWithStatusJSON(
+			http.StatusBadRequest,
+			gin.H{
+				"error":  "not a public key : " + err.Error(),
+				"status": http.StatusBadRequest,
+			},
+		)
+
+		log.Errorf("Invalid JSON payload: %v", err)
+
+		return
+	}
+
+	err = Conf.API.DB.AddKeyHash(hex.EncodeToString(pubKey[:]), c4gh.Description)
 	if err != nil {
 		if strings.Contains(err.Error(), "key hash already exists") {
 			c.AbortWithStatusJSON(
diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go
index f8d309422..503e7e308 100644
--- a/sda/internal/schema/schema.go
+++ b/sda/internal/schema/schema.go
@@ -180,7 +180,7 @@ type Metadata struct {
 	Metadata interface{}
 }
 
-type KeyhashInsertion struct {
-	Hash        string `json:"hash"`
+type C4ghPubKey struct {
+	PubKey      string `json:"pubkey"`
 	Description string `json:"description"`
 }

From b5d77f3377c339923f22c0cee445d000e22de976 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Mon, 21 Oct 2024 12:30:54 +0200
Subject: [PATCH 12/17] Expand tests, validate payload

---
 sda/cmd/api/api_test.go | 45 +++++++++++++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 6 deletions(-)

diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index c65b04c0a..9332893ae 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -1283,13 +1283,13 @@ func (suite *TestSuite) TestListUserFiles() {
 	assert.Equal(suite.T(), 2, len(files))
 }
 
-func (suite *TestSuite) TestAddHashedKeySuccess() {
+func (suite *TestSuite) TestAddC4ghHash() {
 	gin.SetMode(gin.ReleaseMode)
 	assert.NoError(suite.T(), setupJwtAuth())
 	Conf.API.Admins = []string{"dummy"}
 
 	r := gin.Default()
-	r.POST("/c4gh-keys/add", isAdmin(), addHashedKey)
+	r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash)
 	ts := httptest.NewServer(r)
 	defer ts.Close()
 
@@ -1297,8 +1297,8 @@ func (suite *TestSuite) TestAddHashedKeySuccess() {
 	assert.NoError(suite.T(), setupJwtAuth())
 
 	// Create a valid request body
-	keyhash := schema.KeyhashInsertion{
-		Hash:        "somehashedkey",
+	keyhash := schema.C4ghPubKey{
+		PubKey:      "LS0tLS1CRUdJTiBDUllQVDRHSCBQVUJMSUMgS0VZLS0tLS0KdWxGRUF6SmZZNEplUEVDZWd3YmJrVVdLNnZ2SE9SWStqMTRGdVpWVnYwND0KLS0tLS1FTkQgQ1JZUFQ0R0ggUFVCTElDIEtFWS0tLS0tCg==",
 		Description: "Test key description",
 	}
 	body, err := json.Marshal(keyhash)
@@ -1313,15 +1313,21 @@ func (suite *TestSuite) TestAddHashedKeySuccess() {
 	assert.NoError(suite.T(), err)
 	assert.Equal(suite.T(), http.StatusOK, resp.StatusCode)
 	defer resp.Body.Close()
+
+	// Isert pubkey again and expect error
+	resp2, err := client.Do(req)
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), http.StatusConflict, resp2.StatusCode)
+	defer resp2.Body.Close()
 }
 
-func (suite *TestSuite) TestAddHashedKeyInvalidJSON() {
+func (suite *TestSuite) TestAddC4ghHash_emptyJson() {
 	gin.SetMode(gin.ReleaseMode)
 	assert.NoError(suite.T(), setupJwtAuth())
 	Conf.API.Admins = []string{"dummy"}
 
 	r := gin.Default()
-	r.POST("/c4gh-keys/add", isAdmin(), addHashedKey)
+	r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash)
 	ts := httptest.NewServer(r)
 	defer ts.Close()
 
@@ -1341,3 +1347,30 @@ func (suite *TestSuite) TestAddHashedKeyInvalidJSON() {
 	assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode)
 	defer resp.Body.Close()
 }
+
+func (suite *TestSuite) TestAddC4ghHash_notBase64() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+
+	r := gin.Default()
+	r.POST("/c4gh-keys/add", isAdmin(), addC4ghHash)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	client := &http.Client{}
+	assert.NoError(suite.T(), setupJwtAuth())
+
+	// Create an invalid request body
+	body := []byte(`{"pubkey": "asdkjsahfd=", "decription": ""}`)
+
+	req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body))
+	assert.NoError(suite.T(), err)
+	req.Header.Add("Authorization", "Bearer "+suite.Token)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode)
+	defer resp.Body.Close()
+}

From f930065523376e6b96dee3905e87a0f3e197e5d3 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Mon, 21 Oct 2024 12:56:30 +0200
Subject: [PATCH 13/17] Update documentation

---
 sda/cmd/api/api.go | 6 +++---
 sda/cmd/api/api.md | 3 +--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index b821951a6..b0efe1476 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -422,10 +422,10 @@ func listUserFiles(c *gin.Context) {
 	c.JSON(200, files)
 }
 
-// addHashedKey handles the addition of a hashed key to the database.
-// It expects a JSON payload containing the key hash and its description.
+// addC4ghHash handles the addition of a hashed public key to the database.
+// It expects a JSON payload containing the base64 encoded public key and its description.
 // If the JSON payload is invalid, it responds with a 400 Bad Request status.
-// If there is no row update in the database, it responds with a 409 Conflict status
+// If the hash is already in the database, it responds with a 409 Conflict status
 // If the database insertion fails, it responds with a 500 Internal Server Error status.
 // On success, it responds with a 200 OK status.
 func addC4ghHash(c *gin.Context) {
diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 30e0507aa..fa96b4aeb 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -133,10 +133,9 @@ Admin endpoints are only available to a set of whitelisted users specified in th
     Example:
 
     ```bash
-    curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"hash": "key-hex-hash", "description": "this is the key description"}' https://HOSTNAME/kes/hashed
+    curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /PATH/TO/c4gh.pub)"'", "description": "this is the key description"}' https://HOSTNAME/c4gh-keys/add
     ```
 
-
 #### Configure Admin users
 
 The users that should have administrative access can be set in two ways:

From 8cadbb418e87fafb0b82f8d52c77fdf1f6d1a0af Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Mon, 21 Oct 2024 13:36:04 +0200
Subject: [PATCH 14/17] Cleanup db_functions test

---
 sda/internal/database/db_functions_test.go | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go
index ec8a7bd30..2b2235736 100644
--- a/sda/internal/database/db_functions_test.go
+++ b/sda/internal/database/db_functions_test.go
@@ -641,23 +641,19 @@ func (suite *DatabaseTests) TestGetDatasetStatus() {
 	assert.Equal(suite.T(), "deprecated", status)
 }
 
-func (suite *DatabaseTests) TestAddKey() {
+func (suite *DatabaseTests) TestAddKeyHash() {
 	db, err := NewSDAdb(suite.dbConf)
 	assert.NoError(suite.T(), err, "got (%v) when creating new connection", err)
 
 	// Test registering a new key and its description
-	keyHex := `2d2d2d2d2d424547494e204352595054344748205055424c4943204b4559
-2d2d2d2d2d0a65574d394166785761626d775354627657346e736650646b
-432f6953412b3849712b6e516232555a2b6d6f3d0a2d2d2d2d2d454e4420
-4352595054344748205055424c4943204b45592d2d2d2d2d0a`
+	keyHex := `cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23`
 	keyDescription := "this is a test key"
-	err = db.addKeyHash(keyHex, keyDescription)
+	err = db.AddKeyHash(keyHex, keyDescription)
 	assert.NoError(suite.T(), err, "failed to register key in database")
 
 	// Verify that the key was added
 	var exists bool
-	err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.encryption_keys WHERE key_hash=$1 AND description=$2)", keyHex, keyDescription).
-		Scan(&exists)
+	err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.encryption_keys WHERE key_hash=$1 AND description=$2)", keyHex, keyDescription).Scan(&exists)
 	assert.NoError(suite.T(), err, "failed to verify key hash existence")
 	assert.True(suite.T(), exists, "key hash was not added to the database")
 }

From 30664f26983f2c30d26ecfc0ee9fc3c1084fde63 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Mon, 21 Oct 2024 14:30:07 +0200
Subject: [PATCH 15/17] Update API integration tests

---
 .github/integration/sda-s3-integration.yml    |  5 ++-
 .../tests/sda/01_install_dependencies.sh      | 15 +++++++++
 .../integration/tests/sda/09_healthchecks.sh  | 14 --------
 .../integration/tests/sda/10_upload_test.sh   | 23 ++++++-------
 .../tests/sda/11_api-getfiles_test.sh         | 32 +++++++++++++++++--
 5 files changed, 59 insertions(+), 30 deletions(-)
 create mode 100644 .github/integration/tests/sda/01_install_dependencies.sh

diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml
index 7ff6d2ab6..bedb2577e 100644
--- a/.github/integration/sda-s3-integration.yml
+++ b/.github/integration/sda-s3-integration.yml
@@ -289,9 +289,8 @@ services:
       rabbitmq:
         condition: service_healthy
     environment:
-      - BROKER_PASSWORD=ingest
-      - BROKER_USER=ingest
-      - BROKER_ROUTINGKEY=ingest
+      - BROKER_PASSWORD=api
+      - BROKER_USER=api
       - DB_PASSWORD=api
       - DB_USER=api
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
diff --git a/.github/integration/tests/sda/01_install_dependencies.sh b/.github/integration/tests/sda/01_install_dependencies.sh
new file mode 100644
index 000000000..9020696c1
--- /dev/null
+++ b/.github/integration/tests/sda/01_install_dependencies.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+# install tools if missing
+for t in curl expect jq openssh-client postgresql-client xxd; do
+    if [ ! "$(command -v $t)" ]; then
+        if [ "$(id -u)" != 0 ]; then
+            echo "$t is missing, unable to install it"
+            exit 1
+        fi
+
+        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
+        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
+    fi
+done
diff --git a/.github/integration/tests/sda/09_healthchecks.sh b/.github/integration/tests/sda/09_healthchecks.sh
index f1dc369c8..c58c84c8e 100644
--- a/.github/integration/tests/sda/09_healthchecks.sh
+++ b/.github/integration/tests/sda/09_healthchecks.sh
@@ -1,20 +1,6 @@
 #!/bin/sh
 set -e
 
-# install tools if missing
-for t in curl  jq ; do
-    if [ ! "$(command -v $t)" ]; then
-        if [ "$(id -u)" != 0 ]; then
-            echo "$t is missing, unable to install it"
-            exit 1
-        fi
-
-        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
-        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
-    fi
-done
-
-
 # Test the s3inbox's healthchecks, GET /health and HEAD /
 response="$(curl -s -k -LI "http://s3inbox:8000" -o /dev/null -w "%{http_code}\n")"
 if [ "$response" != "200" ]; then
diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh
index 266f73c7b..fb50a4c47 100644
--- a/.github/integration/tests/sda/10_upload_test.sh
+++ b/.github/integration/tests/sda/10_upload_test.sh
@@ -6,18 +6,19 @@ if [ -z "$STORAGETYPE" ]; then
     exit 1
 fi
 
-# install tools if missing
-for t in curl expect jq openssh-client postgresql-client; do
-    if [ ! "$(command -v $t)" ]; then
-        if [ "$(id -u)" != 0 ]; then
-            echo "$t is missing, unable to install it"
-            exit 1
+if [ "$STORAGETYPE" = "posix" ]; then
+    for t in curl jq openssh-client postgresql-client; do
+        if [ ! "$(command -v $t)" ]; then
+            if [ "$(id -u)" != 0 ]; then
+                echo "$t is missing, unable to install it"
+                exit 1
+            fi
+
+            apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
+            apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
         fi
-
-        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
-        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
-    fi
-done
+    done
+fi
 
 cd shared || true
 
diff --git a/.github/integration/tests/sda/11_api-getfiles_test.sh b/.github/integration/tests/sda/11_api-getfiles_test.sh
index fb168da50..a99bb9eaa 100644
--- a/.github/integration/tests/sda/11_api-getfiles_test.sh
+++ b/.github/integration/tests/sda/11_api-getfiles_test.sh
@@ -3,11 +3,39 @@ set -e
 
 # Test the API files endpoint
 token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
-curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token"
-response="$(curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')"
+response="$(curl -s -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')"
 if [ "$response" != "uploaded" ]; then
 	echo "API returned incorrect value, expected ready got: $response"
 	exit 1
 fi
 
+# test inserting a c4gh public key hash
+payload=$(
+	jq -c -n \
+		--arg description "this is the key description" \
+		--arg pubkey "$( base64 -w0 /shared/c4gh.pub.pem)" \
+		'$ARGS.named'
+)
+
+resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")"
+if [ "$resp" != "200" ]; then
+	echo "Error when adding a public key hash, expected 200 got: $resp"
+	exit 1
+fi
+
+# again to verify we get an error
+resp="$(curl -s -k -L  -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")"
+if [ "$resp" != "409" ]; then
+	echo "Error when adding a public key hash, expected 409 got: $resp"
+	exit 1
+fi
+
+manual_hash=$(sed -n '2p' /shared/c4gh.pub.pem | base64 -d -w0 | xxd -c64 -ps)
+
+db_hash=$(psql -U postgres -h postgres -d sda -At -c "SELECT key_hash FROM sda.encryption_keys WHERE description = 'this is the key description';")
+if [ "$db_hash" != "$manual_hash" ]; then
+	echo "wrong hash in the database, expected $manual_hash got $db_hash"
+	exit 1
+fi
+
 echo "api test completed successfully"

From b488b7376914632692bfd6b2ab4fb88d67163736 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Mon, 21 Oct 2024 15:13:14 +0200
Subject: [PATCH 16/17] [db_functions] fail fast if error is expected

---
 sda/internal/database/db_functions.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go
index 3b7975edf..f9cf2eb66 100644
--- a/sda/internal/database/db_functions.go
+++ b/sda/internal/database/db_functions.go
@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"math"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -671,7 +672,7 @@ func (dbs *SDAdb) GetCorrID(user, path string) (string, error) {
 	// 2, 4, 8, 16, 32 seconds between each retry event.
 	for count := 1; count <= RetryTimes; count++ {
 		corrID, err = dbs.getCorrID(user, path)
-		if err == nil {
+		if err == nil || strings.Contains(err.Error(), "sql: no rows in result set") {
 			break
 		}
 		time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second)
@@ -755,7 +756,7 @@ func (dbs *SDAdb) AddKeyHash(keyHash, keyDescription string) error {
 	// 2, 4, 8, 16, 32 seconds between each retry event.
 	for count := 1; count <= RetryTimes; count++ {
 		err = dbs.addKeyHash(keyHash, keyDescription)
-		if err == nil {
+		if err == nil || strings.Contains(err.Error(), "key hash already exists") {
 			break
 		}
 		time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second)

From 1db8a02b334465deafb4e7103ec5b9272b36d256 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 22 Oct 2024 08:23:23 +0200
Subject: [PATCH 17/17] [integration tests] split out posix upload into a
 separate file

---
 .github/integration/sda-posix-integration.yml |   2 +-
 .../integration/tests/sda/10.1_upload_test.sh |  86 +++++++++++
 .../integration/tests/sda/10_upload_test.sh   | 146 ++++++------------
 3 files changed, 135 insertions(+), 99 deletions(-)
 create mode 100644 .github/integration/tests/sda/10.1_upload_test.sh

diff --git a/.github/integration/sda-posix-integration.yml b/.github/integration/sda-posix-integration.yml
index fe88f0563..1433096cc 100644
--- a/.github/integration/sda-posix-integration.yml
+++ b/.github/integration/sda-posix-integration.yml
@@ -182,7 +182,7 @@ services:
     container_name: tester
     command:
       - "bash"
-      - "/tests/sda/10_upload_test.sh"
+      - "/tests/sda/10.1_upload_test.sh"
     depends_on:
       inbox:
         condition: service_started
diff --git a/.github/integration/tests/sda/10.1_upload_test.sh b/.github/integration/tests/sda/10.1_upload_test.sh
new file mode 100644
index 000000000..d7fcea9c6
--- /dev/null
+++ b/.github/integration/tests/sda/10.1_upload_test.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+set -e
+
+if [ -z "$STORAGETYPE" ]; then
+    echo "STORAGETYPE not set, exiting"
+    exit 1
+fi
+
+if [ "$STORAGETYPE" = "s3" ]; then
+    exit 0
+fi
+
+for t in curl jq openssh-client postgresql-client; do
+    if [ ! "$(command -v $t)" ]; then
+        if [ "$(id -u)" != 0 ]; then
+            echo "$t is missing, unable to install it"
+            exit 1
+        fi
+
+        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
+        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
+    fi
+done
+
+cd shared || true
+
+## verify that messages exists in MQ
+URI=http://rabbitmq:15672
+if [ -n "$PGSSLCERT" ]; then
+    URI=https://rabbitmq:15671
+fi
+## empty all queues ##
+for q in accession archived backup completed inbox ingest mappings verified; do
+    curl -s -k -u guest:guest -X DELETE "$URI/api/queues/sda/$q/contents"
+done
+## truncate database
+psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;"
+
+if [ "$STORAGETYPE" = "posix" ]; then
+    for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
+        echo "downloading $file"
+        curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
+        if [ ! -f "$file.c4gh" ]; then
+            yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
+        fi
+
+        sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF
+    put "${file}"
+    dir
+    ls -al
+    exit
+EOF
+    done
+
+    ## reupload a file under a different name
+    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF
+    put NA12878.bam.c4gh NB12878.bam.c4gh
+    dir
+    ls -al
+    exit
+EOF
+
+    ## reupload a file with the same name
+    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF
+    put NA12878.bam.c4gh
+    dir
+    ls -al
+    exit
+EOF
+
+fi
+
+echo "waiting for upload to complete"
+RETRY_TIMES=0
+until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do
+    echo "waiting for upload to complete"
+    RETRY_TIMES=$((RETRY_TIMES + 1))
+    if [ "$RETRY_TIMES" -eq 30 ]; then
+        echo "::error::Time out while waiting for upload to complete"
+        exit 1
+    fi
+    sleep 2
+done
+
+
+echo "files uploaded successfully"
diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh
index fb50a4c47..e20939cf4 100644
--- a/.github/integration/tests/sda/10_upload_test.sh
+++ b/.github/integration/tests/sda/10_upload_test.sh
@@ -6,20 +6,6 @@ if [ -z "$STORAGETYPE" ]; then
     exit 1
 fi
 
-if [ "$STORAGETYPE" = "posix" ]; then
-    for t in curl jq openssh-client postgresql-client; do
-        if [ ! "$(command -v $t)" ]; then
-            if [ "$(id -u)" != 0 ]; then
-                echo "$t is missing, unable to install it"
-                exit 1
-            fi
-
-            apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
-            apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
-        fi
-    done
-fi
-
 cd shared || true
 
 ## verify that messages exists in MQ
@@ -34,57 +20,22 @@ done
 ## truncate database
 psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files CASCADE;"
 
-if [ "$STORAGETYPE" = "posix" ]; then
-    for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
-        echo "downloading $file"
-        curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
-        if [ ! -f "$file.c4gh" ]; then
-            yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
-        fi
-
-        sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF
-    put "${file}"
-    dir
-    ls -al
-    exit
-EOF
-    done
-
-    ## reupload a file under a different name
-    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF
-    put NA12878.bam.c4gh NB12878.bam.c4gh
-    dir
-    ls -al
-    exit
-EOF
-
-    ## reupload a file with the same name
-    sftp -i /shared/keys/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o KbdInteractiveAuthentication=no -o User=dummy@example.com -P 2222 inbox <<-EOF
-    put NA12878.bam.c4gh
-    dir
-    ls -al
-    exit
-EOF
-
-fi
+pip -q install s3cmd
 
-if [ "$STORAGETYPE" = "s3" ]; then
-    pip -q install s3cmd
+for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
+    curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
+    if [ ! -f "$file.c4gh" ]; then
+        yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
+    fi
+    s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/
+done
 
-    for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
-        curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
-        if [ ! -f "$file.c4gh" ]; then
-            yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
-        fi
-        s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/
-    done
+## reupload a file under a different name
+s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh
 
-    ## reupload a file under a different name
-    s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh
+## reupload a file with the same name
+s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/
 
-    ## reupload a file with the same name
-    s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/
-fi
 
 echo "waiting for upload to complete"
 RETRY_TIMES=0
@@ -98,51 +49,50 @@ until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messag
     sleep 2
 done
 
-if [ "$STORAGETYPE" = "s3" ]; then
-    num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
-    if [ "$num_rows" -ne 5 ]; then
-        echo "database queries for register_files failed, expected 5 got $num_rows"
-        exit 1
-    fi
+num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
+if [ "$num_rows" -ne 5 ]; then
+    echo "database queries for register_files failed, expected 5 got $num_rows"
+    exit 1
+fi
 
-    num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
-    if [ "$num_log_rows" -ne 12 ]; then
-        echo "database queries for file_event_logs failed, expected 12 got $num_log_rows"
-        exit 1
-    fi
+num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
+if [ "$num_log_rows" -ne 12 ]; then
+    echo "database queries for file_event_logs failed, expected 12 got $num_log_rows"
+    exit 1
+fi
 
-    ## test with token from OIDC service
-    echo "testing with OIDC token"
-    newToken=$(curl http://oidc:8080/tokens | jq '.[0]')
-    cp s3cfg oidc_s3cfg
-    sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg
+## test with token from OIDC service
+echo "testing with OIDC token"
+newToken=$(curl http://oidc:8080/tokens | jq '.[0]')
+cp s3cfg oidc_s3cfg
+sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg
 
-    s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh
+s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh
 
-    ## verify that messages exists in MQ
+## verify that messages exists in MQ
+echo "waiting for upload to complete"
+RETRY_TIMES=0
+until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do
     echo "waiting for upload to complete"
-    RETRY_TIMES=0
-    until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do
-        echo "waiting for upload to complete"
-        RETRY_TIMES=$((RETRY_TIMES + 1))
-        if [ "$RETRY_TIMES" -eq 30 ]; then
-            echo "::error::Time out while waiting for upload to complete"
-            exit 1
-        fi
-        sleep 2
-    done
-
-    num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
-    if [ "$num_rows" -ne 6 ]; then
-        echo "database queries for register_files failed, expected 6 got $num_rows"
+    RETRY_TIMES=$((RETRY_TIMES + 1))
+    if [ "$RETRY_TIMES" -eq 30 ]; then
+        echo "::error::Time out while waiting for upload to complete"
         exit 1
     fi
+    sleep 2
+done
 
-    num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
-    if [ "$num_log_rows" -ne 14 ]; then
-        echo "database queries for file_event_logs failed, expected 14 got $num_log_rows"
-        exit 1
-    fi
+num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;")
+if [ "$num_rows" -ne 6 ]; then
+    echo "database queries for register_files failed, expected 6 got $num_rows"
+    exit 1
 fi
 
+num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;")
+if [ "$num_log_rows" -ne 14 ]; then
+    echo "database queries for file_event_logs failed, expected 14 got $num_log_rows"
+    exit 1
+fi
+
+
 echo "files uploaded successfully"