From 949ded7d5dd64914d41855c828a1b72de5a9d87c Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Mon, 12 Feb 2024 09:20:08 +0100
Subject: [PATCH 01/16] [api] add handle function to check if user is an admin

---
 sda/cmd/api/api.go            | 19 ++++++++++
 sda/cmd/api/api_test.go       | 67 ++++++++++++++++++++++++++++++++++-
 sda/internal/config/config.go |  1 +
 3 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index 52a7ef83d..e473f2f3e 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"slices"
 	"syscall"
 	"time"
 
@@ -174,3 +175,21 @@ func getFiles(c *gin.Context) {
 	// Return response
 	c.JSON(200, files)
 }
+
+func isAdmin() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		token, err := auth.Authenticate(c.Request)
+		if err != nil {
+			log.Debugln("bad token")
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
+
+			return
+		}
+		if !slices.Contains(Conf.API.Admins, token.Subject()) {
+			log.Debugf("%s is not an admin", token.Subject())
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "not authorized"})
+
+			return
+		}
+	}
+}
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index c3b7b22bf..005f2ab14 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -333,7 +334,7 @@ func (suite *TestSuite) TestReadinessResponse() {
 
 // Initialise configuration and create jwt keys
 func (suite *TestSuite) SetupTest() {
-
+	log.SetLevel(log.DebugLevel)
 	suite.Path = "/tmp/keys/"
 	suite.KeyName = "example.demo"
 
@@ -513,3 +514,67 @@ func (suite *TestSuite) TestAPIGetFiles() {
 func TestApiTestSuite(t *testing.T) {
 	suite.Run(t, new(TestSuite))
 }
+
+func testEndpoint(c *gin.Context) {
+	c.JSON(200, gin.H{"ok": true})
+}
+
+func (suite *TestSuite) TestIsAdmin_NoToken() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("GET", "/", nil)
+	_, router := gin.CreateTestContext(w)
+	router.GET("/", isAdmin(), testEndpoint)
+
+	// no token should not be allowed
+	router.ServeHTTP(w, r)
+	badResponse := w.Result()
+	defer badResponse.Body.Close()
+	b, _ := io.ReadAll(badResponse.Body)
+	assert.Equal(suite.T(), http.StatusUnauthorized, badResponse.StatusCode)
+	assert.Contains(suite.T(), string(b), "no access token supplied")
+}
+func (suite *TestSuite) TestIsAdmin_BadUser() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"foo", "bar"}
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("GET", "/", nil)
+	_, router := gin.CreateTestContext(w)
+	router.GET("/", isAdmin(), testEndpoint)
+
+	// non admin user should not be allowed
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+	router.ServeHTTP(w, r)
+	notAdmin := w.Result()
+	defer notAdmin.Body.Close()
+	b, _ := io.ReadAll(notAdmin.Body)
+	assert.Equal(suite.T(), http.StatusUnauthorized, notAdmin.StatusCode)
+	assert.Contains(suite.T(), string(b), "not authorized")
+}
+func (suite *TestSuite) TestIsAdmin() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"foo", "bar", "dummy"}
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("GET", "/", nil)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/", isAdmin(), testEndpoint)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	b, _ := io.ReadAll(okResponse.Body)
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+	assert.Contains(suite.T(), string(b), "ok")
+}
diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go
index 75324fdf7..6277416c0 100644
--- a/sda/internal/config/config.go
+++ b/sda/internal/config/config.go
@@ -74,6 +74,7 @@ type SyncAPIConf struct {
 }
 
 type APIConf struct {
+	Admins     []string
 	CACert     string
 	ServerCert string
 	ServerKey  string

From bc3051098f808e8abbdcb3a0802540c9e67ac118 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Fri, 19 Jul 2024 12:41:28 +0200
Subject: [PATCH 02/16] [internal/database] add function to get correlation ID

Returns the correlation ID for a `submision_filepath` - `user` combination as long as the file is not part of a dataset.
---
 sda/internal/database/db_functions.go      | 30 ++++++++++++++++
 sda/internal/database/db_functions_test.go | 42 ++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go
index 3c32fe300..ae40adfac 100644
--- a/sda/internal/database/db_functions.go
+++ b/sda/internal/database/db_functions.go
@@ -666,3 +666,33 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) {
 
 	return files, nil
 }
+
+func (dbs *SDAdb) GetCorrID(user, path string) (string, error) {
+	var (
+		corrID string
+		err    error
+	)
+	// 2, 4, 8, 16, 32 seconds between each retry event.
+	for count := 1; count <= RetryTimes; count++ {
+		corrID, err = dbs.getCorrID(user, path)
+		if err == nil {
+			break
+		}
+		time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second)
+	}
+
+	return corrID, err
+}
+func (dbs *SDAdb) getCorrID(user, path string) (string, error) {
+	dbs.checkAndReconnectIfNeeded()
+	db := dbs.DB
+	const query = "SELECT DISTINCT correlation_id FROM sda.file_event_log e RIGHT JOIN sda.files f ON e.file_id = f.id WHERE f.submission_file_path = $1 and f.submission_user = $2 AND NOT EXISTS (SELECT file_id FROM sda.file_dataset WHERE file_id = f.id)"
+
+	var corrID string
+	err := db.QueryRow(query, path, user).Scan(&corrID)
+	if err != nil {
+		return "", err
+	}
+
+	return corrID, nil
+}
diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go
index 3066e030b..6fb774df6 100644
--- a/sda/internal/database/db_functions_test.go
+++ b/sda/internal/database/db_functions_test.go
@@ -468,3 +468,45 @@ func (suite *DatabaseTests) TestGetUserFiles() {
 		assert.Equal(suite.T(), "ready", fileInfo.Status, "incorrect file status")
 	}
 }
+
+func (suite *DatabaseTests) TestGetCorrID() {
+	db, err := NewSDAdb(suite.dbConf)
+	assert.NoError(suite.T(), err, "got (%v) when creating new connection", err)
+
+	filePath := "/testuser/file10.c4gh"
+	user := "testuser"
+
+	fileID, err := db.RegisterFile(filePath, user)
+	assert.NoError(suite.T(), err, "failed to register file in database")
+	err = db.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+	assert.NoError(suite.T(), err, "failed to update satus of file in database")
+
+	corrID, err := db.GetCorrID(user, filePath)
+	assert.NoError(suite.T(), err, "failed to get correlation ID of file in database")
+	assert.Equal(suite.T(), fileID, corrID)
+
+	checksum := fmt.Sprintf("%x", sha256.New().Sum(nil))
+	fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/testuser/file10.c4gh", checksum, 999}
+	err = db.SetArchived(fileInfo, fileID, corrID)
+	assert.NoError(suite.T(), err, "failed to mark file as Archived")
+
+	err = db.SetVerified(fileInfo, fileID, corrID)
+	assert.NoError(suite.T(), err, "failed to mark file as Verified")
+
+	stableID := "TEST:get-corr-id"
+	err = db.SetAccessionID(stableID, fileID)
+	assert.NoError(suite.T(), err, "got (%v) when setting stable ID: %s, %s", err, stableID, fileID)
+
+	diSet := map[string][]string{
+		"dataset-corr-id": {"TEST:get-corr-id"},
+	}
+
+	for di, acs := range diSet {
+		err := db.MapFilesToDataset(di, acs)
+		assert.NoError(suite.T(), err, "failed to map file to dataset")
+	}
+
+	corrID2, err := db.GetCorrID(user, filePath)
+	assert.Error(suite.T(), err, "failed to get correlation ID of file in database")
+	assert.Equal(suite.T(), "", corrID2)
+}

From 50da039be949305a3841c30bc9e8984d93b9e077 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Fri, 19 Jul 2024 12:41:58 +0200
Subject: [PATCH 03/16] [api] Add endpoint to trigger ingestion of a file

---
 sda/cmd/api/api.go      |  43 ++++++++++
 sda/cmd/api/api.md      |  16 +++-
 sda/cmd/api/api_test.go | 171 +++++++++++++++++++++++++++++++++++++++-
 3 files changed, 226 insertions(+), 4 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index e473f2f3e..b423f2970 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"os"
@@ -16,6 +17,7 @@ import (
 	"github.com/neicnordic/sensitive-data-archive/internal/broker"
 	"github.com/neicnordic/sensitive-data-archive/internal/config"
 	"github.com/neicnordic/sensitive-data-archive/internal/database"
+	"github.com/neicnordic/sensitive-data-archive/internal/schema"
 	"github.com/neicnordic/sensitive-data-archive/internal/userauth"
 	log "github.com/sirupsen/logrus"
 )
@@ -71,6 +73,8 @@ func setup(config *config.Config) *http.Server {
 	r := gin.Default()
 	r.GET("/ready", readinessResponse)
 	r.GET("/files", getFiles)
+	// admin endpoints below here
+	r.POST("/file/ingest", isAdmin(), ingestFile) // start ingestion of a file
 
 	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
 
@@ -193,3 +197,42 @@ func isAdmin() gin.HandlerFunc {
 		}
 	}
 }
+
+func ingestFile(c *gin.Context) {
+	var ingest schema.IngestionTrigger
+	if err := c.BindJSON(&ingest); err != nil {
+		c.AbortWithStatusJSON(
+			http.StatusBadRequest,
+			gin.H{
+				"error":  "json decoding : " + err.Error(),
+				"status": http.StatusBadRequest,
+			},
+		)
+
+		return
+	}
+
+	ingest.Type = "ingest"
+	marshaledMsg, _ := json.Marshal(&ingest)
+	if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil {
+		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+
+		return
+	}
+
+	corrID, err := Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
+	err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "ingest", marshaledMsg)
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
+	c.Status(http.StatusOK)
+}
\ No newline at end of file
diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 36da9cf16..7c1564e47 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -16,4 +16,18 @@ Endpoints:
       $ curl 'https://server/files' -H "Authorization: Bearer $token"
     [{"inboxPath":"requester_demo.org/data/file1.c4gh","fileStatus":"uploaded","createAt":"2023-11-13T10:12:43.144242Z"}] 
     ```
-     If the `token` is invalid, 401 is returned.
\ No newline at end of file
+     If the `token` is invalid, 401 is returned.
+
+### Admin endpoints
+
+Admin endpoints are only available to a set of whitelisted users specified in the application config.
+
+- `/file/ingest`
+  - accepts `POST` requests with JSON data with the format: `{"filepath": "</PATH/TO/FILE/IN/INBOX>", "user": "<USERNAME>"}`
+  - triggers the ingestion of the file.
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/ingest
+    ```
\ No newline at end of file
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index 005f2ab14..d569286e4 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"database/sql"
 	"encoding/json"
 	"fmt"
@@ -30,6 +31,7 @@ import (
 )
 
 var dbPort, mqPort, OIDCport int
+var BrokerAPI string
 
 func TestMain(m *testing.M) {
 	if _, err := os.Stat("/.dockerenv"); err == nil {
@@ -107,10 +109,10 @@ func TestMain(m *testing.M) {
 	}
 
 	mqPort, _ = strconv.Atoi(rabbitmq.GetPort("5672/tcp"))
-	mqHostAndPort := rabbitmq.GetHostPort("15672/tcp")
+	BrokerAPI = rabbitmq.GetHostPort("15672/tcp")
 
 	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest(http.MethodGet, "http://"+mqHostAndPort+"/api/users", http.NoBody)
+	req, err := http.NewRequest(http.MethodPut, "http://"+BrokerAPI+"/api/queues/%2F/ingest", http.NoBody)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -333,7 +335,7 @@ func (suite *TestSuite) TestReadinessResponse() {
 }
 
 // Initialise configuration and create jwt keys
-func (suite *TestSuite) SetupTest() {
+func (suite *TestSuite) SetupSuite() {
 	log.SetLevel(log.DebugLevel)
 	suite.Path = "/tmp/keys/"
 	suite.KeyName = "example.demo"
@@ -375,6 +377,16 @@ func (suite *TestSuite) SetupTest() {
 	Conf.API.DB, err = database.NewSDAdb(Conf.Database)
 	assert.NoError(suite.T(), err)
 
+	Conf.Broker = broker.MQConf{
+		Host:     "localhost",
+		Port:     mqPort,
+		User:     "guest",
+		Password: "guest",
+		Exchange: "",
+	}
+	Conf.API.MQ, err = broker.NewMQ(Conf.Broker)
+	assert.NoError(suite.T(), err)
+
 }
 
 func (suite *TestSuite) TestDatabasePingCheck() {
@@ -578,3 +590,156 @@ func (suite *TestSuite) TestIsAdmin() {
 	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
 	assert.Contains(suite.T(), string(b), "ok")
 }
+
+func (suite *TestSuite) TestIngestFile() {
+	user := "dummy"
+	filePath := "/inbox/dummy/file10.c4gh"
+
+	fileID, err := Conf.API.DB.RegisterFile(filePath, user)
+	assert.NoError(suite.T(), err, "failed to register file in database")
+	err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+	assert.NoError(suite.T(), err, "failed to update satus of file in database")
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	type ingest struct {
+		FilePath string `json:"filepath"`
+		User     string `json:"user"`
+	}
+	ingestMsg, _ := json.Marshal(ingest{User: user, FilePath: filePath})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg))
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/file/ingest", ingestFile)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	// verify that the message shows up in the queue
+	time.Sleep(10*time.Second) // this is needed to ensure we don't get any false negatives
+	client := http.Client{Timeout: 5 * time.Second}
+	req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/%2F/ingest", http.NoBody)
+	req.SetBasicAuth("guest", "guest")
+	res, err := client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	var data struct {
+		MessagesReady int `json:"messages_ready"`
+	}
+	body, err := io.ReadAll(res.Body)
+	res.Body.Close()
+	assert.NoError(suite.T(), err, "failed to read response from broker")
+	err = json.Unmarshal(body, &data)
+	assert.NoError(suite.T(), err, "failed to unmarshal response")
+	assert.Equal(suite.T(), 1, data.MessagesReady)
+}
+
+func (suite *TestSuite) TestIngestFile_NoUser() {
+	user := "dummy"
+	filePath := "/inbox/dummy/file10.c4gh"
+
+	fileID, err := Conf.API.DB.RegisterFile(filePath, user)
+	assert.NoError(suite.T(), err, "failed to register file in database")
+	err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+	assert.NoError(suite.T(), err, "failed to update satus of file in database")
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	type ingest struct {
+		FilePath string `json:"filepath"`
+		User     string `json:"user"`
+	}
+	ingestMsg, _ := json.Marshal(ingest{User: "", FilePath: filePath})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg))
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/file/ingest", ingestFile)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
+}
+func (suite *TestSuite) TestIngestFile_WrongUser() {
+	user := "dummy"
+	filePath := "/inbox/dummy/file10.c4gh"
+
+	fileID, err := Conf.API.DB.RegisterFile(filePath, user)
+	assert.NoError(suite.T(), err, "failed to register file in database")
+	err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+	assert.NoError(suite.T(), err, "failed to update satus of file in database")
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	type ingest struct {
+		FilePath string `json:"filepath"`
+		User     string `json:"user"`
+	}
+	ingestMsg, _ := json.Marshal(ingest{User: "foo", FilePath: filePath})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg))
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/file/ingest", ingestFile)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	b, _ := io.ReadAll(okResponse.Body)
+	assert.Equal(suite.T(), http.StatusInternalServerError, okResponse.StatusCode)
+	assert.Contains(suite.T(), string(b), "sql: no rows in result set")
+}
+
+func (suite *TestSuite) TestIngestFile_WrongFilePath() {
+	user := "dummy"
+	filePath := "/inbox/dummy/file10.c4gh"
+
+	fileID, err := Conf.API.DB.RegisterFile(filePath, user)
+	assert.NoError(suite.T(), err, "failed to register file in database")
+	err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+	assert.NoError(suite.T(), err, "failed to update satus of file in database")
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	type ingest struct {
+		FilePath string `json:"filepath"`
+		User     string `json:"user"`
+	}
+	ingestMsg, _ := json.Marshal(ingest{User: "dummy", FilePath: "bad/path"})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("POST", "/file/ingest", bytes.NewBuffer(ingestMsg))
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/file/ingest", isAdmin(), ingestFile)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	b, _ := io.ReadAll(okResponse.Body)
+	assert.Equal(suite.T(), http.StatusInternalServerError, okResponse.StatusCode)
+	assert.Contains(suite.T(), string(b), "sql: no rows in result set")
+}

From b65c66d9788be8258756648c85449369d15309cb Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Fri, 19 Jul 2024 12:42:24 +0200
Subject: [PATCH 04/16] [Schemas] username and filepath must have a minimum
 length

---
 sda/schemas/federated/ingestion-trigger.json | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sda/schemas/federated/ingestion-trigger.json b/sda/schemas/federated/ingestion-trigger.json
index 87bb270e6..120ea3ae2 100644
--- a/sda/schemas/federated/ingestion-trigger.json
+++ b/sda/schemas/federated/ingestion-trigger.json
@@ -99,6 +99,7 @@
             "type": "string",
             "title": "The username",
             "description": "The username",
+            "minLength": 2,
             "examples": [
                 "user.name@central-ega.eu"
             ]
@@ -108,6 +109,7 @@
             "type": "string",
             "title": "The new filepath",
             "description": "The new filepath",
+            "minLength": 2,
             "examples": [
                 "/ega/inbox/user.name@central-ega.eu/the-file.c4gh"
             ]

From 8cf4c58bea0a87d261ce0ae6b0561be2079a3420 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 23 Jul 2024 14:01:10 +0200
Subject: [PATCH 05/16] [api][tests] use prebuild MQ image

---
 sda/cmd/api/api_test.go | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index d569286e4..a12180d27 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -95,8 +95,8 @@ func TestMain(m *testing.M) {
 
 	// pulls an image, creates a container based on it and runs it
 	rabbitmq, err := pool.RunWithOptions(&dockertest.RunOptions{
-		Repository: "rabbitmq",
-		Tag:        "3-management-alpine",
+		Repository: "ghcr.io/neicnordic/sensitive-data-archive",
+		Tag:        "v0.3.89-rabbitmq",
 	}, func(config *docker.HostConfig) {
 		// set AutoRemove to true so that stopped container goes away by itself
 		config.AutoRemove = true
@@ -105,14 +105,17 @@ func TestMain(m *testing.M) {
 		}
 	})
 	if err != nil {
+		if err := pool.Purge(postgres); err != nil {
+			log.Fatalf("Could not purge resource: %s", err)
+		}
 		log.Fatalf("Could not start resource: %s", err)
 	}
 
 	mqPort, _ = strconv.Atoi(rabbitmq.GetPort("5672/tcp"))
 	BrokerAPI = rabbitmq.GetHostPort("15672/tcp")
 
-	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest(http.MethodPut, "http://"+BrokerAPI+"/api/queues/%2F/ingest", http.NoBody)
+	client := http.Client{Timeout: 30 * time.Second}
+	req, err := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/", http.NoBody)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -233,7 +236,8 @@ func (suite *TestSuite) TestShutdown() {
 		Port:     mqPort,
 		User:     "guest",
 		Password: "guest",
-		Exchange: "amq.default",
+		Exchange: "sda",
+		Vhost:    "/sda",
 	}
 	Conf.API.MQ, err = broker.NewMQ(Conf.Broker)
 	assert.NoError(suite.T(), err)
@@ -274,6 +278,7 @@ func (suite *TestSuite) TestReadinessResponse() {
 		User:     "guest",
 		Password: "guest",
 		Exchange: "amq.default",
+		Vhost:    "/sda",
 	}
 	Conf.API.MQ, err = broker.NewMQ(Conf.Broker)
 	assert.NoError(suite.T(), err)
@@ -382,7 +387,8 @@ func (suite *TestSuite) SetupSuite() {
 		Port:     mqPort,
 		User:     "guest",
 		Password: "guest",
-		Exchange: "",
+		Exchange: "sda",
+		Vhost:    "/sda",
 	}
 	Conf.API.MQ, err = broker.NewMQ(Conf.Broker)
 	assert.NoError(suite.T(), err)
@@ -624,9 +630,9 @@ func (suite *TestSuite) TestIngestFile() {
 	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
 
 	// verify that the message shows up in the queue
-	time.Sleep(10*time.Second) // this is needed to ensure we don't get any false negatives
+	time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives
 	client := http.Client{Timeout: 5 * time.Second}
-	req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/%2F/ingest", http.NoBody)
+	req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/ingest", http.NoBody)
 	req.SetBasicAuth("guest", "guest")
 	res, err := client.Do(req)
 	assert.NoError(suite.T(), err, "failed to query broker")

From c8cd1389c8956d125ca54c9e930f1d8b5dbe80be Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 23 Jul 2024 14:01:44 +0200
Subject: [PATCH 06/16] [tests][api] Ensure DB and MQ connection are up before
 each test starts

---
 sda/cmd/api/api_test.go | 48 +++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index a12180d27..ccc6c22d3 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -271,29 +271,6 @@ func (suite *TestSuite) TestReadinessResponse() {
 	ts := httptest.NewServer(r)
 	defer ts.Close()
 
-	Conf = &config.Config{}
-	Conf.Broker = broker.MQConf{
-		Host:     "localhost",
-		Port:     mqPort,
-		User:     "guest",
-		Password: "guest",
-		Exchange: "amq.default",
-		Vhost:    "/sda",
-	}
-	Conf.API.MQ, err = broker.NewMQ(Conf.Broker)
-	assert.NoError(suite.T(), err)
-
-	Conf.Database = database.DBConf{
-		Host:     "localhost",
-		Port:     dbPort,
-		User:     "postgres",
-		Password: "rootpasswd",
-		Database: "sda",
-		SslMode:  "disable",
-	}
-	Conf.API.DB, err = database.NewSDAdb(Conf.Database)
-	assert.NoError(suite.T(), err)
-
 	res, err := http.Get(ts.URL + "/ready")
 	assert.NoError(suite.T(), err)
 	assert.Equal(suite.T(), http.StatusOK, res.StatusCode)
@@ -395,6 +372,31 @@ func (suite *TestSuite) SetupSuite() {
 
 }
 
+func (suite *TestSuite) SetupTest() {
+	log.Print("Setup DB for my test")
+	Conf.Database = database.DBConf{
+		Host:     "localhost",
+		Port:     dbPort,
+		User:     "postgres",
+		Password: "rootpasswd",
+		Database: "sda",
+		SslMode:  "disable",
+	}
+	Conf.API.DB, err = database.NewSDAdb(Conf.Database)
+	assert.NoError(suite.T(), err)
+
+	Conf.Broker = broker.MQConf{
+		Host:     "localhost",
+		Port:     mqPort,
+		User:     "guest",
+		Password: "guest",
+		Exchange: "sda",
+		Vhost:    "/sda",
+	}
+	Conf.API.MQ, err = broker.NewMQ(Conf.Broker)
+	assert.NoError(suite.T(), err)
+}
+
 func (suite *TestSuite) TestDatabasePingCheck() {
 	emptyDB := database.SDAdb{}
 	assert.Error(suite.T(), checkDB(&emptyDB, 1*time.Second), "nil DB should fail")

From b85827bae026f1a61c2d7e852e9c6266349da5ae Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 23 Jul 2024 10:19:20 +0200
Subject: [PATCH 07/16] [api] add endpoint to assign accessionID to a file

---
 sda/cmd/api/api.go      |  58 ++++++++++++++
 sda/cmd/api/api.md      |  12 ++-
 sda/cmd/api/api_test.go | 165 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 234 insertions(+), 1 deletion(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index b423f2970..4bee106d2 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -75,6 +75,7 @@ func setup(config *config.Config) *http.Server {
 	r.GET("/files", getFiles)
 	// admin endpoints below here
 	r.POST("/file/ingest", isAdmin(), ingestFile) // start ingestion of a file
+	r.POST("/file/accession", isAdmin(), setAccession) // assign accession ID to a file
 
 	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
 
@@ -234,5 +235,62 @@ func ingestFile(c *gin.Context) {
 		return
 	}
 
+	c.Status(http.StatusOK)
+}
+
+func setAccession(c *gin.Context) {
+	var accession schema.IngestionAccession
+	if err := c.BindJSON(&accession); err != nil {
+		c.AbortWithStatusJSON(
+			http.StatusBadRequest,
+			gin.H{
+				"error":  "json decoding : " + err.Error(),
+				"status": http.StatusBadRequest,
+			},
+		)
+
+		return
+	}
+
+	corrID, err := Conf.API.DB.GetCorrID(accession.User, accession.FilePath)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+
+		return
+	}
+
+	fileInfo, err := Conf.API.DB.GetFileInfo(corrID)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
+	accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileInfo.DecryptedChecksum}}
+	accession.Type = "accession"
+	marshaledMsg, err := json.Marshal(&accession)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+	if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+
+		return
+	}
+
+	err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "accession", marshaledMsg)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
 	c.Status(http.StatusOK)
 }
\ No newline at end of file
diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 7c1564e47..682c4fb12 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -30,4 +30,14 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 
     ```bash
     curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/ingest
-    ```
\ No newline at end of file
+    ```
+
+- `/file/accession`
+  - accepts `POST` requests with JSON data with the format: `{"accession_id": "<FILE_ACCESSION>", "filepath": "</PATH/TO/FILE/IN/INBOX>", "user": "<USERNAME>"}`
+  - assigns accession ID to the file.
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "my-id-01", "filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/accession
+    ```
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index ccc6c22d3..9af0f1d73 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"database/sql"
 	"encoding/json"
 	"fmt"
@@ -751,3 +752,167 @@ func (suite *TestSuite) TestIngestFile_WrongFilePath() {
 	assert.Equal(suite.T(), http.StatusInternalServerError, okResponse.StatusCode)
 	assert.Contains(suite.T(), string(b), "sql: no rows in result set")
 }
+
+func (suite *TestSuite) TestSetAccession() {
+	user := "dummy"
+	filePath := "/inbox/dummy/file11.c4gh"
+
+	fileID, err := Conf.API.DB.RegisterFile(filePath, user)
+	assert.NoError(suite.T(), err, "failed to register file in database")
+	err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+	assert.NoError(suite.T(), err, "failed to update satus of file in database")
+
+	encSha := sha256.New()
+	_, err = encSha.Write([]byte("Checksum"))
+	assert.NoError(suite.T(), err)
+
+	decSha := sha256.New()
+	_, err = decSha.Write([]byte("DecryptedChecksum"))
+	assert.NoError(suite.T(), err)
+
+	fileInfo := database.FileInfo{
+		Checksum:          fmt.Sprintf("%x", encSha.Sum(nil)),
+		Size:              1000,
+		Path:              filePath,
+		DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)),
+		DecryptedSize:     948,
+	}
+	err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID)
+	assert.NoError(suite.T(), err, "failed to mark file as Archived")
+
+	err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID)
+	assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err)
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	type accession struct {
+		AccessionID string `json:"accession_id"`
+		FilePath    string `json:"filepath"`
+		User        string `json:"user"`
+	}
+	aID := "API:accession-id-01"
+	accessionMsg, _ := json.Marshal(accession{AccessionID: aID, FilePath: filePath, User: user})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg))
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/file/accession", isAdmin(), setAccession)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	// verify that the message shows up in the queue
+	time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives
+	client := http.Client{Timeout: 5 * time.Second}
+	req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody)
+	req.SetBasicAuth("guest", "guest")
+	res, err := client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	var data struct {
+		MessagesReady int `json:"messages_ready"`
+	}
+	body, err := io.ReadAll(res.Body)
+	res.Body.Close()
+	assert.NoError(suite.T(), err, "failed to read response from broker")
+	err = json.Unmarshal(body, &data)
+	assert.NoError(suite.T(), err, "failed to unmarshal response")
+	assert.Equal(suite.T(), 1, data.MessagesReady)
+}
+
+func (suite *TestSuite) TestSetAccession_WrongUser() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	type accession struct {
+		AccessionID string `json:"accession_id"`
+		FilePath    string `json:"filepath"`
+		User        string `json:"user"`
+	}
+	aID := "API:accession-id-01"
+	accessionMsg, _ := json.Marshal(accession{AccessionID: aID, FilePath: "/inbox/dummy/file11.c4gh", User: "fooBar"})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg))
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/file/accession", isAdmin(), setAccession)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
+
+	// verify that the message shows up in the queue
+	time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives
+	client := http.Client{Timeout: 5 * time.Second}
+	req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody)
+	req.SetBasicAuth("guest", "guest")
+	res, err := client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	var data struct {
+		MessagesReady int `json:"messages_ready"`
+	}
+	body, err := io.ReadAll(res.Body)
+	res.Body.Close()
+	assert.NoError(suite.T(), err, "failed to read response from broker")
+	err = json.Unmarshal(body, &data)
+	assert.NoError(suite.T(), err, "failed to unmarshal response")
+	assert.Equal(suite.T(), 1, data.MessagesReady)
+}
+
+func (suite *TestSuite) TestSetAccession_WrongFormat() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/federated"
+
+	type accession struct {
+		AccessionID string `json:"accession_id"`
+		FilePath    string `json:"filepath"`
+		User        string `json:"user"`
+	}
+	aID := "API:accession-id-01"
+	accessionMsg, _ := json.Marshal(accession{AccessionID: aID, FilePath: "/inbox/dummy/file11.c4gh", User: "dummy"})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg))
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/file/accession", isAdmin(), setAccession)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
+
+	// verify that the message shows up in the queue
+	time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives
+	client := http.Client{Timeout: 5 * time.Second}
+	req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody)
+	req.SetBasicAuth("guest", "guest")
+	res, err := client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	var data struct {
+		MessagesReady int `json:"messages_ready"`
+	}
+	body, err := io.ReadAll(res.Body)
+	res.Body.Close()
+	assert.NoError(suite.T(), err, "failed to read response from broker")
+	err = json.Unmarshal(body, &data)
+	assert.NoError(suite.T(), err, "failed to unmarshal response")
+	assert.Equal(suite.T(), 1, data.MessagesReady)
+}

From 6fc048d17f794ec0fb5d036ae795c5d7f43e8807 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 23 Jul 2024 11:12:45 +0200
Subject: [PATCH 08/16] [api] add endpoint to create a dataset

---
 sda/cmd/api/api.go      |  41 ++++++++++++++++
 sda/cmd/api/api.md      |  10 ++++
 sda/cmd/api/api_test.go | 101 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 152 insertions(+)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index 4bee106d2..917cae177 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -76,6 +76,7 @@ func setup(config *config.Config) *http.Server {
 	// admin endpoints below here
 	r.POST("/file/ingest", isAdmin(), ingestFile) // start ingestion of a file
 	r.POST("/file/accession", isAdmin(), setAccession) // assign accession ID to a file
+	r.POST("/dataset/create", isAdmin(), createDataset) // maps a set of files to a dataset
 
 	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
 
@@ -292,5 +293,45 @@ func setAccession(c *gin.Context) {
 		return
 	}
 
+	c.Status(http.StatusOK)
+}
+
+func createDataset(c *gin.Context) {
+	var dataset schema.DatasetMapping
+	if err := c.BindJSON(&dataset); err != nil {
+		c.AbortWithStatusJSON(
+			http.StatusBadRequest,
+			gin.H{
+				"error":  "json decoding : " + err.Error(),
+				"status": http.StatusBadRequest,
+			},
+		)
+
+		return
+	}
+
+	dataset.Type = "mapping"
+	marshaledMsg, err := json.Marshal(&dataset)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusConflict, err.Error())
+
+		return
+	}
+	if err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+
+		return
+	}
+
+	err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "mappings", marshaledMsg)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
 	c.Status(http.StatusOK)
 }
\ No newline at end of file
diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 682c4fb12..c5e31b843 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -41,3 +41,13 @@ Admin endpoints are only available to a set of whitelisted users specified in th
     ```bash
     curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "my-id-01", "filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/accession
     ```
+
+- `/dataset/create`
+  - accepts `POST` requests with JSON data with the format: `{"accession_ids": ["<FILE_ACCESSION_01>", "<FILE_ACCESSION_02>"], "dataset_id": "<DATASET_01>"}`
+  - creates a datset from the list of accession IDs and the dataset ID.
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_idd": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01"}' https://HOSTNAME/dataset/create
+    ```
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index 9af0f1d73..d70b6b742 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -916,3 +916,104 @@ func (suite *TestSuite) TestSetAccession_WrongFormat() {
 	assert.NoError(suite.T(), err, "failed to unmarshal response")
 	assert.Equal(suite.T(), 1, data.MessagesReady)
 }
+
+func (suite *TestSuite) TestCreateDataset() {
+	user := "dummy"
+	filePath := "/inbox/dummy/file12.c4gh"
+
+	fileID, err := Conf.API.DB.RegisterFile(filePath, user)
+	assert.NoError(suite.T(), err, "failed to register file in database")
+	err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+	assert.NoError(suite.T(), err, "failed to update satus of file in database")
+
+	encSha := sha256.New()
+	_, err = encSha.Write([]byte("Checksum"))
+	assert.NoError(suite.T(), err)
+
+	decSha := sha256.New()
+	_, err = decSha.Write([]byte("DecryptedChecksum"))
+	assert.NoError(suite.T(), err)
+
+	fileInfo := database.FileInfo{
+		Checksum:          fmt.Sprintf("%x", encSha.Sum(nil)),
+		Size:              1000,
+		Path:              filePath,
+		DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)),
+		DecryptedSize:     948,
+	}
+	err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID)
+	assert.NoError(suite.T(), err, "failed to mark file as Archived")
+
+	err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID)
+	assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err)
+
+	err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID)
+	assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err)
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	type dataset struct {
+		AccessionIDs []string `json:"accession_ids"`
+		DatasetID    string   `json:"dataset_id"`
+	}
+	accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11", "API:accession-id-12", "API:accession-id-13"}, DatasetID: "API:dataset-01"})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg))
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/dataset/create", isAdmin(), createDataset)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	// verify that the message shows up in the queue
+	time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives
+	client := http.Client{Timeout: 5 * time.Second}
+	req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/mappings", http.NoBody)
+	req.SetBasicAuth("guest", "guest")
+	res, err := client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	var data struct {
+		MessagesReady int `json:"messages_ready"`
+	}
+	body, err := io.ReadAll(res.Body)
+	res.Body.Close()
+	assert.NoError(suite.T(), err, "failed to read response from broker")
+	assert.NoError(suite.T(), json.Unmarshal(body, &data), "failed to unmarshal response")
+	assert.Equal(suite.T(), 1, data.MessagesReady)
+}
+
+func (suite *TestSuite) TestCreateDataset_BadFormat() {
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/federated"
+
+	type dataset struct {
+		AccessionIDs []string `json:"accession_ids"`
+		DatasetID    string   `json:"dataset_id"`
+	}
+	accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11", "API:accession-id-12", "API:accession-id-13"}, DatasetID: "API:dataset-01"})
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg))
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/dataset/create", isAdmin(), createDataset)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	response := w.Result()
+	defer response.Body.Close()
+
+	assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode)
+}
\ No newline at end of file

From 2a2b84083d5156a2520b938304bb5d02dc11aca6 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 23 Jul 2024 14:37:22 +0200
Subject: [PATCH 09/16] [api] add endpoint to release dataset

---
 sda/cmd/api/api.go      | 40 ++++++++++++++++++---
 sda/cmd/api/api.md      | 10 ++++++
 sda/cmd/api/api_test.go | 78 ++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 123 insertions(+), 5 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index 917cae177..04cb2b877 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/signal"
 	"slices"
+	"strings"
 	"syscall"
 	"time"
 
@@ -74,9 +75,10 @@ func setup(config *config.Config) *http.Server {
 	r.GET("/ready", readinessResponse)
 	r.GET("/files", getFiles)
 	// admin endpoints below here
-	r.POST("/file/ingest", isAdmin(), ingestFile) // start ingestion of a file
-	r.POST("/file/accession", isAdmin(), setAccession) // assign accession ID to a file
-	r.POST("/dataset/create", isAdmin(), createDataset) // maps a set of files to a dataset
+	r.POST("/file/ingest", isAdmin(), ingestFile)                  // start ingestion of a file
+	r.POST("/file/accession", isAdmin(), setAccession)             // assign accession ID to a file
+	r.POST("/dataset/create", isAdmin(), createDataset)            // maps a set of files to a dataset
+	r.POST("/dataset/release/*dataset", isAdmin(), releaseDataset) // Releases a dataset to be accessible
 
 	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
 
@@ -334,4 +336,34 @@ func createDataset(c *gin.Context) {
 	}
 
 	c.Status(http.StatusOK)
-}
\ No newline at end of file
+}
+
+func releaseDataset(c *gin.Context) {
+	datasetMsg := schema.DatasetRelease{
+		Type:      "release",
+		DatasetID: strings.TrimPrefix(c.Param("dataset"), "/"),
+	}
+	marshaledMsg, err := json.Marshal(&datasetMsg)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+	if err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-release.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+
+		return
+	}
+
+	err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "mappings", marshaledMsg)
+	if err != nil {
+		log.Debugln(err.Error())
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
+	c.Status(http.StatusOK)
+}
diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index c5e31b843..452f7088c 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -51,3 +51,13 @@ Admin endpoints are only available to a set of whitelisted users specified in th
     ```bash
     curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_idd": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01"}' https://HOSTNAME/dataset/create
     ```
+
+- `/dataset/release/*dataset`
+  - accepts `POST` requests with the dataset name as last part of the path`
+  - releases a dataset so that it can be downloaded.
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -X POST  https://HOSTNAME/dataset/release/my-dataset-01
+    ```
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index d70b6b742..2d2e75c81 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -1016,4 +1016,80 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() {
 	defer response.Body.Close()
 
 	assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode)
-}
\ No newline at end of file
+}
+
+func (suite *TestSuite) TestReleaseDataset() {
+	// purge the queue so that the test passes when all tests are run as well as when run standalone.
+	client := http.Client{Timeout: 30 * time.Second}
+	req, err := http.NewRequest(http.MethodDelete, "http://"+BrokerAPI+"/api/queues/sda/mappings/contents", http.NoBody)
+	assert.NoError(suite.T(), err, "failed to generate query")
+	req.SetBasicAuth("guest", "guest")
+	res, err := client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	res.Body.Close()
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/dataset/release/API:dataset-01", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/dataset/release/*dataset", isAdmin(), releaseDataset)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	// verify that the message shows up in the queue
+	time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives
+	req, _ = http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/mappings", http.NoBody)
+	req.SetBasicAuth("guest", "guest")
+	res, err = client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	var data struct {
+		MessagesReady int `json:"messages_ready"`
+	}
+	body, err := io.ReadAll(res.Body)
+	res.Body.Close()
+	assert.NoError(suite.T(), err, "failed to read response from broker")
+	err = json.Unmarshal(body, &data)
+	assert.NoError(suite.T(), err, "failed to unmarshal response")
+	assert.Equal(suite.T(), 1, data.MessagesReady)
+}
+
+func (suite *TestSuite) TestReleaseDataset_NoDataset() {
+	// purge the queue so that the test passes when all tests are run as well as when run standalone.
+	client := http.Client{Timeout: 30 * time.Second}
+	req, err := http.NewRequest(http.MethodDelete, "http://"+BrokerAPI+"/api/queues/sda/mappings/contents", http.NoBody)
+	assert.NoError(suite.T(), err, "failed to generate query")
+	req.SetBasicAuth("guest", "guest")
+	res, err := client.Do(req)
+	assert.NoError(suite.T(), err, "failed to query broker")
+	res.Body.Close()
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+	Conf.Broker.SchemasPath = "../../schemas/isolated"
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, "/dataset/release/", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.POST("/dataset/release/*dataset", isAdmin(), releaseDataset)
+
+	// admin user should be allowed
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
+}

From c5b0c51519815257b00dc35a262213dbd9cde67b Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 23 Jul 2024 14:38:33 +0200
Subject: [PATCH 10/16] [schemas][isolated] any identifier is viable for
 dataset actions

---
 sda/schemas/isolated/dataset-deprecate.json | 30 ++++++++++++++++++++-
 sda/schemas/isolated/dataset-release.json   | 30 ++++++++++++++++++++-
 2 files changed, 58 insertions(+), 2 deletions(-)
 mode change 120000 => 100644 sda/schemas/isolated/dataset-deprecate.json
 mode change 120000 => 100644 sda/schemas/isolated/dataset-release.json

diff --git a/sda/schemas/isolated/dataset-deprecate.json b/sda/schemas/isolated/dataset-deprecate.json
deleted file mode 120000
index c3fd79a98..000000000
--- a/sda/schemas/isolated/dataset-deprecate.json
+++ /dev/null
@@ -1 +0,0 @@
-../federated/dataset-deprecate.json
\ No newline at end of file
diff --git a/sda/schemas/isolated/dataset-deprecate.json b/sda/schemas/isolated/dataset-deprecate.json
new file mode 100644
index 000000000..0e09965a1
--- /dev/null
+++ b/sda/schemas/isolated/dataset-deprecate.json
@@ -0,0 +1,29 @@
+{
+    "title": "JSON schema for Local EGA dataset deprecation message interface",
+    "$id": "https://github.com/neicnordic/sensitive-data-archive/tree/master/sda/schemas/federated/dataset-deprecate.json",
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "type": "object",
+    "required": [
+        "type",
+        "dataset_id"
+    ],
+    "additionalProperties": true,
+    "properties": {
+        "type": {
+            "$id": "#/properties/type",
+            "type": "string",
+            "title": "The message type",
+            "description": "The message type",
+            "const": "deprecate"
+        },
+        "dataset_id": {
+            "$id": "#/properties/dataset_id",
+            "type": "string",
+            "title": "The Accession identifier for the dataset",
+            "description": "The Accession identifier for the dataset",
+            "examples": [
+                "anyidentifier"
+            ]
+        }
+    }
+}
\ No newline at end of file
diff --git a/sda/schemas/isolated/dataset-release.json b/sda/schemas/isolated/dataset-release.json
deleted file mode 120000
index e22bb197c..000000000
--- a/sda/schemas/isolated/dataset-release.json
+++ /dev/null
@@ -1 +0,0 @@
-../federated/dataset-release.json
\ No newline at end of file
diff --git a/sda/schemas/isolated/dataset-release.json b/sda/schemas/isolated/dataset-release.json
new file mode 100644
index 000000000..14e7f9839
--- /dev/null
+++ b/sda/schemas/isolated/dataset-release.json
@@ -0,0 +1,29 @@
+{
+    "title": "JSON schema for Local EGA dataset release message interface",
+    "$id": "https://github.com/neicnordic/sensitive-data-archive/tree/master/sda/schemas/federated/dataset-release.json",
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "type": "object",
+    "required": [
+        "type",
+        "dataset_id"
+    ],
+    "additionalProperties": true,
+    "properties": {
+        "type": {
+            "$id": "#/properties/type",
+            "type": "string",
+            "title": "The message type",
+            "description": "The message type",
+            "const": "release"
+        },
+        "dataset_id": {
+            "$id": "#/properties/dataset_id",
+            "type": "string",
+            "title": "The Accession identifier for the dataset",
+            "description": "The Accession identifier for the dataset",
+            "examples": [
+                "anyidentifier"
+            ]
+        }
+    }
+}
\ No newline at end of file

From be7330fcbe9bc9b0296d003649339a7705fa7fc0 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Wed, 24 Jul 2024 08:31:56 +0200
Subject: [PATCH 11/16] [schemas] dataset must have a minimum length

---
 sda/schemas/isolated/dataset-release.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sda/schemas/isolated/dataset-release.json b/sda/schemas/isolated/dataset-release.json
index 14e7f9839..b696f9b06 100644
--- a/sda/schemas/isolated/dataset-release.json
+++ b/sda/schemas/isolated/dataset-release.json
@@ -23,7 +23,8 @@
             "description": "The Accession identifier for the dataset",
             "examples": [
                 "anyidentifier"
-            ]
+            ],
+            "minLength": 2
         }
     }
 }
\ No newline at end of file

From 5b9b7e7f4164534f338f027a44846283b687261d Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Wed, 24 Jul 2024 09:07:42 +0200
Subject: [PATCH 12/16] Cleanup the readme

---
 sda/cmd/api/api.md | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 452f7088c..8d9c97183 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -1,22 +1,25 @@
 # API
+
 The API service provides data submitters with functionality to control
-their submissions. Users are authenticated with a JWT. 
+their submissions. Users are authenticated with a JWT.
 
 ## Service Description
 
 Endpoints:
-  - `/files`
-  
-    1. Parses and validates the JWT token against the public keys, either locally provisioned or from OIDC JWK endpoints.
-    2. The `sub` field from the token is extracted and used as the user's identifier
-    3. All files belonging to this user are extracted from the database, together with their latest status and creation date
-     
-     Example:
-      ```bash
-      $ curl 'https://server/files' -H "Authorization: Bearer $token"
-    [{"inboxPath":"requester_demo.org/data/file1.c4gh","fileStatus":"uploaded","createAt":"2023-11-13T10:12:43.144242Z"}] 
-    ```
-     If the `token` is invalid, 401 is returned.
+
+- `/files`
+  1. Parses and validates the JWT token against the public keys, either locally provisioned or from OIDC JWK endpoints.
+  2. The `sub` field from the token is extracted and used as the user's identifier
+  3. All files belonging to this user are extracted from the database, together with their latest status and creation date
+
+    Example:
+
+    ```bash
+    $ curl 'https://server/files' -H "Authorization: Bearer $token"
+  [{"inboxPath":"requester_demo.org/data/file1.c4gh","fileStatus":"uploaded","createAt":"2023-11-13T10:12:43.144242Z"}] 
+  ```
+
+  If the `token` is invalid, 401 is returned.
 
 ### Admin endpoints
 

From 521ee81bfc9098b4bbab4877f6835c570f1f51c7 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 30 Jul 2024 07:49:18 +0200
Subject: [PATCH 13/16] [api][tests] remove obsolete comments

---
 sda/cmd/api/api_test.go | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index 2d2e75c81..791f4ad9b 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -374,7 +374,6 @@ func (suite *TestSuite) SetupSuite() {
 }
 
 func (suite *TestSuite) SetupTest() {
-	log.Print("Setup DB for my test")
 	Conf.Database = database.DBConf{
 		Host:     "localhost",
 		Port:     dbPort,
@@ -591,7 +590,6 @@ func (suite *TestSuite) TestIsAdmin() {
 	_, router := gin.CreateTestContext(w)
 	router.GET("/", isAdmin(), testEndpoint)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -626,7 +624,6 @@ func (suite *TestSuite) TestIngestFile() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/file/ingest", ingestFile)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -676,7 +673,6 @@ func (suite *TestSuite) TestIngestFile_NoUser() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/file/ingest", ingestFile)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -708,7 +704,6 @@ func (suite *TestSuite) TestIngestFile_WrongUser() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/file/ingest", ingestFile)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -744,7 +739,6 @@ func (suite *TestSuite) TestIngestFile_WrongFilePath() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/file/ingest", isAdmin(), ingestFile)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -803,7 +797,6 @@ func (suite *TestSuite) TestSetAccession() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/file/accession", isAdmin(), setAccession)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -848,7 +841,6 @@ func (suite *TestSuite) TestSetAccession_WrongUser() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/file/accession", isAdmin(), setAccession)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -893,7 +885,6 @@ func (suite *TestSuite) TestSetAccession_WrongFormat() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/file/accession", isAdmin(), setAccession)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -968,7 +959,6 @@ func (suite *TestSuite) TestCreateDataset() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/dataset/create", isAdmin(), createDataset)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -1010,7 +1000,6 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/dataset/create", isAdmin(), createDataset)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	response := w.Result()
 	defer response.Body.Close()
@@ -1041,7 +1030,6 @@ func (suite *TestSuite) TestReleaseDataset() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/dataset/release/*dataset", isAdmin(), releaseDataset)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
@@ -1087,7 +1075,6 @@ func (suite *TestSuite) TestReleaseDataset_NoDataset() {
 	_, router := gin.CreateTestContext(w)
 	router.POST("/dataset/release/*dataset", isAdmin(), releaseDataset)
 
-	// admin user should be allowed
 	router.ServeHTTP(w, r)
 	okResponse := w.Result()
 	defer okResponse.Body.Close()

From 2be829fb9c91af26fa95856b2cc627465dbb02c7 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 30 Jul 2024 08:00:04 +0200
Subject: [PATCH 14/16] [api] return correct error on CorrID failures

Failure to look up the Correlation ID can be 400 or 500 depending on where the error originated from.
---
 sda/cmd/api/api.go      | 15 ++++++++++++---
 sda/cmd/api/api_test.go |  4 ++--
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index 04cb2b877..0c5b754a6 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -226,7 +226,12 @@ func ingestFile(c *gin.Context) {
 
 	corrID, err := Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath)
 	if err != nil {
-		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		switch {
+		case corrID == "":
+			c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+		default:
+			c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		}
 
 		return
 	}
@@ -257,8 +262,12 @@ func setAccession(c *gin.Context) {
 
 	corrID, err := Conf.API.DB.GetCorrID(accession.User, accession.FilePath)
 	if err != nil {
-		log.Debugln(err.Error())
-		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+		switch {
+		case corrID == "":
+			c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
+		default:
+			c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+		}
 
 		return
 	}
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index 791f4ad9b..0b6676286 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -708,7 +708,7 @@ func (suite *TestSuite) TestIngestFile_WrongUser() {
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
 	b, _ := io.ReadAll(okResponse.Body)
-	assert.Equal(suite.T(), http.StatusInternalServerError, okResponse.StatusCode)
+	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
 	assert.Contains(suite.T(), string(b), "sql: no rows in result set")
 }
 
@@ -743,7 +743,7 @@ func (suite *TestSuite) TestIngestFile_WrongFilePath() {
 	okResponse := w.Result()
 	defer okResponse.Body.Close()
 	b, _ := io.ReadAll(okResponse.Body)
-	assert.Equal(suite.T(), http.StatusInternalServerError, okResponse.StatusCode)
+	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
 	assert.Contains(suite.T(), string(b), "sql: no rows in result set")
 }
 

From 2b82cdd7480e3e306f4c2a6f1ba6a24025a75201 Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 30 Jul 2024 08:07:10 +0200
Subject: [PATCH 15/16] [api] no need to check for error on JSON marshaling

---
 sda/cmd/api/api.go | 24 +++---------------------
 1 file changed, 3 insertions(+), 21 deletions(-)

diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go
index 0c5b754a6..fc5e4b9d9 100644
--- a/sda/cmd/api/api.go
+++ b/sda/cmd/api/api.go
@@ -282,13 +282,7 @@ func setAccession(c *gin.Context) {
 
 	accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileInfo.DecryptedChecksum}}
 	accession.Type = "accession"
-	marshaledMsg, err := json.Marshal(&accession)
-	if err != nil {
-		log.Debugln(err.Error())
-		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
-
-		return
-	}
+	marshaledMsg, _ := json.Marshal(&accession)
 	if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil {
 		log.Debugln(err.Error())
 		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
@@ -322,13 +316,7 @@ func createDataset(c *gin.Context) {
 	}
 
 	dataset.Type = "mapping"
-	marshaledMsg, err := json.Marshal(&dataset)
-	if err != nil {
-		log.Debugln(err.Error())
-		c.AbortWithStatusJSON(http.StatusConflict, err.Error())
-
-		return
-	}
+	marshaledMsg, _ := json.Marshal(&dataset)
 	if err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil {
 		log.Debugln(err.Error())
 		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())
@@ -352,13 +340,7 @@ func releaseDataset(c *gin.Context) {
 		Type:      "release",
 		DatasetID: strings.TrimPrefix(c.Param("dataset"), "/"),
 	}
-	marshaledMsg, err := json.Marshal(&datasetMsg)
-	if err != nil {
-		log.Debugln(err.Error())
-		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
-
-		return
-	}
+	marshaledMsg, _ := json.Marshal(&datasetMsg)
 	if err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-release.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil {
 		log.Debugln(err.Error())
 		c.AbortWithStatusJSON(http.StatusBadRequest, err.Error())

From aee57cc06f4447176673b147fb236e01cf82276e Mon Sep 17 00:00:00 2001
From: Joakim Bygdell <joakim.bygdell@nbis.se>
Date: Tue, 30 Jul 2024 08:12:26 +0200
Subject: [PATCH 16/16] [api] add descriptions of error codes to the readme

---
 sda/cmd/api/api.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md
index 8d9c97183..5014ffb94 100644
--- a/sda/cmd/api/api.md
+++ b/sda/cmd/api/api.md
@@ -29,6 +29,12 @@ Admin endpoints are only available to a set of whitelisted users specified in th
   - accepts `POST` requests with JSON data with the format: `{"filepath": "</PATH/TO/FILE/IN/INBOX>", "user": "<USERNAME>"}`
   - triggers the ingestion of the file.
 
+- Error codes
+  - `200` Query execute ok.
+  - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination.
+  - `401` User is not in the list of admins.
+  - `500` Internal error due to DB or MQ failures.
+
     Example:
 
     ```bash
@@ -39,6 +45,12 @@ Admin endpoints are only available to a set of whitelisted users specified in th
   - accepts `POST` requests with JSON data with the format: `{"accession_id": "<FILE_ACCESSION>", "filepath": "</PATH/TO/FILE/IN/INBOX>", "user": "<USERNAME>"}`
   - assigns accession ID to the file.
 
+- Error codes
+  - `200` Query execute ok.
+  - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination.
+  - `401` User is not in the list of admins.
+  - `500` Internal error due to DB or MQ failures.
+
     Example:
 
     ```bash
@@ -49,6 +61,12 @@ Admin endpoints are only available to a set of whitelisted users specified in th
   - accepts `POST` requests with JSON data with the format: `{"accession_ids": ["<FILE_ACCESSION_01>", "<FILE_ACCESSION_02>"], "dataset_id": "<DATASET_01>"}`
   - creates a datset from the list of accession IDs and the dataset ID.
 
+- Error codes
+  - `200` Query execute ok.
+  - `400` Error due to bad payload.
+  - `401` User is not in the list of admins.
+  - `500` Internal error due to DB or MQ failures.
+
     Example:
 
     ```bash
@@ -59,6 +77,12 @@ Admin endpoints are only available to a set of whitelisted users specified in th
   - accepts `POST` requests with the dataset name as last part of the path`
   - releases a dataset so that it can be downloaded.
 
+- Error codes
+  - `200` Query execute ok.
+  - `400` Error due to bad payload.
+  - `401` User is not in the list of admins.
+  - `500` Internal error due to DB or MQ failures.
+
     Example:
 
     ```bash