Merge pull request #956 from neicnordic/feature/admin-api-2

API admin part 2

sda/cmd/api/api.go

@@ -79,6 +79,8 @@ func setup(config *config.Config) *http.Server {
 	r.POST("/file/accession", isAdmin(), setAccession)             // assign accession ID to a file
 	r.POST("/dataset/create", isAdmin(), createDataset)            // maps a set of files to a dataset
 	r.POST("/dataset/release/*dataset", isAdmin(), releaseDataset) // Releases a dataset to be accessible
+	r.GET("/users", isAdmin(), listActiveUsers)                    // Lists all users
+	r.GET("/users/:username/files", isAdmin(), listUserFiles)      // Lists all unmapped files for a user
 
 	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
 
@@ -358,3 +360,30 @@ func releaseDataset(c *gin.Context) {
 
 	c.Status(http.StatusOK)
 }
+
+func listActiveUsers(c *gin.Context) {
+	users, err := Conf.API.DB.ListActiveUsers()
+	if err != nil {
+		log.Debugln("ListActiveUsers failed")
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+	c.JSON(http.StatusOK, users)
+}
+
+func listUserFiles(c *gin.Context) {
+	username := c.Param("username")
+	username = strings.TrimPrefix(username, "/")
+	username = strings.TrimSuffix(username, "/files")
+	log.Debugln(username)
+	files, err := Conf.API.DB.GetUserFiles(strings.ReplaceAll(username, "@", "_"))
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
+	c.Writer.Header().Set("Content-Type", "application/json")
+	c.JSON(200, files)
+}

sda/cmd/api/api.md

@@ -32,7 +32,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
@@ -48,7 +48,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
@@ -64,7 +64,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
@@ -80,11 +80,41 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
 
     ```bash
     curl -H "Authorization: Bearer $token" -X POST  https://HOSTNAME/dataset/release/my-dataset-01
     ```
+
+- `/users`
+  - accepts `GET` requests`
+  - Returns all users with active uploads as a JSON array
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -X GET  https://HOSTNAME/users
+    ```
+
+- Error codes
+  - `200` Query execute ok.
+  - `401` Token user is not in the list of admins.
+  - `500` Internal error due to DB failure.
+
+- `/users/:username/files`
+  - accepts `GET` requests`
+  - Returns all files for a user with active uploads as a JSON array
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -X GET  https://HOSTNAME/users/[email protected]/files
+    ```
+
+- Error codes
+  - `200` Query execute ok.
+  - `401` Token user is not in the list of admins.
+  - `500` Internal error due to DB failure.

sda/cmd/api/api_test.go

@@ -385,6 +385,9 @@ func (suite *TestSuite) SetupTest() {
 	Conf.API.DB, err = database.NewSDAdb(Conf.Database)
 	assert.NoError(suite.T(), err)
 
+	_, err = Conf.API.DB.DB.Exec("TRUNCATE sda.files CASCADE")
+	assert.NoError(suite.T(), err)
+
 	Conf.Broker = broker.MQConf{
 		Host:     "localhost",
 		Port:     mqPort,
@@ -1080,3 +1083,103 @@ func (suite *TestSuite) TestReleaseDataset_NoDataset() {
 	defer okResponse.Body.Close()
 	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
 }
+
+func (suite *TestSuite) TestListActiveUsers() {
+	testUsers := []string{"User-A", "User-B", "User-C"}
+	for _, user := range testUsers {
+		for i := 0; i < 3; i++ {
+			fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user)
+			if err != nil {
+				suite.FailNow("failed to register file in database")
+			}
+
+			err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+			if err != nil {
+				suite.FailNow("failed to update satus of file in database")
+			}
+
+			stableID := fmt.Sprintf("accession_%s_0%d", user, i)
+			err = Conf.API.DB.SetAccessionID(stableID, fileID)
+			if err != nil {
+				suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+			}
+		}
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_User-A_00", "accession_User-A_01", "accession_User-A_02"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodGet, "/users", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/users", isAdmin(), listActiveUsers)
+
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	var users []string
+	err = json.NewDecoder(okResponse.Body).Decode(&users)
+	assert.NoError(suite.T(), err, "failed to list users from DB")
+	assert.Equal(suite.T(), []string{"User-B", "User-C"}, users)
+}
+
+func (suite *TestSuite) TestListUserFiles() {
+	testUsers := []string{"user_example.org", "User-B", "User-C"}
+	for _, user := range testUsers {
+		for i := 0; i < 5; i++ {
+			fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user)
+			if err != nil {
+				suite.FailNow("failed to register file in database")
+			}
+
+			err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+			if err != nil {
+				suite.FailNow("failed to update satus of file in database")
+			}
+
+			stableID := fmt.Sprintf("accession_%s_0%d", user, i)
+			err = Conf.API.DB.SetAccessionID(stableID, fileID)
+			if err != nil {
+				suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+			}
+		}
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodGet, "/users/[email protected]/files", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/users/:username/files", isAdmin(), listUserFiles)
+
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	files := []database.SubmissionFileInfo{}
+	err = json.NewDecoder(okResponse.Body).Decode(&files)
+	assert.NoError(suite.T(), err, "failed to list users from DB")
+	assert.Equal(suite.T(), 2, len(files))
+}

sda/internal/database/db_functions.go

@@ -635,14 +635,9 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) {
 	db := dbs.DB
 
 	// select all files of the user, each one annotated with its latest event
-	const query = "SELECT f.submission_file_path, e.event, f.created_at " +
-		"FROM sda.files f " +
-		"LEFT JOIN ( " +
-		"SELECT DISTINCT ON (file_id) file_id, started_at, event " +
-		"FROM sda.file_event_log " +
-		"ORDER BY file_id, started_at DESC" +
-		") e ON f.id = e.file_id " +
-		"WHERE f.submission_user = $1; "
+	const query = "SELECT f.submission_file_path, e.event, f.created_at FROM sda.files f " +
+		"LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " +
+		"AND f.id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id); "
 
 	// nolint:rowserrcheck
 	rows, err := db.Query(query, userID)
@@ -696,3 +691,30 @@ func (dbs *SDAdb) getCorrID(user, path string) (string, error) {
 
 	return corrID, nil
 }
+
+func (dbs *SDAdb) ListActiveUsers() ([]string, error) {
+	dbs.checkAndReconnectIfNeeded()
+	db := dbs.DB
+
+	var users []string
+	rows, err := db.Query("SELECT DISTINCT submission_user FROM sda.files WHERE id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id) ORDER BY submission_user ASC;")
+	if err != nil {
+		return nil, err
+	}
+	if rows.Err() != nil {
+		return nil, rows.Err()
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var user string
+		err := rows.Scan(&user)
+		if err != nil {
+			return nil, err
+		}
+
+		users = append(users, user)
+	}
+
+	return users, nil
+}

sda/internal/database/db_functions_test.go

@@ -510,3 +510,62 @@ func (suite *DatabaseTests) TestGetCorrID() {
 	assert.Error(suite.T(), err, "failed to get correlation ID of file in database")
 	assert.Equal(suite.T(), "", corrID2)
 }
+
+func (suite *DatabaseTests) TestListActiveUsers() {
+	db, err := NewSDAdb(suite.dbConf)
+	assert.NoError(suite.T(), err, "got (%v) when creating new connection", err)
+	testCases := 5
+	testUsers := []string{"User-A", "User-B", "User-C", "User-D"}
+
+	for _, user := range testUsers {
+		for i := 0; i < testCases; i++ {
+			filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i)
+			fileID, err := db.RegisterFile(filePath, user)
+			if err != nil {
+				suite.FailNow("Failed to register file")
+			}
+			err = db.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+			if err != nil {
+				suite.FailNow("Failed to update file event log")
+			}
+
+			corrID, err := db.GetCorrID(user, filePath)
+			if err != nil {
+				suite.FailNow("Failed to get CorrID for file")
+			}
+			assert.Equal(suite.T(), fileID, corrID)
+
+			checksum := fmt.Sprintf("%x", sha256.New().Sum(nil))
+			fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999}
+			err = db.SetArchived(fileInfo, fileID, corrID)
+			if err != nil {
+				suite.FailNow("failed to mark file as Archived")
+			}
+
+			err = db.SetVerified(fileInfo, fileID, corrID)
+			if err != nil {
+				suite.FailNow("failed to mark file as Verified")
+			}
+
+			stableID := fmt.Sprintf("accession_%s_0%d", user, i)
+			err = db.SetAccessionID(stableID, fileID)
+			if err != nil {
+				suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+			}
+		}
+	}
+
+	err = db.MapFilesToDataset("test-dataset-01", []string{"accession_User-A_00", "accession_User-A_01", "accession_User-A_02"})
+	if err != nil {
+		suite.FailNow("failed to map files§ to dataset")
+	}
+
+	err = db.MapFilesToDataset("test-dataset-02", []string{"accession_User-C_00", "accession_User-C_01", "accession_User-C_02", "accession_User-C_03", "accession_User-C_04"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+
+	userList, err := db.ListActiveUsers()
+	assert.NoError(suite.T(), err, "failed to list users from DB")
+	assert.Equal(suite.T(), 3, len(userList))
+}

sda/internal/database/db_test.go

@@ -116,6 +116,10 @@ func (suite *DatabaseTests) SetupTest() {
 		ClientKey:  "",
 	}
 
+	db, err := NewSDAdb(suite.dbConf)
+	assert.Nil(suite.T(), err, "got %v when creating new connection", err)
+	_, err = db.DB.Exec("TRUNCATE sda.files CASCADE")
+	assert.NoError(suite.T(), err)
 }
 
 func (suite *DatabaseTests) TearDownTest() {}