V2 of the sync service for Bigpicture

sda/cmd/api/api.go

@@ -79,6 +79,8 @@ func setup(config *config.Config) *http.Server {
 	r.POST("/file/accession", isAdmin(), setAccession)             // assign accession ID to a file
 	r.POST("/dataset/create", isAdmin(), createDataset)            // maps a set of files to a dataset
 	r.POST("/dataset/release/*dataset", isAdmin(), releaseDataset) // Releases a dataset to be accessible
+	r.GET("/users", isAdmin(), listActiveUsers)                    // Lists all users
+	r.GET("/users/:username/files", isAdmin(), listUserFiles)      // Lists all unmapped files for a user
 
 	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
 
@@ -358,3 +360,30 @@ func releaseDataset(c *gin.Context) {
 
 	c.Status(http.StatusOK)
 }
+
+func listActiveUsers(c *gin.Context) {
+	users, err := Conf.API.DB.ListActiveUsers()
+	if err != nil {
+		log.Debugln("ListActiveUsers failed")
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+	c.JSON(http.StatusOK, users)
+}
+
+func listUserFiles(c *gin.Context) {
+	username := c.Param("username")
+	username = strings.TrimPrefix(username, "/")
+	username = strings.TrimSuffix(username, "/files")
+	log.Debugln(username)
+	files, err := Conf.API.DB.GetUserFiles(strings.ReplaceAll(username, "@", "_"))
+	if err != nil {
+		c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
+
+		return
+	}
+
+	c.Writer.Header().Set("Content-Type", "application/json")
+	c.JSON(200, files)
+}

sda/cmd/api/api.md

@@ -32,7 +32,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
@@ -48,7 +48,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
@@ -64,7 +64,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
@@ -80,11 +80,41 @@ Admin endpoints are only available to a set of whitelisted users specified in th
 - Error codes
   - `200` Query execute ok.
   - `400` Error due to bad payload.
-  - `401` User is not in the list of admins.
+  - `401` Token user is not in the list of admins.
   - `500` Internal error due to DB or MQ failures.
 
     Example:
 
     ```bash
     curl -H "Authorization: Bearer $token" -X POST  https://HOSTNAME/dataset/release/my-dataset-01
     ```
+
+- `/users`
+  - accepts `GET` requests`
+  - Returns all users with active uploads as a JSON array
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -X GET  https://HOSTNAME/users
+    ```
+
+- Error codes
+  - `200` Query execute ok.
+  - `401` Token user is not in the list of admins.
+  - `500` Internal error due to DB failure.
+
+- `/users/:username/files`
+  - accepts `GET` requests`
+  - Returns all files for a user with active uploads as a JSON array
+
+    Example:
+
+    ```bash
+    curl -H "Authorization: Bearer $token" -X GET  https://HOSTNAME/users/[email protected]/files
+    ```
+
+- Error codes
+  - `200` Query execute ok.
+  - `401` Token user is not in the list of admins.
+  - `500` Internal error due to DB failure.

sda/cmd/api/api_test.go

@@ -385,6 +385,9 @@ func (suite *TestSuite) SetupTest() {
 	Conf.API.DB, err = database.NewSDAdb(Conf.Database)
 	assert.NoError(suite.T(), err)
 
+	_, err = Conf.API.DB.DB.Exec("TRUNCATE sda.files CASCADE")
+	assert.NoError(suite.T(), err)
+
 	Conf.Broker = broker.MQConf{
 		Host:     "localhost",
 		Port:     mqPort,
@@ -1080,3 +1083,103 @@ func (suite *TestSuite) TestReleaseDataset_NoDataset() {
 	defer okResponse.Body.Close()
 	assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode)
 }
+
+func (suite *TestSuite) TestListActiveUsers() {
+	testUsers := []string{"User-A", "User-B", "User-C"}
+	for _, user := range testUsers {
+		for i := 0; i < 3; i++ {
+			fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user)
+			if err != nil {
+				suite.FailNow("failed to register file in database")
+			}
+
+			err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+			if err != nil {
+				suite.FailNow("failed to update satus of file in database")
+			}
+
+			stableID := fmt.Sprintf("accession_%s_0%d", user, i)
+			err = Conf.API.DB.SetAccessionID(stableID, fileID)
+			if err != nil {
+				suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+			}
+		}
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_User-A_00", "accession_User-A_01", "accession_User-A_02"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodGet, "/users", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/users", isAdmin(), listActiveUsers)
+
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	var users []string
+	err = json.NewDecoder(okResponse.Body).Decode(&users)
+	assert.NoError(suite.T(), err, "failed to list users from DB")
+	assert.Equal(suite.T(), []string{"User-B", "User-C"}, users)
+}
+
+func (suite *TestSuite) TestListUserFiles() {
+	testUsers := []string{"user_example.org", "User-B", "User-C"}
+	for _, user := range testUsers {
+		for i := 0; i < 5; i++ {
+			fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user)
+			if err != nil {
+				suite.FailNow("failed to register file in database")
+			}
+
+			err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}")
+			if err != nil {
+				suite.FailNow("failed to update satus of file in database")
+			}
+
+			stableID := fmt.Sprintf("accession_%s_0%d", user, i)
+			err = Conf.API.DB.SetAccessionID(stableID, fileID)
+			if err != nil {
+				suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
+			}
+		}
+	}
+
+	err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"})
+	if err != nil {
+		suite.FailNow("failed to map files to dataset")
+	}
+
+	gin.SetMode(gin.ReleaseMode)
+	assert.NoError(suite.T(), setupJwtAuth())
+	Conf.API.Admins = []string{"dummy"}
+
+	// Mock request and response holders
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodGet, "/users/[email protected]/files", http.NoBody)
+	r.Header.Add("Authorization", "Bearer "+suite.Token)
+
+	_, router := gin.CreateTestContext(w)
+	router.GET("/users/:username/files", isAdmin(), listUserFiles)
+
+	router.ServeHTTP(w, r)
+	okResponse := w.Result()
+	defer okResponse.Body.Close()
+	assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)
+
+	files := []database.SubmissionFileInfo{}
+	err = json.NewDecoder(okResponse.Body).Decode(&files)
+	assert.NoError(suite.T(), err, "failed to list users from DB")
+	assert.Equal(suite.T(), 2, len(files))
+}