Skip to content

Commit

Permalink
Merge pull request #1119 from neicnordic/feature/api-list-datasets
Browse files Browse the repository at this point in the history
[API] list datasets
  • Loading branch information
jbygdell authored Nov 28, 2024
2 parents 6ec4cf4 + e35692a commit 84b1534
Show file tree
Hide file tree
Showing 8 changed files with 540 additions and 2 deletions.
10 changes: 10 additions & 0 deletions .github/integration/sda/rbac.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@
"path": "/c4gh-keys/*",
"action": "(GET)|(POST)|(PUT)"
},
{
"role": "submission",
"path": "/datasets/*",
"action": "GET"
},
{
"role": "submission",
"path": "/file/ingest",
Expand All @@ -25,6 +30,11 @@
"path": "/users/:username/files",
"action": "GET"
},
{
"role": "*",
"path": "/datasets",
"action": "GET"
},
{
"role": "*",
"path": "/files",
Expand Down
10 changes: 10 additions & 0 deletions .github/integration/tests/sda/40_mapper_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -197,4 +197,14 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select count(id) from sda
sleep 2
done


## Use API to list the datasets
token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
resp="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/datasets/list" | jq '. | length')"
if [ "$resp" -ne 2 ]; then
echo "Error when listing key hash, expected 2 entries got: $resp"
exit 1
fi


echo "mapping test completed successfully"
43 changes: 43 additions & 0 deletions sda/cmd/api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ func setup(config *config.Config) *http.Server {
r := gin.Default()
r.GET("/ready", readinessResponse)
r.GET("/files", rbac(e), getFiles)
r.GET("/datasets", rbac(e), listDatasets)
// admin endpoints below here
r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) // Adds a key hash to the database
r.GET("/c4gh-keys/list", rbac(e), listC4ghHashes) // Lists key hashes in the database
Expand All @@ -104,6 +105,8 @@ func setup(config *config.Config) *http.Server {
r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file
r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset
r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible
r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status
r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specififc user
r.GET("/users", rbac(e), listActiveUsers) // Lists all users
r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user
cfg := &tls.Config{MinVersion: tls.VersionTLS12}
Expand Down Expand Up @@ -599,3 +602,43 @@ func deprecateC4ghHash(c *gin.Context) {
return
}
}

func listAllDatasets(c *gin.Context) {
datasets, err := Conf.API.DB.ListDatasets()
if err != nil {
log.Errorf("ListAllDatasets failed, reason: %s", err.Error())
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())

return
}
c.JSON(http.StatusOK, datasets)
}

func listUserDatasets(c *gin.Context) {
username := strings.TrimPrefix(c.Param("username"), "/")
datasets, err := Conf.API.DB.ListUserDatasets(username)
if err != nil {
log.Errorf("ListUserDatasets failed, reason: %s", err.Error())
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())

return
}
c.JSON(http.StatusOK, datasets)
}

func listDatasets(c *gin.Context) {
token, err := auth.Authenticate(c.Request)
if err != nil {
c.JSON(401, err.Error())

return
}
datasets, err := Conf.API.DB.ListUserDatasets(token.Subject())
if err != nil {
log.Errorf("ListDatasets failed, reason: %s", err.Error())
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())

return
}
c.JSON(http.StatusOK, datasets)
}
55 changes: 53 additions & 2 deletions sda/cmd/api/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,23 @@ Endpoints:

If the `token` is invalid, 401 is returned.

- `/datasets`
- accepts `GET` requests
- Returns all datasets, along with their status and last modified timestamp, for which the user has submitted data.

- Error codes
- `200` Query execute ok.
- `400` Error due to bad payload.
- `401` Token user is not in the list of admins.
- `500` Internal error due to DB failures.

Example:

```bash
$curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/datasets
[{"DatasetID":"EGAD74900000101","Status":"deprecated","Timestamp":"2024-11-05T11:31:16.81475Z"}]
```

### Admin endpoints

Admin endpoints are only available to a set of whitelisted users specified in the application config.
Expand Down Expand Up @@ -89,8 +106,42 @@ Admin endpoints are only available to a set of whitelisted users specified in th
curl -H "Authorization: Bearer $token" -X POST https://HOSTNAME/dataset/release/my-dataset-01
```

- `/datasets/list`
- accepts `GET` requests
- Returns all datasets together with their status and last modified timestamp.

- Error codes
- `200` Query execute ok.
- `400` Error due to bad payload.
- `401` Token user is not in the list of admins.
- `500` Internal error due to DB failures.

Example:

```bash
$curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/datasets/list
[{"DatasetID":"EGAD74900000101","Status":"deprecated","Timestamp":"2024-11-05T11:31:16.81475Z"},{"DatasetID":"SYNC-001-12345","Status":"registered","Timestamp":"2024-11-05T11:31:16.965226Z"}]
```

- `/datasets/list/:username`
- accepts `GET` requests with the username name as last part of the path`
- Returns all datasets, along with their status and last modified timestamp,for which the user has submitted data.
- Error codes
- `200` Query execute ok.
- `400` Error due to bad payload.
- `401` Token user is not in the list of admins.
- `500` Internal error due to DB failures.
Example:
```bash
curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/datasets/list/submission-user
[{"DatasetID":"EGAD74900000101","Status":"deprecated","Timestamp":"2024-11-05T11:31:16.81475Z"}]
```

- `/users`
- accepts `GET` requests`
- accepts `GET` requests
- Returns all users with active uploads as a JSON array

Example:
Expand All @@ -105,7 +156,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th
- `500` Internal error due to DB failure.

- `/users/:username/files`
- accepts `GET` requests`
- accepts `GET` requests
- Returns all files (that are not part of a dataset) for a user with active uploads as a JSON array

Example:
Expand Down
172 changes: 172 additions & 0 deletions sda/cmd/api/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1769,3 +1769,175 @@ func (suite *TestSuite) TestDeprecateC4ghHash_wrongHash() {
assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode)
defer resp.Body.Close()
}

func (suite *TestSuite) TestListDatasets() {
for i := 0; i < 5; i++ {
fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy")
if err != nil {
suite.FailNow("failed to register file in database")
}

stableID := fmt.Sprintf("accession_%s_0%d", "dummy", i)
err = Conf.API.DB.SetAccessionID(stableID, fileID)
if err != nil {
suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
}
}

err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_dummy_00", "accession_dummy_01", "accession_dummy_02"})
if err != nil {
suite.FailNow("failed to map files to dataset")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}

err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_dummy_03", "accession_dummy_04"})
if err != nil {
suite.FailNow("failed to map files to dataset")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}

gin.SetMode(gin.ReleaseMode)
assert.NoError(suite.T(), setupJwtAuth())

// Mock request and response holders
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/datasets/list", http.NoBody)
r.Header.Add("Authorization", "Bearer "+suite.Token)

_, router := gin.CreateTestContext(w)
router.GET("/datasets/list", listAllDatasets)
router.GET("/dataset/list", listAllDatasets)

router.ServeHTTP(w, r)
okResponse := w.Result()
defer okResponse.Body.Close()
assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)

datasets := []database.DatasetInfo{}
err = json.NewDecoder(okResponse.Body).Decode(&datasets)
assert.NoError(suite.T(), err, "failed to list datasets from DB")
assert.Equal(suite.T(), 2, len(datasets))
assert.Equal(suite.T(), "released", datasets[1].Status)
assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status))
}

func (suite *TestSuite) TestListUserDatasets() {
for i := 0; i < 5; i++ {
fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@"))
if err != nil {
suite.FailNow("failed to register file in database")
}

stableID := fmt.Sprintf("accession_%s_0%d", "user_example.org", i)
err = Conf.API.DB.SetAccessionID(stableID, fileID)
if err != nil {
suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
}
}

err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"})
if err != nil {
suite.FailNow("failed to map files to dataset")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}

err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"})
if err != nil {
suite.FailNow("failed to map files to dataset")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}

gin.SetMode(gin.ReleaseMode)
assert.NoError(suite.T(), setupJwtAuth())

// Mock request and response holders
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/datasets/list/[email protected]", http.NoBody)
r.Header.Add("Authorization", "Bearer "+suite.Token)

_, router := gin.CreateTestContext(w)
router.GET("/datasets/list/:username", listUserDatasets)

router.ServeHTTP(w, r)
okResponse := w.Result()
defer okResponse.Body.Close()
assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)

datasets := []database.DatasetInfo{}
err = json.NewDecoder(okResponse.Body).Decode(&datasets)
assert.NoError(suite.T(), err, "failed to list datasets from DB")
assert.Equal(suite.T(), 2, len(datasets))
assert.Equal(suite.T(), "released", datasets[1].Status)
assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status))
}

func (suite *TestSuite) TestListDatasetsAsUser() {
for i := 0; i < 5; i++ {
fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), suite.User)
if err != nil {
suite.FailNow("failed to register file in database")
}

stableID := fmt.Sprintf("accession_user_example.org_0%d", i)
err = Conf.API.DB.SetAccessionID(stableID, fileID)
if err != nil {
suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID)
}
}

err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"})
if err != nil {
suite.FailNow("failed to map files to dataset")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}

err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"})
if err != nil {
suite.FailNow("failed to map files to dataset")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}
if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil {
suite.FailNow("failed to update dataset event")
}

gin.SetMode(gin.ReleaseMode)
assert.NoError(suite.T(), setupJwtAuth())

// Mock request and response holders
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/datasets", http.NoBody)
r.Header.Add("Authorization", "Bearer "+suite.Token)

_, router := gin.CreateTestContext(w)
router.GET("/datasets", listDatasets)

router.ServeHTTP(w, r)
okResponse := w.Result()
defer okResponse.Body.Close()
assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode)

datasets := []database.DatasetInfo{}
err = json.NewDecoder(okResponse.Body).Decode(&datasets)
assert.NoError(suite.T(), err, "failed to list datasets from DB")
assert.Equal(suite.T(), 2, len(datasets))
assert.Equal(suite.T(), "released", datasets[1].Status)
assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status))
}
6 changes: 6 additions & 0 deletions sda/internal/database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,12 @@ type SubmissionFileInfo struct {
CreateAt string `json:"createAt"`
}

type DatasetInfo struct {
DatasetID string `json:"datasetID"`
Status string `json:"status"`
Timestamp string `json:"timeStamp"`
}

// SchemaName is the name of the remote database schema to query
var SchemaName = "sda"

Expand Down
Loading

0 comments on commit 84b1534

Please sign in to comment.