Skip to content

Commit

Permalink
Merge pull request #150 from pixlise/development
Browse files Browse the repository at this point in the history 
Public Access
  • Loading branch information
@RyanStonebraker
RyanStonebraker authored May 18, 2023
2 parents 07a6762 + 27dd523 commit 1091222
Show file tree
Hide file tree
Showing 38 changed files with 1,060 additions and 262 deletions.
4 changes: 3 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@

## What is it?

PIXLISE Core is the API and data management processes for the PIXLISE platform.
PIXLISE Core is the API and data management processes for the PIXLISE platform.

PIXLISE is deployed to https://www.pixlise.org

## Building

Expand Down
33 changes: 31 additions & 2 deletions api/endpoints/DataExpression.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,12 @@ import (
func registerDataExpressionHandler(router *apiRouter.ApiObjectRouter) {
const pathPrefix = "data-expression"

router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataExpressionList)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataExpressionList)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), dataExpressionPost)

router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), dataExpressionPut)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataAnalysis), dataExpressionDelete)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataExpressionGet)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataExpressionGet)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+"/execution-stat", idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), dataExpressionExecutionStatPut)

router.AddShareHandler(handlers.MakeEndpointPath(shareURLRoot+"/"+pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteSharedExpression), dataExpressionShare)
Expand Down Expand Up @@ -82,10 +82,27 @@ func dataExpressionList(params handlers.ApiHandlerParams) (interface{}, error) {
return nil, err
}

isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis]

result := map[string]expressions.DataExpressionWire{}

publicObjectsAuth, err := permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser)
if err != nil {
return result, err
}

// We're sending them back in a different struct for legacy reasons
for _, item := range items {
if isPublicUser {
isExpressionPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Expressions, item.ID)
if err != nil {
return result, err
}

if !isExpressionPublic {
continue
}
}
resultItem := toWire(item)
result[resultItem.ID] = resultItem
}
Expand All @@ -98,6 +115,18 @@ func dataExpressionGet(params handlers.ApiHandlerParams) (interface{}, error) {
itemID := params.PathParams[idIdentifier]
strippedID, _ := utils.StripSharedItemIDPrefix(itemID)

isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis]
if isPublicUser {
isExpressionPublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectExpression, strippedID)
if err != nil {
return nil, err
}

if !isExpressionPublic {
return nil, api.MakeBadRequestError(errors.New("expression is not public"))
}
}

// Get expression
expr, err := params.Svcs.Expressions.GetExpression(strippedID, true)
if err != nil {
Expand Down
49 changes: 46 additions & 3 deletions api/endpoints/DataModule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ func registerDataModuleHandler(router *apiRouter.ApiObjectRouter) {
const pathPrefix = "data-module"

// Listing
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataModuleList)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataModuleList)
// Getting an individual module
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier, idVersion), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataModuleGet)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier, idVersion), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataModuleGet)
// Adding a new module
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), dataModulePost)
// Adding a new version for a module
Expand All @@ -50,13 +50,56 @@ func registerDataModuleHandler(router *apiRouter.ApiObjectRouter) {
}

func dataModuleList(params handlers.ApiHandlerParams) (interface{}, error) {
return params.Svcs.Expressions.ListModules(true)
filteredModules := modules.DataModuleWireLookup{}

isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis]
allModules, err := params.Svcs.Expressions.ListModules(true)
if err != nil {
return filteredModules, err
}

if isPublicUser {
publicObjectsAuth, err := permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser)
if err != nil {
return nil, err
}

// Filter out any modules that are not public
for _, mod := range allModules {
isModPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Modules, mod.ID)
if err != nil {
return nil, err
}

if isModPublic {
fmt.Println("MOD IS PUBLIC, ADDING", mod.ID)
filteredModules[mod.ID] = mod
}
}
} else {
// No filtering needed
filteredModules = allModules
}

return filteredModules, nil
}

func dataModuleGet(params handlers.ApiHandlerParams) (interface{}, error) {
modID := params.PathParams[idIdentifier]
version := params.PathParams[idVersion]

isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis]
if isPublicUser {
isModulePublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectModule, modID)
if err != nil {
return nil, err
}

if !isModulePublic {
return nil, api.MakeBadRequestError(errors.New("module is not public"))
}
}

var ver *modules.SemanticVersion

if len(version) > 0 {
Expand Down
51 changes: 44 additions & 7 deletions api/endpoints/Dataset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,28 +75,28 @@ var allowedQueryNames = map[string]bool{

func registerDatasetHandler(router *apiRouter.ApiObjectRouter) {
// Listing datasets (tiles screen)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetListing)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetListing)

// Creating datasets
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataset), datasetCreatePost)

// Regeneration/manual editing of datasets
// Setting/getting meta fields
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/meta", datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetCustomMetaGet)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/meta", datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetCustomMetaGet)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/meta", datasetIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataset), datasetCustomMetaPut)

// Reprocess
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/reprocess", datasetIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermReadDataAnalysis), datasetReprocess)

// Adding/viewing/removing extra images (eg WATSON)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetCustomImagesList)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetCustomImageGet)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetCustomImagesList)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetCustomImageGet)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataset), datasetCustomImagesPost)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataset), datasetCustomImagesPut)
router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataset), datasetCustomImagesDelete)

// Streaming from S3
router.AddCacheControlledStreamHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/"+handlers.UrlStreamDownloadIndicator, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetFileStream)
router.AddCacheControlledStreamHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/"+handlers.UrlStreamDownloadIndicator, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetFileStream)
}

func readDataSetData(svcs *services.APIServices, s3Path string) (datasetModel.DatasetConfig, error) {
Expand Down Expand Up @@ -306,11 +306,48 @@ func datasetListing(params handlers.ApiHandlerParams) (interface{}, error) {
return nil, err
}

datasetsAuth := permission.DatasetsAuth{}
publicObjectsAuth := permission.PublicObjectsAuth{}
isSuperAdmin := params.UserInfo.Permissions[permission.PermSuperAdmin]
isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis]

if !isSuperAdmin {
datasetsAuthPath := filepaths.GetDatasetsAuthPath()
datasetsAuth, err = permission.ReadDatasetsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, datasetsAuthPath)
if err != nil {
return nil, err
}

publicObjectsAuth, err = permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser)
if err != nil {
return nil, err
}
}

userAllowedGroups := permission.GetAccessibleGroups(params.UserInfo.Permissions)

for _, item := range dataSets.Datasets {
isPublic := false
if !isSuperAdmin {
isPublic, err = permission.CheckAndUpdatePublicDataset(params.Svcs.FS, params.Svcs.Config.ConfigBucket, item.DatasetID, datasetsAuth)
if err != nil {
return nil, err
}
}

if isPublicUser {
isDatasetPublicWithObjects, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Datasets, item.DatasetID)
if err != nil {
return nil, err
}

if !isDatasetPublicWithObjects {
continue
}
}

// Check that the user is allowed to see this dataset based on group permissions
if !userAllowedGroups[item.Group] {
if !userAllowedGroups[item.Group] && !isPublic {
continue
}

Expand Down Expand Up @@ -352,7 +389,7 @@ func datasetFileStream(params handlers.ApiHandlerStreamParams) (*s3.GetObjectOut
statuscode := 200

// Due to newly implemented group permissions, we now need to download the dataset summary to check the group is allowable
summary, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, datasetID)
summary, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID)
if err != nil {
return nil, "", "", "", http.StatusInternalServerError, err
}
Expand Down
10 changes: 10 additions & 0 deletions api/endpoints/DatasetCustomisation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (

"github.com/pixlise/core/v3/api/filepaths"
"github.com/pixlise/core/v3/api/handlers"
"github.com/pixlise/core/v3/api/permission"
)

// NOTE: No registration function here, this sits in the dataset registration function, shares paths with it. Only separated
Expand Down Expand Up @@ -119,6 +120,15 @@ func datasetCustomImagesList(params handlers.ApiHandlerParams) (interface{}, err
datasetID := params.PathParams[datasetIdentifier]
imgType := params.PathParams[customImageTypeIdentifier]

isSuperAdmin := params.UserInfo.Permissions[permission.PermSuperAdmin]

if !isSuperAdmin {
_, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID)
if err != nil {
return nil, err
}
}

if !isValidCustomImageType(imgType) {
return nil, api.MakeBadRequestError(fmt.Errorf("Invalid custom image type: \"%v\"", imgType))
}
Expand Down
28 changes: 28 additions & 0 deletions api/endpoints/DatasetCustomisation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/pixlise/core/v3/core/awsutil"
"github.com/pixlise/core/v3/core/pixlUser"
)

const artifactManualUploadBucket = "Manual-Uploads"
Expand Down Expand Up @@ -148,6 +149,15 @@ func Example_datasetCustomImagesList_rgbu() {

svcs := MakeMockSvcs(&mockS3, nil, nil, nil)
svcs.Config.ManualUploadBucket = artifactManualUploadBucket
mockUser := pixlUser.UserInfo{
Name: "Niko Bellic",
UserID: "600f2a0806b6c70071d3d174",
Permissions: map[string]bool{
"read:data-analysis": true,
"access:super-admin": true,
},
}
svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser}
apiRouter := MakeRouter(svcs)

req, _ := http.NewRequest("GET", "/dataset/images/abc-111/rgbu", nil)
Expand Down Expand Up @@ -195,6 +205,15 @@ func Example_datasetCustomImagesList_unaligned() {

svcs := MakeMockSvcs(&mockS3, nil, nil, nil)
svcs.Config.ManualUploadBucket = artifactManualUploadBucket
mockUser := pixlUser.UserInfo{
Name: "Niko Bellic",
UserID: "600f2a0806b6c70071d3d174",
Permissions: map[string]bool{
"read:data-analysis": true,
"access:super-admin": true,
},
}
svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser}
apiRouter := MakeRouter(svcs)

req, _ := http.NewRequest("GET", "/dataset/images/abc-111/unaligned", nil)
Expand Down Expand Up @@ -247,6 +266,15 @@ func Example_datasetCustomImagesList_matched() {

svcs := MakeMockSvcs(&mockS3, nil, nil, nil)
svcs.Config.ManualUploadBucket = artifactManualUploadBucket
mockUser := pixlUser.UserInfo{
Name: "Niko Bellic",
UserID: "600f2a0806b6c70071d3d174",
Permissions: map[string]bool{
"read:data-analysis": true,
"access:super-admin": true,
},
}
svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser}
apiRouter := MakeRouter(svcs)

req, _ := http.NewRequest("GET", "/dataset/images/abc-111/matched", nil)
Expand Down
24 changes: 22 additions & 2 deletions api/endpoints/Dataset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,8 @@ func Example_datasetHandler_List() {
"access:the-group": true,
"access:groupie": true,
"access:another-group": true,
"access:super-admin": true,
"read:data-analysis": true,
},
}
svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser}
Expand Down Expand Up @@ -406,7 +408,7 @@ func Example_datasetHandler_List() {
// }
// ]
//
// Permissions left: 2
// Permissions left: 4
// 200
// [
// {
Expand Down Expand Up @@ -584,17 +586,33 @@ func Example_datasetHandler_Stream_BadGroup_403() {
"pseudo_intensities": 441,
"detector_config": "PIXL"
}`

const publicDatasetsJSON = `{
"590340": {
"dataset_id": "590340",
"public": false,
"public_release_utc_time_sec": 0,
"sol": ""
}
}`

var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
{
Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/590340/summary.json"),
},
{
Bucket: aws.String("config-bucket"), Key: aws.String("PixliseConfig/datasets-auth.json"),
},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
{
Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))),
},
{
Body: ioutil.NopCloser(bytes.NewReader([]byte(publicDatasetsJSON))),
},
}

svcs := MakeMockSvcs(&mockS3, nil, nil, nil)
Expand All @@ -603,7 +621,9 @@ func Example_datasetHandler_Stream_BadGroup_403() {
"600f2a0806b6c70071d3d174",
"[email protected]",
map[string]bool{
"access:the-group": true,
"access:the-group": true,
"access:super-admin": true,
"read:data-analysis": true,
},
}
svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser}
Expand Down
4 changes: 2 additions & 2 deletions api/endpoints/ElementSet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,10 @@ type elementSetSummaryLookup map[string]elementSetSummary
func registerElementSetHandler(router *apiRouter.ApiObjectRouter) {
const pathPrefix = "element-set"

router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), elementSetList)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), elementSetList)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), elementSetPost)

router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), elementSetGet)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), elementSetGet)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), elementSetPut)
router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataAnalysis), elementSetDelete)

Expand Down
Loading

0 comments on commit 1091222

Please sign in to comment.