diff --git a/README.md b/README.md index e3925c2d..24a286b1 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ ## What is it? -PIXLISE Core is the API and data management processes for the PIXLISE platform. +PIXLISE Core is the API and data management processes for the PIXLISE platform. + +PIXLISE is deployed to https://www.pixlise.org ## Building diff --git a/api/endpoints/DataExpression.go b/api/endpoints/DataExpression.go index 084931da..da6b3e78 100644 --- a/api/endpoints/DataExpression.go +++ b/api/endpoints/DataExpression.go @@ -41,12 +41,12 @@ import ( func registerDataExpressionHandler(router *apiRouter.ApiObjectRouter) { const pathPrefix = "data-expression" - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataExpressionList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataExpressionList) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), dataExpressionPost) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), dataExpressionPut) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataAnalysis), dataExpressionDelete) - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataExpressionGet) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataExpressionGet) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+"/execution-stat", idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), dataExpressionExecutionStatPut) router.AddShareHandler(handlers.MakeEndpointPath(shareURLRoot+"/"+pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteSharedExpression), dataExpressionShare) @@ -82,10 +82,27 @@ func dataExpressionList(params handlers.ApiHandlerParams) (interface{}, error) { return nil, err } + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + result := map[string]expressions.DataExpressionWire{} + publicObjectsAuth, err := permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return result, err + } + // We're sending them back in a different struct for legacy reasons for _, item := range items { + if isPublicUser { + isExpressionPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Expressions, item.ID) + if err != nil { + return result, err + } + + if !isExpressionPublic { + continue + } + } resultItem := toWire(item) result[resultItem.ID] = resultItem } @@ -98,6 +115,18 @@ func dataExpressionGet(params handlers.ApiHandlerParams) (interface{}, error) { itemID := params.PathParams[idIdentifier] strippedID, _ := utils.StripSharedItemIDPrefix(itemID) + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + if isPublicUser { + isExpressionPublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectExpression, strippedID) + if err != nil { + return nil, err + } + + if !isExpressionPublic { + return nil, api.MakeBadRequestError(errors.New("expression is not public")) + } + } + // Get expression expr, err := params.Svcs.Expressions.GetExpression(strippedID, true) if err != nil { diff --git a/api/endpoints/DataModule.go b/api/endpoints/DataModule.go index ba1554b3..d14b9f37 100644 --- a/api/endpoints/DataModule.go +++ b/api/endpoints/DataModule.go @@ -39,9 +39,9 @@ func registerDataModuleHandler(router *apiRouter.ApiObjectRouter) { const pathPrefix = "data-module" // Listing - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataModuleList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataModuleList) // Getting an individual module - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier, idVersion), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), dataModuleGet) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier, idVersion), apiRouter.MakeMethodPermission("GET", permission.PermPublic), dataModuleGet) // Adding a new module router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), dataModulePost) // Adding a new version for a module @@ -50,13 +50,56 @@ func registerDataModuleHandler(router *apiRouter.ApiObjectRouter) { } func dataModuleList(params handlers.ApiHandlerParams) (interface{}, error) { - return params.Svcs.Expressions.ListModules(true) + filteredModules := modules.DataModuleWireLookup{} + + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + allModules, err := params.Svcs.Expressions.ListModules(true) + if err != nil { + return filteredModules, err + } + + if isPublicUser { + publicObjectsAuth, err := permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return nil, err + } + + // Filter out any modules that are not public + for _, mod := range allModules { + isModPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Modules, mod.ID) + if err != nil { + return nil, err + } + + if isModPublic { + fmt.Println("MOD IS PUBLIC, ADDING", mod.ID) + filteredModules[mod.ID] = mod + } + } + } else { + // No filtering needed + filteredModules = allModules + } + + return filteredModules, nil } func dataModuleGet(params handlers.ApiHandlerParams) (interface{}, error) { modID := params.PathParams[idIdentifier] version := params.PathParams[idVersion] + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + if isPublicUser { + isModulePublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectModule, modID) + if err != nil { + return nil, err + } + + if !isModulePublic { + return nil, api.MakeBadRequestError(errors.New("module is not public")) + } + } + var ver *modules.SemanticVersion if len(version) > 0 { diff --git a/api/endpoints/Dataset.go b/api/endpoints/Dataset.go index 649b2f46..b8a70556 100644 --- a/api/endpoints/Dataset.go +++ b/api/endpoints/Dataset.go @@ -75,28 +75,28 @@ var allowedQueryNames = map[string]bool{ func registerDatasetHandler(router *apiRouter.ApiObjectRouter) { // Listing datasets (tiles screen) - router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetListing) + router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetListing) // Creating datasets router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataset), datasetCreatePost) // Regeneration/manual editing of datasets // Setting/getting meta fields - router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/meta", datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetCustomMetaGet) + router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/meta", datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetCustomMetaGet) router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/meta", datasetIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataset), datasetCustomMetaPut) // Reprocess router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/reprocess", datasetIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermReadDataAnalysis), datasetReprocess) // Adding/viewing/removing extra images (eg WATSON) - router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetCustomImagesList) - router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetCustomImageGet) + router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetCustomImagesList) + router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetCustomImageGet) router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataset), datasetCustomImagesPost) router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataset), datasetCustomImagesPut) router.AddJSONHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/images", datasetIdentifier, customImageTypeIdentifier, customImageIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataset), datasetCustomImagesDelete) // Streaming from S3 - router.AddCacheControlledStreamHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/"+handlers.UrlStreamDownloadIndicator, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), datasetFileStream) + router.AddCacheControlledStreamHandler(handlers.MakeEndpointPath(datasetPathPrefix+"/"+handlers.UrlStreamDownloadIndicator, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), datasetFileStream) } func readDataSetData(svcs *services.APIServices, s3Path string) (datasetModel.DatasetConfig, error) { @@ -306,11 +306,48 @@ func datasetListing(params handlers.ApiHandlerParams) (interface{}, error) { return nil, err } + datasetsAuth := permission.DatasetsAuth{} + publicObjectsAuth := permission.PublicObjectsAuth{} + isSuperAdmin := params.UserInfo.Permissions[permission.PermSuperAdmin] + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + + if !isSuperAdmin { + datasetsAuthPath := filepaths.GetDatasetsAuthPath() + datasetsAuth, err = permission.ReadDatasetsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, datasetsAuthPath) + if err != nil { + return nil, err + } + + publicObjectsAuth, err = permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return nil, err + } + } + userAllowedGroups := permission.GetAccessibleGroups(params.UserInfo.Permissions) for _, item := range dataSets.Datasets { + isPublic := false + if !isSuperAdmin { + isPublic, err = permission.CheckAndUpdatePublicDataset(params.Svcs.FS, params.Svcs.Config.ConfigBucket, item.DatasetID, datasetsAuth) + if err != nil { + return nil, err + } + } + + if isPublicUser { + isDatasetPublicWithObjects, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Datasets, item.DatasetID) + if err != nil { + return nil, err + } + + if !isDatasetPublicWithObjects { + continue + } + } + // Check that the user is allowed to see this dataset based on group permissions - if !userAllowedGroups[item.Group] { + if !userAllowedGroups[item.Group] && !isPublic { continue } @@ -352,7 +389,7 @@ func datasetFileStream(params handlers.ApiHandlerStreamParams) (*s3.GetObjectOut statuscode := 200 // Due to newly implemented group permissions, we now need to download the dataset summary to check the group is allowable - summary, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, datasetID) + summary, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) if err != nil { return nil, "", "", "", http.StatusInternalServerError, err } diff --git a/api/endpoints/DatasetCustomisation.go b/api/endpoints/DatasetCustomisation.go index 47f39811..bdc4471d 100644 --- a/api/endpoints/DatasetCustomisation.go +++ b/api/endpoints/DatasetCustomisation.go @@ -35,6 +35,7 @@ import ( "github.com/pixlise/core/v3/api/filepaths" "github.com/pixlise/core/v3/api/handlers" + "github.com/pixlise/core/v3/api/permission" ) // NOTE: No registration function here, this sits in the dataset registration function, shares paths with it. Only separated @@ -119,6 +120,15 @@ func datasetCustomImagesList(params handlers.ApiHandlerParams) (interface{}, err datasetID := params.PathParams[datasetIdentifier] imgType := params.PathParams[customImageTypeIdentifier] + isSuperAdmin := params.UserInfo.Permissions[permission.PermSuperAdmin] + + if !isSuperAdmin { + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + } + if !isValidCustomImageType(imgType) { return nil, api.MakeBadRequestError(fmt.Errorf("Invalid custom image type: \"%v\"", imgType)) } diff --git a/api/endpoints/DatasetCustomisation_test.go b/api/endpoints/DatasetCustomisation_test.go index a60ba395..deeed55e 100644 --- a/api/endpoints/DatasetCustomisation_test.go +++ b/api/endpoints/DatasetCustomisation_test.go @@ -26,6 +26,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/pixlise/core/v3/core/awsutil" + "github.com/pixlise/core/v3/core/pixlUser" ) const artifactManualUploadBucket = "Manual-Uploads" @@ -148,6 +149,15 @@ func Example_datasetCustomImagesList_rgbu() { svcs := MakeMockSvcs(&mockS3, nil, nil, nil) svcs.Config.ManualUploadBucket = artifactManualUploadBucket + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:data-analysis": true, + "access:super-admin": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) req, _ := http.NewRequest("GET", "/dataset/images/abc-111/rgbu", nil) @@ -195,6 +205,15 @@ func Example_datasetCustomImagesList_unaligned() { svcs := MakeMockSvcs(&mockS3, nil, nil, nil) svcs.Config.ManualUploadBucket = artifactManualUploadBucket + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:data-analysis": true, + "access:super-admin": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) req, _ := http.NewRequest("GET", "/dataset/images/abc-111/unaligned", nil) @@ -247,6 +266,15 @@ func Example_datasetCustomImagesList_matched() { svcs := MakeMockSvcs(&mockS3, nil, nil, nil) svcs.Config.ManualUploadBucket = artifactManualUploadBucket + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:data-analysis": true, + "access:super-admin": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) req, _ := http.NewRequest("GET", "/dataset/images/abc-111/matched", nil) diff --git a/api/endpoints/Dataset_test.go b/api/endpoints/Dataset_test.go index 057b8fe8..9d01e30a 100644 --- a/api/endpoints/Dataset_test.go +++ b/api/endpoints/Dataset_test.go @@ -267,6 +267,8 @@ func Example_datasetHandler_List() { "access:the-group": true, "access:groupie": true, "access:another-group": true, + "access:super-admin": true, + "read:data-analysis": true, }, } svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} @@ -406,7 +408,7 @@ func Example_datasetHandler_List() { // } // ] // - // Permissions left: 2 + // Permissions left: 4 // 200 // [ // { @@ -584,17 +586,33 @@ func Example_datasetHandler_Stream_BadGroup_403() { "pseudo_intensities": 441, "detector_config": "PIXL" }` + + const publicDatasetsJSON = `{ + "590340": { + "dataset_id": "590340", + "public": false, + "public_release_utc_time_sec": 0, + "sol": "" + } +}` + var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/590340/summary.json"), }, + { + Bucket: aws.String("config-bucket"), Key: aws.String("PixliseConfig/datasets-auth.json"), + }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))), }, + { + Body: ioutil.NopCloser(bytes.NewReader([]byte(publicDatasetsJSON))), + }, } svcs := MakeMockSvcs(&mockS3, nil, nil, nil) @@ -603,7 +621,9 @@ func Example_datasetHandler_Stream_BadGroup_403() { "600f2a0806b6c70071d3d174", "niko@rockstar.com", map[string]bool{ - "access:the-group": true, + "access:the-group": true, + "access:super-admin": true, + "read:data-analysis": true, }, } svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} diff --git a/api/endpoints/ElementSet.go b/api/endpoints/ElementSet.go index 87a7bf0f..015a8160 100644 --- a/api/endpoints/ElementSet.go +++ b/api/endpoints/ElementSet.go @@ -86,10 +86,10 @@ type elementSetSummaryLookup map[string]elementSetSummary func registerElementSetHandler(router *apiRouter.ApiObjectRouter) { const pathPrefix = "element-set" - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), elementSetList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), elementSetList) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), elementSetPost) - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), elementSetGet) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), elementSetGet) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), elementSetPut) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataAnalysis), elementSetDelete) diff --git a/api/endpoints/HelpersFor_test.go b/api/endpoints/HelpersFor_test.go index dd47e9da..1348e198 100644 --- a/api/endpoints/HelpersFor_test.go +++ b/api/endpoints/HelpersFor_test.go @@ -54,10 +54,12 @@ func (m MockJWTReader) GetUserInfo(*http.Request) (pixlUser.UserInfo, error) { } //This user id is real don't change it.... return pixlUser.UserInfo{ - Name: "Niko Bellic", - UserID: "600f2a0806b6c70071d3d174", - Email: "niko@spicule.co.uk", - Permissions: map[string]bool{}, + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Email: "niko@spicule.co.uk", + Permissions: map[string]bool{ + "read:data-analysis": true, + }, }, nil } @@ -177,8 +179,8 @@ func checkResult(t *testing.T, resp *httptest.ResponseRecorder, expectedStatus i gotRespBody := resp.Body.String() if gotRespBody != expectedBody { - t.Errorf("Bad resp body:\n%v", gotRespBody) - t.Errorf("vs expected body:\n%v", expectedBody) + t.Errorf("Bad resp body:\n|%v|", gotRespBody) + t.Errorf("vs expected body:\n|%v|", expectedBody) } } diff --git a/api/endpoints/PrometheusMiddleware.go b/api/endpoints/PrometheusMiddleware.go new file mode 100644 index 00000000..baf50afb --- /dev/null +++ b/api/endpoints/PrometheusMiddleware.go @@ -0,0 +1,30 @@ +package endpoints + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "net/http" + "time" +) + +var ( + httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_response_time_seconds", + Help: "Duration of HTTP requests.", + }, []string{"path"}) + httpRequests = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "Number of HTTP requests.", + }, []string{"path"}) +) + +func PrometheusMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + duration := time.Since(start) + path := r.URL.Path + httpDuration.WithLabelValues(path).Observe(duration.Seconds()) + httpRequests.WithLabelValues(path).Inc() + }) +} diff --git a/api/endpoints/Quantification.go b/api/endpoints/Quantification.go index cf7dd6bd..919a3560 100644 --- a/api/endpoints/Quantification.go +++ b/api/endpoints/Quantification.go @@ -62,7 +62,7 @@ func registerQuantificationHandler(router *apiRouter.ApiObjectRouter) { router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadPiquantJobs), quantificationJobAdminList) // Normal users can access this - what quants are available and in-progress - router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), quantificationList) + router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), quantificationList) // Multi-quant comparison router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix+"/comparison-for-roi", datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermReadDataAnalysis), multiQuantificationComparisonPost) @@ -80,7 +80,7 @@ func registerQuantificationHandler(router *apiRouter.ApiObjectRouter) { router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix+"/combine", datasetIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermCreateQuantification), quantificationCombine) // Accessing individual quant - router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), quantificationGet) + router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), quantificationGet) // Deleting a quant router.AddJSONHandler(handlers.MakeEndpointPath(quantURLPathPrefix, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataAnalysis), quantificationDelete) @@ -95,7 +95,7 @@ func registerQuantificationHandler(router *apiRouter.ApiObjectRouter) { router.AddShareHandler(handlers.MakeEndpointPath(shareURLRoot+"/"+quantURLPathPrefix, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteSharedQuantification), quantificationShare) // Streaming quant files from S3 (map command) - router.AddStreamHandler(handlers.MakeEndpointPath(quantURLPathPrefix+"/"+handlers.UrlStreamDownloadIndicator, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), quantificationFileStream) + router.AddStreamHandler(handlers.MakeEndpointPath(quantURLPathPrefix+"/"+handlers.UrlStreamDownloadIndicator, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), quantificationFileStream) // Streaming log files from S3 router.AddStreamHandler(handlers.MakeEndpointPath(quantURLPathPrefix+"/log/"+handlers.UrlStreamDownloadIndicator, datasetIdentifier, idIdentifier, quantLogIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), quantificationLogFileStream) diff --git a/api/endpoints/QuantificationLastRunDownload.go b/api/endpoints/QuantificationLastRunDownload.go index 0e1ec944..05899d39 100644 --- a/api/endpoints/QuantificationLastRunDownload.go +++ b/api/endpoints/QuantificationLastRunDownload.go @@ -33,7 +33,7 @@ func quantificationLastRunFileStream(params handlers.ApiHandlerStreamParams) (*s datasetID := params.PathParams[datasetIdentifier] // Check if user has rights for this dataset - _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, datasetID) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) if err != nil { return nil, "", err } diff --git a/api/endpoints/QuantificationLogDownload.go b/api/endpoints/QuantificationLogDownload.go index 489fc152..adf96a03 100644 --- a/api/endpoints/QuantificationLogDownload.go +++ b/api/endpoints/QuantificationLogDownload.go @@ -35,7 +35,7 @@ func quantificationLogFileStream(params handlers.ApiHandlerStreamParams) (*s3.Ge // First, check if the user is allowed to access the given dataset datasetID := params.PathParams[datasetIdentifier] - _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, datasetID) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) if err != nil { return nil, "", err } diff --git a/api/endpoints/QuantificationRetrieve.go b/api/endpoints/QuantificationRetrieve.go index 57694a76..66b41237 100644 --- a/api/endpoints/QuantificationRetrieve.go +++ b/api/endpoints/QuantificationRetrieve.go @@ -18,6 +18,7 @@ package endpoints import ( + "errors" "path" "sort" "strings" @@ -109,7 +110,12 @@ func (a ByJobID) Less(i, j int) bool { return a[i].JobID < a[j].JobID } func quantificationList(params handlers.ApiHandlerParams) (interface{}, error) { datasetID := params.PathParams[datasetIdentifier] - summaries, availableQuantIds, err := listQuantsForUser(params.Svcs, datasetID, params.UserInfo.UserID) + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + + summaries, availableQuantIds, err := listQuantsForUser(params.Svcs, datasetID, params.UserInfo.UserID, isPublicUser) + if err != nil { + return nil, err + } // Also list in-progress quantifications processing, err := quantModel.ListQuantJobsForDataset(params.Svcs, params.UserInfo.UserID, params.PathParams[datasetIdentifier]) @@ -153,7 +159,7 @@ func quantificationList(params handlers.ApiHandlerParams) (interface{}, error) { return &QuantListingResponse{Summaries: summaries, BlessedQuant: blessItem}, nil } -func listQuantsForUser(svcs *services.APIServices, datasetID string, userID string) ([]quantModel.JobSummaryItem, map[string]bool, error) { +func listQuantsForUser(svcs *services.APIServices, datasetID string, userID string, isPublicUser bool) ([]quantModel.JobSummaryItem, map[string]bool, error) { userQuantSummaryPrefixedPath := filepaths.GetUserQuantPath(userID, datasetID, filepaths.QuantSummaryFilePrefix) sharedQuantSummaryPrefixedPath := filepaths.GetSharedQuantPath(datasetID, filepaths.QuantSummaryFilePrefix) @@ -210,7 +216,23 @@ func listQuantsForUser(svcs *services.APIServices, datasetID string, userID stri } } + publicObjectsAuth, err := permission.GetPublicObjectsAuth(svcs.FS, svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return summaries, availableQuantIds, err + } + for summary := range summariesCh { + // If we're a public user, check if the quant is in the public set and filter out if not + if isPublicUser { + isQuantPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Quantifications, summary.JobID) + if err != nil { + return summaries, availableQuantIds, err + } + + if !isQuantPublic { + continue + } + } summaries = append(summaries, summary) availableQuantIds[summary.JobID] = true } @@ -229,12 +251,24 @@ func quantificationGet(params handlers.ApiHandlerParams) (interface{}, error) { // First, check if the user is allowed to access the given dataset datasetID := params.PathParams[datasetIdentifier] - _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, datasetID) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) if err != nil { return nil, err } jobID := params.PathParams[idIdentifier] + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + + if isPublicUser { + isQuantPublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectQuantification, jobID) + if err != nil { + return nil, err + } + + if !isQuantPublic { + return nil, api.MakeBadRequestError(errors.New("quantification is not public")) + } + } requestJobID := jobID @@ -277,16 +311,30 @@ func quantificationFileStream(params handlers.ApiHandlerStreamParams) (*s3.GetOb // First, check if the user is allowed to access the given dataset datasetID := params.PathParams[datasetIdentifier] - _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, datasetID) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) if err != nil { return nil, "", err } + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + jobID := params.PathParams[idIdentifier] + if isPublicUser { + isQuantPublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectQuantification, jobID) + if err != nil { + return nil, "", err + } + + if !isQuantPublic { + return nil, "", api.MakeBadRequestError(errors.New("quantification is not public")) + } + } + fileName := filepaths.MakeQuantDataFileName(jobID) binPath := filepaths.GetUserQuantPath(params.UserInfo.UserID, datasetID, fileName) strippedID, isSharedReq := utils.StripSharedItemIDPrefix(jobID) + if isSharedReq { jobID = strippedID // New job ID! diff --git a/api/endpoints/QuantificationRetrieve_test.go b/api/endpoints/QuantificationRetrieve_test.go index d59ceecf..3e688c3c 100644 --- a/api/endpoints/QuantificationRetrieve_test.go +++ b/api/endpoints/QuantificationRetrieve_test.go @@ -865,6 +865,15 @@ func Test_quantHandler_Get(t *testing.T) { "pseudo_intensities": 441, "detector_config": "PIXL" }` + + const publicDatasetsJSON = `{ + "rtt-456": { + "dataset_id": "rtt-456", + "public": false, + "public_release_utc_time_sec": 0, + "sol": "" + } +}` var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() @@ -873,34 +882,25 @@ func Test_quantHandler_Get(t *testing.T) { Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), }, { - Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), + Bucket: aws.String(UsersBucketForUnitTest), Key: aws.String("UserContent/600f2a0806b6c70071d3d174/rtt-456/Quantifications/summary-job1.json"), // 4 }, { - Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), + Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), // 1 }, { - Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), + Bucket: aws.String("config-bucket"), Key: aws.String("PixliseConfig/datasets-auth.json"), // 1 }, { - Bucket: aws.String(UsersBucketForUnitTest), Key: aws.String("UserContent/600f2a0806b6c70071d3d174/rtt-456/Quantifications/summary-job1.json"), + Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), // 2 }, { - Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), + Bucket: aws.String("config-bucket"), Key: aws.String("PixliseConfig/datasets-auth.json"), // 2 }, { - Bucket: aws.String(UsersBucketForUnitTest), Key: aws.String("UserContent/600f2a0806b6c70071d3d174/rtt-456/Quantifications/summary-job1.json"), - }, - { - Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), + Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), // 3 }, { - Bucket: aws.String(UsersBucketForUnitTest), Key: aws.String("UserContent/shared/rtt-456/Quantifications/summary-job7.json"), - }, - { - Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/rtt-456/summary.json"), - }, - { - Bucket: aws.String(UsersBucketForUnitTest), Key: aws.String("UserContent/shared/rtt-456/Quantifications/summary-job7.json"), + Bucket: aws.String("config-bucket"), Key: aws.String("PixliseConfig/datasets-auth.json"), // 2 }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ @@ -908,100 +908,61 @@ func Test_quantHandler_Get(t *testing.T) { Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))), }, { - Body: ioutil.NopCloser(bytes.NewReader([]byte("bad json"))), + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{ + "shared": false, + "params": { + "pmcsCount": 93, + "name": "my test quant", + "dataBucket": "dev-pixlise-data", + "datasetPath": "Datasets/rtt-456/5x5dataset.bin", + "datasetID": "rtt-456", + "jobBucket": "dev-pixlise-piquant-jobs", + "detectorConfig": "PIXL", + "elements": [ + "Sc", + "Cr" + ], + "parameters": "-q,pPIETXCFsr -b,0,12,60,910,280,16", + "runTimeSec": 120, + "coresPerNode": 6, + "startUnixTime": 1589948988, + "creator": { + "name": "peternemere", + "user_id": "600f2a0806b6c70071d3d174", + "email": "" + }, + "roiID": "ZcH49SYZ", + "elementSetID": "", + "quantMode": "AB" + }, + "jobId": "job1", + "status": "complete", + "message": "Nodes ran: 1", + "endUnixTime": 1589949035, + "outputFilePath": "UserContent/user-1/rtt-456/Quantifications", + "piquantLogList": [ + "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_stdout.log", + "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_threads.log" + ] + }`))), }, - nil, { - Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))), + Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))), // 1 }, - nil, { - Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))), + Body: ioutil.NopCloser(bytes.NewReader([]byte(publicDatasetsJSON))), // 1 }, { - Body: ioutil.NopCloser(bytes.NewReader([]byte(`{ - "shared": false, - "params": { - "pmcsCount": 93, - "name": "my test quant", - "dataBucket": "dev-pixlise-data", - "datasetPath": "Datasets/rtt-456/5x5dataset.bin", - "datasetID": "rtt-456", - "jobBucket": "dev-pixlise-piquant-jobs", - "detectorConfig": "PIXL", - "elements": [ - "Sc", - "Cr" - ], - "parameters": "-q,pPIETXCFsr -b,0,12,60,910,280,16", - "runTimeSec": 120, - "coresPerNode": 6, - "startUnixTime": 1589948988, - "creator": { - "name": "peternemere", - "user_id": "600f2a0806b6c70071d3d174", - "email": "" - }, - "roiID": "ZcH49SYZ", - "elementSetID": "", - "quantMode": "AB" - }, - "jobId": "job1", - "status": "complete", - "message": "Nodes ran: 1", - "endUnixTime": 1589949035, - "outputFilePath": "UserContent/user-1/rtt-456/Quantifications", - "piquantLogList": [ - "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_stdout.log", - "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_threads.log" - ] -}`))), + Body: ioutil.NopCloser(bytes.NewReader([]byte("bad json"))), // 2 }, { - Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))), + Body: ioutil.NopCloser(bytes.NewReader([]byte(publicDatasetsJSON))), // 2 }, - nil, { - Body: ioutil.NopCloser(bytes.NewReader([]byte(summaryJSON))), + Body: ioutil.NopCloser(bytes.NewReader([]byte("bad json"))), // 3 }, { - Body: ioutil.NopCloser(bytes.NewReader([]byte(`{ - "shared": false, - "params": { - "pmcsCount": 93, - "name": "my test quant", - "dataBucket": "dev-pixlise-data", - "datasetPath": "Datasets/rtt-456/5x5dataset.bin", - "datasetID": "rtt-456", - "jobBucket": "dev-pixlise-piquant-jobs", - "detectorConfig": "PIXL", - "elements": [ - "Sc", - "Cr" - ], - "parameters": "-q,pPIETXCFsr -b,0,12,60,910,280,16", - "runTimeSec": 120, - "coresPerNode": 6, - "startUnixTime": 1589948988, - "creator": { - "name": "peternemere", - "user_id": "600f2a0806b6c70071d3d174", - "email": "" - }, - "roiID": "ZcH49SYZ", - "elementSetID": "" - }, - "elements": ["Sc", "Cr"], - "jobId": "job7", - "status": "complete", - "message": "Nodes ran: 1", - "endUnixTime": 1589949035, - "outputFilePath": "UserContent/user-1/rtt-456/Quantifications", - "piquantLogList": [ - "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_stdout.log", - "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_threads.log" - ] -}`))), + Body: ioutil.NopCloser(bytes.NewReader([]byte(publicDatasetsJSON))), // 2 }, } @@ -1011,51 +972,17 @@ func Test_quantHandler_Get(t *testing.T) { Name: "Niko Bellic", UserID: "600f2a0806b6c70071d3d174", Permissions: map[string]bool{ - "access:wrong-group": true, + "read:data-analysis": true, + "access:the-group": true, }, } svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) - // Dataset summary file wrong group, ERROR - NO ACCESS + // File found, OK req, _ := http.NewRequest("GET", "/quantification/rtt-456/job1", nil) resp := executeRequest(req, apiRouter.Router) - checkResult(t, resp, 403, `dataset rtt-456 not permitted -`) - mockUser = pixlUser.UserInfo{ - Name: "Niko Bellic", - UserID: "600f2a0806b6c70071d3d174", - Permissions: map[string]bool{ - "access:the-group": true, - }, - } - - // Dataset summary has different group, ACCESS DENIED - req, _ = http.NewRequest("GET", "/quantification/rtt-456/job1", nil) - resp = executeRequest(req, apiRouter.Router) - - checkResult(t, resp, 500, `failed to verify dataset group permission -`) - - // Failed to parse summary JSON, ERROR - req, _ = http.NewRequest("GET", "/quantification/rtt-456/job1", nil) - resp = executeRequest(req, apiRouter.Router) - - checkResult(t, resp, 404, `rtt-456 not found -`) - - // File not found, ERROR - req, _ = http.NewRequest("GET", "/quantification/rtt-456/job1", nil) - resp = executeRequest(req, apiRouter.Router) - - checkResult(t, resp, 404, `job1 not found -`) - - // File found, OK - req, _ = http.NewRequest("GET", "/quantification/rtt-456/job1", nil) - resp = executeRequest(req, apiRouter.Router) - checkResult(t, resp, 200, `{ "summary": { "shared": false, @@ -1102,64 +1029,46 @@ func Test_quantHandler_Get(t *testing.T) { } `) - // Shared file not found, ERROR - req, _ = http.NewRequest("GET", "/quantification/rtt-456/shared-job7", nil) + mockUser = pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:data-analysis": true, + "access:wrong-group": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} + apiRouter = MakeRouter(svcs) + + // Dataset summary file wrong group, ERROR - NO ACCESS + req, _ = http.NewRequest("GET", "/quantification/rtt-456/job1", nil) resp = executeRequest(req, apiRouter.Router) - checkResult(t, resp, 404, `job7 not found + checkResult(t, resp, 403, `dataset rtt-456 not permitted `) - // Shared file found, OK - req, _ = http.NewRequest("GET", "/quantification/rtt-456/shared-job7", nil) + // Dataset summary has different group, ACCESS DENIED + req, _ = http.NewRequest("GET", "/quantification/rtt-456/job1", nil) resp = executeRequest(req, apiRouter.Router) - checkResult(t, resp, 200, `{ - "summary": { - "shared": false, - "params": { - "pmcsCount": 93, - "name": "my test quant", - "dataBucket": "dev-pixlise-data", - "datasetPath": "Datasets/rtt-456/5x5dataset.bin", - "datasetID": "rtt-456", - "jobBucket": "dev-pixlise-piquant-jobs", - "detectorConfig": "PIXL", - "elements": [ - "Sc", - "Cr" - ], - "parameters": "-q,pPIETXCFsr -b,0,12,60,910,280,16", - "runTimeSec": 120, - "coresPerNode": 6, - "startUnixTime": 1589948988, - "creator": { - "name": "Peter N", - "user_id": "600f2a0806b6c70071d3d174", - "email": "peter@pixlise.org" - }, - "roiID": "ZcH49SYZ", - "elementSetID": "", - "piquantVersion": "", - "quantMode": "", - "comments": "", - "roiIDs": [] - }, - "elements": [ - "Sc", - "Cr" - ], - "jobId": "job7", - "status": "complete", - "message": "Nodes ran: 1", - "endUnixTime": 1589949035, - "outputFilePath": "UserContent/user-1/rtt-456/Quantifications", - "piquantLogList": [ - "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_stdout.log", - "https://dev-pixlise-piquant-jobs.s3.us-east-1.amazonaws.com/Jobs/UC2Bchyz/piquant-logs/node00001.pmcs_threads.log" - ] - }, - "url": "https:///quantification/download/rtt-456/shared-job7" -} + checkResult(t, resp, 500, `failed to verify dataset group permission +`) + mockUser = pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:data-analysis": true, + "access:the-group": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} + apiRouter = MakeRouter(svcs) + + // Failed to parse summary JSON, ERROR + req, _ = http.NewRequest("GET", "/quantification/rtt-456/job1", nil) + resp = executeRequest(req, apiRouter.Router) + + checkResult(t, resp, 500, `failed to verify dataset group permission `) }) } @@ -1213,7 +1122,8 @@ func Example_quant_Stream_OK() { Name: "Niko Bellic", UserID: "600f2a0806b6c70071d3d174", Permissions: map[string]bool{ - "access:groupie": true, + "access:groupie": true, + "read:data-analysis": true, }, } svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} diff --git a/api/endpoints/QuantificationZStack.go b/api/endpoints/QuantificationZStack.go index 248448a5..90ff79bd 100644 --- a/api/endpoints/QuantificationZStack.go +++ b/api/endpoints/QuantificationZStack.go @@ -43,7 +43,7 @@ func quantificationCombineListSave(params handlers.ApiHandlerParams) (interface{ } // Make sure this dataset exists already by loading its summary file - _, err = permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, datasetID) + _, err = permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) if err != nil { return nil, api.MakeBadRequestError(err) } diff --git a/api/endpoints/QuantificationZStack_test.go b/api/endpoints/QuantificationZStack_test.go index 99073d21..2c695f79 100644 --- a/api/endpoints/QuantificationZStack_test.go +++ b/api/endpoints/QuantificationZStack_test.go @@ -140,6 +140,15 @@ func Example_quantHandler_ZStackSave() { var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() + const publicDatasetsJSON = `{ + "rtt-456": { + "dataset_id": "rtt-456", + "public": false, + "public_release_utc_time_sec": 0, + "sol": "" + } +}` + mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/datasetThatDoesntExist/summary.json"), @@ -147,6 +156,9 @@ func Example_quantHandler_ZStackSave() { { Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/datasetNoPermission/summary.json"), }, + { + Bucket: aws.String("config-bucket"), Key: aws.String("PixliseConfig/datasets-auth.json"), + }, { Bucket: aws.String(DatasetsBucketForUnitTest), Key: aws.String("Datasets/dataset123/summary.json"), }, @@ -175,6 +187,9 @@ func Example_quantHandler_ZStackSave() { "detector_config": "PIXL" }`))), }, + { + Body: ioutil.NopCloser(bytes.NewReader([]byte(publicDatasetsJSON))), // 2 + }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(`{ "dataset_id": "dataset123", @@ -215,7 +230,8 @@ func Example_quantHandler_ZStackSave() { Name: "Niko Bellic", UserID: "600f2a0806b6c70071d3d174", Permissions: map[string]bool{ - "access:GroupB": true, + "read:data-analysis": true, + "access:GroupB": true, }, } svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} diff --git a/api/endpoints/RGBMix.go b/api/endpoints/RGBMix.go index 5a51a71d..e568acd0 100644 --- a/api/endpoints/RGBMix.go +++ b/api/endpoints/RGBMix.go @@ -79,7 +79,7 @@ type RGBMixLookup map[string]RGBMix func registerRGBMixHandler(router *apiRouter.ApiObjectRouter) { const pathPrefix = "rgb-mix" - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), rgbMixList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("GET", permission.PermPublic), rgbMixList) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), rgbMixPost) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), rgbMixPut) @@ -134,13 +134,29 @@ func getRGBMixByID(svcs *services.APIServices, ID string, s3PathFrom string, mar return result, nil } -func getRGBMixListing(svcs *services.APIServices, s3PathFrom string, sharedFile bool, outMap *RGBMixLookup) error { +func getRGBMixListing(svcs *services.APIServices, s3PathFrom string, sharedFile bool, outMap *RGBMixLookup, isPublicUser bool) error { items, err := readRGBMixData(svcs, s3PathFrom) if err != nil { return err } + publicObjectsAuth, err := permission.GetPublicObjectsAuth(svcs.FS, svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return err + } + for id, item := range items { + if isPublicUser { + isRGBMixPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.RGBMixes, id) + if err != nil { + return err + } + + if !isRGBMixPublic { + continue + } + } + // We modify the ids of shared items, so if passed to GET/PUT/DELETE we know this refers to something that's shared saveID := id if sharedFile { @@ -157,14 +173,16 @@ func getRGBMixListing(svcs *services.APIServices, s3PathFrom string, sharedFile func rgbMixList(params handlers.ApiHandlerParams) (interface{}, error) { items := RGBMixLookup{} + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + // Get user item summaries - err := getRGBMixListing(params.Svcs, filepaths.GetRGBMixPath(params.UserInfo.UserID), false, &items) + err := getRGBMixListing(params.Svcs, filepaths.GetRGBMixPath(params.UserInfo.UserID), false, &items, isPublicUser) if err != nil { return nil, err } // Get shared item summaries (into same map) - err = getRGBMixListing(params.Svcs, filepaths.GetRGBMixPath(pixlUser.ShareUserID), true, &items) + err = getRGBMixListing(params.Svcs, filepaths.GetRGBMixPath(pixlUser.ShareUserID), true, &items, isPublicUser) if err != nil { return nil, err } diff --git a/api/endpoints/ROI.go b/api/endpoints/ROI.go index 326a9e02..7754f267 100644 --- a/api/endpoints/ROI.go +++ b/api/endpoints/ROI.go @@ -47,7 +47,7 @@ type roiHandler struct { func registerROIHandler(router *apiRouter.ApiObjectRouter) { const pathPrefix = "roi" - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), roiList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), roiList) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), roiPost) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier, "bulk"), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), roiBulkPost) @@ -64,14 +64,16 @@ func roiList(params handlers.ApiHandlerParams) (interface{}, error) { rois := roiModel.ROILookup{} + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + // Get user item summaries - err := roiModel.GetROIs(params.Svcs, params.UserInfo.UserID, datasetID, &rois) + err := roiModel.GetROIs(params.Svcs, params.UserInfo.UserID, datasetID, &rois, isPublicUser) if err != nil { return nil, err } // Get shared item summaries (into same map) - err = roiModel.GetROIs(params.Svcs, pixlUser.ShareUserID, datasetID, &rois) + err = roiModel.GetROIs(params.Svcs, pixlUser.ShareUserID, datasetID, &rois, isPublicUser) if err != nil { return nil, err } diff --git a/api/endpoints/Tags.go b/api/endpoints/Tags.go index 51c15e3c..57fdbfa4 100644 --- a/api/endpoints/Tags.go +++ b/api/endpoints/Tags.go @@ -44,7 +44,7 @@ type tagHandler struct { func registerTagHandler(router *apiRouter.ApiObjectRouter) { const pathPrefix = "tags" - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadDataAnalysis), tagList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), tagList) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteDataAnalysis), tagPost) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWriteDataAnalysis), tagDelete) } diff --git a/api/endpoints/ViewState.go b/api/endpoints/ViewState.go index e71c9559..ed986999 100644 --- a/api/endpoints/ViewState.go +++ b/api/endpoints/ViewState.go @@ -44,27 +44,29 @@ func registerViewStateHandler(router *apiRouter.ApiObjectRouter) { const pathPrefix = "view-state" const savedURIPath = "/saved" const collectionURIPath = "/collections" + const publicPath = "/public" // "Current" view state, as saved by widgets as we go along, and the GET call to retrieve it - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadPIXLISESettings), viewStateList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), viewStateList) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWritePIXLISESettings), viewStatePut) // Saved view states - these are named copies of a view state, with CRUD calls - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadPIXLISESettings), savedViewStateGet) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), savedViewStateGet) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWritePIXLISESettings), savedViewStatePut) // Renaming a view state - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier, idIdentifier)+"/references", apiRouter.MakeMethodPermission("GET", permission.PermReadPIXLISESettings), savedViewStateGetReferencedIDs) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier, idIdentifier)+"/references", apiRouter.MakeMethodPermission("GET", permission.PermPublic), savedViewStateGetReferencedIDs) //router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier, idIdentifier)+"/rename", apiRouter.MakeMethodPermission("POST", permission.PermWritePIXLISESettings), savedViewStateRenamePost) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWritePIXLISESettings), savedViewStateDelete) - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadPIXLISESettings), savedViewStateList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+savedURIPath, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), savedViewStateList) // Collections (of saved view states) - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+collectionURIPath, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadPIXLISESettings), viewStateCollectionList) - router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+collectionURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermReadPIXLISESettings), viewStateCollectionGet) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+collectionURIPath, datasetIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), viewStateCollectionList) + router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+collectionURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("GET", permission.PermPublic), viewStateCollectionGet) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+collectionURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWritePIXLISESettings), viewStateCollectionPut) router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+collectionURIPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("DELETE", permission.PermWritePIXLISESettings), viewStateCollectionDelete) // Sharing the above + router.AddJSONHandler(handlers.MakeEndpointPath(shareURLRoot+"/"+pathPrefix+publicPath, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWritePIXLISESettings), viewStateCollectionPostPublic) router.AddShareHandler(handlers.MakeEndpointPath(shareURLRoot+"/"+pathPrefix, datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWritePIXLISESettings), viewStateShare) router.AddShareHandler(handlers.MakeEndpointPath(shareURLRoot+"/"+pathPrefix+"-collection", datasetIdentifier, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWritePIXLISESettings), viewStateCollectionShare) } @@ -76,6 +78,15 @@ func viewStateList(params handlers.ApiHandlerParams) (interface{}, error) { datasetID := params.PathParams[datasetIdentifier] // It's a get, we don't care about the body... + isPublicUser := !params.UserInfo.Permissions[permission.PermReadDataAnalysis] + if isPublicUser { + // Verify user has access to dataset (need to do this now that permissions are on a per-dataset basis) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + } + state := defaultWholeViewState() // If user is NOT resetting, overwrite what we just made with the stored view state. If anything is not in there, it retains the default state diff --git a/api/endpoints/ViewStateCollection.go b/api/endpoints/ViewStateCollection.go index a6518b74..589bd859 100644 --- a/api/endpoints/ViewStateCollection.go +++ b/api/endpoints/ViewStateCollection.go @@ -30,6 +30,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/pixlise/core/v3/api/filepaths" "github.com/pixlise/core/v3/api/handlers" + "github.com/pixlise/core/v3/api/permission" "github.com/pixlise/core/v3/api/services" "github.com/pixlise/core/v3/core/api" "github.com/pixlise/core/v3/core/pixlUser" @@ -77,15 +78,44 @@ type workspaceCollectionListItem struct { func viewStateCollectionList(params handlers.ApiHandlerParams) (interface{}, error) { datasetID := params.PathParams[datasetIdentifier] + + isPublicUser := !params.UserInfo.Permissions[permission.PermReadPIXLISESettings] + publicObjectsAuth := permission.PublicObjectsAuth{} + + if isPublicUser { + // Verify user has access to dataset (need to do this now that permissions are on a per-dataset basis) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + + publicObjectsAuth, err = permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return nil, err + } + } + userList, _ := getViewStateCollectionListing(params.Svcs, datasetID, params.UserInfo.UserID) sharedList, _ := getViewStateCollectionListing(params.Svcs, datasetID, pixlUser.ShareUserID) result := []workspaceCollectionListItem{} - for _, item := range userList { - result = append(result, item) + + // If user is not public, return their own collections + if !isPublicUser { + result = append(result, userList...) } for _, item := range sharedList { + if isPublicUser { + isCollectionPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Collections, item.Name) + if err != nil { + return nil, err + } + + if !isCollectionPublic { + continue + } + } item.Name = utils.SharedItemIDPrefix + item.Name result = append(result, item) } @@ -128,6 +158,25 @@ func viewStateCollectionGet(params handlers.ApiHandlerParams) (interface{}, erro datasetID := params.PathParams[datasetIdentifier] collectionID := params.PathParams[idIdentifier] + isPublicUser := !params.UserInfo.Permissions[permission.PermReadPIXLISESettings] + + if isPublicUser { + // Verify user has access to dataset (need to do this now that permissions are on a per-dataset basis) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + + isCollectionPublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectCollection, collectionID) + if err != nil { + return nil, err + } + + if !isCollectionPublic { + return nil, api.MakeBadRequestError(errors.New("workspace is not public")) + } + } + s3Path := filepaths.GetCollectionPath(params.UserInfo.UserID, datasetID, collectionID) strippedID, isSharedReq := utils.StripSharedItemIDPrefix(collectionID) if isSharedReq { @@ -181,23 +230,172 @@ func loadViewStates(params handlers.ApiHandlerParams, viewStateIDs []string) (ma for _, viewStateID := range viewStateIDs { s3Path := filepaths.GetWorkspacePath(params.UserInfo.UserID, datasetID, viewStateID) + // If this is a shared view state, load it from the shared user's workspace + strippedID, isShared := utils.StripSharedItemIDPrefix(viewStateID) + if isShared { + s3Path = filepaths.GetWorkspacePath(pixlUser.ShareUserID, datasetID, strippedID) + } + // Set up a default view state to read into loadedWorkspace := Workspace{ ViewState: defaultWholeViewState(), } + err := params.Svcs.FS.ReadJSON(params.Svcs.Config.UsersBucket, s3Path, &loadedWorkspace, false) if err != nil { - return nil, api.MakeNotFoundError(viewStateID) + return nil, api.MakeNotFoundError(strippedID) } applyQuantByROIFallback(&loadedWorkspace.ViewState.Quantification) - result[viewStateID] = loadedWorkspace.ViewState + result[strippedID] = loadedWorkspace.ViewState } return result, nil } +func viewStateCollectionPostPublic(params handlers.ApiHandlerParams) (interface{}, error) { + datasetID := params.PathParams[datasetIdentifier] + collectionID := params.PathParams[idIdentifier] + + strippedID, isSharedReq := utils.StripSharedItemIDPrefix(collectionID) + if !isSharedReq { + return nil, api.MakeBadRequestError(errors.New("can't make non-shared collections public")) + } + + // Verify user has access to dataset (need to do this now that permissions are on a per-dataset basis) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + + // Verify the dataset is public before sharing contained objects + isDatasetPublic, err := permission.CheckIsPublicDataset(params.Svcs.FS, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + + if !isDatasetPublic { + return nil, api.MakeBadRequestError(errors.New("cannot share a collection from a non-public dataset")) + } + + // Verify user has access to collection + s3SharedPath := filepaths.GetCollectionPath(pixlUser.ShareUserID, datasetID, strippedID) + collection, err := getCollection(params, strippedID, s3SharedPath, true) + if err != nil { + return nil, err + } + + // Get all existing public objects + publicObjectsPath := filepaths.GetPublicObjectsPath() + publicObjects, err := permission.ReadPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, publicObjectsPath) + if err != nil { + // Assume the file doesn't exist yet + log.Printf("No public objects file found, creating new one") + publicObjects = permission.PublicObjectsAuth{} + } + + // Get all view states in collection + states, err := loadViewStates(params, collection.ViewStateIDs) + if err != nil { + return nil, err + } + + // Get flat lists of all objects in view states + collectionObjects := publicObjects + + // Ensure all lists are initialised + if collectionObjects.Expressions == nil { + collectionObjects.Expressions = []string{} + } + if collectionObjects.Modules == nil { + collectionObjects.Modules = []string{} + } + if collectionObjects.ROIs == nil { + collectionObjects.ROIs = []string{} + } + if collectionObjects.RGBMixes == nil { + collectionObjects.RGBMixes = []string{} + } + if collectionObjects.Quantifications == nil { + collectionObjects.Quantifications = []string{} + } + if collectionObjects.Workspaces == nil { + collectionObjects.Workspaces = []string{} + } + if collectionObjects.Collections == nil { + collectionObjects.Collections = []string{} + } + if collectionObjects.Datasets == nil { + collectionObjects.Datasets = []string{} + } + + // Add dataset to public objects + if !utils.StringInSlice(datasetID, collectionObjects.Datasets) { + collectionObjects.Datasets = append(collectionObjects.Datasets, datasetID) + } + + // Add collection to public objects + if !utils.StringInSlice(strippedID, collectionObjects.Collections) { + collectionObjects.Collections = append(collectionObjects.Collections, strippedID) + } + + for viewStateID, state := range states { + + // Add view state to public objects + if !utils.StringInSlice(viewStateID, collectionObjects.Workspaces) { + collectionObjects.Workspaces = append(collectionObjects.Workspaces, viewStateID) + } + + // Add all objects in view state to public objects + referencedIDs := state.getReferencedIDs() + + for _, roi := range referencedIDs.ROIs { + if !utils.StringInSlice(roi.ID, collectionObjects.ROIs) { + collectionObjects.ROIs = append(collectionObjects.ROIs, roi.ID) + } + } + + for _, expression := range referencedIDs.Expressions { + if !utils.StringInSlice(expression.ID, collectionObjects.Expressions) { + collectionObjects.Expressions = append(collectionObjects.Expressions, expression.ID) + } + + strippedID, _ := utils.StripSharedItemIDPrefix(expression.ID) + expr, err := params.Svcs.Expressions.GetExpression(strippedID, true) + if err != nil { + continue + } + + // Still need to check expression module references even if it is already public because + // references may have changed since it was made public + for _, module := range expr.ModuleReferences { + if !utils.StringInSlice(module.ModuleID, collectionObjects.Modules) { + collectionObjects.Modules = append(collectionObjects.Modules, module.ModuleID) + } + } + } + + for _, rgbmixes := range referencedIDs.RGBMixes { + if !utils.StringInSlice(rgbmixes.ID, collectionObjects.RGBMixes) { + collectionObjects.RGBMixes = append(collectionObjects.RGBMixes, rgbmixes.ID) + } + } + + if !utils.StringInSlice(referencedIDs.Quant.ID, collectionObjects.Quantifications) { + collectionObjects.Quantifications = append(collectionObjects.Quantifications, referencedIDs.Quant.ID) + } + } + + // Save public objects + err = params.Svcs.FS.WriteJSON(params.Svcs.Config.ConfigBucket, publicObjectsPath, collectionObjects) + if err != nil { + return nil, err + } + + return collectionObjects, nil +} + func viewStateCollectionPut(params handlers.ApiHandlerParams) (interface{}, error) { datasetID := params.PathParams[datasetIdentifier] collectionID := params.PathParams[idIdentifier] @@ -284,12 +482,12 @@ func viewStateCollectionShare(params handlers.ApiHandlerParams) (interface{}, er datasetID := params.PathParams[datasetIdentifier] collectionID := params.PathParams[idIdentifier] - s3Path := filepaths.GetCollectionPath(params.UserInfo.UserID, datasetID, collectionID) _, isSharedReq := utils.StripSharedItemIDPrefix(collectionID) if isSharedReq { - return nil, fmt.Errorf("Cannot share a shared ID") + return nil, fmt.Errorf("cannot share a shared ID") } + s3Path := filepaths.GetCollectionPath(params.UserInfo.UserID, datasetID, collectionID) collectionContents, err := getCollection(params, collectionID, s3Path, true) if err != nil { return nil, err diff --git a/api/endpoints/ViewStateCollection_test.go b/api/endpoints/ViewStateCollection_test.go index 08405204..cce45ced 100644 --- a/api/endpoints/ViewStateCollection_test.go +++ b/api/endpoints/ViewStateCollection_test.go @@ -64,6 +64,15 @@ func Example_viewStateHandler_ListCollections() { } svcs := MakeMockSvcs(&mockS3, nil, nil, nil) + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} + apiRouter := MakeRouter(svcs) // Exists, success @@ -184,6 +193,16 @@ func Test_viewStateHandler_GetCollection(t *testing.T) { svcs := MakeMockSvcs(&mockS3, nil, nil, nil) svcs.Users = pixlUser.MakeUserDetailsLookup(mt.Client, "unit_test") + + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} + apiRouter := MakeRouter(svcs) // Doesn't exist, should fail @@ -546,6 +565,15 @@ func Example_viewStateHandler_GetCollectionShared() { } svcs := MakeMockSvcs(&mockS3, nil, nil, nil) + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} + apiRouter := MakeRouter(svcs) // Doesn't exist, should fail diff --git a/api/endpoints/ViewStateModel.go b/api/endpoints/ViewStateModel.go index 419c03bc..72e7acc9 100644 --- a/api/endpoints/ViewStateModel.go +++ b/api/endpoints/ViewStateModel.go @@ -496,6 +496,8 @@ Quantification.AppliedQuantID func (state wholeViewState) getReferencedIDs() viewStateReferencedIDs { // Unfortunately this has to be manually coded... can't exactly search for field names or something to identify them + // Modules? + // We use maps here for uniqueness roiIDs := map[string]bool{} expressionIDs := map[string]bool{} diff --git a/api/endpoints/ViewStateWorkspace.go b/api/endpoints/ViewStateWorkspace.go index 2a0f920e..496ac8e6 100644 --- a/api/endpoints/ViewStateWorkspace.go +++ b/api/endpoints/ViewStateWorkspace.go @@ -29,6 +29,7 @@ import ( "github.com/pixlise/core/v3/api/filepaths" "github.com/pixlise/core/v3/api/handlers" + "github.com/pixlise/core/v3/api/permission" "github.com/pixlise/core/v3/api/services" "github.com/pixlise/core/v3/core/api" "github.com/pixlise/core/v3/core/pixlUser" @@ -77,8 +78,27 @@ func savedViewStateList(params handlers.ApiHandlerParams) (interface{}, error) { sharedList, _ := getViewStateListing(params.Svcs, datasetID, pixlUser.ShareUserID) // We don't check for errors because path may not exist yet and thats a valid scenario, just don't return any in that case + isPublicUser := !params.UserInfo.Permissions[permission.PermReadPIXLISESettings] + result := []workspaceSummary{} + publicObjectsAuth, err := permission.GetPublicObjectsAuth(params.Svcs.FS, params.Svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return result, err + } + for _, item := range userList { + // If public user, check if workspace is in public set + if isPublicUser { + isWorkspacePublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.Workspaces, item.ID) + if err != nil { + return result, err + } + + if !isWorkspacePublic { + continue + } + } + result = append(result, item) } @@ -134,6 +154,15 @@ func savedViewStateGet(params handlers.ApiHandlerParams) (interface{}, error) { datasetID := params.PathParams[datasetIdentifier] viewStateID := params.PathParams[idIdentifier] + isPublicUser := !params.UserInfo.Permissions[permission.PermReadPIXLISESettings] + if isPublicUser { + // Verify user has access to dataset (need to do this now that permissions are on a per-dataset basis) + _, err := permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + } + s3Path := filepaths.GetWorkspacePath(params.UserInfo.UserID, datasetID, viewStateID) strippedID, isSharedReq := utils.StripSharedItemIDPrefix(viewStateID) @@ -307,6 +336,24 @@ func savedViewStateGetReferencedIDs(params handlers.ApiHandlerParams) (interface datasetID := params.PathParams[datasetIdentifier] viewStateID := params.PathParams[idIdentifier] s3Path := filepaths.GetWorkspacePath(params.UserInfo.UserID, datasetID, viewStateID) + isPublicUser := !params.UserInfo.Permissions[permission.PermReadPIXLISESettings] + + if isPublicUser { + isWorkspacePublic, err := permission.CheckIsObjectPublic(params.Svcs.FS, params.Svcs.Config.ConfigBucket, permission.PublicObjectWorkspace, viewStateID) + if err != nil { + return nil, err + } + + if !isWorkspacePublic { + return nil, api.MakeBadRequestError(errors.New("workspace is not public")) + } + + // Verify user has access to dataset (need to do this now that permissions are on a per-dataset basis) + _, err = permission.UserCanAccessDatasetWithSummaryDownload(params.Svcs.FS, params.UserInfo, params.Svcs.Config.DatasetsBucket, params.Svcs.Config.ConfigBucket, datasetID) + if err != nil { + return nil, err + } + } // Read the file in state := Workspace{ @@ -332,13 +379,14 @@ func savedViewStateGetReferencedIDs(params handlers.ApiHandlerParams) (interface rois := roiModel.ROILookup{} // Get user item summaries - err := roiModel.GetROIs(params.Svcs, params.UserInfo.UserID, datasetID, &rois) + // We don't have to check permissions here, as we've already checked them above + err := roiModel.GetROIs(params.Svcs, params.UserInfo.UserID, datasetID, &rois, false) if err != nil { params.Svcs.Log.Errorf("Failed to load user ROIs file, returned items may not have name/creator. Error: %v", err) } // Get shared item summaries (into same map) - err = roiModel.GetROIs(params.Svcs, pixlUser.ShareUserID, datasetID, &rois) + err = roiModel.GetROIs(params.Svcs, pixlUser.ShareUserID, datasetID, &rois, false) if err != nil { params.Svcs.Log.Errorf("Failed to load shared ROIs file, returned items may not have name/creator. Error: %v", err) } @@ -484,6 +532,18 @@ func viewStateShare(params handlers.ApiHandlerParams) (interface{}, error) { return viewStateID + " shared", nil } +func checkIsBuiltinID(id string) bool { + ignoredIDs := []string{"AllPoints", "SelectedPoints", "RemainingPoints", "rgbmix-runtime-exploratory"} + + for _, ignoredID := range ignoredIDs { + if id == ignoredID { + return true + } + } + + return false +} + // Saved view state helpers func autoShareNonSharedItems(svcs *services.APIServices, ids viewStateReferencedIDs, datasetID string, userID string) (map[string]string, error) { idRemap := map[string]string{} @@ -493,7 +553,7 @@ func autoShareNonSharedItems(svcs *services.APIServices, ids viewStateReferenced unsharedRGBMixIDs := []string{} for _, item := range ids.ROIs { - if !strings.HasPrefix(item.ID, utils.SharedItemIDPrefix) { + if !strings.HasPrefix(item.ID, utils.SharedItemIDPrefix) && !checkIsBuiltinID(item.ID) { unsharedROIIDs = append(unsharedROIIDs, item.ID) } } @@ -505,7 +565,7 @@ func autoShareNonSharedItems(svcs *services.APIServices, ids viewStateReferenced } for _, item := range ids.RGBMixes { - if !strings.HasPrefix(item.ID, utils.SharedItemIDPrefix) { + if !strings.HasPrefix(item.ID, utils.SharedItemIDPrefix) && !checkIsBuiltinID(item.ID) { unsharedRGBMixIDs = append(unsharedRGBMixIDs, item.ID) } } diff --git a/api/endpoints/ViewStateWorkspace_test.go b/api/endpoints/ViewStateWorkspace_test.go index 130c8581..de422bd7 100644 --- a/api/endpoints/ViewStateWorkspace_test.go +++ b/api/endpoints/ViewStateWorkspace_test.go @@ -257,6 +257,14 @@ func Test_viewStateHandler_ListSaved(t *testing.T) { svcs := MakeMockSvcs(&mockS3, nil, nil, nil) svcs.Users = pixlUser.MakeUserDetailsLookup(mt.Client, "unit_test") + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) // None @@ -486,6 +494,14 @@ func Test_viewStateHandler_GetSaved(t *testing.T) { svcs := MakeMockSvcs(&mockS3, nil, nil, nil) svcs.Users = pixlUser.MakeUserDetailsLookup(mt.Client, "unit_test") + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) // Doesn't exist, should fail @@ -678,6 +694,14 @@ func Example_viewStateHandler_GetSaved_ROIQuantFallbackCheck() { } svcs := MakeMockSvcs(&mockS3, nil, nil, nil) + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) // Doesn't exist, should fail @@ -806,6 +830,14 @@ func Example_viewStateHandler_GetSavedShared() { } svcs := MakeMockSvcs(&mockS3, nil, nil, nil) + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) // Doesn't exist, should fail @@ -1590,6 +1622,14 @@ func Test_viewStateHandler_GetReferencedIDs(t *testing.T) { svcs.Expressions = db + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:pixlise-settings": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} apiRouter := MakeRouter(svcs) // User file not there, should say not found diff --git a/api/endpoints/ViewState_test.go b/api/endpoints/ViewState_test.go index 5f931be5..89ba0bcc 100644 --- a/api/endpoints/ViewState_test.go +++ b/api/endpoints/ViewState_test.go @@ -32,6 +32,7 @@ import ( "github.com/pixlise/core/v3/core/awsutil" "github.com/pixlise/core/v3/core/logger" "github.com/pixlise/core/v3/core/notifications" + "github.com/pixlise/core/v3/core/pixlUser" "github.com/pixlise/core/v3/core/timestamper" "go.mongodb.org/mongo-driver/mongo/integration/mtest" ) @@ -316,6 +317,15 @@ func Example_viewStateHandler_List() { svcs := MakeMockSvcs(&mockS3, nil, nil, nil) apiRouter := MakeRouter(svcs) + mockUser := pixlUser.UserInfo{ + Name: "Niko Bellic", + UserID: "600f2a0806b6c70071d3d174", + Permissions: map[string]bool{ + "read:data-analysis": true, + }, + } + svcs.JWTReader = MockJWTReader{InfoToReturn: &mockUser} + // Various bits should return in the response... req, _ := http.NewRequest("GET", "/view-state/TheDataSetID", nil) resp := executeRequest(req, apiRouter.Router) diff --git a/api/filepaths/filepaths.go b/api/filepaths/filepaths.go index b340fc7d..b17fbf6d 100644 --- a/api/filepaths/filepaths.go +++ b/api/filepaths/filepaths.go @@ -149,6 +149,14 @@ func GetDatasetListPath() string { return GetConfigFilePath("datasets.json") } +func GetDatasetsAuthPath() string { + return GetConfigFilePath("datasets-auth.json") +} + +func GetPublicObjectsPath() string { + return GetConfigFilePath("public-objects.json") +} + // Config contains the docker container to use for PIQUANT. Separate from config.json because users can configure this in UI const PiquantVersionFileName = "piquant-version.json" diff --git a/api/permission/access-models.go b/api/permission/access-models.go new file mode 100644 index 00000000..dffcd7bd --- /dev/null +++ b/api/permission/access-models.go @@ -0,0 +1,59 @@ +// Licensed to NASA JPL under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. NASA JPL licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package permission + +// This is a list of all the objects that are publicly accessible. This is used to +// determine whether a user has access to an object or not. If the object is not in +// this list, then the user must have access to it in order to see it. +type PublicObjectsAuth struct { + Datasets []string // This is a list of all datasets that are public AND have public objects in them + ROIs []string + Expressions []string + Modules []string + RGBMixes []string + Quantifications []string + Collections []string + Workspaces []string +} + +// DatasetAuthInfo - Structure of dataset auth JSON files +// This is used to check whether an individual dataset CAN be public or not +type DatasetAuthInfo struct { + DatasetID string `json:"dataset_id"` + Public bool `json:"public"` + PublicReleaseUTCTimeSec int64 `json:"public_release_utc_time_sec"` + Sol string `json:"sol"` +} + +// DatasetsAuth - Structure of dataset auth JSON files +// This is used to check the public status of all datasets +type DatasetsAuth map[string]DatasetAuthInfo + +// These enums keep track of the different types of objects that can be public +type PublicObjectEnumType int64 + +const ( + PublicObjectDataset PublicObjectEnumType = iota + PublicObjectROI + PublicObjectExpression + PublicObjectModule + PublicObjectRGBMix + PublicObjectQuantification + PublicObjectCollection + PublicObjectWorkspace +) diff --git a/api/permission/permissions.go b/api/permission/permissions.go index e6420004..87c6f752 100644 --- a/api/permission/permissions.go +++ b/api/permission/permissions.go @@ -24,11 +24,14 @@ import ( "fmt" "net/http" "strings" + "time" + "github.com/pixlise/core/v3/api/filepaths" "github.com/pixlise/core/v3/core/api" datasetModel "github.com/pixlise/core/v3/core/dataset" "github.com/pixlise/core/v3/core/fileaccess" "github.com/pixlise/core/v3/core/pixlUser" + "github.com/pixlise/core/v3/core/utils" ) // Public endpoints, mainly for getting the API version @@ -118,6 +121,9 @@ const PermWriteSharedAnnotation = "write:shared-annotation" // Sharing expressions const PermWriteSharedExpression = "write:shared-expression" +// Super Admin - not a real permission and mainly used to bypass tests +const PermSuperAdmin = "access:super-admin" + // Get all groups that are accessible by the list of permissions provided. This means // basically returning what's after access: in each permission func GetAccessibleGroups(permissions map[string]bool) map[string]bool { @@ -135,18 +141,146 @@ func GetAccessibleGroups(permissions map[string]bool) map[string]bool { return result } +func ReadPublicObjectsAuth(fs fileaccess.FileAccess, configBucket string, s3Path string) (PublicObjectsAuth, error) { + publicObjectsAuth := PublicObjectsAuth{} + + err := fs.ReadJSON(configBucket, s3Path, &publicObjectsAuth, true) + return publicObjectsAuth, err +} + +func ReadDatasetsAuth(fs fileaccess.FileAccess, configBucket string, s3Path string) (DatasetsAuth, error) { + datasetsAuth := DatasetsAuth{} + err := fs.ReadJSON(configBucket, s3Path, &datasetsAuth, true) + return datasetsAuth, err +} + +func CheckAndUpdatePublicDataset(fs fileaccess.FileAccess, configBucket string, datasetID string, datasetsAuth DatasetsAuth) (bool, error) { + isPublic := false + datasetsAuthPath := filepaths.GetDatasetsAuthPath() + + // Check if it's in the public dict + if datasetInfo, ok := datasetsAuth[datasetID]; ok { + isPublic = datasetInfo.Public + if !isPublic { + // Check if it's past the date where the dataset should be released public + if datasetInfo.PublicReleaseUTCTimeSec > 0 { + isPublic = time.Now().Unix() > datasetInfo.PublicReleaseUTCTimeSec + + // If it's now public, update the public flag in the dict + if isPublic { + datasetInfo.Public = true + err := fs.WriteJSON(configBucket, datasetsAuthPath, datasetsAuth) + if err != nil { + return isPublic, err + } + } + } + } + } + + return isPublic, nil +} + +// Check if the dataset CAN be public +func CheckIsPublicDataset(fs fileaccess.FileAccess, configBucket string, datasetID string) (bool, error) { + isPublic := false + + datasetsAuthPath := filepaths.GetDatasetsAuthPath() + datasetsAuth, err := ReadDatasetsAuth(fs, configBucket, datasetsAuthPath) + if err != nil { + return isPublic, err + } + + return CheckAndUpdatePublicDataset(fs, configBucket, datasetID, datasetsAuth) +} + +func GetPublicObjectsAuth(fs fileaccess.FileAccess, configBucket string, isPublicUser bool) (PublicObjectsAuth, error) { + publicObjectsAuth := PublicObjectsAuth{} + if !isPublicUser { + return publicObjectsAuth, nil + } + + publicObjectsPath := filepaths.GetPublicObjectsPath() + publicObjectsAuth, err := ReadPublicObjectsAuth(fs, configBucket, publicObjectsPath) + if err != nil { + return publicObjectsAuth, err + } + + return publicObjectsAuth, nil +} + +// Check if the dataset is both public and has shared objects in it +func CheckIsPublicDatasetWithSharedObjects(fs fileaccess.FileAccess, configBucket string, datasetID string) (bool, error) { + publicObjectsAuth, err := GetPublicObjectsAuth(fs, configBucket, true) + if err != nil { + return false, err + } + + return utils.StringInSlice(datasetID, publicObjectsAuth.Datasets), nil +} + +func CheckIsObjectPublic(fs fileaccess.FileAccess, configBucket string, objectType PublicObjectEnumType, objectID string) (bool, error) { + publicObjectsPath := filepaths.GetPublicObjectsPath() + publicObjectsAuth, err := ReadPublicObjectsAuth(fs, configBucket, publicObjectsPath) + if err != nil { + return false, err + } + + switch objectType { + case PublicObjectDataset: + return CheckIsObjectInPublicSet(publicObjectsAuth.Datasets, objectID) + case PublicObjectROI: + return CheckIsObjectInPublicSet(publicObjectsAuth.ROIs, objectID) + case PublicObjectExpression: + return CheckIsObjectInPublicSet(publicObjectsAuth.Expressions, objectID) + case PublicObjectModule: + return CheckIsObjectInPublicSet(publicObjectsAuth.Modules, objectID) + case PublicObjectRGBMix: + return CheckIsObjectInPublicSet(publicObjectsAuth.RGBMixes, objectID) + case PublicObjectQuantification: + return CheckIsObjectInPublicSet(publicObjectsAuth.Quantifications, objectID) + case PublicObjectCollection: + return CheckIsObjectInPublicSet(publicObjectsAuth.Collections, objectID) + case PublicObjectWorkspace: + return CheckIsObjectInPublicSet(publicObjectsAuth.Workspaces, objectID) + default: + return false, errors.New("unknown object type") + } +} + +func CheckIsObjectInPublicSet(publicObjectsList []string, objectID string) (bool, error) { + strippedObjectID, _ := utils.StripSharedItemIDPrefix(objectID) + + for _, publicObjectID := range publicObjectsList { + strippedPublicObjectID, _ := utils.StripSharedItemIDPrefix(publicObjectID) + if strippedPublicObjectID == strippedObjectID { + return true, nil + } + } + + return false, nil +} + // Returns nil if user CAN access it, otherwise a api.StatusError with the right HTTP error code -func UserCanAccessDataset(userInfo pixlUser.UserInfo, summary datasetModel.SummaryFileData) error { +func UserCanAccessDataset(userInfo pixlUser.UserInfo, summary datasetModel.SummaryFileData, fs fileaccess.FileAccess, configBucket string) error { userAllowedGroups := GetAccessibleGroups(userInfo.Permissions) if !userAllowedGroups[summary.Group] { - // User is not allowed to see this - return api.MakeStatusError(http.StatusForbidden, fmt.Errorf("dataset %v not permitted", summary.DatasetID)) + isPublic, err := CheckIsPublicDataset(fs, configBucket, summary.DatasetID) + if err != nil { + return err + } else if isPublic { + // Public dataset, anyone can access it + return nil + } else { + // User is not allowed to see this + return api.MakeStatusError(http.StatusForbidden, fmt.Errorf("dataset %v not permitted", summary.DatasetID)) + } } return nil } // Checking if the user can access a given dataset - use this if you don't already have summary info downloaded -func UserCanAccessDatasetWithSummaryDownload(fs fileaccess.FileAccess, userInfo pixlUser.UserInfo, dataBucket string, datasetID string) (datasetModel.SummaryFileData, error) { +func UserCanAccessDatasetWithSummaryDownload(fs fileaccess.FileAccess, userInfo pixlUser.UserInfo, dataBucket string, configBucket string, datasetID string) (datasetModel.SummaryFileData, error) { summary, err := datasetModel.ReadDataSetSummary(fs, dataBucket, datasetID) if err != nil { if fs.IsNotFoundError(err) { @@ -156,5 +290,5 @@ func UserCanAccessDatasetWithSummaryDownload(fs fileaccess.FileAccess, userInfo } } - return summary, UserCanAccessDataset(userInfo, summary) + return summary, UserCanAccessDataset(userInfo, summary, fs, configBucket) } diff --git a/core/downloader/parallel-loader.go b/core/downloader/parallel-loader.go index 7abe51de..1433d3fc 100644 --- a/core/downloader/parallel-loader.go +++ b/core/downloader/parallel-loader.go @@ -107,7 +107,7 @@ func DownloadFiles( svcs.Log.Debugf(" Downloading ROIs for datasetID: %v, roiID: %v", datasetID, roiLoadUserID) roisLoaded := roiModel.ROILookup{} - err := roiModel.GetROIs(svcs, roiLoadUserID, datasetID, &roisLoaded) + err := roiModel.GetROIs(svcs, roiLoadUserID, datasetID, &roisLoaded, false) mu.Lock() defer mu.Unlock() diff --git a/core/roiModel/roi.go b/core/roiModel/roi.go index 1ba58bca..6feab0cb 100644 --- a/core/roiModel/roi.go +++ b/core/roiModel/roi.go @@ -24,6 +24,7 @@ import ( "strconv" "github.com/pixlise/core/v3/api/filepaths" + "github.com/pixlise/core/v3/api/permission" "github.com/pixlise/core/v3/api/services" "github.com/pixlise/core/v3/core/api" datasetModel "github.com/pixlise/core/v3/core/dataset" @@ -185,7 +186,7 @@ func ReadROIData(svcs *services.APIServices, s3Path string) (ROILookup, error) { return itemLookup, err } -func GetROIs(svcs *services.APIServices, userID string, datasetID string, outMap *ROILookup) error { +func GetROIs(svcs *services.APIServices, userID string, datasetID string, outMap *ROILookup, isPublicUser bool) error { s3Path := filepaths.GetROIPath(userID, datasetID) items, err := ReadROIData(svcs, s3Path) @@ -195,8 +196,24 @@ func GetROIs(svcs *services.APIServices, userID string, datasetID string, outMap sharedFile := userID == pixlUser.ShareUserID + publicObjectsAuth, err := permission.GetPublicObjectsAuth(svcs.FS, svcs.Config.ConfigBucket, isPublicUser) + if err != nil { + return err + } + // Run through and just return summary info for id, item := range items { + if isPublicUser { + isROIPublic, err := permission.CheckIsObjectInPublicSet(publicObjectsAuth.ROIs, id) + if err != nil { + return err + } + + if !isROIPublic { + continue + } + } + toSave := ROISavedItem{ ROIItem: item.ROIItem, APIObjectItem: &pixlUser.APIObjectItem{ diff --git a/data-import/internal/data-converters/pixlfm/import.go b/data-import/internal/data-converters/pixlfm/import.go index 8df689be..ff4acbe0 100644 --- a/data-import/internal/data-converters/pixlfm/import.go +++ b/data-import/internal/data-converters/pixlfm/import.go @@ -328,6 +328,10 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data log, ) + if err != nil { + return nil, "", err + } + return data, importPath, nil } diff --git a/data-import/internal/importerutils/fileReadHelpers.go b/data-import/internal/importerutils/fileReadHelpers.go index d6d6a390..3c2273bd 100644 --- a/data-import/internal/importerutils/fileReadHelpers.go +++ b/data-import/internal/importerutils/fileReadHelpers.go @@ -122,10 +122,23 @@ func MakeFMDatasetOutput( return nil, errors.New("Failed to determine dataset RTT") } + isEM := false + // Ensure it matches what we're expecting // We allow for missing 0's at the start because for a while we imported RTTs as ints, so older dataset RTTs // were coming in as eg 76481028, while we now read them as 076481028 - if meta.RTT != datasetIDExpected && meta.RTT != "0"+datasetIDExpected { + // NOTE: it looks like EM datasets are generated with the RTT: 000000453 + // so if this is the RTT we don't do the check + if meta.RTT == "000000453" { + isEM = true + if datasetIDExpected == meta.RTT { + return nil, fmt.Errorf("Read RTT %v, need expected dataset ID to be different", meta.RTT) + } else { + // Set the RTT to the expected ID, we're importing some kind of test dataset + // so use something else, otherwise we would overwrite them all the time + meta.RTT = datasetIDExpected + } + } else if meta.RTT != datasetIDExpected && meta.RTT != "0"+datasetIDExpected { return nil, fmt.Errorf("Expected dataset ID %v, read %v", datasetIDExpected, meta.RTT) } @@ -134,7 +147,7 @@ func MakeFMDatasetOutput( // Depending on the SOL we may override the group and detector, as we have some test datasets that came // from the EM and have special characters as first part of SOL - if len(meta.SOL) > 0 && (meta.SOL[0] == 'D' || meta.SOL[0] == 'C') { + if isEM || len(meta.SOL) > 0 && (meta.SOL[0] == 'D' || meta.SOL[0] == 'C') { detectorConfig = "PIXL-EM-E2E" group = "PIXL-EM" } diff --git a/go.mod b/go.mod index f53fce9d..ca8a6a20 100644 --- a/go.mod +++ b/go.mod @@ -8,16 +8,16 @@ require ( github.com/aws/aws-sdk-go v1.44.159 github.com/aws/aws-secretsmanager-caching-go v1.1.0 github.com/getsentry/sentry-go v0.16.0 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/orcaman/concurrent-map v1.0.0 github.com/pixlise/diffraction-peak-detection/v2 v2.0.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.15.1 go.mongodb.org/mongo-driver v1.11.1 golang.org/x/image v0.2.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/protobuf v1.30.0 gopkg.in/auth0.v4 v4.7.0 gopkg.in/square/go-jose.v2 v2.6.0 k8s.io/api v0.26.0 @@ -55,8 +55,8 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.39.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect @@ -65,7 +65,7 @@ require ( github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect golang.org/x/crypto v0.4.0 // indirect golang.org/x/net v0.9.0 // indirect - golang.org/x/oauth2 v0.3.0 // indirect + golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/term v0.7.0 // indirect diff --git a/go.sum b/go.sum index 5c220ef7..14da74b5 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -124,6 +126,7 @@ github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrD github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -158,13 +161,19 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -243,6 +252,8 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -329,6 +340,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/auth0.v4 v4.7.0 h1:RrFItkglPN6xPtWXe8cAo5J8fX+drTK35A1eCw8Td+c= gopkg.in/auth0.v4 v4.7.0/go.mod h1:6ZOcoQequCmURgwJnGIX09/51deRWVRpUaUP8p1Jbpk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/lambdas/data-import/main.go b/internal/lambdas/data-import/main.go index 310532e4..e73e2fa4 100644 --- a/internal/lambdas/data-import/main.go +++ b/internal/lambdas/data-import/main.go @@ -89,6 +89,10 @@ func HandleRequest(ctx context.Context, event awsutil.Event) (string, error) { fmt.Printf("ImportForTrigger: \"%v\"\n", record.SNS.Message) result, err := importer.ImportForTrigger([]byte(record.SNS.Message), envName, configBucket, datasetBucket, manualBucket, nil, remoteFS) + if err != nil { + return "", err + } + defer result.Logger.Close() if len(result.WhatChanged) > 0 { diff --git a/internal/pixlise-api/apiMain.go b/internal/pixlise-api/apiMain.go index 58261376..6bf55265 100644 --- a/internal/pixlise-api/apiMain.go +++ b/internal/pixlise-api/apiMain.go @@ -160,7 +160,9 @@ func main() { JwtValidator: jwtReader.Validator, } - router.Router.Use(authware.Middleware, logware.Middleware) + promware := endpoints.PrometheusMiddleware + + router.Router.Use(authware.Middleware, logware.Middleware, promware) // Now also log this to the world... svcs.Log.Infof("API version \"%v\" started...", services.ApiVersion)