diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2269059b..49ecb1a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,12 +62,14 @@ jobs: with: go-version: '1.18.0' - name: Test - run: make test + run: echo $PIXLISE_API_TEST_ZENODO_URI && make test env: PIXLISE_API_TEST_AUTH0_CLIENT_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} PIXLISE_API_TEST_AUTH0_DOMAIN: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} PIXLISE_API_TEST_AUTH0_SECRET: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID }} + PIXLISE_API_TEST_ZENODO_URI: ${{ secrets.ZENODO_URI }} + PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: us-east-1 AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/api/config/config.go b/api/config/config.go index a5f6baa9..da490b30 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -81,6 +81,9 @@ type APIConfig struct { UsersBucket string + ZenodoURI string + ZenodoAccessToken string + // Vars not set by environment NodeCountOverride int32 MaxQuantNodes int32 diff --git a/api/endpoints/DataExpression.go b/api/endpoints/DataExpression.go index 0544ae08..084931da 100644 --- a/api/endpoints/DataExpression.go +++ b/api/endpoints/DataExpression.go @@ -50,6 +50,7 @@ func registerDataExpressionHandler(router *apiRouter.ApiObjectRouter) { router.AddJSONHandler(handlers.MakeEndpointPath(pathPrefix+"/execution-stat", idIdentifier), apiRouter.MakeMethodPermission("PUT", permission.PermWriteDataAnalysis), dataExpressionExecutionStatPut) router.AddShareHandler(handlers.MakeEndpointPath(shareURLRoot+"/"+pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteSharedExpression), dataExpressionShare) + router.AddShareHandler(handlers.MakeEndpointPath(shareURLRoot+"/doi/"+pathPrefix, idIdentifier), apiRouter.MakeMethodPermission("POST", permission.PermWriteSharedExpression), publishDataExpressionToZenodo) } func toWire(expr expressions.DataExpression) expressions.DataExpressionWire { @@ -69,6 +70,7 @@ func toWire(expr expressions.DataExpression) expressions.DataExpressionWire { ModuleReferences: expr.ModuleReferences, APIObjectItem: &orig, RecentExecStats: expr.RecentExecStats, + DOIMetadata: expr.DOIMetadata, } return resultItem } @@ -278,6 +280,26 @@ func dataExpressionShare(params handlers.ApiHandlerParams) (interface{}, error) return sharedIDs[0], nil } +func publishDataExpressionToZenodo(params handlers.ApiHandlerParams) (interface{}, error) { + expressionID := params.PathParams[idIdentifier] + + // Get the uploaded zip data + zipData, err := ioutil.ReadAll(params.Request.Body) + if err != nil { + return nil, err + } + + zenodoURI := params.Svcs.Config.ZenodoURI + zenodoToken := params.Svcs.Config.ZenodoAccessToken + + expression, err := params.Svcs.Expressions.PublishExpressionToZenodo(expressionID, zipData, zenodoURI, zenodoToken) + if err != nil { + return nil, err + } + + return expression, nil +} + func shareExpressions(svcs *services.APIServices, userID string, expressionIDs []string) ([]string, error) { generatedIDs := []string{} diff --git a/api/endpoints/DataExpression_test.go b/api/endpoints/DataExpression_test.go index b713b040..81bf482a 100644 --- a/api/endpoints/DataExpression_test.go +++ b/api/endpoints/DataExpression_test.go @@ -26,6 +26,7 @@ import ( "github.com/pixlise/core/v3/core/awsutil" expressionDB "github.com/pixlise/core/v3/core/expressions/database" "github.com/pixlise/core/v3/core/expressions/expressions" + zenodoModels "github.com/pixlise/core/v3/core/expressions/zenodo-models" "github.com/pixlise/core/v3/core/pixlUser" "github.com/pixlise/core/v3/core/timestamper" "go.mongodb.org/mongo-driver/bson" @@ -64,11 +65,13 @@ func makeExprDBList(idx int, includeSource bool) bson.D { 340.5, 1234568888, }, + zenodoModels.DOIMetadata{}, }, { "def456", "Iron Error", "element(\"Fe\", \"err\")", "PIXLANG", "comments for def456 expression", []string{}, []expressions.ModuleReference{}, makeOrigin("999", "Peter N", "niko@spicule.co.uk", false, 1668100001, 1668100001), nil, + zenodoModels.DOIMetadata{}, }, { "ghi789", "Iron %", "element(\"Fe\", \"%\")", "PIXLANG", "", []string{}, []expressions.ModuleReference{}, @@ -78,6 +81,7 @@ func makeExprDBList(idx int, includeSource bool) bson.D { 20, 1234568999, }, + zenodoModels.DOIMetadata{}, }, // Same as first item, but with real user id and different ID { @@ -88,6 +92,7 @@ func makeExprDBList(idx int, includeSource bool) bson.D { 340, 1234568888, }, + zenodoModels.DOIMetadata{}, }, // Same as above item, but shared { @@ -98,6 +103,7 @@ func makeExprDBList(idx int, includeSource bool) bson.D { 340, 1234568888, }, + zenodoModels.DOIMetadata{}, }, } @@ -249,6 +255,20 @@ func Test_dataExpressionHandler_List_OK(t *testing.T) { ], "runtimeMs": 340.5, "mod_unix_time_sec": 1234568888 + }, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" } }, "def456": { @@ -265,7 +285,21 @@ func Test_dataExpressionHandler_List_OK(t *testing.T) { "email": "peter@spicule.co.uk" }, "create_unix_time_sec": 1668100001, - "mod_unix_time_sec": 1668100001 + "mod_unix_time_sec": 1668100001, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" + } }, "shared-ghi789": { "id": "shared-ghi789", @@ -289,6 +323,20 @@ func Test_dataExpressionHandler_List_OK(t *testing.T) { ], "runtimeMs": 20, "mod_unix_time_sec": 1234568999 + }, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" } } } @@ -438,6 +486,20 @@ func Test_dataExpressionHandler_Get_OK(t *testing.T) { ], "runtimeMs": 340.5, "mod_unix_time_sec": 1234568888 + }, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" } } `) @@ -467,6 +529,20 @@ func Test_dataExpressionHandler_Get_OK(t *testing.T) { ], "runtimeMs": 20, "mod_unix_time_sec": 1234568999 + }, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" } } `) @@ -547,7 +623,21 @@ func Test_dataExpressionHandler_Post(t *testing.T) { "email": "niko@spicule.co.uk" }, "create_unix_time_sec": 1668142579, - "mod_unix_time_sec": 1668142579 + "mod_unix_time_sec": 1668142579, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" + } } `) }) @@ -662,7 +752,21 @@ func Test_dataExpressionHandler_Put(t *testing.T) { "email": "niko@spicule.co.uk" }, "create_unix_time_sec": 1668100000, - "mod_unix_time_sec": 1668100000 + "mod_unix_time_sec": 1668100000, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" + } } `) @@ -807,7 +911,21 @@ func Test_dataExpressionHandler_Put_NoSourceCode(t *testing.T) { "email": "niko@spicule.co.uk" }, "create_unix_time_sec": 1668100000, - "mod_unix_time_sec": 1668100002 + "mod_unix_time_sec": 1668100002, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" + } } `) @@ -850,7 +968,21 @@ func Test_dataExpressionHandler_Put_NoSourceCode(t *testing.T) { "email": "niko@spicule.co.uk" }, "create_unix_time_sec": 1668100000, - "mod_unix_time_sec": 1668100003 + "mod_unix_time_sec": 1668100003, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" + } } `) }) @@ -962,7 +1094,21 @@ func Test_dataExpressionHandler_Put_Shared(t *testing.T) { "email": "niko@spicule.co.uk" }, "create_unix_time_sec": 1668100000, - "mod_unix_time_sec": 1668100004 + "mod_unix_time_sec": 1668100004, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" + } } `) }) diff --git a/api/endpoints/DataModule.go b/api/endpoints/DataModule.go index e57ff344..ba1554b3 100644 --- a/api/endpoints/DataModule.go +++ b/api/endpoints/DataModule.go @@ -76,6 +76,9 @@ func dataModulePost(params handlers.ApiHandlerParams) (interface{}, error) { return nil, err } + // A new DOI is published to Zenodo if the "publish_doi" query parameter is true + publishDOI := params.Request.URL.Query().Get("publish_doi") == "true" + var req modules.DataModuleInput err = json.Unmarshal(body, &req) if err != nil { @@ -90,12 +93,15 @@ func dataModulePost(params handlers.ApiHandlerParams) (interface{}, error) { return modules.DataModuleSpecificVersionWire{}, api.MakeBadRequestError(errors.New("Source code field cannot be empty")) } - return params.Svcs.Expressions.CreateModule(req, params.UserInfo) + return params.Svcs.Expressions.CreateModule(req, params.UserInfo, publishDOI) } func dataModulePut(params handlers.ApiHandlerParams) (interface{}, error) { modID := params.PathParams[idIdentifier] + // A new DOI is published to Zenodo if the "publish_doi" query parameter is true + publishDOI := params.Request.URL.Query().Get("publish_doi") == "true" + body, err := ioutil.ReadAll(params.Request.Body) if err != nil { return nil, err @@ -107,5 +113,5 @@ func dataModulePut(params handlers.ApiHandlerParams) (interface{}, error) { return nil, api.MakeBadRequestError(err) } - return params.Svcs.Expressions.AddModuleVersion(modID, req) + return params.Svcs.Expressions.AddModuleVersion(modID, req, publishDOI) } diff --git a/api/endpoints/DataModule_test.go b/api/endpoints/DataModule_test.go index f6e9fe05..b2a5a140 100644 --- a/api/endpoints/DataModule_test.go +++ b/api/endpoints/DataModule_test.go @@ -115,7 +115,21 @@ func Test_Module_Listing(t *testing.T) { "A" ], "comments": "Module 1", - "mod_unix_time_sec": 1234567891 + "mod_unix_time_sec": 1234567891, + "doiMetadata": { + "title": "", + "creators": null, + "description": "", + "keywords": "", + "notes": "", + "relatedIdentifiers": null, + "contributors": null, + "references": "", + "version": "", + "doi": "", + "doiBadge": "", + "doiLink": "" + } } ] } diff --git a/api/endpoints/ViewStateWorkspace_test.go b/api/endpoints/ViewStateWorkspace_test.go index 5965cee1..130c8581 100644 --- a/api/endpoints/ViewStateWorkspace_test.go +++ b/api/endpoints/ViewStateWorkspace_test.go @@ -31,6 +31,7 @@ import ( "github.com/pixlise/core/v3/core/awsutil" expressionDB "github.com/pixlise/core/v3/core/expressions/database" "github.com/pixlise/core/v3/core/expressions/expressions" + zenodoModels "github.com/pixlise/core/v3/core/expressions/zenodo-models" "github.com/pixlise/core/v3/core/pixlUser" "github.com/pixlise/core/v3/core/timestamper" "go.mongodb.org/mongo-driver/bson" @@ -1364,11 +1365,13 @@ func makeExprDBItem(idx int, useCallerUserId bool) bson.D { "abc123", "Temp data", "housekeeping(\"something\")", "PIXLANG", "comments for abc123 expression", []string{}, []expressions.ModuleReference{}, makeOrigin(ownerID, "Niko", "niko@spicule.co.uk", false, 1668100000, 1668100000), nil, + zenodoModels.DOIMetadata{}, }, { "expr1", "Calcium weight%", "element(\"Ca\", \"%\")", "PIXLANG", "comments for expr1", []string{}, []expressions.ModuleReference{}, makeOrigin(ownerID2, "Peter N", "peter@spicule.co.uk", false, 1668100001, 1668100001), nil, + zenodoModels.DOIMetadata{}, }, } diff --git a/core/expressions/database/db_expressions.go b/core/expressions/database/db_expressions.go index a0f9c96b..fa90d11a 100644 --- a/core/expressions/database/db_expressions.go +++ b/core/expressions/database/db_expressions.go @@ -24,7 +24,10 @@ import ( "github.com/pixlise/core/v3/core/api" "github.com/pixlise/core/v3/core/expressions/expressions" + "github.com/pixlise/core/v3/core/expressions/zenodo" + zenodoModels "github.com/pixlise/core/v3/core/expressions/zenodo-models" "github.com/pixlise/core/v3/core/pixlUser" + "github.com/pixlise/core/v3/core/utils" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo/options" @@ -126,6 +129,7 @@ func (e *ExpressionDB) CreateExpression(input expressions.DataExpressionInput, c CreatedUnixTimeSec: nowUnix, ModifiedUnixTimeSec: nowUnix, }, + DOIMetadata: zenodoModels.DOIMetadata{}, // RecentExecStats is blank at this point! } @@ -143,7 +147,7 @@ func (e *ExpressionDB) CreateExpression(input expressions.DataExpressionInput, c // Replaces the existing expression with the new one // This assumes a GetExpression was required already to validate user permissions to this expression, etc // therefore the prevUnixTime should be available. This way we can preserve the creation time but set a new -// modified time now. Also now requires the existing expression shared and source code field! +// modified time now. Also now requires the existing expressƒion shared and source code field! func (e *ExpressionDB) UpdateExpression( expressionID string, input expressions.DataExpressionInput, @@ -196,6 +200,7 @@ func (e *ExpressionDB) UpdateExpression( CreatedUnixTimeSec: createdUnixTimeSec, ModifiedUnixTimeSec: nowUnix, }, + DOIMetadata: input.DOIMetadata, // Expression was edited, so any previous RecentExecStats are no longer valid, so blank! } @@ -244,3 +249,55 @@ func (e *ExpressionDB) DeleteExpression(expressionID string) error { return nil } + +func (e *ExpressionDB) PublishExpressionToZenodo(expressionID string, zipData []byte, zenodoURI string, zenodoToken string) (expressions.DataExpression, error) { + result := expressions.DataExpression{} + + strippedID, _ := utils.StripSharedItemIDPrefix(expressionID) + exprResult := e.Expressions.FindOne(context.TODO(), bson.M{"_id": strippedID}) + + if exprResult.Err() != nil { + return result, exprResult.Err() + } + + // Read the expression item + err := exprResult.Decode(&result) + if err != nil { + return result, err + } + + // Verify the expression exists and is shared before publishing + if result.ID == strippedID && result.Origin.Shared { + + deposition, err := zenodo.PublishExpressionZipToZenodo(result, zipData, zenodoURI, zenodoToken) + if err != nil { + return result, err + } + + // Update the expression with the DOI + filter := bson.D{{"_id", strippedID}} + + // Add the returned DOI links to the stored metadata + result.DOIMetadata.DOI = deposition.DOI + result.DOIMetadata.DOILink = deposition.Links.DOI + result.DOIMetadata.DOIBadge = deposition.Links.Badge + + update := bson.D{ + {"$set", bson.D{ + {"doiMetadata", result.DOIMetadata}, + }}, + } + + updResult, err := e.Expressions.UpdateOne(context.TODO(), filter, update) + if err != nil { + return result, err + } + + // Make sure it worked + if updResult.MatchedCount != 1 || updResult.ModifiedCount != 1 { + return result, api.MakeNotFoundError(strippedID) + } + } + + return result, nil +} diff --git a/core/expressions/database/db_modules.go b/core/expressions/database/db_modules.go index 94953fc2..e5f87da0 100644 --- a/core/expressions/database/db_modules.go +++ b/core/expressions/database/db_modules.go @@ -23,6 +23,7 @@ import ( "fmt" "github.com/pixlise/core/v3/core/expressions/modules" + "github.com/pixlise/core/v3/core/expressions/zenodo" "github.com/pixlise/core/v3/core/pixlUser" "github.com/pixlise/core/v3/core/utils" "go.mongodb.org/mongo-driver/bson" @@ -194,6 +195,7 @@ func (e *ExpressionDB) GetModule(moduleID string, version *modules.SemanticVersi Tags: ver.Tags, Comments: ver.Comments, TimeStampUnixSec: ver.TimeStampUnixSec, + DOIMetadata: ver.DOIMetadata, }, }, } @@ -204,6 +206,7 @@ func (e *ExpressionDB) GetModule(moduleID string, version *modules.SemanticVersi func (e *ExpressionDB) CreateModule( input modules.DataModuleInput, creator pixlUser.UserInfo, + publishDOI bool, ) (modules.DataModuleSpecificVersionWire, error) { nowUnix := e.Svcs.TimeStamper.GetTimeNowSec() modId := e.Svcs.IDGen.GenObjectID() @@ -240,6 +243,29 @@ func (e *ExpressionDB) CreateModule( Tags: input.Tags, Comments: "Initial version", TimeStampUnixSec: nowUnix, + DOIMetadata: input.DOIMetadata, + } + + if publishDOI { + deposition, err := zenodo.PublishModuleToZenodo(modules.DataModuleSpecificVersionWire{ + DataModule: &mod, + Version: modules.DataModuleVersionSourceWire{ + SourceCode: input.SourceCode, + DataModuleVersionWire: &modules.DataModuleVersionWire{ + Version: modules.SemanticVersionToString(ver.Version), + Tags: ver.Tags, + Comments: ver.Comments, + TimeStampUnixSec: ver.TimeStampUnixSec, + }, + }, + }, e.Svcs.Config.ZenodoURI, e.Svcs.Config.ZenodoAccessToken) + if err != nil { + e.Svcs.Log.Errorf("Failed to publish new module to Zenodo: %v. Error: %v", modId, err) + } + + ver.DOIMetadata.DOI = deposition.DOI + ver.DOIMetadata.DOIBadge = deposition.Links.Badge + ver.DOIMetadata.DOILink = deposition.Links.DOI } insertResult, err = e.ModuleVersions.InsertOne(context.TODO(), ver) @@ -258,6 +284,7 @@ func (e *ExpressionDB) CreateModule( Tags: ver.Tags, Comments: ver.Comments, TimeStampUnixSec: ver.TimeStampUnixSec, + DOIMetadata: ver.DOIMetadata, }, } @@ -269,8 +296,8 @@ func (e *ExpressionDB) CreateModule( return result, err } -func (e *ExpressionDB) getLatestVersion(moduleID string) (modules.SemanticVersion, error) { - result := modules.SemanticVersion{} +func (e *ExpressionDB) getLatestModule(moduleID string) (modules.DataModuleVersion, error) { + result := modules.DataModuleVersion{} ctx := context.TODO() cursor, err := e.ModuleVersions.Aggregate(ctx, bson.A{ @@ -298,14 +325,15 @@ func (e *ExpressionDB) getLatestVersion(moduleID string) (modules.SemanticVersio err = cursor.Decode(&ver) } - result = ver.Version - //ver := bson.D{} - //err = cursor.Decode(&ver) + return ver, err +} - return result, err +func (e *ExpressionDB) getLatestVersion(moduleID string) (modules.SemanticVersion, error) { + version, err := e.getLatestModule(moduleID) + return version.Version, err } -func (e *ExpressionDB) AddModuleVersion(moduleID string, input modules.DataModuleVersionInput) (modules.DataModuleSpecificVersionWire, error) { +func (e *ExpressionDB) AddModuleVersion(moduleID string, input modules.DataModuleVersionInput, publishDOI bool) (modules.DataModuleSpecificVersionWire, error) { if e.Modules == nil { return modules.DataModuleSpecificVersionWire{}, errors.New("AddModuleVersion: Mongo not connected") } @@ -350,6 +378,31 @@ func (e *ExpressionDB) AddModuleVersion(moduleID string, input modules.DataModul Tags: input.Tags, Comments: input.Comments, TimeStampUnixSec: nowUnix, + DOIMetadata: input.DOIMetadata, + } + + if publishDOI { + deposition, err := zenodo.PublishModuleToZenodo(modules.DataModuleSpecificVersionWire{ + DataModule: &mod, + Version: modules.DataModuleVersionSourceWire{ + SourceCode: input.SourceCode, + DataModuleVersionWire: &modules.DataModuleVersionWire{ + Version: modules.SemanticVersionToString(verRec.Version), + Tags: verRec.Tags, + Comments: verRec.Comments, + TimeStampUnixSec: verRec.TimeStampUnixSec, + DOIMetadata: verRec.DOIMetadata, + }, + }, + }, e.Svcs.Config.ZenodoURI, e.Svcs.Config.ZenodoAccessToken) + if err != nil { + e.Svcs.Log.Errorf("Failed to publish new version of module to Zenodo: %v. Error: %v", moduleID, err) + return modules.DataModuleSpecificVersionWire{}, err + } + + verRec.DOIMetadata.DOI = deposition.DOI + verRec.DOIMetadata.DOIBadge = deposition.Links.Badge + verRec.DOIMetadata.DOILink = deposition.Links.DOI } insertResult, err := e.ModuleVersions.InsertOne(context.TODO(), verRec) @@ -368,6 +421,7 @@ func (e *ExpressionDB) AddModuleVersion(moduleID string, input modules.DataModul Tags: verRec.Tags, Comments: verRec.Comments, TimeStampUnixSec: verRec.TimeStampUnixSec, + DOIMetadata: verRec.DOIMetadata, }, } diff --git a/core/expressions/database/module_addversion_test.go b/core/expressions/database/module_addversion_test.go index 547323f5..59d311d5 100644 --- a/core/expressions/database/module_addversion_test.go +++ b/core/expressions/database/module_addversion_test.go @@ -70,7 +70,7 @@ func Test_Module_DB_AddVersion_NoModule(t *testing.T) { Comments: "My comment", Tags: []string{"The best"}, } - _, err := db.AddModuleVersion("mod123", input) + _, err := db.AddModuleVersion("mod123", input, false) if err == nil { t.Error("Expected error") @@ -149,7 +149,7 @@ func Test_Module_DB_AddVersion_OK(t *testing.T) { Comments: "My comment", Tags: []string{"The best"}, } - result, err := db.AddModuleVersion("mod123", input) + result, err := db.AddModuleVersion("mod123", input, false) if err != nil { t.Error(err) diff --git a/core/expressions/database/module_create_test.go b/core/expressions/database/module_create_test.go index d9dac31d..85531c81 100644 --- a/core/expressions/database/module_create_test.go +++ b/core/expressions/database/module_create_test.go @@ -68,7 +68,7 @@ func Test_Module_DB_Create(t *testing.T) { } user := pixlUser.UserInfo{Name: "Peter N", UserID: "999", Email: "peter@pixlise.org"} - _, err := db.CreateModule(input, user) + _, err := db.CreateModule(input, user, false) if err != nil { t.Error(err) diff --git a/core/expressions/db-interface.go b/core/expressions/db-interface.go index ea123aaf..04091869 100644 --- a/core/expressions/db-interface.go +++ b/core/expressions/db-interface.go @@ -39,12 +39,13 @@ type ExpressionDB interface { ) (expressions.DataExpression, error) StoreExpressionRecentRunStats(expressionID string, stats expressions.DataExpressionExecStats) error DeleteExpression(expressionID string) error + PublishExpressionToZenodo(expressionID string, zipData []byte, zenodoURI string, zenodoToken string) (expressions.DataExpression, error) // Module storage ListModules(retrieveUpdatedUserInfo bool) (modules.DataModuleWireLookup, error) GetModule(moduleID string, version *modules.SemanticVersion, retrieveUpdatedUserInfo bool) (modules.DataModuleSpecificVersionWire, error) - CreateModule(input modules.DataModuleInput, creator pixlUser.UserInfo) (modules.DataModuleSpecificVersionWire, error) - AddModuleVersion(moduleID string, input modules.DataModuleVersionInput) (modules.DataModuleSpecificVersionWire, error) + CreateModule(input modules.DataModuleInput, creator pixlUser.UserInfo, publishDOI bool) (modules.DataModuleSpecificVersionWire, error) + AddModuleVersion(moduleID string, input modules.DataModuleVersionInput, publishDOI bool) (modules.DataModuleSpecificVersionWire, error) // Helper to decide what kind of error was returned IsNotFoundError(err error) bool diff --git a/core/expressions/expressions/expressions.go b/core/expressions/expressions/expressions.go index b8597da7..fbce8995 100644 --- a/core/expressions/expressions/expressions.go +++ b/core/expressions/expressions/expressions.go @@ -19,6 +19,7 @@ package expressions import ( + zenodoModels "github.com/pixlise/core/v3/core/expressions/zenodo-models" "github.com/pixlise/core/v3/core/pixlUser" ) @@ -30,6 +31,7 @@ type DataExpressionInput struct { Comments string `json:"comments"` Tags []string `json:"tags"` ModuleReferences []ModuleReference `json:"moduleReferences,omitempty" bson:"moduleReferences,omitempty"` + DOIMetadata zenodoModels.DOIMetadata } // Stats related to executing an expression. We get these from the UI when it runs @@ -59,6 +61,7 @@ type DataExpression struct { Origin pixlUser.APIObjectItem `json:"origin"` // NOTE: if modifying below, ensure it's in sync with ExpressionDB StoreExpressionRecentRunStats() RecentExecStats *DataExpressionExecStats `json:"recentExecStats,omitempty" bson:"recentExecStats,omitempty"` + DOIMetadata zenodoModels.DOIMetadata `json:"doiMetadata,omitempty" bson:"doiMetadata,omitempty"` } func (a DataExpression) SetTimes(userID string, t int64) { @@ -85,4 +88,5 @@ type DataExpressionWire struct { *pixlUser.APIObjectItem // NOTE: if modifying below, ensure it's in sync with ExpressionDB StoreExpressionRecentRunStats() RecentExecStats *DataExpressionExecStats `json:"recentExecStats,omitempty" bson:"recentExecStats,omitempty"` + DOIMetadata zenodoModels.DOIMetadata `json:"doiMetadata,omitempty" bson:"doiMetadata,omitempty"` } diff --git a/core/expressions/modules/modules.go b/core/expressions/modules/modules.go index 629701a0..d927e715 100644 --- a/core/expressions/modules/modules.go +++ b/core/expressions/modules/modules.go @@ -24,6 +24,7 @@ import ( "strconv" "strings" + zenodoModels "github.com/pixlise/core/v3/core/expressions/zenodo-models" "github.com/pixlise/core/v3/core/pixlUser" ) @@ -59,18 +60,20 @@ import ( // What users send in POST type DataModuleInput struct { - Name string `json:"name"` // Editable name - SourceCode string `json:"sourceCode"` // The module executable code - Comments string `json:"comments"` // Editable comments - Tags []string `json:"tags"` // Any tags for this version + Name string `json:"name"` // Editable name + SourceCode string `json:"sourceCode"` // The module executable code + Comments string `json:"comments"` // Editable comments + Tags []string `json:"tags"` // Any tags for this version + DOIMetadata zenodoModels.DOIMetadata `json:"doiMetadata,omitempty" bson:"doiMetadata,omitempty"` } // And what we get in PUT for new versions being uploaded type DataModuleVersionInput struct { - SourceCode string `json:"sourceCode"` // The module executable code - Comments string `json:"comments"` // Editable comments - Tags []string `json:"tags"` // Any tags for this version - VersionUpdate string `json:"versionupdate"` // What are we updating? patch, minor or major. Anything else = error + SourceCode string `json:"sourceCode"` // The module executable code + Comments string `json:"comments"` // Editable comments + Tags []string `json:"tags"` // Any tags for this version + VersionUpdate string `json:"versionupdate"` // What are we updating? patch, minor or major. Anything else = error + DOIMetadata zenodoModels.DOIMetadata `json:"doiMetadata,omitempty" bson:"doiMetadata,omitempty"` } type SemanticVersion struct { @@ -108,13 +111,14 @@ func SemanticVersionFromString(v string) (SemanticVersion, error) { // Stored version of a module type DataModuleVersion struct { - ID string `json:"-" bson:"_id"` // Use as Mongo ID - ModuleID string `json:"moduleID"` // The ID of the module we belong to - SourceCode string `json:"sourceCode"` - Version SemanticVersion `json:"version"` - Tags []string `json:"tags"` - Comments string `json:"comments"` - TimeStampUnixSec int64 `json:"mod_unix_time_sec"` + ID string `json:"-" bson:"_id"` // Use as Mongo ID + ModuleID string `json:"moduleID"` // The ID of the module we belong to + SourceCode string `json:"sourceCode"` + Version SemanticVersion `json:"version"` + Tags []string `json:"tags"` + Comments string `json:"comments"` + TimeStampUnixSec int64 `json:"mod_unix_time_sec"` + DOIMetadata zenodoModels.DOIMetadata `json:"doiMetadata,omitempty" bson:"doiMetadata,omitempty"` } // Stored module object itself @@ -127,10 +131,11 @@ type DataModule struct { // What we send out to users - notice versions only contains version numbers & tags type DataModuleVersionWire struct { - Version string `json:"version"` - Tags []string `json:"tags"` - Comments string `json:"comments"` - TimeStampUnixSec int64 `json:"mod_unix_time_sec"` + Version string `json:"version"` + Tags []string `json:"tags"` + Comments string `json:"comments"` + TimeStampUnixSec int64 `json:"mod_unix_time_sec"` + DOIMetadata zenodoModels.DOIMetadata `json:"doiMetadata,omitempty" bson:"doiMetadata,omitempty"` } // As above, but with source field diff --git a/core/expressions/zenodo-models/zenodo-models.go b/core/expressions/zenodo-models/zenodo-models.go new file mode 100644 index 00000000..0cdd11fa --- /dev/null +++ b/core/expressions/zenodo-models/zenodo-models.go @@ -0,0 +1,230 @@ +package zenodoModels + +// Structures we store +type DOIRelatedIdentifier struct { + Identifier string `json:"identifier"` + Relation string `json:"relation"` +} + +type DOICreator struct { + Name string `json:"name"` + Affiliation string `json:"affiliation"` + Orcid string `json:"orcid"` +} + +type DOIContributor struct { + Name string `json:"name"` + Affiliation string `json:"affiliation"` + Orcid string `json:"orcid"` + Type string `json:"type"` +} + +type DOIMetadata struct { + Title string `json:"title"` + Creators []DOICreator `json:"creators"` + Description string `json:"description"` + Keywords string `json:"keywords"` + Notes string `json:"notes"` + RelatedIdentifiers []DOIRelatedIdentifier `json:"relatedIdentifiers"` + Contributors []DOIContributor `json:"contributors"` + References string `json:"references"` + Version string `json:"version"` + DOI string `json:"doi"` + DOIBadge string `json:"doiBadge"` + DOILink string `json:"doiLink"` +} + +// Structures received from Zenodo +type ZenodoPublishResponse struct { + ConceptDOI string `json:"conceptdoi"` + ConceptRecID string `json:"conceptrecid"` + Created string `json:"created"` + DOI string `json:"doi"` + DOIURL string `json:"doi_url"` + + Files []struct { + Checksum string `json:"checksum"` + Filename string `json:"filename"` + Filesize int `json:"filesize"` + ID string `json:"id"` + + Links struct { + Download string `json:"download"` + Self string `json:"self"` + } `json:"links"` + } `json:"files"` + + ID int `json:"id"` + + Links struct { + Badge string `json:"badge"` + Bucket string `json:"bucket"` + ConceptBadge string `json:"conceptbadge"` + ConceptDOI string `json:"conceptdoi"` + DOI string `json:"doi"` + Latest string `json:"latest"` + LatestHTML string `json:"latest_html"` + Record string `json:"record"` + RecordHTML string `json:"record_html"` + } `json:"links"` + + Metadata struct { + AccessRight string `json:"access_right"` + + Communities []struct { + Identifier string `json:"identifier"` + } `json:"communities"` + + Creators []struct { + Name string `json:"name"` + } `json:"creators"` + + Description string `json:"description"` + DOI string `json:"doi"` + License string `json:"license"` + + PrereserveDOI struct { + DOI string `json:"doi"` + RecID int `json:"recid"` + } `json:"prereserve_doi"` + + PublicationDate string `json:"publication_date"` + Title string `json:"title"` + UploadType string `json:"upload_type"` + } `json:"metadata"` + + Modified string `json:"modified"` + Owner int `json:"owner"` + + RecordID int `json:"record_id"` + State string `json:"state"` + + Submitted bool `json:"submitted"` + Title string `json:"title"` +} + +type ZenodoDepositionMetadata struct { + AccessRight string `json:"access_right"` + + Communities []struct { + Identifier string `json:"identifier"` + } `json:"communities"` + + Creators []struct { + Name string `json:"name"` + Affiliation string `json:"affiliation"` + } `json:"creators"` + + Description string `json:"description"` + DOI string `json:"doi"` + License string `json:"license"` + + PrereserveDOI struct { + DOI string `json:"doi"` + RecID int `json:"recid"` + } `json:"prereserve_doi"` + + PublicationDate string `json:"publication_date"` + Title string `json:"title"` + UploadType string `json:"upload_type"` +} + +type ZenodoMetaResponse struct { + ConceptRecID string `json:"conceptrecid"` + Created string `json:"created"` + DOI string `json:"doi"` + DOIURL string `json:"doi_url"` + + Files []struct { + Checksum string `json:"checksum"` + Filename string `json:"filename"` + Filesize int `json:"filesize"` + ID string `json:"id"` + + Links struct { + Download string `json:"download"` + Self string `json:"self"` + } `json:"links"` + } `json:"files"` + + ID int `json:"id"` + + Links struct { + Badge string `json:"badge"` + Bucket string `json:"bucket"` + ConceptBadge string `json:"conceptbadge"` + ConceptDOI string `json:"conceptdoi"` + DOI string `json:"doi"` + Latest string `json:"latest"` + LatestHTML string `json:"latest_html"` + Record string `json:"record"` + RecordHTML string `json:"record_html"` + } `json:"links"` + + Metadata ZenodoDepositionMetadata `json:"metadata"` + + Modified string `json:"modified"` + Owner int `json:"owner"` + + RecordID int `json:"record_id"` + State string `json:"state"` + + Submitted bool `json:"submitted"` + Title string `json:"title"` +} + +type ZenodoDepositionResponse struct { + ConceptRecID string `json:"conceptrecid"` + Created string `json:"created"` + + Files []struct { + Links struct { + Download string `json:"download"` + } `json:"links"` + } `json:"files"` + + ID int `json:"id"` + Links struct { + Bucket string `json:"bucket"` + Discard string `json:"discard"` + Edit string `json:"edit"` + Files string `json:"files"` + HTML string `json:"html"` + LatestDraft string `json:"latest_draft"` + LatestDraftHTML string `json:"latest_draft_html"` + Publish string `json:"publish"` + Self string `json:"self"` + } + + Meta struct { + PrereserveDOI struct { + DOI string `json:"doi"` + RecID int `json:"recid"` + } `json:"prereserve_doi"` + } `json:"metadata"` + + Owner int `json:"owner"` + RecordID int `json:"record_id"` + State string `json:"state"` + Submitted bool `json:"submitted"` + Title string `json:"title"` +} + +type ZenodoFileUploadResponse struct { + Key string `json:"key"` + Mimetype string `json:"mimetype"` + Checksum string `json:"checksum"` + VersionID string `json:"version_id"` + Size int `json:"size"` + Created string `json:"created"` + Updated string `json:"updated"` + + Links struct { + Self string `json:"self"` + Version string `json:"version"` + Uploads string `json:"uploads"` + } `json:"links"` + + IsHead bool `json:"is_head"` + DeleteMarker bool `json:"delete_marker"` +} diff --git a/core/expressions/zenodo/zenodo_publisher.go b/core/expressions/zenodo/zenodo_publisher.go new file mode 100644 index 00000000..feac2cb5 --- /dev/null +++ b/core/expressions/zenodo/zenodo_publisher.go @@ -0,0 +1,337 @@ +// Licensed to NASA JPL under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. NASA JPL licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package zenodo + +import ( + "bytes" + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "strings" + + "github.com/pixlise/core/v3/core/expressions/expressions" + "github.com/pixlise/core/v3/core/expressions/modules" + zenodoModels "github.com/pixlise/core/v3/core/expressions/zenodo-models" +) + +func createEmptyDeposition(zenodoURI string, accessToken string) (*zenodoModels.ZenodoDepositionResponse, error) { + emptyResponse := zenodoModels.ZenodoDepositionResponse{} + + depositionsURL := zenodoURI + "/api/deposit/depositions?access_token=" + accessToken + + resp, err := http.Post(depositionsURL, "application/json", bytes.NewBuffer([]byte("{}"))) + if err != nil { + return &emptyResponse, err + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return &emptyResponse, err + } + + response := zenodoModels.ZenodoDepositionResponse{} + err = json.Unmarshal(body, &response) + if err != nil { + return &emptyResponse, err + } + + return &response, nil +} + +func uploadFileContentsToZenodo(deposition zenodoModels.ZenodoDepositionResponse, filename string, contents *bytes.Buffer, accessToken string) (*zenodoModels.ZenodoFileUploadResponse, error) { + emptyResponse := zenodoModels.ZenodoFileUploadResponse{} + + uploadUrl := deposition.Links.Bucket + "/" + filename + "?access_token=" + accessToken + putReq, err := http.NewRequest("PUT", uploadUrl, contents) + if err != nil { + return &emptyResponse, err + } + + putReq.Header.Set("Content-Type", "application/octet-stream") + + putResponse, err := http.DefaultClient.Do(putReq) + if err != nil { + return &emptyResponse, err + } + + defer putResponse.Body.Close() + putBody, err := ioutil.ReadAll(putResponse.Body) + if err != nil { + return &emptyResponse, err + } + + fileUploadResponse := zenodoModels.ZenodoFileUploadResponse{} + err = json.Unmarshal(putBody, &fileUploadResponse) + if err != nil { + return &emptyResponse, err + } + + return &fileUploadResponse, nil +} + +func uploadModuleToZenodo(deposition zenodoModels.ZenodoDepositionResponse, module modules.DataModuleSpecificVersionWire, accessToken string) (*zenodoModels.ZenodoFileUploadResponse, error) { + zenodoResponse := zenodoModels.ZenodoFileUploadResponse{} + + filename := module.DataModule.ID + ".json" + jsonContents, err := json.Marshal(module) + if err != nil { + return &zenodoResponse, err + } + + fileUploadResponse, err := uploadFileContentsToZenodo(deposition, filename, bytes.NewBuffer([]byte(jsonContents)), accessToken) + if err != nil { + return &zenodoResponse, err + } + + return fileUploadResponse, nil +} + +func uploadExpressionZipToZenodo(deposition zenodoModels.ZenodoDepositionResponse, filename string, zipFile []byte, accessToken string) (*zenodoModels.ZenodoFileUploadResponse, error) { + zenodoResponse := zenodoModels.ZenodoFileUploadResponse{} + + fileUploadResponse, err := uploadFileContentsToZenodo(deposition, filename, bytes.NewBuffer([]byte(zipFile)), accessToken) + if err != nil { + return &zenodoResponse, err + } + + return fileUploadResponse, nil +} + +func uploadMetadataToDeposition(deposition zenodoModels.ZenodoDepositionResponse, metadata map[string]interface{}, accessToken string) (*zenodoModels.ZenodoMetaResponse, error) { + emptyResponse := zenodoModels.ZenodoMetaResponse{} + + metadataJson, err := json.Marshal(metadata) + if err != nil { + return &emptyResponse, err + } + + depositionMetaURL := deposition.Links.LatestDraft + "?access_token=" + accessToken + depositionMetaReq, err := http.NewRequest("PUT", depositionMetaURL, bytes.NewBuffer([]byte(metadataJson))) + if err != nil { + return &emptyResponse, err + } + + depositionMetaReq.Header.Set("Content-Type", "application/json") + metaResponse, err := http.DefaultClient.Do(depositionMetaReq) + if err != nil { + return &emptyResponse, err + } + + defer metaResponse.Body.Close() + + metaObject, err := ioutil.ReadAll(metaResponse.Body) + if err != nil { + return &emptyResponse, err + } + + metaResponseObj := zenodoModels.ZenodoMetaResponse{} + err = json.Unmarshal(metaObject, &metaResponseObj) + if err != nil { + return &emptyResponse, err + } + + return &metaResponseObj, nil +} + +func addMetadataToDeposition(deposition zenodoModels.ZenodoDepositionResponse, doiMetadata zenodoModels.DOIMetadata, accessToken string) (*zenodoModels.ZenodoMetaResponse, error) { + // API fails if any empty keys are included, so we need to remove them + + metadata := map[string]interface{}{ + "metadata": map[string]interface{}{ + "title": doiMetadata.Title, + "upload_type": "software", + "description": doiMetadata.Description, + }, + } + + if doiMetadata.Creators != nil && len(doiMetadata.Creators) > 0 { + metadata["metadata"].(map[string]interface{})["creators"] = []map[string]interface{}{} + + for _, creator := range doiMetadata.Creators { + creatorMap := map[string]interface{}{ + "name": creator.Name, + } + + if creator.Affiliation != "" { + creatorMap["affiliation"] = creator.Affiliation + } + + if creator.Orcid != "" { + creatorMap["orcid"] = creator.Orcid + } + + metadata["metadata"].(map[string]interface{})["creators"] = append(metadata["metadata"].(map[string]interface{})["creators"].([]map[string]interface{}), creatorMap) + } + } + + if doiMetadata.Keywords != "" { + metadata["metadata"].(map[string]interface{})["keywords"] = strings.Split(doiMetadata.Keywords, ",") + } + + if doiMetadata.Notes != "" { + metadata["metadata"].(map[string]interface{})["notes"] = doiMetadata.Notes + } + + if doiMetadata.RelatedIdentifiers != nil && len(doiMetadata.RelatedIdentifiers) > 0 { + metadata["metadata"].(map[string]interface{})["related_identifiers"] = []map[string]interface{}{} + + for _, relatedIdentifier := range doiMetadata.RelatedIdentifiers { + relatedID := map[string]interface{}{ + "identifier": relatedIdentifier.Identifier, + "relation": relatedIdentifier.Relation, + } + metadata["metadata"].(map[string]interface{})["related_identifiers"] = append(metadata["metadata"].(map[string]interface{})["related_identifiers"].([]map[string]interface{}), relatedID) + } + } + + if doiMetadata.Contributors != nil && len(doiMetadata.Contributors) > 0 { + metadata["metadata"].(map[string]interface{})["contributors"] = []map[string]interface{}{} + + for _, contributor := range doiMetadata.Contributors { + contributorMap := map[string]interface{}{ + "name": contributor.Name, + } + + if contributor.Affiliation != "" { + contributorMap["affiliation"] = contributor.Affiliation + } + + if contributor.Type != "" { + contributorMap["type"] = contributor.Type + } + + if contributor.Orcid != "" { + contributorMap["orcid"] = contributor.Orcid + } + + metadata["metadata"].(map[string]interface{})["contributors"] = append(metadata["metadata"].(map[string]interface{})["contributors"].([]map[string]interface{}), contributorMap) + } + } + + if doiMetadata.References != "" { + metadata["metadata"].(map[string]interface{})["references"] = strings.Split(doiMetadata.References, ",") + } + + if doiMetadata.Version != "" { + metadata["metadata"].(map[string]interface{})["version"] = doiMetadata.Version + } + + return uploadMetadataToDeposition(deposition, metadata, accessToken) +} + +func publishDeposition(deposition zenodoModels.ZenodoDepositionResponse, accessToken string) (*zenodoModels.ZenodoPublishResponse, error) { + publishURL := deposition.Links.Publish + "?access_token=" + accessToken + publishReq, err := http.NewRequest(http.MethodPost, publishURL, bytes.NewBuffer([]byte("{}"))) + if err != nil { + return nil, err + } + + publishReq.Header.Set("Content-Type", "application/json") + publishResponse, err := http.DefaultClient.Do(publishReq) + if err != nil { + return nil, err + } + + defer publishResponse.Body.Close() + + publishBody, err := ioutil.ReadAll(publishResponse.Body) + if err != nil { + return nil, err + } + + zenodoResponse := zenodoModels.ZenodoPublishResponse{} + err = json.Unmarshal(publishBody, &zenodoResponse) + if err != nil { + return nil, err + } + + return &zenodoResponse, nil +} + +func PublishModuleToZenodo(module modules.DataModuleSpecificVersionWire, zenodoURI string, zenodoToken string) (*zenodoModels.ZenodoPublishResponse, error) { + zenodoResponse := zenodoModels.ZenodoPublishResponse{} + + if zenodoURI == "" { + return &zenodoResponse, errors.New("ZENODO_URI not found") + } + + if zenodoToken == "" { + return &zenodoResponse, errors.New("ZENODO_ACCESS_TOKEN not found") + } + + deposition, err := createEmptyDeposition(zenodoURI, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + _, err = uploadModuleToZenodo(*deposition, module, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + _, err = addMetadataToDeposition(*deposition, module.Version.DOIMetadata, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + publishResponse, err := publishDeposition(*deposition, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + zenodoResponse = *publishResponse + return &zenodoResponse, nil +} + +func PublishExpressionZipToZenodo(expression expressions.DataExpression, zipFile []byte, zenodoURI string, zenodoToken string) (*zenodoModels.ZenodoPublishResponse, error) { + zenodoResponse := zenodoModels.ZenodoPublishResponse{} + + if zenodoURI == "" { + return &zenodoResponse, errors.New("ZENODO_URI not found") + } + + if zenodoToken == "" { + return &zenodoResponse, errors.New("ZENODO_ACCESS_TOKEN not found") + } + + deposition, err := createEmptyDeposition(zenodoURI, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + filename := expression.ID + ".zip" + _, err = uploadExpressionZipToZenodo(*deposition, filename, zipFile, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + _, err = addMetadataToDeposition(*deposition, expression.DOIMetadata, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + publishResponse, err := publishDeposition(*deposition, zenodoToken) + if err != nil { + return &zenodoResponse, err + } + + zenodoResponse = *publishResponse + return &zenodoResponse, nil +} diff --git a/core/expressions/zenodo/zenodo_publisher_test_integration.go b/core/expressions/zenodo/zenodo_publisher_test_integration.go new file mode 100644 index 00000000..91fddc54 --- /dev/null +++ b/core/expressions/zenodo/zenodo_publisher_test_integration.go @@ -0,0 +1,166 @@ +// Licensed to NASA JPL under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. NASA JPL licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package zenodo + +import ( + "bytes" + "encoding/json" + "os" + "testing" + + "github.com/pixlise/core/v3/api/config" + "github.com/pixlise/core/v3/api/services" + "github.com/pixlise/core/v3/core/awsutil" + "github.com/pixlise/core/v3/core/expressions/modules" + "github.com/pixlise/core/v3/core/logger" + "github.com/pixlise/core/v3/core/pixlUser" +) + +func makeMockSvcs(idGen services.IDGenerator) services.APIServices { + cfg := config.APIConfig{} + + return services.APIServices{ + Config: cfg, + Log: &logger.NullLogger{}, + SNS: &awsutil.MockSNS{}, + IDGen: idGen, + } +} + +func setTestZenodoConfig(svcs *services.APIServices) { + svcs.Config.ZenodoURI = os.Getenv("PIXLISE_API_TEST_ZENODO_URI") + svcs.Config.ZenodoAccessToken = os.Getenv("PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN") + + if len(svcs.Config.ZenodoURI) <= 0 || len(svcs.Config.ZenodoAccessToken) <= 0 { + panic("Missing one or more env vars for testing: PIXLISE_API_TEST_ZENODO_URI, PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN") + } +} + +func Test_create_empty_deposition(t *testing.T) { + idGen := services.MockIDGenerator{ + IDs: []string{"expr111"}, + } + + svcs := makeMockSvcs(&idGen) + setTestZenodoConfig(&svcs) + + deposition, err := createEmptyDeposition(svcs.Config.ZenodoURI, svcs.Config.ZenodoAccessToken) + if err != nil { + t.Errorf("Failed to create empty deposition: %v", err) + } + + if deposition == nil { + t.Errorf("Deposition is nil") + } else if deposition.Links.Bucket == "" { + t.Errorf("Deposition.Links.Bucket is empty") + } else if deposition.Links.LatestDraft == "" { + t.Errorf("Deposition.Links.LatestDraft is empty") + } else if deposition.Links.Publish == "" { + t.Errorf("Deposition.Links.Publish is empty") + } +} + +func Test_upload_file_to_deposition(t *testing.T) { + idGen := services.MockIDGenerator{ + IDs: []string{"expr123"}, + } + + svcs := makeMockSvcs(&idGen) + setTestZenodoConfig(&svcs) + + deposition, err := createEmptyDeposition(svcs.Config.ZenodoURI, svcs.Config.ZenodoAccessToken) + if err != nil { + t.Errorf("Failed to create empty deposition: %v", err) + } + + data := map[string]string{ + "data": "this is a test", + } + + filename := "test.json" + jsonContents, err := json.Marshal(data) + if err != nil { + t.Errorf("Failed to marshal test data: %v", err) + } + + fileUploadResponse, err := uploadFileContentsToZenodo(*deposition, filename, bytes.NewBuffer([]byte(jsonContents)), svcs.Config.ZenodoAccessToken) + if err != nil { + t.Errorf("Failed to upload test file contents to Zenodo: %v", err) + } + + if fileUploadResponse == nil { + t.Errorf("File upload response is nil") + } else if fileUploadResponse.Key == "" { + t.Errorf("File upload response.Key is empty, probably malformed data") + } + + testModule := modules.DataModuleSpecificVersionWire{ + DataModule: &modules.DataModule{ID: "mod123", + Name: "TestModule", + Comments: "This is a test", + Origin: pixlUser.APIObjectItem{ + Shared: true, + Creator: pixlUser.UserInfo{Name: "Ryan S", UserID: "333", Email: "ryan@pixlise.org"}, + CreatedUnixTimeSec: 1234567889, + ModifiedUnixTimeSec: 1234567892, + }, + }, + Version: modules.DataModuleVersionSourceWire{ + SourceCode: "element(\"Ca\", \"%\", \"A\")", + DataModuleVersionWire: &modules.DataModuleVersionWire{ + Version: "2.1.43", + Tags: []string{"latest", "experimental", "test"}, + Comments: "This is a test", + TimeStampUnixSec: 1234567891, + }, + }, + } + + fileUploadResponse, err = uploadModuleToZenodo(*deposition, testModule, svcs.Config.ZenodoAccessToken) + if err != nil { + t.Errorf("Failed to upload test module to Zenodo: %v", err) + } + + if fileUploadResponse == nil { + t.Errorf("File upload response is nil for test module") + } else if fileUploadResponse.Key == "" { + t.Errorf("File upload response.Key is empty for test module, probably malformed file data") + } + + metadataResponse, err := addMetadataToDeposition(*deposition, testModule.Version.DOIMetadata, svcs.Config.ZenodoAccessToken) + if err != nil { + t.Errorf("Failed to add metadata to deposition: %v", err) + } + + if metadataResponse == nil { + t.Errorf("Metadata response is nil") + } else if metadataResponse.ConceptRecID == "" { + t.Errorf("Metadata response.ConceptRecID is empty, probably malformed metadata") + } + + publishResponse, err := publishDeposition(*deposition, svcs.Config.ZenodoAccessToken) + if err != nil { + t.Errorf("Failed to publish deposition: %v", err) + } + + if publishResponse == nil { + t.Errorf("Publish response is nil") + } else if !publishResponse.Submitted { + t.Errorf("Publish response.Submitted is false, test data was not published!") + } +} diff --git a/core/quantModel/runnerKubernetes.go b/core/quantModel/runnerKubernetes.go index 715fde1a..2ebcf8a9 100644 --- a/core/quantModel/runnerKubernetes.go +++ b/core/quantModel/runnerKubernetes.go @@ -105,7 +105,11 @@ func (r *kubernetesRunner) runPiquant(piquantDockerImage string, params PiquantP func getPodObject(paramsStr string, params PiquantParams, dockerImage string, jobid, namespace string, creator pixlUser.UserInfo, length int) *apiv1.Pod { sec := apiv1.LocalObjectReference{Name: "api-auth"} + application := "piquant-runner" parts := strings.Split(params.PMCListName, ".") + node := parts[0] + name := fmt.Sprintf("piquant-%s", params.Command) + instance := fmt.Sprintf("%s-%s", name, node) // Set the serviceaccount for the piquant pods based on namespace // Piquant Fit commands will run in the same namespace and share a service account // Piquant Map commands (jobs) will run in the piquant-map namespace with a more limited service account @@ -121,12 +125,16 @@ func getPodObject(paramsStr string, params PiquantParams, dockerImage string, jo Name: jobid + "-" + parts[0], Namespace: namespace, Labels: map[string]string{ - "pixlise/application": "piquant-runner", - "piquant/command": params.Command, - "app": parts[0], - "owner": creator.UserID, - "jobid": jobid, - "numberofpods": strconv.Itoa(length), + "pixlise.org/application": application, + "pixlise.org/environment": params.RunTimeEnv, + "app.kubernetes.io/name": name, + "app.kubernetes.io/instance": instance, + "app.kubernetes.io/component": application, + "piquant/command": params.Command, + "app": node, + "owner": creator.UserID, + "jobid": jobid, + "numberofpods": strconv.Itoa(length), }, }, Spec: apiv1.PodSpec{ diff --git a/internal/cmdline-tools/mongo-test/main.go b/internal/cmdline-tools/mongo-test/main.go index 2d068e26..386f8489 100644 --- a/internal/cmdline-tools/mongo-test/main.go +++ b/internal/cmdline-tools/mongo-test/main.go @@ -208,6 +208,7 @@ func runTests(db *expressionDB.ExpressionDB, userDB *pixlUser.UserDetailsLookup, Tags: []string{"v1"}, }, userInfoPeter, + false, ) verifyResult( @@ -225,6 +226,7 @@ func runTests(db *expressionDB.ExpressionDB, userDB *pixlUser.UserDetailsLookup, Tags: []string{"v2"}, }, userInfoPeter, + false, ) verifyResult( @@ -279,6 +281,7 @@ func runTests(db *expressionDB.ExpressionDB, userDB *pixlUser.UserDetailsLookup, Comments: "module one A", Tags: []string{"v1", "v1.1"}, }, + false, ) verifyResult( @@ -423,6 +426,7 @@ func runTests(db *expressionDB.ExpressionDB, userDB *pixlUser.UserDetailsLookup, Tags: []string{"v1", "v1.1"}, VersionUpdate: "minor", }, + false, ) verifyResult( @@ -441,6 +445,7 @@ func runTests(db *expressionDB.ExpressionDB, userDB *pixlUser.UserDetailsLookup, Tags: []string{"v1", "v1.1"}, VersionUpdate: "major", }, + false, ) verifyResult(