Merge pull request #248 from pixlise/development

Release v4.15.0

api/coreg/import.go

@@ -39,7 +39,7 @@ func StartCoregImport(triggerUrl string, hctx wsHelpers.HandlerContext) (string,
 
 	// Start an image coreg import job (this is a Lambda function)
 	// Once it completes, we have the data we need, so we can treat it as a "normal" image importing task
-	jobStatus, err := job.AddJob("coreg", protos.JobStatus_JT_IMPORT_IMAGE, triggerUrl, uint32(hctx.Svcs.Config.ImportJobMaxTimeSec), hctx.Svcs.MongoDB, hctx.Svcs.IDGen, hctx.Svcs.TimeStamper, hctx.Svcs.Log, i.sendUpdate)
+	jobStatus, err := job.AddJob("coreg", hctx.SessUser.User.Id, protos.JobStatus_JT_IMPORT_IMAGE, triggerUrl, fmt.Sprintf("Import: %v", path.Base(triggerUrl)), []string{}, uint32(hctx.Svcs.Config.ImportJobMaxTimeSec), hctx.Svcs.MongoDB, hctx.Svcs.IDGen, hctx.Svcs.TimeStamper, hctx.Svcs.Log, i.sendUpdate)
 	jobId := ""
 	if jobStatus != nil {
 		jobId = jobStatus.JobId

api/dataimport/for-trigger.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pixlise/core/v4/api/dataimport/internal/datasetArchive"
 	"github.com/pixlise/core/v4/api/dbCollections"
 	"github.com/pixlise/core/v4/api/job"
+	"github.com/pixlise/core/v4/api/specialUserIds"
 	"github.com/pixlise/core/v4/core/fileaccess"
 	"github.com/pixlise/core/v4/core/logger"
 	"github.com/pixlise/core/v4/core/timestamper"
@@ -183,6 +184,7 @@ func completeJobState(jobId string, success bool, scanId string, message string,
 				EndUnixTimeSec:        now,
 				OutputFilePath:        outputFilePath,
 				OtherLogFiles:         otherLogFiles,
+				RequestorUserId:       specialUserIds.PIXLISESystemUserId, // We don't have a requestor ID to write, we're an auto import
 			}
 
 			insertResult, err := coll.InsertOne(ctx, jobStatus, opt)

api/dataimport/for-trigger_test.go

@@ -362,7 +362,7 @@ func Example_importForTrigger_OCS_DatasetEdit() {
 	// <nil>|{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"}
 }
 
-func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32) {
+func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32, beamLocLBLFileName string, beamVersion uint32) {
 	// Ensure these log msgs appeared...
 	requiredLogs := []string{
 		"Downloading archived zip files...",
@@ -380,6 +380,12 @@ func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database,
 		"No auto-share destination found, so only importing user will be able to access this dataset.",
 	}
 
+	if len(beamLocLBLFileName) > 0 {
+		requiredLogs = append(requiredLogs, fmt.Sprintf("WARNING: Beam location LBL file (%v) could not be read. Beam Version version is assumed to be: %v", beamLocLBLFileName, beamVersion))
+	} else if beamVersion > 0 {
+		requiredLogs = append(requiredLogs, fmt.Sprintf("Saving as beam geometry tool version: %v", beamVersion))
+	}
+
 	for _, msg := range requiredLogs {
 		fmt.Printf("Logged \"%v\": %v\n", msg, log.LogContains(msg))
 	}
@@ -412,7 +418,7 @@ func Example_importForTrigger_Manual_JPL() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234", 3)
+	printManualOKLogOutput(log, db, "test1234", 3, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -445,7 +451,7 @@ func Example_importForTrigger_Manual_SBU() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234sbu", 4)
+	printManualOKLogOutput(log, db, "test1234sbu", 4, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -478,7 +484,7 @@ func Example_ImportForTrigger_Manual_SBU_NoAutoShare() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234sbu", 4)
+	printManualOKLogOutput(log, db, "test1234sbu", 4, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -529,7 +535,41 @@ func Example_importForTrigger_Manual_EM() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "048300551", 3)
+	printManualOKLogOutput(log, db, "048300551", 3, "PE__0125_0678031418_000RXL_N004171204830055100910__J01.LBL", 2)
+
+	// Output:
+	// Errors: <nil>, changes: unknown, isUpdate: false
+	// Logged "Downloading archived zip files...": true
+	// Logged "Downloaded 0 zip files, unzipped 0 files": true
+	// Logged "No zip files found in archive, dataset may have been manually uploaded. Trying to download...": true
+	// Logged "Dataset 048300551 downloaded 3 files from manual upload area": true
+	// Logged "Downloading pseudo-intensity ranges...": true
+	// Logged "Downloading user customisation files...": true
+	// Logged "Reading 1261 files from spectrum directory...": false
+	// Logged "Reading spectrum [1135/1260] 90%": false
+	// Logged "PMC 1261 has 4 MSA/spectrum entries": false
+	// Logged "WARNING: No main context image determined": false
+	// Logged "Diffraction db saved successfully": true
+	// Logged "Warning: No import.json found, defaults will be used": false
+	// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
+	// Logged "WARNING: Beam location LBL file (PE__0125_0678031418_000RXL_N004171204830055100910__J01.LBL) could not be read. Beam Version version is assumed to be: 2": true
+	// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
+}
+
+// Import a breadboard dataset from manual uploaded zip file
+func Example_importForTrigger_Manual_EM_WithBeamV2() {
+	remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("ManualEM_Beamv2_OK", specialUserIds.PIXLISESystemUserId, "PIXLFMGroupId")
+
+	trigger := `{
+	"datasetID": "048300551",
+	"jobID": "dataimport-unittest048300551"
+}`
+
+	result, err := ImportForTrigger([]byte(trigger), envName, configBucket, datasetBucket, manualBucket, db, log, remoteFS)
+
+	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
+
+	printManualOKLogOutput(log, db, "048300551", 3, "", 2)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -546,6 +586,7 @@ func Example_importForTrigger_Manual_EM() {
 	// Logged "Diffraction db saved successfully": true
 	// Logged "Warning: No import.json found, defaults will be used": false
 	// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
+	// Logged "Saving as beam geometry tool version: 2": true
 	// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
 }
 

api/dataimport/internal/converters/pixlfm/import.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels"
 	"github.com/pixlise/core/v4/api/dataimport/internal/importerutils"
@@ -118,6 +119,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 
 	// Allocate everything needed (empty, if we find & load stuff, great, but we still need the data struct for the last step)
 	beamLookup := dataConvertModels.BeamLocationByPMC{}
+	beamToolVersion := 0
 	hkData := dataConvertModels.HousekeepingData{}
 	locSpectraLookup := dataConvertModels.DetectorSampleByPMC{}
 	bulkMaxSpectraLookup := dataConvertModels.DetectorSampleByPMC{}
@@ -214,10 +216,38 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 			for file, beamCsvMeta := range latestVersionFoundPaths {
 				if beamCsvMeta.ProdType == "RXL" {
 					// If files don't conform, don't read...
-					beamLookup, err = importerutils.ReadBeamLocationsFile(filepath.Join(pathToSubdir, file), true, 1, log)
+					beamFilePath := filepath.Join(pathToSubdir, file)
+					beamLookup, err = importerutils.ReadBeamLocationsFile(beamFilePath, true, 1, log)
 					if err != nil {
 						return nil, "", err
 					} else {
+						// Import worked, find out what version of the beam geometry tool generated this by looking in the LBL file
+						lblPath := beamFilePath[0:len(beamFilePath)-3] + "LBL"
+						lblFileBytes, err := os.ReadFile(lblPath)
+						if err != nil {
+							//return nil, "", fmt.Errorf("Failed to read LBL file: %v. Beam geometry version could not be determined.", lblPath)
+							beamToolVersion = 2 // If we don't have an LBL file but it's a "recent" copy of the data, it has to be using beam geometry v2...
+							log.Errorf("WARNING: Beam location LBL file (%v) could not be read. Beam Version version is assumed to be: %v", filepath.Base(lblPath), beamToolVersion)
+						} else {
+							// Read each line and find the beam geometry version
+							lblLines := strings.Split(string(lblFileBytes), "\n")
+							searchStr := "Beam Geometry Tool Version "
+
+							for _, line := range lblLines {
+								if pos := strings.Index(line, searchStr); pos > -1 {
+									verPos := pos + len(searchStr)
+
+									log.Infof("Beam Geometry Tool version text found: %v", line[verPos:])
+
+									verChar := line[verPos : verPos+1]
+									r, _ := utf8.DecodeRuneInString(verChar)
+									beamToolVersion = int(r) - '0'
+									log.Infof("Saving as beam geometry tool version: %v", beamToolVersion)
+									break
+								}
+							}
+						}
+
 						// Found it, why keep looping?
 						break
 					}
@@ -358,7 +388,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 		datasetIDExpected,
 		p.overrideInstrument,
 		p.overrideDetector,
-		1, // TODO: Retrieve beam version and set it here!
+		uint32(beamToolVersion),
 		log,
 	)
 

api/dataimport/internal/output/output.go

@@ -335,11 +335,7 @@ func (s *PIXLISEDataSaver) Save(
 	// as we switched to storing them in DB (to allow import of other images with a corresponding set of beam locations)
 	// Redundant, but this is how it evolved...
 	for idx, imgItem := range exp.AlignedContextImages {
-		beamVer := data.BeamVersion
-		if beamVer < 1 {
-			beamVer = 1
-		}
-		err := beamLocation.ImportBeamLocationToDB(path.Join(data.DatasetID, imgItem.Image), data.Instrument, data.DatasetID, beamVer, idx, &exp, db, jobLog)
+		err := beamLocation.ImportBeamLocationToDB(path.Join(data.DatasetID, imgItem.Image), data.Instrument, data.DatasetID, data.BeamVersion, idx, &exp, db, jobLog)
 		if err != nil {
 			return fmt.Errorf("Failed to import beam locations for image %v into DB. Error: %v", imgItem.Image, err)
 		}
@@ -368,7 +364,7 @@ func (s *PIXLISEDataSaver) Save(
 		prevSavedScan = nil
 	}
 
-	summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId)
+	summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId, jobLog)
 
 	jobLog.Infof("Writing summary to DB for %v...", summaryData.Id)
 
@@ -380,8 +376,8 @@ func (s *PIXLISEDataSaver) Save(
 	if err != nil {
 		jobLog.Errorf("Failed to write summary to DB: %v", err)
 		return err
-	} else if result.UpsertedCount != 1 {
-		jobLog.Errorf("Expected summary write to create 1 upsert, got: %v", result.UpsertedCount)
+	} else if result.UpsertedCount != 1 && result.ModifiedCount != 1 {
+		jobLog.Errorf("Expected summary write to create 1 upsert, got: %v upsert, %v modified", result.UpsertedCount, result.ModifiedCount)
 	}
 
 	// Set ownership

api/dataimport/internal/output/summary.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels"
+	"github.com/pixlise/core/v4/core/logger"
 	protos "github.com/pixlise/core/v4/generated-protos"
 )
 
@@ -33,7 +34,8 @@ func makeSummaryFileContent(
 	meta dataConvertModels.FileMetaData,
 	//fileSize int,
 	creationUnixTimeSec int64,
-	creatorUserId string) *protos.ScanItem {
+	creatorUserId string,
+	jobLog logger.ILogger) *protos.ScanItem {
 	contextImgCount := len(exp.AlignedContextImages) + len(exp.UnalignedContextImages) + len(exp.MatchedAlignedContextImages)
 	tiffContextImgCount := 0
 
@@ -117,16 +119,21 @@ func makeSummaryFileContent(
 
 		// Add new time at the end
 		s.PreviousImportTimesUnixSec = append(s.PreviousImportTimesUnixSec, prevSavedScan.TimestampUnixSec)
+		jobLog.Infof(" Added new previous time stamp entry: %v, total previous timestamps: %v", prevSavedScan.TimestampUnixSec, len(s.PreviousImportTimesUnixSec))
 	}
 
 	// Save a time stamp for completion
 	if isComplete {
+		jobLog.Infof(" Detected completed dataset...")
+
 		// If previous scan was also complete, DON'T update the time stamp, just preserve it
 		if prevSavedScan != nil && prevSavedScan.CompleteTimeStampUnixSec > 0 {
 			s.CompleteTimeStampUnixSec = prevSavedScan.CompleteTimeStampUnixSec
+			jobLog.Infof(" Preserved previous CompleteTimeStampUnixSec of %v", s.CompleteTimeStampUnixSec)
 		} else {
 			// We must've just completed now, so save the time
 			s.CompleteTimeStampUnixSec = uint32(creationUnixTimeSec)
+			jobLog.Infof(" Setting CompleteTimeStampUnixSec=%v", creationUnixTimeSec)
 		}
 	}
 
@@ -135,9 +142,16 @@ func makeSummaryFileContent(
 		s.Tags = prevSavedScan.Tags
 		s.Description = prevSavedScan.Description
 
+		descSnippet := s.Description
+		if len(descSnippet) > 30 {
+			descSnippet = descSnippet[0:30] + "..."
+		}
+		jobLog.Infof(" Preserved previous description=\"%v\"...", descSnippet)
+
 		if len(prevSavedScan.Description) > 0 {
 			// User has entered a description so perserve the title too
 			s.Title = prevSavedScan.Title
+			jobLog.Infof(" Preserved previous title=\"%v\"", s.Title)
 		}
 	}
 

api/dataimport/test-data/ManualEM_Beamv2_OK/config-bucket/DatasetConfig/StandardPseudoIntensities.csv

@@ -0,0 +1,23 @@
+Name, StartChannel, EndChannel
+Na, 122, 142
+Mg, 148, 170
+Al, 176, 200
+Si, 207, 233
+P, 241, 268
+S, 277, 306
+Cl, 315, 346
+K, 400, 435
+Ca, 446, 483
+Ti, 547, 588
+Ce, 588, 630
+Cr, 658, 703
+Mn, 718, 765
+Fe, 780, 829
+Ni, 913, 966
+Ge, 1211, 1271
+As, 1293, 1355
+Zn, 1057, 1113
+Sr, 1742, 1814
+Y, 1840, 1914
+Zr, 1938, 2014
+Ba, 542, 582

api/dataimport/test-data/ManualEM_Beamv2_OK/dataset-bucket/.gitignore

@@ -0,0 +1,2 @@
+Scans/
+Images/

api/dataimport/test-data/ManualEM_Beamv2_OK/manual-bucket/UploadedDatasets/048300551/creator.json

@@ -0,0 +1,5 @@
+{
+    "name": "peternemere",
+    "user_id": "peter123",
+    "email": "[email protected]"
+}

api/dataimport/test-data/ManualEM_Beamv2_OK/manual-bucket/UploadedDatasets/048300551/detector.json

@@ -0,0 +1,3 @@
+{
+    "detector": "pixl-em"
+}

api/job/jobWatcher.go

@@ -22,7 +22,20 @@ var activeJobLock = sync.Mutex{}
 // Expected to be called by API to create the initial record of a job. It can then trigger it however it needs to
 // (eg AWS lambda or running PIQUANT nodes) and this sticks around monitoring the DB entry for changes, calling
 // the sendUpdate callback function on change. Returns the snapshot of the "added" job that was saved
-func AddJob(idPrefix string, jobType protos.JobStatus_JobType, jobItemId string, jobTimeoutSec uint32, db *mongo.Database, idgen idgen.IDGenerator, ts timestamper.ITimeStamper, logger logger.ILogger, sendUpdate func(*protos.JobStatus)) (*protos.JobStatus, error) {
+
+func AddJob(
+	idPrefix string,
+	requestorUserId string,
+	jobType protos.JobStatus_JobType,
+	jobItemId string,
+	jobName string,
+	elementList []string, // optional, only set if it's a quant!
+	jobTimeoutSec uint32,
+	db *mongo.Database,
+	idgen idgen.IDGenerator,
+	ts timestamper.ITimeStamper,
+	logger logger.ILogger,
+	sendUpdate func(*protos.JobStatus)) (*protos.JobStatus, error) {
 	// Generate a new job Id that this job will write to
 	// which we also return to the caller, so they can track what happens
 	// with this async task
@@ -40,6 +53,8 @@ func AddJob(idPrefix string, jobType protos.JobStatus_JobType, jobItemId string,
 		OtherLogFiles:    []string{},
 		JobType:          jobType,
 		JobItemId:        jobItemId,
+		Name:             jobName,
+		Elements:         elementList,
 	}
 
 	if _, ok := activeJobs[jobId]; ok {

api/job/statusChanges.go

@@ -34,6 +34,11 @@ func UpdateJob(jobId string, status protos.JobStatus_Status, message string, log
 		logger.Errorf("Failed to read existing job status when writing UpdateJob %v: %v", jobId, err)
 	} else {
 		jobStatus.StartUnixTimeSec = existingStatus.StartUnixTimeSec
+		jobStatus.JobType = existingStatus.JobType
+		jobStatus.JobItemId = existingStatus.JobItemId
+		jobStatus.RequestorUserId = existingStatus.RequestorUserId
+		jobStatus.Name = existingStatus.Name
+		jobStatus.Elements = existingStatus.Elements
 	}
 
 	replaceResult, err := coll.ReplaceOne(ctx, filter, jobStatus, opt)
@@ -86,6 +91,11 @@ func CompleteJob(jobId string, success bool, message string, outputFilePath stri
 	} else {
 		jobStatus.LogId = existingStatus.LogId
 		jobStatus.StartUnixTimeSec = existingStatus.StartUnixTimeSec
+		jobStatus.JobType = existingStatus.JobType
+		jobStatus.JobItemId = existingStatus.JobItemId
+		jobStatus.RequestorUserId = existingStatus.RequestorUserId
+		jobStatus.Name = existingStatus.Name
+		jobStatus.Elements = existingStatus.Elements
 	}
 
 	replaceResult, err := coll.ReplaceOne(ctx, filter, jobStatus, opt)

api/notificationSender/notifications.go

@@ -114,9 +114,11 @@ func (n *NotificationSender) sendNotification(sourceId string, topicId string, n
 					uiNotificationUsers = append(uiNotificationUsers, userId)
 				}
 
+				/* Removed because it's really not that helpful but spams logs heaps because lots of users don't have notifications on!
 				if method == protos.NotificationMethod_NOTIF_NONE {
-					n.log.Infof("Skipping notification of topic: %v to user %v because they have this topic turned off", topicId, userId)
+					n.log.Debugf("Skipping notification of topic: %v to user %v because they have this topic turned off", topicId, userId)
 				}
+				*/
 			}
 		}
 	}