Merge branch 'development' into feature/quant-tracker

api/dataimport/for-trigger_test.go

@@ -362,7 +362,7 @@ func Example_importForTrigger_OCS_DatasetEdit() {
 	// <nil>|{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"}
 }
 
-func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32) {
+func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32, beamLocLBLFileName string, beamVersion uint32) {
 	// Ensure these log msgs appeared...
 	requiredLogs := []string{
 		"Downloading archived zip files...",
@@ -380,6 +380,12 @@ func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database,
 		"No auto-share destination found, so only importing user will be able to access this dataset.",
 	}
 
+	if len(beamLocLBLFileName) > 0 {
+		requiredLogs = append(requiredLogs, fmt.Sprintf("WARNING: Beam location LBL file (%v) could not be read. Beam Version version is assumed to be: %v", beamLocLBLFileName, beamVersion))
+	} else if beamVersion > 0 {
+		requiredLogs = append(requiredLogs, fmt.Sprintf("Saving as beam geometry tool version: %v", beamVersion))
+	}
+
 	for _, msg := range requiredLogs {
 		fmt.Printf("Logged \"%v\": %v\n", msg, log.LogContains(msg))
 	}
@@ -412,7 +418,7 @@ func Example_importForTrigger_Manual_JPL() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234", 3)
+	printManualOKLogOutput(log, db, "test1234", 3, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -445,7 +451,7 @@ func Example_importForTrigger_Manual_SBU() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234sbu", 4)
+	printManualOKLogOutput(log, db, "test1234sbu", 4, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -478,7 +484,7 @@ func Example_ImportForTrigger_Manual_SBU_NoAutoShare() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234sbu", 4)
+	printManualOKLogOutput(log, db, "test1234sbu", 4, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -529,7 +535,41 @@ func Example_importForTrigger_Manual_EM() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "048300551", 3)
+	printManualOKLogOutput(log, db, "048300551", 3, "PE__0125_0678031418_000RXL_N004171204830055100910__J01.LBL", 2)
+
+	// Output:
+	// Errors: <nil>, changes: unknown, isUpdate: false
+	// Logged "Downloading archived zip files...": true
+	// Logged "Downloaded 0 zip files, unzipped 0 files": true
+	// Logged "No zip files found in archive, dataset may have been manually uploaded. Trying to download...": true
+	// Logged "Dataset 048300551 downloaded 3 files from manual upload area": true
+	// Logged "Downloading pseudo-intensity ranges...": true
+	// Logged "Downloading user customisation files...": true
+	// Logged "Reading 1261 files from spectrum directory...": false
+	// Logged "Reading spectrum [1135/1260] 90%": false
+	// Logged "PMC 1261 has 4 MSA/spectrum entries": false
+	// Logged "WARNING: No main context image determined": false
+	// Logged "Diffraction db saved successfully": true
+	// Logged "Warning: No import.json found, defaults will be used": false
+	// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
+	// Logged "WARNING: Beam location LBL file (PE__0125_0678031418_000RXL_N004171204830055100910__J01.LBL) could not be read. Beam Version version is assumed to be: 2": true
+	// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
+}
+
+// Import a breadboard dataset from manual uploaded zip file
+func Example_importForTrigger_Manual_EM_WithBeamV2() {
+	remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("ManualEM_Beamv2_OK", specialUserIds.PIXLISESystemUserId, "PIXLFMGroupId")
+
+	trigger := `{
+	"datasetID": "048300551",
+	"jobID": "dataimport-unittest048300551"
+}`
+
+	result, err := ImportForTrigger([]byte(trigger), envName, configBucket, datasetBucket, manualBucket, db, log, remoteFS)
+
+	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
+
+	printManualOKLogOutput(log, db, "048300551", 3, "", 2)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -546,6 +586,7 @@ func Example_importForTrigger_Manual_EM() {
 	// Logged "Diffraction db saved successfully": true
 	// Logged "Warning: No import.json found, defaults will be used": false
 	// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
+	// Logged "Saving as beam geometry tool version: 2": true
 	// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
 }
 

api/dataimport/internal/converters/pixlfm/import.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels"
 	"github.com/pixlise/core/v4/api/dataimport/internal/importerutils"
@@ -118,6 +119,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 
 	// Allocate everything needed (empty, if we find & load stuff, great, but we still need the data struct for the last step)
 	beamLookup := dataConvertModels.BeamLocationByPMC{}
+	beamToolVersion := 0
 	hkData := dataConvertModels.HousekeepingData{}
 	locSpectraLookup := dataConvertModels.DetectorSampleByPMC{}
 	bulkMaxSpectraLookup := dataConvertModels.DetectorSampleByPMC{}
@@ -214,10 +216,38 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 			for file, beamCsvMeta := range latestVersionFoundPaths {
 				if beamCsvMeta.ProdType == "RXL" {
 					// If files don't conform, don't read...
-					beamLookup, err = importerutils.ReadBeamLocationsFile(filepath.Join(pathToSubdir, file), true, 1, log)
+					beamFilePath := filepath.Join(pathToSubdir, file)
+					beamLookup, err = importerutils.ReadBeamLocationsFile(beamFilePath, true, 1, log)
 					if err != nil {
 						return nil, "", err
 					} else {
+						// Import worked, find out what version of the beam geometry tool generated this by looking in the LBL file
+						lblPath := beamFilePath[0:len(beamFilePath)-3] + "LBL"
+						lblFileBytes, err := os.ReadFile(lblPath)
+						if err != nil {
+							//return nil, "", fmt.Errorf("Failed to read LBL file: %v. Beam geometry version could not be determined.", lblPath)
+							beamToolVersion = 2 // If we don't have an LBL file but it's a "recent" copy of the data, it has to be using beam geometry v2...
+							log.Errorf("WARNING: Beam location LBL file (%v) could not be read. Beam Version version is assumed to be: %v", filepath.Base(lblPath), beamToolVersion)
+						} else {
+							// Read each line and find the beam geometry version
+							lblLines := strings.Split(string(lblFileBytes), "\n")
+							searchStr := "Beam Geometry Tool Version "
+
+							for _, line := range lblLines {
+								if pos := strings.Index(line, searchStr); pos > -1 {
+									verPos := pos + len(searchStr)
+
+									log.Infof("Beam Geometry Tool version text found: %v", line[verPos:])
+
+									verChar := line[verPos : verPos+1]
+									r, _ := utf8.DecodeRuneInString(verChar)
+									beamToolVersion = int(r) - '0'
+									log.Infof("Saving as beam geometry tool version: %v", beamToolVersion)
+									break
+								}
+							}
+						}
+
 						// Found it, why keep looping?
 						break
 					}
@@ -358,7 +388,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 		datasetIDExpected,
 		p.overrideInstrument,
 		p.overrideDetector,
-		1, // TODO: Retrieve beam version and set it here!
+		uint32(beamToolVersion),
 		log,
 	)
 

api/dataimport/internal/output/output.go

@@ -335,11 +335,7 @@ func (s *PIXLISEDataSaver) Save(
 	// as we switched to storing them in DB (to allow import of other images with a corresponding set of beam locations)
 	// Redundant, but this is how it evolved...
 	for idx, imgItem := range exp.AlignedContextImages {
-		beamVer := data.BeamVersion
-		if beamVer < 1 {
-			beamVer = 1
-		}
-		err := beamLocation.ImportBeamLocationToDB(path.Join(data.DatasetID, imgItem.Image), data.Instrument, data.DatasetID, beamVer, idx, &exp, db, jobLog)
+		err := beamLocation.ImportBeamLocationToDB(path.Join(data.DatasetID, imgItem.Image), data.Instrument, data.DatasetID, data.BeamVersion, idx, &exp, db, jobLog)
 		if err != nil {
 			return fmt.Errorf("Failed to import beam locations for image %v into DB. Error: %v", imgItem.Image, err)
 		}
@@ -368,7 +364,7 @@ func (s *PIXLISEDataSaver) Save(
 		prevSavedScan = nil
 	}
 
-	summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId)
+	summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId, jobLog)
 
 	jobLog.Infof("Writing summary to DB for %v...", summaryData.Id)
 
@@ -380,8 +376,8 @@ func (s *PIXLISEDataSaver) Save(
 	if err != nil {
 		jobLog.Errorf("Failed to write summary to DB: %v", err)
 		return err
-	} else if result.UpsertedCount != 1 {
-		jobLog.Errorf("Expected summary write to create 1 upsert, got: %v", result.UpsertedCount)
+	} else if result.UpsertedCount != 1 && result.ModifiedCount != 1 {
+		jobLog.Errorf("Expected summary write to create 1 upsert, got: %v upsert, %v modified", result.UpsertedCount, result.ModifiedCount)
 	}
 
 	// Set ownership

api/dataimport/internal/output/summary.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels"
+	"github.com/pixlise/core/v4/core/logger"
 	protos "github.com/pixlise/core/v4/generated-protos"
 )
 
@@ -33,7 +34,8 @@ func makeSummaryFileContent(
 	meta dataConvertModels.FileMetaData,
 	//fileSize int,
 	creationUnixTimeSec int64,
-	creatorUserId string) *protos.ScanItem {
+	creatorUserId string,
+	jobLog logger.ILogger) *protos.ScanItem {
 	contextImgCount := len(exp.AlignedContextImages) + len(exp.UnalignedContextImages) + len(exp.MatchedAlignedContextImages)
 	tiffContextImgCount := 0
 
@@ -117,16 +119,21 @@ func makeSummaryFileContent(
 
 		// Add new time at the end
 		s.PreviousImportTimesUnixSec = append(s.PreviousImportTimesUnixSec, prevSavedScan.TimestampUnixSec)
+		jobLog.Infof(" Added new previous time stamp entry: %v, total previous timestamps: %v", prevSavedScan.TimestampUnixSec, len(s.PreviousImportTimesUnixSec))
 	}
 
 	// Save a time stamp for completion
 	if isComplete {
+		jobLog.Infof(" Detected completed dataset...")
+
 		// If previous scan was also complete, DON'T update the time stamp, just preserve it
 		if prevSavedScan != nil && prevSavedScan.CompleteTimeStampUnixSec > 0 {
 			s.CompleteTimeStampUnixSec = prevSavedScan.CompleteTimeStampUnixSec
+			jobLog.Infof(" Preserved previous CompleteTimeStampUnixSec of %v", s.CompleteTimeStampUnixSec)
 		} else {
 			// We must've just completed now, so save the time
 			s.CompleteTimeStampUnixSec = uint32(creationUnixTimeSec)
+			jobLog.Infof(" Setting CompleteTimeStampUnixSec=%v", creationUnixTimeSec)
 		}
 	}
 
@@ -135,9 +142,16 @@ func makeSummaryFileContent(
 		s.Tags = prevSavedScan.Tags
 		s.Description = prevSavedScan.Description
 
+		descSnippet := s.Description
+		if len(descSnippet) > 30 {
+			descSnippet = descSnippet[0:30] + "..."
+		}
+		jobLog.Infof(" Preserved previous description=\"%v\"...", descSnippet)
+
 		if len(prevSavedScan.Description) > 0 {
 			// User has entered a description so perserve the title too
 			s.Title = prevSavedScan.Title
+			jobLog.Infof(" Preserved previous title=\"%v\"", s.Title)
 		}
 	}
 

api/dataimport/test-data/ManualEM_Beamv2_OK/config-bucket/DatasetConfig/StandardPseudoIntensities.csv

@@ -0,0 +1,23 @@
+Name, StartChannel, EndChannel
+Na, 122, 142
+Mg, 148, 170
+Al, 176, 200
+Si, 207, 233
+P, 241, 268
+S, 277, 306
+Cl, 315, 346
+K, 400, 435
+Ca, 446, 483
+Ti, 547, 588
+Ce, 588, 630
+Cr, 658, 703
+Mn, 718, 765
+Fe, 780, 829
+Ni, 913, 966
+Ge, 1211, 1271
+As, 1293, 1355
+Zn, 1057, 1113
+Sr, 1742, 1814
+Y, 1840, 1914
+Zr, 1938, 2014
+Ba, 542, 582

api/dataimport/test-data/ManualEM_Beamv2_OK/dataset-bucket/.gitignore

@@ -0,0 +1,2 @@
+Scans/
+Images/

api/dataimport/test-data/ManualEM_Beamv2_OK/manual-bucket/UploadedDatasets/048300551/creator.json

@@ -0,0 +1,5 @@
+{
+    "name": "peternemere",
+    "user_id": "peter123",
+    "email": "[email protected]"
+}

api/dataimport/test-data/ManualEM_Beamv2_OK/manual-bucket/UploadedDatasets/048300551/detector.json

@@ -0,0 +1,3 @@
+{
+    "detector": "pixl-em"
+}