From 67fcbfba9e681eecdc83bb0b8d2736cf1841c606 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 13 Nov 2024 10:35:29 +1000 Subject: [PATCH] EM IMPORT: Exclude images that have no beam locations, fix SOL to be current date --- .../dataimportHelpers/beamLocation.go | 24 +++++--- .../dataimportHelpers/beamLocation_test.go | 57 ++++++++++--------- .../converters/jplbreadboard/import.go | 2 +- .../internal/converters/pixlem/import.go | 11 +++- .../internal/converters/pixlfm/import.go | 2 +- .../internal/converters/soff/import.go | 2 +- .../beam-geom-v3-csv-importer/main.go | 22 ++----- 7 files changed, 60 insertions(+), 60 deletions(-) diff --git a/api/dataimport/dataimportHelpers/beamLocation.go b/api/dataimport/dataimportHelpers/beamLocation.go index 77d5befb..db81d677 100644 --- a/api/dataimport/dataimportHelpers/beamLocation.go +++ b/api/dataimport/dataimportHelpers/beamLocation.go @@ -31,11 +31,11 @@ import ( ) // ReadBeamLocationsFile - Reads beam location CSV. Old style (expectMultipleIJ=false) or new multi-image IJ coord CSVs. -func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, error) { +func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, []int32, error) { // Find the first row that has the start of data we're interested in! file, err := os.Open(beamPath) if err != nil { - return nil, err + return nil, []int32{}, err } defer file.Close() @@ -51,7 +51,7 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC } if lineNo > 4 { - return nil, fmt.Errorf("Failed to find header row of beam location data") + return nil, []int32{}, fmt.Errorf("Failed to find header row of beam location data") } } @@ -61,16 +61,17 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC } rows, err := ReadCSV(beamPath, lineNo, ',', jobLog) if err != nil { - return nil, err + return nil, []int32{}, err } return parseBeamLocations(rows, expectMultipleIJ, mainImagePMC, ignoreColumns) } -func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, error) { +func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, []int32, error) { + pmcs := []int32{} headerLookup, geom_corrIdx, err := parseBeamLocationHeaders(rows[0], expectMultipleIJ, mainImagePMC, ignoreColumns) if err != nil { - return nil, err + return nil, pmcs, err } // Read in each row and store based on the header lookup we made @@ -79,16 +80,21 @@ func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int for line, row := range rows[1:] { pmc, locData, err := parseBeamLocationRow(row, headerLookup, geom_corrIdx) if err != nil { - return nil, fmt.Errorf("line [%v] - ERROR: %v", line, err) + return nil, pmcs, fmt.Errorf("line [%v] - ERROR: %v", line, err) } if _, ok := result[pmc]; ok { - return nil, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc) + return nil, pmcs, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc) } result[pmc] = locData } - return result, nil + // Also save a list of PMCs who had ijs + for _, item := range headerLookup { + pmcs = append(pmcs, item.pmc) + } + + return result, pmcs, nil } type pmcColIdxs struct { diff --git a/api/dataimport/dataimportHelpers/beamLocation_test.go b/api/dataimport/dataimportHelpers/beamLocation_test.go index 7841ff05..01e09af5 100644 --- a/api/dataimport/dataimportHelpers/beamLocation_test.go +++ b/api/dataimport/dataimportHelpers/beamLocation_test.go @@ -127,53 +127,56 @@ func Example_parseBeamLocationRow() { } func Example_parseBeamLocations() { - data, err := parseBeamLocations([][]string{[]string{"PMC", "x", "y", "z", "image_i", "image_j"}, []string{"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{}) - fmt.Printf("%v|%v\n", data, err) + data, pmcs, err := parseBeamLocations([][]string{{"PMC", "x", "y", "z", "image_i", "image_j"}, {"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{}) + fmt.Printf("%v|%v|%v\n", data, pmcs, err) - data, err = parseBeamLocations([][]string{ - []string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"}, - []string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"}, - []string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"}, + data, pmcs, err = parseBeamLocations([][]string{ + {"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"}, + {"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"}, + {"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"}, }, true, 333, []string{}) - fmt.Printf("%v|%v\n", data, err) + fmt.Printf("%v|%v|%v\n", data, pmcs, err) - data, err = parseBeamLocations([][]string{ - []string{"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"}, - []string{"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"}, - []string{"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"}, + data, pmcs, err = parseBeamLocations([][]string{ + {"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"}, + {"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"}, + {"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"}, }, true, 333, []string{}) - fmt.Printf("%v|%v\n", data, err) + fmt.Printf("%v|%v|%v\n", data, pmcs, err) - data, err = parseBeamLocations([][]string{ - []string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"}, - []string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"}, - []string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"}, + data, pmcs, err = parseBeamLocations([][]string{ + {"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"}, + {"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"}, + {"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"}, }, true, 333, []string{}) - fmt.Printf("%v|%v\n", data, err) + fmt.Printf("%v|%v|%v\n", data, pmcs, err) // Output: - // map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]| - // map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]| - // map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]| - // map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]| + // map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]|[222]| + // map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]| + // map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]| + // map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]| } // This is kind of redunant, was already tested elsewhere, but this is an easy point to add a file and run the test to make sure it imports! func Example_ReadBeamLocationsFile() { - rxl, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{}) - fmt.Printf("%v\nrxl: %v\n", err, len(rxl)) + rxl, pmcs, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{}) + fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl)) - rxl, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{}) - fmt.Printf("%v\nrxl: %v\n", err, len(rxl)) + rxl, pmcs, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{}) + fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl)) - rxl, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{}) - fmt.Printf("%v\nrxl: %v\n", err, len(rxl)) + rxl, pmcs, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{}) + fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl)) // Output: // + // pmcs: [4 457 5 458] // rxl: 3 // + // pmcs: [4 457 5 458] // rxl: 3 // Unexpected count of i/j columns + // pmcs: [] // rxl: 0 } diff --git a/api/dataimport/internal/converters/jplbreadboard/import.go b/api/dataimport/internal/converters/jplbreadboard/import.go index 04a660e6..282add21 100644 --- a/api/dataimport/internal/converters/jplbreadboard/import.go +++ b/api/dataimport/internal/converters/jplbreadboard/import.go @@ -100,7 +100,7 @@ func (m MSATestData) Import(importPath string, pseudoIntensityRangesPath string, if params.MsaBeamParams == "" && params.BeamFile != "" { jobLog.Infof(" Reading Beam Locations: \"%v\", using minimum context image PMC detected: %v\n", params.BeamFile, minContextPMC) - beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog) + beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog) if err != nil { return nil, "", err } diff --git a/api/dataimport/internal/converters/pixlem/import.go b/api/dataimport/internal/converters/pixlem/import.go index ddf27cd3..debc5f64 100644 --- a/api/dataimport/internal/converters/pixlem/import.go +++ b/api/dataimport/internal/converters/pixlem/import.go @@ -23,6 +23,7 @@ import ( "path/filepath" "strconv" "strings" + "time" dataImportHelpers "github.com/pixlise/core/v4/api/dataimport/dataimportHelpers" "github.com/pixlise/core/v4/api/dataimport/internal/converters/jplbreadboard" @@ -214,14 +215,14 @@ func importEMData(creatorId string, rtt string, beamLocPath string, hkPath strin minContextPMC := importerutils.GetMinimumContextPMC(contextImgsPerPMC) // Read Beams - beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger) + beamLookup, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger) if err != nil { return nil, err } // Remove any images which don't have beam locations for pmc, img := range contextImgsPerPMC { - if _, ok := beamLookup[pmc]; !ok { + if !utils.ItemInSlice(pmc, ijPMCs) { logger.Infof("Excluding image due to not having beam locations: %v", img) delete(contextImgsPerPMC, pmc) } @@ -240,7 +241,11 @@ func importEMData(creatorId string, rtt string, beamLocPath string, hkPath strin site := "000" drive := "0000" product := "???" - sol := "D000" + + // Use current date encoded as a test sol + // A=2017, 'A' is 65 ascii + sol := fmt.Sprintf("%v%v", string(65+time.Now().Year()-2017), time.Now().YearDay()) + ftype := "??" // PE producer := "J" version := "01" diff --git a/api/dataimport/internal/converters/pixlfm/import.go b/api/dataimport/internal/converters/pixlfm/import.go index 02b458ad..49a85cbf 100644 --- a/api/dataimport/internal/converters/pixlfm/import.go +++ b/api/dataimport/internal/converters/pixlfm/import.go @@ -218,7 +218,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data if beamCsvMeta.ProdType == "RXL" { // If files don't conform, don't read... beamFilePath := filepath.Join(pathToSubdir, file) - beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log) + beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log) if err != nil { return nil, "", err } else { diff --git a/api/dataimport/internal/converters/soff/import.go b/api/dataimport/internal/converters/soff/import.go index 5b22446b..26bbf177 100644 --- a/api/dataimport/internal/converters/soff/import.go +++ b/api/dataimport/internal/converters/soff/import.go @@ -148,7 +148,7 @@ func (s *SOFFImport) Import(importPath string, pseudoIntensityRangesPath string, } // Read each one - beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log) + beamLookup, _, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log) if err != nil { return nil, "", err } diff --git a/internal/cmd-line-tools/beam-geom-v3-csv-importer/main.go b/internal/cmd-line-tools/beam-geom-v3-csv-importer/main.go index 2ca22a41..6d4eac3b 100644 --- a/internal/cmd-line-tools/beam-geom-v3-csv-importer/main.go +++ b/internal/cmd-line-tools/beam-geom-v3-csv-importer/main.go @@ -134,30 +134,16 @@ func main() { // Find out what PMCs we have ij's for, and find the corresponding image file name to import for // this way we can import into ImageBeamLocations using the file name, and insert an entry for v3 - beamLocs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{}) + beamLocs, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{}) if err != nil { log.Fatalln(err) } - for _, beam := range beamLocs { - // They should all be the same so only checking first one - // NOTE: Also ensure we don't have any images stored for PMCs that we don't have beam data for! - validPMCs := []int32{} - for imgPMC := range beam.IJ { - if _, ok := pmcImageLookup[imgPMC]; !ok { - log.Fatalf("Failed to find image for ij PMC: %v", imgPMC) - } else { - validPMCs = append(validPMCs, imgPMC) - } - } - - for pmc := range pmcImageLookup { - if !utils.ItemInSlice(pmc, validPMCs) { - delete(pmcImageLookup, pmc) - } + for pmc := range pmcImageLookup { + if !utils.ItemInSlice(pmc, ijPMCs) { + delete(pmcImageLookup, pmc) } - break } s3Path := fmt.Sprintf("Scans/%v/dataset.bin", scanId)