Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

EM IMPORT: Exclude images that have no beam locations, fix SOL to be … #359

Merged
merged 1 commit into from
Nov 13, 2024
Merged

EM IMPORT: Exclude images that have no beam locations, fix SOL to be … #359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions api/dataimport/dataimportHelpers/beamLocation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ import (
)

// ReadBeamLocationsFile - Reads beam location CSV. Old style (expectMultipleIJ=false) or new multi-image IJ coord CSVs.
func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, error) {
func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, []int32, error) {
// Find the first row that has the start of data we're interested in!
file, err := os.Open(beamPath)
if err != nil {
return nil, err
return nil, []int32{}, err
}
defer file.Close()

Expand All @@ -51,7 +51,7 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC
}

if lineNo > 4 {
return nil, fmt.Errorf("Failed to find header row of beam location data")
return nil, []int32{}, fmt.Errorf("Failed to find header row of beam location data")
}
}

Expand All @@ -61,16 +61,17 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC
}
rows, err := ReadCSV(beamPath, lineNo, ',', jobLog)
if err != nil {
return nil, err
return nil, []int32{}, err
}

return parseBeamLocations(rows, expectMultipleIJ, mainImagePMC, ignoreColumns)
}

func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, error) {
func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, []int32, error) {
pmcs := []int32{}
headerLookup, geom_corrIdx, err := parseBeamLocationHeaders(rows[0], expectMultipleIJ, mainImagePMC, ignoreColumns)
if err != nil {
return nil, err
return nil, pmcs, err
}

// Read in each row and store based on the header lookup we made
Expand All @@ -79,16 +80,21 @@ func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int
for line, row := range rows[1:] {
pmc, locData, err := parseBeamLocationRow(row, headerLookup, geom_corrIdx)
if err != nil {
return nil, fmt.Errorf("line [%v] - ERROR: %v", line, err)
return nil, pmcs, fmt.Errorf("line [%v] - ERROR: %v", line, err)
}
if _, ok := result[pmc]; ok {
return nil, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc)
return nil, pmcs, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc)
}

result[pmc] = locData
}

return result, nil
// Also save a list of PMCs who had ijs
for _, item := range headerLookup {
pmcs = append(pmcs, item.pmc)
}

return result, pmcs, nil
}

type pmcColIdxs struct {
Expand Down
57 changes: 30 additions & 27 deletions api/dataimport/dataimportHelpers/beamLocation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,53 +127,56 @@ func Example_parseBeamLocationRow() {
}

func Example_parseBeamLocations() {
data, err := parseBeamLocations([][]string{[]string{"PMC", "x", "y", "z", "image_i", "image_j"}, []string{"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{})
fmt.Printf("%v|%v\n", data, err)
data, pmcs, err := parseBeamLocations([][]string{{"PMC", "x", "y", "z", "image_i", "image_j"}, {"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{})
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

data, err = parseBeamLocations([][]string{
[]string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
[]string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"},
[]string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"},
data, pmcs, err = parseBeamLocations([][]string{
{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"},
{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"},
}, true, 333, []string{})
fmt.Printf("%v|%v\n", data, err)
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

data, err = parseBeamLocations([][]string{
[]string{"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
[]string{"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"},
[]string{"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"},
data, pmcs, err = parseBeamLocations([][]string{
{"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
{"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"},
{"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"},
}, true, 333, []string{})
fmt.Printf("%v|%v\n", data, err)
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

data, err = parseBeamLocations([][]string{
[]string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"},
[]string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"},
[]string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"},
data, pmcs, err = parseBeamLocations([][]string{
{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"},
{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"},
{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"},
}, true, 333, []string{})
fmt.Printf("%v|%v\n", data, err)
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

// Output:
// map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]|<nil>
// map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
// map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]|[222]|<nil>
// map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
}

// This is kind of redunant, was already tested elsewhere, but this is an easy point to add a file and run the test to make sure it imports!
func Example_ReadBeamLocationsFile() {
rxl, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
rxl, pmcs, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))

rxl, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
rxl, pmcs, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))

rxl, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
rxl, pmcs, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))

// Output:
// <nil>
// pmcs: [4 457 5 458]
// rxl: 3
// <nil>
// pmcs: [4 457 5 458]
// rxl: 3
// Unexpected count of i/j columns
// pmcs: []
// rxl: 0
}
2 changes: 1 addition & 1 deletion api/dataimport/internal/converters/jplbreadboard/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func (m MSATestData) Import(importPath string, pseudoIntensityRangesPath string,

if params.MsaBeamParams == "" && params.BeamFile != "" {
jobLog.Infof(" Reading Beam Locations: \"%v\", using minimum context image PMC detected: %v\n", params.BeamFile, minContextPMC)
beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog)
beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog)
if err != nil {
return nil, "", err
}
Expand Down
11 changes: 8 additions & 3 deletions api/dataimport/internal/converters/pixlem/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"path/filepath"
"strconv"
"strings"
"time"

dataImportHelpers "github.com/pixlise/core/v4/api/dataimport/dataimportHelpers"
"github.com/pixlise/core/v4/api/dataimport/internal/converters/jplbreadboard"
Expand Down Expand Up @@ -214,14 +215,14 @@
minContextPMC := importerutils.GetMinimumContextPMC(contextImgsPerPMC)

// Read Beams
beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger)
beamLookup, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger)
if err != nil {
return nil, err
}

// Remove any images which don't have beam locations
for pmc, img := range contextImgsPerPMC {
if _, ok := beamLookup[pmc]; !ok {
if !utils.ItemInSlice(pmc, ijPMCs) {
logger.Infof("Excluding image due to not having beam locations: %v", img)
delete(contextImgsPerPMC, pmc)
}
Expand All @@ -240,7 +241,11 @@
site := "000"
drive := "0000"
product := "???"
sol := "D000"

// Use current date encoded as a test sol
// A=2017, 'A' is 65 ascii
sol := fmt.Sprintf("%v%v", string(65+time.Now().Year()-2017), time.Now().YearDay())

Check failure on line 247 in api/dataimport/internal/converters/pixlem/import.go

View workflow job for this annotation

GitHub Actions / feature / test 

conversion from int to string yields a string of one rune, not a string of digits (did you mean fmt.Sprint(x)?)

Check failure on line 247 in api/dataimport/internal/converters/pixlem/import.go

View workflow job for this annotation

GitHub Actions / feature / test 

conversion from int to string yields a string of one rune, not a string of digits (did you mean fmt.Sprint(x)?)

ftype := "??" // PE
producer := "J"
version := "01"
Expand Down
2 changes: 1 addition & 1 deletion api/dataimport/internal/converters/pixlfm/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
if beamCsvMeta.ProdType == "RXL" {
// If files don't conform, don't read...
beamFilePath := filepath.Join(pathToSubdir, file)
beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log)
beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log)
if err != nil {
return nil, "", err
} else {
Expand Down
2 changes: 1 addition & 1 deletion api/dataimport/internal/converters/soff/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func (s *SOFFImport) Import(importPath string, pseudoIntensityRangesPath string,
}

// Read each one
beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log)
beamLookup, _, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log)
if err != nil {
return nil, "", err
}
Expand Down
22 changes: 4 additions & 18 deletions internal/cmd-line-tools/beam-geom-v3-csv-importer/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,30 +134,16 @@ func main() {

// Find out what PMCs we have ij's for, and find the corresponding image file name to import for
// this way we can import into ImageBeamLocations using the file name, and insert an entry for v3
beamLocs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{})
beamLocs, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{})

if err != nil {
log.Fatalln(err)
}

for _, beam := range beamLocs {
// They should all be the same so only checking first one
// NOTE: Also ensure we don't have any images stored for PMCs that we don't have beam data for!
validPMCs := []int32{}
for imgPMC := range beam.IJ {
if _, ok := pmcImageLookup[imgPMC]; !ok {
log.Fatalf("Failed to find image for ij PMC: %v", imgPMC)
} else {
validPMCs = append(validPMCs, imgPMC)
}
}

for pmc := range pmcImageLookup {
if !utils.ItemInSlice(pmc, validPMCs) {
delete(pmcImageLookup, pmc)
}
for pmc := range pmcImageLookup {
if !utils.ItemInSlice(pmc, ijPMCs) {
delete(pmcImageLookup, pmc)
}
break
}

s3Path := fmt.Sprintf("Scans/%v/dataset.bin", scanId)
Expand Down
Loading