From 28b115ed310301edc2daa8fb0adedbb4e2bb6005 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 18 Jun 2024 16:40:01 +1000 Subject: [PATCH] Added more logging to importer to determine why we didnt end up making the new fields in scan items --- api/dataimport/internal/output/output.go | 6 +++--- api/dataimport/internal/output/summary.go | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/api/dataimport/internal/output/output.go b/api/dataimport/internal/output/output.go index 07757757..40bedbea 100644 --- a/api/dataimport/internal/output/output.go +++ b/api/dataimport/internal/output/output.go @@ -368,7 +368,7 @@ func (s *PIXLISEDataSaver) Save( prevSavedScan = nil } - summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId) + summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId, jobLog) jobLog.Infof("Writing summary to DB for %v...", summaryData.Id) @@ -380,8 +380,8 @@ func (s *PIXLISEDataSaver) Save( if err != nil { jobLog.Errorf("Failed to write summary to DB: %v", err) return err - } else if result.UpsertedCount != 1 { - jobLog.Errorf("Expected summary write to create 1 upsert, got: %v", result.UpsertedCount) + } else if result.UpsertedCount != 1 && result.ModifiedCount != 1 { + jobLog.Errorf("Expected summary write to create 1 upsert, got: %v upsert, %v modified", result.UpsertedCount, result.ModifiedCount) } // Set ownership diff --git a/api/dataimport/internal/output/summary.go b/api/dataimport/internal/output/summary.go index 87dc041f..aca85676 100644 --- a/api/dataimport/internal/output/summary.go +++ b/api/dataimport/internal/output/summary.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels" + "github.com/pixlise/core/v4/core/logger" protos "github.com/pixlise/core/v4/generated-protos" ) @@ -33,7 +34,8 @@ func makeSummaryFileContent( meta dataConvertModels.FileMetaData, //fileSize int, creationUnixTimeSec int64, - creatorUserId string) *protos.ScanItem { + creatorUserId string, + jobLog logger.ILogger) *protos.ScanItem { contextImgCount := len(exp.AlignedContextImages) + len(exp.UnalignedContextImages) + len(exp.MatchedAlignedContextImages) tiffContextImgCount := 0 @@ -117,16 +119,21 @@ func makeSummaryFileContent( // Add new time at the end s.PreviousImportTimesUnixSec = append(s.PreviousImportTimesUnixSec, prevSavedScan.TimestampUnixSec) + jobLog.Infof(" Added new previous time stamp entry: %v, total previous timestamps: %v", prevSavedScan.TimestampUnixSec, len(s.PreviousImportTimesUnixSec)) } // Save a time stamp for completion if isComplete { + jobLog.Infof(" Detected completed dataset...") + // If previous scan was also complete, DON'T update the time stamp, just preserve it if prevSavedScan != nil && prevSavedScan.CompleteTimeStampUnixSec > 0 { s.CompleteTimeStampUnixSec = prevSavedScan.CompleteTimeStampUnixSec + jobLog.Infof(" Preserved previous CompleteTimeStampUnixSec of %v", s.CompleteTimeStampUnixSec) } else { // We must've just completed now, so save the time s.CompleteTimeStampUnixSec = uint32(creationUnixTimeSec) + jobLog.Infof(" Setting CompleteTimeStampUnixSec=%v", creationUnixTimeSec) } } @@ -135,9 +142,16 @@ func makeSummaryFileContent( s.Tags = prevSavedScan.Tags s.Description = prevSavedScan.Description + descSnippet := s.Description + if len(descSnippet) > 30 { + descSnippet = descSnippet[0:30] + "..." + } + jobLog.Infof(" Preserved previous description=\"%v\"...", descSnippet) + if len(prevSavedScan.Description) > 0 { // User has entered a description so perserve the title too s.Title = prevSavedScan.Title + jobLog.Infof(" Preserved previous title=\"%v\"", s.Title) } }