From d5e6d3224f54c07a1563c593b47485e69367729c Mon Sep 17 00:00:00 2001 From: Joey Kleingers Date: Thu, 19 Sep 2024 11:42:32 -0400 Subject: [PATCH 1/4] Adjust filter to re-read the headers from the header line every time. Signed-off-by: Joey Kleingers --- .../src/SimplnxCore/Filters/ReadCSVFileFilter.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp index 61dc99c403..a410b5e208 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp @@ -466,15 +466,12 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d } else if(headerMode == ReadCSVData::HeaderMode::LINE) { - if(readCSVData.headersLine != s_HeaderCache[s_InstanceId].HeadersLine) + // We have to read the headers every time because it's possible that the header line is the same but the data in the file still was changed underneath the covers + IFilter::PreflightResult result = readHeaders(readCSVData.inputFilePath, readCSVData.headersLine, s_HeaderCache[s_InstanceId]); + if(result.outputActions.invalid()) { - IFilter::PreflightResult result = readHeaders(readCSVData.inputFilePath, readCSVData.headersLine, s_HeaderCache[s_InstanceId]); - if(result.outputActions.invalid()) - { - return result; - } + return result; } - headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, readCSVData.delimiters, readCSVData.consecutiveDelimiters); } From 07159c64b9f9117e2acf516b6f470297bc81c227 Mon Sep 17 00:00:00 2001 From: Joey Kleingers Date: Thu, 19 Sep 2024 15:11:18 -0400 Subject: [PATCH 2/4] Adjust the filter to only re-read the file when the following occurs: 1. File path changes 2. File is modified 3. Headers mode is set to LINE and the header line has changed. Signed-off-by: Joey Kleingers --- .../SimplnxCore/Filters/ReadCSVFileFilter.cpp | 94 +++++++++++-------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp index a410b5e208..c883dfa669 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp @@ -38,6 +38,7 @@ struct ReadCSVFileFilterCache usize TotalLines = 0; usize HeadersLine = 0; std::string Headers; + fs::file_time_type LastModifiedTime; }; std::atomic_int32_t s_InstanceId = 0; @@ -323,6 +324,47 @@ IFilter::PreflightResult readHeaders(const std::string& inputFilePath, usize hea return {}; } +Result<> cacheHeaders(const ReadCSVData& readCsvData) +{ + std::fstream in(readCsvData.inputFilePath.c_str(), std::ios_base::in); + if(!in.is_open()) + { + return {MakeErrorResult(to_underlying(IssueCodes::FILE_NOT_OPEN), fmt::format("Could not open file for reading: {}", readCsvData.inputFilePath)), {}}; + } + + usize currentLine = 0; + while(!in.eof()) + { + std::string line; + std::getline(in, line); + currentLine++; + + if(currentLine == readCsvData.headersLine) + { + s_HeaderCache[s_InstanceId].Headers = line; + s_HeaderCache[s_InstanceId].HeadersLine = readCsvData.headersLine; + break; + } + } + + return {}; +} + +Result<> cacheFullFile(const ReadCSVData& readCsvData) +{ + s_HeaderCache[s_InstanceId].FilePath = readCsvData.inputFilePath; + auto result = cacheHeaders(readCsvData); + if(result.invalid()) + { + return result; + } + + s_HeaderCache[s_InstanceId].TotalLines = nx::core::FileUtilities::LinesInFile(readCsvData.inputFilePath); + s_HeaderCache[s_InstanceId].LastModifiedTime = fs::last_write_time(readCsvData.inputFilePath); + + return {}; +} + } // namespace namespace nx::core @@ -431,54 +473,28 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d } StringVector headers; - if(readCSVData.inputFilePath != s_HeaderCache[s_InstanceId].FilePath) + auto lastModifiedTime = fs::last_write_time(readCSVData.inputFilePath); + if(readCSVData.inputFilePath != s_HeaderCache[s_InstanceId].FilePath || lastModifiedTime > s_HeaderCache[s_InstanceId].LastModifiedTime) { - int64 lineCount = nx::core::FileUtilities::LinesInFile(inputFilePath); - if(lineCount < 0) - { - return {MakeErrorResult(to_underlying(IssueCodes::FILE_NOT_OPEN), fmt::format("Could not open file for reading: {}", inputFilePath)), {}}; - } - std::fstream in(inputFilePath.c_str(), std::ios_base::in); - if(!in.is_open()) + // File path changed or file was modified + auto result = cacheFullFile(readCSVData); + if(result.invalid()) { - return {MakeErrorResult(to_underlying(IssueCodes::FILE_NOT_OPEN), fmt::format("Could not open file for reading: {}", inputFilePath)), {}}; + return {ConvertResultTo(ConvertResult(std::move(result)), {})}; } - - s_HeaderCache[s_InstanceId].FilePath = readCSVData.inputFilePath; - - usize currentLine = 0; - while(!in.eof()) - { - std::string line; - std::getline(in, line); - currentLine++; - - if(headerMode == ReadCSVData::HeaderMode::LINE && currentLine == readCSVData.headersLine) - { - s_HeaderCache[s_InstanceId].Headers = line; - s_HeaderCache[s_InstanceId].HeadersLine = readCSVData.headersLine; - break; - } - } - - headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, readCSVData.delimiters, readCSVData.consecutiveDelimiters); - s_HeaderCache[s_InstanceId].TotalLines = lineCount; } - else if(headerMode == ReadCSVData::HeaderMode::LINE) + else if(headerMode == ReadCSVData::HeaderMode::LINE && readCSVData.headersLine != s_HeaderCache[s_InstanceId].HeadersLine) { - // We have to read the headers every time because it's possible that the header line is the same but the data in the file still was changed underneath the covers - IFilter::PreflightResult result = readHeaders(readCSVData.inputFilePath, readCSVData.headersLine, s_HeaderCache[s_InstanceId]); - if(result.outputActions.invalid()) + // We are in header line mode and the header line number changed + auto result = cacheHeaders(readCSVData); + if(result.invalid()) { - return result; + return {ConvertResultTo(ConvertResult(std::move(result)), {})}; } - headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, readCSVData.delimiters, readCSVData.consecutiveDelimiters); } - if(headerMode == ReadCSVData::HeaderMode::CUSTOM) - { - headers = readCSVData.customHeaders; - } + headers = (headerMode == ReadCSVData::HeaderMode::LINE) ? StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, readCSVData.delimiters, readCSVData.consecutiveDelimiters) : + readCSVData.customHeaders; usize totalLines = s_HeaderCache[s_InstanceId].TotalLines; From 03536b75b3e026e2c2b64aeccb8a477dc2effec2 Mon Sep 17 00:00:00 2001 From: Joey Kleingers Date: Thu, 19 Sep 2024 16:09:19 -0400 Subject: [PATCH 3/4] Clang format. Signed-off-by: Joey Kleingers --- .../SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp index c883dfa669..b7df830cf1 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp @@ -346,7 +346,7 @@ Result<> cacheHeaders(const ReadCSVData& readCsvData) break; } } - + return {}; } From 2af93919e13f5e87758dd3b820f2a3771387fe26 Mon Sep 17 00:00:00 2001 From: Joey Kleingers Date: Fri, 20 Sep 2024 08:55:36 -0400 Subject: [PATCH 4/4] Fix Windows crash. Signed-off-by: Joey Kleingers --- .../SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp index b7df830cf1..06f65f1ed5 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp @@ -329,7 +329,7 @@ Result<> cacheHeaders(const ReadCSVData& readCsvData) std::fstream in(readCsvData.inputFilePath.c_str(), std::ios_base::in); if(!in.is_open()) { - return {MakeErrorResult(to_underlying(IssueCodes::FILE_NOT_OPEN), fmt::format("Could not open file for reading: {}", readCsvData.inputFilePath)), {}}; + return MakeErrorResult(to_underlying(IssueCodes::FILE_NOT_OPEN), fmt::format("Could not open file for reading: {}", readCsvData.inputFilePath)); } usize currentLine = 0;