From b6fb270ce632d5b14d9e3cf0a5dce1d88f58f770 Mon Sep 17 00:00:00 2001 From: Joey Kleingers Date: Thu, 19 Dec 2024 11:38:26 -0500 Subject: [PATCH] ENH: ReadCSVFile can now read data into existing data groups. Signed-off-by: Joey Kleingers --- .../SimplnxCore/docs/ReadCSVFileFilter.md | 2 +- .../SimplnxCore/Filters/ReadCSVFileFilter.cpp | 52 +++++++++++-------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/Plugins/SimplnxCore/docs/ReadCSVFileFilter.md b/src/Plugins/SimplnxCore/docs/ReadCSVFileFilter.md index 9aa9636b1e..064a248e34 100644 --- a/src/Plugins/SimplnxCore/docs/ReadCSVFileFilter.md +++ b/src/Plugins/SimplnxCore/docs/ReadCSVFileFilter.md @@ -46,7 +46,7 @@ The user can select the tuple dimensions that will be applied to the imported ar ![Input Text File Field](Images/Read_CSV_8.png) -The imported arrays can be stored in either an existing attribute matrix or a new attribute matrix can be created. +The imported arrays can be stored in an existing attribute matrix or data group, or a new attribute matrix can be created. ![Input Text File Field](Images/Read_CSV_9.png) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp index 46ecb0f9cc..4f94cefb6f 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadCSVFileFilter.cpp @@ -13,6 +13,7 @@ #include "simplnx/Parameters/AttributeMatrixSelectionParameter.hpp" #include "simplnx/Parameters/BoolParameter.hpp" #include "simplnx/Parameters/DataGroupCreationParameter.hpp" +#include "simplnx/Parameters/DataGroupSelectionParameter.hpp" #include "simplnx/Parameters/DynamicTableParameter.hpp" #include "simplnx/Parameters/ReadCSVFileParameter.hpp" #include "simplnx/Utilities/FileUtilities.hpp" @@ -87,7 +88,7 @@ Result validateExistingGroup(const DataPath& groupPath, const Dat { if(groupPath.empty()) { - return {MakeErrorResult(to_underlying(IssueCodes::EMPTY_EXISTING_DG), "'Existing Attribute Matrix' - Data path is empty.")}; + return {MakeErrorResult(to_underlying(IssueCodes::EMPTY_EXISTING_DG), "'Existing Data Group or Attribute Matrix' - Data path is empty.")}; } const auto& selectedGroup = dataStructure.getDataRefAs(groupPath); @@ -426,10 +427,11 @@ Parameters ReadCSVFileFilter::parameters() const tableInfo.setRowsInfo(DynamicTableInfo::StaticVectorInfo({"Dim 0"})); params.insertSeparator(Parameters::Separator{"Attribute Matrix Options"}); - params.insertLinkableParameter( - std::make_unique(k_UseExistingGroup_Key, "Use Existing Attribute Matrix", "Store the imported CSV data arrays in an existing attribute matrix.", false)); - params.insert(std::make_unique(k_SelectedAttributeMatrixPath_Key, "Existing Attribute Matrix", - "Store the imported CSV data arrays in this existing attribute matrix.", DataPath{})); + params.insertLinkableParameter(std::make_unique(k_UseExistingGroup_Key, "Use Existing Data Group or Attribute Matrix", + "Store the imported CSV data arrays in an existing data group or attribute matrix.", false)); + params.insert(std::make_unique(k_SelectedAttributeMatrixPath_Key, "Existing Data Group or Attribute Matrix", + "Store the imported CSV data arrays in an existing data group or attribute matrix.", DataPath{}, + DataGroupSelectionParameter::AllowedTypes{BaseGroup::GroupType::AttributeMatrix, BaseGroup::GroupType::DataGroup})); params.insert(std::make_unique(k_CreatedDataGroup_Key, "New Attribute Matrix", "Store the imported CSV data arrays in a newly created attribute matrix.", DataPath{{"Imported Data"}})); @@ -457,8 +459,8 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d const std::atomic_bool& shouldCancel) const { auto readCSVData = filterArgs.value(k_ReadCSVData_Key); - auto useExistingAM = filterArgs.value(k_UseExistingGroup_Key); - auto selectedAM = filterArgs.value(k_SelectedAttributeMatrixPath_Key); + auto useExistingGroupOrAM = filterArgs.value(k_UseExistingGroup_Key); + auto selectedGroupOrAM = filterArgs.value(k_SelectedAttributeMatrixPath_Key); auto createdDataAM = filterArgs.value(k_CreatedDataGroup_Key); std::string inputFilePath = readCSVData.inputFilePath; @@ -601,7 +603,7 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d std::string errMsg = fmt::format("Error: The current tuple dimensions ({}) has 0 total tuples. At least 1 tuple is required.", tupleDimsStr, tupleTotal, totalImportedLines); return {MakeErrorResult(to_underlying(IssueCodes::INCORRECT_TUPLES), errMsg), {}}; } - else if(tupleTotal > totalImportedLines && !useExistingAM) + else if(tupleTotal > totalImportedLines && !useExistingGroupOrAM) { std::string tupleDimsStr = tupleDimsToString(readCSVData.tupleDims); std::string errMsg = fmt::format("Error: The current tuple dimensions ({}) has {} total tuples, but this is larger than the total number of available lines to import ({}).", tupleDimsStr, @@ -611,14 +613,14 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d // Validate the existing/created group DataPath groupPath; - if(useExistingAM) + if(useExistingGroupOrAM) { - Result result = validateExistingGroup(selectedAM, dataStructure, headers); + Result result = validateExistingGroup(selectedGroupOrAM, dataStructure, headers); if(result.invalid()) { return {std::move(result)}; } - groupPath = selectedAM; + groupPath = selectedGroupOrAM; } else { @@ -634,16 +636,19 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d // Create the arrays std::vector tupleDims(readCSVData.tupleDims.size()); std::transform(readCSVData.tupleDims.begin(), readCSVData.tupleDims.end(), tupleDims.begin(), [](usize d) { return d; }); - if(useExistingAM) + if(useExistingGroupOrAM) { - const auto& am = dataStructure.getDataRefAs(groupPath); - tupleDims = am.getShape(); + const auto* am = dataStructure.getDataAs(groupPath); + if(am != nullptr) + { + tupleDims = am->getShape(); - auto totalLinesRead = std::accumulate(tupleDims.begin(), tupleDims.end(), static_cast(1), std::multiplies<>()); + auto totalLinesRead = std::accumulate(tupleDims.begin(), tupleDims.end(), static_cast(1), std::multiplies<>()); - std::string msg = fmt::format("The Array Tuple Dimensions ({}) will be ignored and the Existing Attribute Matrix tuple dimensions ({}) will be used. The total number of lines read will be {}.", - fmt::join(readCSVData.tupleDims, "x"), fmt::join(tupleDims, "x"), totalLinesRead); - resultOutputActions.warnings().push_back(Warning{to_underlying(IssueCodes::IGNORED_TUPLE_DIMS), msg}); + std::string msg = fmt::format("The Array Tuple Dimensions ({}) will be ignored and the Existing Attribute Matrix tuple dimensions ({}) will be used. The total number of lines read will be {}.", + fmt::join(readCSVData.tupleDims, "x"), fmt::join(tupleDims, "x"), totalLinesRead); + resultOutputActions.warnings().push_back(Warning{to_underlying(IssueCodes::IGNORED_TUPLE_DIMS), msg}); + } } for(usize i = 0; i < headers.size(); i++) @@ -671,7 +676,7 @@ Result<> ReadCSVFileFilter::executeImpl(DataStructure& dataStructure, const Argu { auto readCSVData = filterArgs.value(k_ReadCSVData_Key); auto useExistingGroup = filterArgs.value(k_UseExistingGroup_Key); - auto selectedDataGroup = filterArgs.value(k_SelectedAttributeMatrixPath_Key); + auto selectedDataGroupOrAM = filterArgs.value(k_SelectedAttributeMatrixPath_Key); auto createdDataGroup = filterArgs.value(k_CreatedDataGroup_Key); std::string inputFilePath = readCSVData.inputFilePath; @@ -691,7 +696,7 @@ Result<> ReadCSVFileFilter::executeImpl(DataStructure& dataStructure, const Argu DataPath groupPath = createdDataGroup; if(useExistingGroup) { - groupPath = selectedDataGroup; + groupPath = selectedDataGroupOrAM; } Result parsersResult = createParsers(dataTypes, skippedArrays, groupPath, headers, dataStructure); @@ -716,8 +721,11 @@ Result<> ReadCSVFileFilter::executeImpl(DataStructure& dataStructure, const Argu usize numTuples = std::accumulate(readCSVData.tupleDims.cbegin(), readCSVData.tupleDims.cend(), static_cast(1), std::multiplies<>()); if(useExistingGroup) { - const AttributeMatrix& am = dataStructure.getDataRefAs(groupPath); - numTuples = std::accumulate(am.getShape().cbegin(), am.getShape().cend(), static_cast(1), std::multiplies<>()); + const AttributeMatrix* am = dataStructure.getDataAs(groupPath); + if(am != nullptr) + { + numTuples = std::accumulate(am->getShape().cbegin(), am->getShape().cend(), static_cast(1), std::multiplies<>()); + } } usize lineNum = startImportRow; for(usize i = 0; i < numTuples && !in.eof(); i++)