diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportTextDataFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportTextDataFilter.cpp index c64886cbec..fde9502642 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportTextDataFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportTextDataFilter.cpp @@ -63,6 +63,7 @@ enum class IssueCodes HEADER_LINE_OUT_OF_RANGE = -120, START_IMPORT_ROW_OUT_OF_RANGE = -121, EMPTY_HEADERS = -122, + EMPTY_DELIMITERS = -123, IGNORED_TUPLE_DIMS = -200 }; @@ -288,8 +289,7 @@ std::string tupleDimsToString(const std::vector& tupleDims) } //------------------------------------------------------------------------------ -IFilter::PreflightResult readHeaders(const std::string& inputFilePath, usize headersLineNum, ImportTextDataFilterCache& headerCache, bool useTab, bool useSemicolon, bool useSpace, bool useComma, - bool useConsecutive) +IFilter::PreflightResult readHeaders(const std::string& inputFilePath, usize headersLineNum, ImportTextDataFilterCache& headerCache) { std::fstream in(inputFilePath.c_str(), std::ios_base::in); if(!in.is_open()) @@ -417,8 +417,12 @@ IFilter::PreflightResult ImportTextDataFilter::preflightImpl(const DataStructure return {ConvertResultTo(std::move(ConvertResult(std::move(csvResult))), {}), {}}; } + if(textImporterData.delimiters.empty()) + { + return {MakeErrorResult(to_underlying(IssueCodes::EMPTY_DELIMITERS), "No delimiters have been chosen. Please choose at least one delimiter."), {}}; + } + StringVector headers; - std::vector delimiters = CreateDelimitersVector(textImporterData.tabAsDelimiter, textImporterData.semicolonAsDelimiter, textImporterData.commaAsDelimiter, textImporterData.spaceAsDelimiter); if(textImporterData.inputFilePath != s_HeaderCache[s_InstanceId].FilePath) { std::fstream in(inputFilePath.c_str(), std::ios_base::in); @@ -443,23 +447,21 @@ IFilter::PreflightResult ImportTextDataFilter::preflightImpl(const DataStructure } } - headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, delimiters, textImporterData.consecutiveDelimiters); + headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, textImporterData.delimiters, textImporterData.consecutiveDelimiters); s_HeaderCache[s_InstanceId].TotalLines = lineCount; } else if(headerMode == TextImporterData::HeaderMode::LINE) { if(textImporterData.headersLine != s_HeaderCache[s_InstanceId].HeadersLine) { - IFilter::PreflightResult result = - readHeaders(textImporterData.inputFilePath, textImporterData.headersLine, s_HeaderCache[s_InstanceId], textImporterData.tabAsDelimiter, textImporterData.semicolonAsDelimiter, - textImporterData.spaceAsDelimiter, textImporterData.commaAsDelimiter, textImporterData.consecutiveDelimiters); + IFilter::PreflightResult result = readHeaders(textImporterData.inputFilePath, textImporterData.headersLine, s_HeaderCache[s_InstanceId]); if(result.outputActions.invalid()) { return result; } } - headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, delimiters, textImporterData.consecutiveDelimiters); + headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, textImporterData.delimiters, textImporterData.consecutiveDelimiters); } if(headerMode == TextImporterData::HeaderMode::CUSTOM) @@ -637,8 +639,7 @@ Result<> ImportTextDataFilter::executeImpl(DataStructure& dataStructure, const A DataPath createdDataGroup = filterArgs.value(k_CreatedDataGroup_Key); std::string inputFilePath = textImporterData.inputFilePath; - std::vector delimiters = CreateDelimitersVector(textImporterData.tabAsDelimiter, textImporterData.semicolonAsDelimiter, textImporterData.commaAsDelimiter, textImporterData.spaceAsDelimiter); - StringVector headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, delimiters, textImporterData.consecutiveDelimiters); + StringVector headers = StringUtilities::split(s_HeaderCache[s_InstanceId].Headers, textImporterData.delimiters, textImporterData.consecutiveDelimiters); DataTypeVector dataTypes = textImporterData.dataTypes; std::vector skippedArrays = textImporterData.skippedArrayMask; bool consecutiveDelimiters = textImporterData.consecutiveDelimiters; @@ -688,7 +689,7 @@ Result<> ImportTextDataFilter::executeImpl(DataStructure& dataStructure, const A return {}; } - Result<> parsingResult = parseLine(in, parsersResult.value(), headers, delimiters, consecutiveDelimiters, lineNum, startImportRow); + Result<> parsingResult = parseLine(in, parsersResult.value(), headers, textImporterData.delimiters, consecutiveDelimiters, lineNum, startImportRow); if(parsingResult.invalid()) { return std::move(parsingResult); diff --git a/src/Plugins/ComplexCore/test/ImportTextDataTest.cpp b/src/Plugins/ComplexCore/test/ImportTextDataTest.cpp index 847544782c..4a555cf19d 100644 --- a/src/Plugins/ComplexCore/test/ImportTextDataTest.cpp +++ b/src/Plugins/ComplexCore/test/ImportTextDataTest.cpp @@ -35,6 +35,7 @@ constexpr int32 k_EmptyNames = -116; constexpr int32 k_HeaderLineOutOfRange = -120; constexpr int32 k_StartImportRowOutOfRange = -121; constexpr int32 k_EmptyHeaders = -122; +constexpr int32 k_EmptyDelimiters = -123; constexpr int32 k_FileDoesNotExist = -300; } // namespace @@ -80,9 +81,9 @@ void CreateTestDataFile(const fs::path& inputFilePath, nonstd::span } // ----------------------------------------------------------------------------- -Arguments createArguments(const std::string& inputFilePath, usize startImportRow, TextImporterData::HeaderMode headerMode, usize headersLine, const std::vector& customHeaders, - const std::vector& dataTypes, const std::vector& skippedArrayMask, const std::vector& tupleDims, nonstd::span values, - const std::string& newGroupName) +Arguments createArguments(const std::string& inputFilePath, usize startImportRow, TextImporterData::HeaderMode headerMode, usize headersLine, const std::vector& delimiters, + const std::vector& customHeaders, const std::vector& dataTypes, const std::vector& skippedArrayMask, const std::vector& tupleDims, + nonstd::span values, const std::string& newGroupName) { Arguments args; @@ -91,7 +92,7 @@ Arguments createArguments(const std::string& inputFilePath, usize startImportRow data.customHeaders = customHeaders; data.dataTypes = dataTypes; data.startImportRow = startImportRow; - data.commaAsDelimiter = true; + data.delimiters = delimiters; data.headersLine = headersLine; data.headerMode = headerMode; data.tupleDims = tupleDims; @@ -118,7 +119,8 @@ void TestCase_TestPrimitives(nonstd::span values) ImportTextDataFilter filter; DataStructure dataStructure; - Arguments args = createArguments(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {arrayName}, {GetDataType()}, {false}, {static_cast(values.size())}, values, newGroupName); + Arguments args = + createArguments(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {arrayName}, {GetDataType()}, {false}, {static_cast(values.size())}, values, newGroupName); // Create the test input data file CreateTestDataFile(k_TestInput, values, {arrayName}); @@ -160,7 +162,8 @@ void TestCase_TestPrimitives_Error(nonstd::span values, int32 expec ImportTextDataFilter filter; DataStructure dataStructure; - Arguments args = createArguments(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {arrayName}, {GetDataType()}, {false}, {static_cast(values.size())}, values, newGroupName); + Arguments args = + createArguments(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {arrayName}, {GetDataType()}, {false}, {static_cast(values.size())}, values, newGroupName); // Create the test input data file fs::create_directories(k_TestInput.parent_path()); @@ -178,14 +181,14 @@ void TestCase_TestPrimitives_Error(nonstd::span values, int32 expec } // ----------------------------------------------------------------------------- -void TestCase_TestImporterData_Error(const std::string& inputFilePath, usize startImportRow, TextImporterData::HeaderMode headerMode, usize headersLine, const std::vector& headers, - const std::vector& dataTypes, const std::vector& skippedArrayMask, const std::vector& tupleDims, nonstd::span values, - int32 expectedErrorCode) +void TestCase_TestImporterData_Error(const std::string& inputFilePath, usize startImportRow, TextImporterData::HeaderMode headerMode, usize headersLine, const std::vector& delimiters, + const std::vector& headers, const std::vector& dataTypes, const std::vector& skippedArrayMask, const std::vector& tupleDims, + nonstd::span values, int32 expectedErrorCode) { std::string newGroupName = "New Group"; ImportTextDataFilter filter; DataStructure dataStructure; - Arguments args = createArguments(inputFilePath, startImportRow, headerMode, headersLine, headers, dataTypes, skippedArrayMask, tupleDims, values, newGroupName); + Arguments args = createArguments(inputFilePath, startImportRow, headerMode, headersLine, delimiters, headers, dataTypes, skippedArrayMask, tupleDims, values, newGroupName); // Execute the filter and check the result auto executeResult = filter.execute(dataStructure, args); @@ -243,7 +246,8 @@ TEST_CASE("ComplexCore::ImportTextDataFilter (Case 2): Valid filter execution - ImportTextDataFilter filter; DataStructure dataStructure; std::vector values = {"0"}; - Arguments args = createArguments(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {arrayName}, {DataType::int8}, {true}, {static_cast(values.size())}, values, newGroupName); + Arguments args = + createArguments(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {arrayName}, {DataType::int8}, {true}, {static_cast(values.size())}, values, newGroupName); // Create the test input data file CreateTestDataFile(k_TestInput, values, {arrayName}); @@ -387,51 +391,52 @@ TEST_CASE("ComplexCore::ImportTextDataFilter (Case 5): Invalid filter execution std::vector tupleDims = {static_cast(v.size())}; // Empty input file path - TestCase_TestImporterData_Error("", 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_EmptyFile); + TestCase_TestImporterData_Error("", 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_EmptyFile); // Input file does not exist fs::path tmp_file = fs::temp_directory_path() / "ThisFileDoesNotExist.txt"; - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_FileDoesNotExist); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_FileDoesNotExist); // Start Import Row Out-of-Range - TestCase_TestImporterData_Error(k_TestInput.string(), 0, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_StartImportRowOutOfRange); - TestCase_TestImporterData_Error(k_TestInput.string(), 500, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_StartImportRowOutOfRange); + TestCase_TestImporterData_Error(k_TestInput.string(), 0, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_StartImportRowOutOfRange); + TestCase_TestImporterData_Error(k_TestInput.string(), 500, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_StartImportRowOutOfRange); // Header Line Number Out-of-Range - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 0, {}, {DataType::int8}, {false}, tupleDims, v, k_HeaderLineOutOfRange); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 600, {}, {DataType::int8}, {false}, tupleDims, v, k_HeaderLineOutOfRange); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 3, {}, {DataType::int8}, {false}, tupleDims, v, k_HeaderLineOutOfRange); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 0, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_HeaderLineOutOfRange); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 600, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_HeaderLineOutOfRange); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 3, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_HeaderLineOutOfRange); // Empty array headers tmp_file = fs::temp_directory_path() / "BlankLines.txt"; v = {std::to_string(std::numeric_limits::min()), "", std::to_string(std::numeric_limits::max())}; CreateTestDataFile(tmp_file, v, {"Array"}); - TestCase_TestImporterData_Error(tmp_file.string(), 4, TextImporterData::HeaderMode::LINE, 3, {}, {DataType::int8}, {false}, {static_cast(v.size())}, v, k_EmptyHeaders); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {}, {DataType::int8}, {false}, {static_cast(v.size())}, v, k_EmptyHeaders); + TestCase_TestImporterData_Error(tmp_file.string(), 4, TextImporterData::HeaderMode::LINE, 3, {','}, {}, {DataType::int8}, {false}, {static_cast(v.size())}, v, k_EmptyHeaders); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {}, {DataType::int8}, {false}, {static_cast(v.size())}, v, k_EmptyHeaders); fs::remove(tmp_file); v = {std::to_string(std::numeric_limits::min()), std::to_string(std::numeric_limits::max())}; // Incorrect Data Type Count - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {}, {false}, tupleDims, v, k_IncorrectDataTypeCount); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8, DataType::int32}, {false}, tupleDims, v, k_IncorrectDataTypeCount); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {}, {false}, tupleDims, v, k_IncorrectDataTypeCount); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {DataType::int8, DataType::int32}, {false}, tupleDims, v, + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {}, {false}, tupleDims, v, k_IncorrectDataTypeCount); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8, DataType::int32}, {false}, tupleDims, v, k_IncorrectDataTypeCount); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {}, {false}, tupleDims, v, k_IncorrectDataTypeCount); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {DataType::int8, DataType::int32}, {false}, tupleDims, v, k_IncorrectDataTypeCount); // Incorrect Skipped Array Mask Count - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {}, tupleDims, v, k_IncorrectMaskCount); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false, false}, tupleDims, v, k_IncorrectMaskCount); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {DataType::int8}, {}, tupleDims, v, k_IncorrectMaskCount); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {DataType::int8}, {false, false}, tupleDims, v, k_IncorrectMaskCount); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {}, tupleDims, v, k_IncorrectMaskCount); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false, false}, tupleDims, v, k_IncorrectMaskCount); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {DataType::int8}, {}, tupleDims, v, k_IncorrectMaskCount); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {DataType::int8}, {false, false}, tupleDims, v, k_IncorrectMaskCount); // Empty Header Names - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {""}, {DataType::int8}, {false}, tupleDims, v, k_EmptyNames); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {""}, {DataType::int8}, {false}, tupleDims, v, k_EmptyNames); // Duplicate Header Names tmp_file = fs::temp_directory_path() / "DuplicateHeaders.txt"; std::vector duplicateHeaders = {"Custom Array", "Custom Array"}; CreateTestDataFile(tmp_file, v, duplicateHeaders); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, duplicateHeaders, {DataType::int8, DataType::int8}, {false, false}, tupleDims, v, k_DuplicateNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, duplicateHeaders, {DataType::int8, DataType::int8}, {false, false}, tupleDims, v, + k_DuplicateNames); fs::remove(tmp_file); // Illegal Header Names @@ -439,35 +444,39 @@ TEST_CASE("ComplexCore::ImportTextDataFilter (Case 5): Invalid filter execution std::vector illegalHeaders = {"Illegal/Header"}; CreateTestDataFile(tmp_file, v, illegalHeaders); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); illegalHeaders = {"Illegal\\Header"}; CreateTestDataFile(tmp_file, v, illegalHeaders); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); illegalHeaders = {"Illegal&Header"}; CreateTestDataFile(tmp_file, v, illegalHeaders); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); illegalHeaders = {"Illegal:Header"}; CreateTestDataFile(tmp_file, v, illegalHeaders); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); - TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::LINE, 1, {','}, {}, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); + TestCase_TestImporterData_Error(tmp_file.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, illegalHeaders, {DataType::int8}, {false}, tupleDims, v, k_IllegalNames); fs::remove(tmp_file); // Incorrect Tuple Dimensions - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {DataType::int8}, {false}, {0}, v, k_IncorrectTuples); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {DataType::int8}, {false}, {30}, v, k_IncorrectTuples); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {DataType::int8}, {false}, {30, 2}, v, k_IncorrectTuples); - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array"}, {DataType::int8}, {false}, {30, 5, 7}, v, k_IncorrectTuples); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {DataType::int8}, {false}, {0}, v, k_IncorrectTuples); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {DataType::int8}, {false}, {30}, v, k_IncorrectTuples); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {DataType::int8}, {false}, {30, 2}, v, k_IncorrectTuples); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array"}, {DataType::int8}, {false}, {30, 5, 7}, v, k_IncorrectTuples); // Inconsistent Columns - TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {"Custom Array", "Custom Array2"}, {DataType::int8, DataType::int8}, {false, false}, tupleDims, v, - k_InconsistentCols); + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {','}, {"Custom Array", "Custom Array2"}, {DataType::int8, DataType::int8}, {false, false}, + tupleDims, v, k_InconsistentCols); + + // Empty Delimiters + TestCase_TestImporterData_Error(k_TestInput.string(), 2, TextImporterData::HeaderMode::CUSTOM, 1, {}, {"Custom Array", "Custom Array2"}, {DataType::int8, DataType::int8}, {false, false}, tupleDims, + v, k_EmptyDelimiters); } TEST_CASE("ComplexCore::ImportTextDataFilter (Case 6): Invalid filter execution - Blank Lines") diff --git a/src/Plugins/ComplexCore/wrapping/python/complexpy.cpp b/src/Plugins/ComplexCore/wrapping/python/complexpy.cpp index b43e15a886..65c7dd8a16 100644 --- a/src/Plugins/ComplexCore/wrapping/python/complexpy.cpp +++ b/src/Plugins/ComplexCore/wrapping/python/complexpy.cpp @@ -443,10 +443,7 @@ PYBIND11_MODULE(complex, mod) textImporterData.def_readwrite("headers_line", &TextImporterData::headersLine); textImporterData.def_readwrite("header_mode", &TextImporterData::headerMode); textImporterData.def_readwrite("tuple_dims", &TextImporterData::tupleDims); - textImporterData.def_readwrite("tab_as_delimiter", &TextImporterData::tabAsDelimiter); - textImporterData.def_readwrite("semicolon_as_delimiter", &TextImporterData::semicolonAsDelimiter); - textImporterData.def_readwrite("comma_as_delimiter", &TextImporterData::commaAsDelimiter); - textImporterData.def_readwrite("space_as_delimiter", &TextImporterData::spaceAsDelimiter); + textImporterData.def_readwrite("delimiters", &TextImporterData::delimiters); textImporterData.def_readwrite("consecutive_delimiters", &TextImporterData::consecutiveDelimiters); textImporterData.def("__repr__", [](const TextImporterData& self) { return "TextImporterData()"; }); diff --git a/src/Plugins/OrientationAnalysis/test/ExportGBCDGMTFileTest.cpp b/src/Plugins/OrientationAnalysis/test/ExportGBCDGMTFileTest.cpp index 83abb61198..e9b3603a99 100644 --- a/src/Plugins/OrientationAnalysis/test/ExportGBCDGMTFileTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ExportGBCDGMTFileTest.cpp @@ -105,7 +105,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDGMTFileFilter", "[OrientationAnalysis] data.dataTypes = {DataType::float32, DataType::float32, DataType::float32}; data.skippedArrayMask = {false, false, false}; data.startImportRow = 2; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {3751}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); @@ -125,7 +125,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDGMTFileFilter", "[OrientationAnalysis] data.dataTypes = {DataType::float32, DataType::float32, DataType::float32}; data.skippedArrayMask = {false, false, false}; data.startImportRow = 2; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {3751}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); @@ -191,7 +191,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDGMTFileFilter", "[OrientationAnalysis] data.dataTypes = {DataType::float32, DataType::float32, DataType::float32}; data.skippedArrayMask = {false, false, false}; data.startImportRow = 2; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {3751}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); @@ -211,7 +211,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDGMTFileFilter", "[OrientationAnalysis] data.dataTypes = {DataType::float32, DataType::float32, DataType::float32}; data.skippedArrayMask = {false, false, false}; data.startImportRow = 2; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {3751}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); @@ -277,7 +277,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDGMTFileFilter", "[OrientationAnalysis] data.dataTypes = {DataType::float32, DataType::float32, DataType::float32}; data.skippedArrayMask = {false, false, false}; data.startImportRow = 2; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {3751}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); @@ -297,7 +297,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDGMTFileFilter", "[OrientationAnalysis] data.dataTypes = {DataType::float32, DataType::float32, DataType::float32}; data.skippedArrayMask = {false, false, false}; data.startImportRow = 2; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {3751}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); diff --git a/src/Plugins/OrientationAnalysis/test/ExportGBCDTriangleDataTest.cpp b/src/Plugins/OrientationAnalysis/test/ExportGBCDTriangleDataTest.cpp index 2c9f3f04f6..c286e652d8 100644 --- a/src/Plugins/OrientationAnalysis/test/ExportGBCDTriangleDataTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ExportGBCDTriangleDataTest.cpp @@ -97,7 +97,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDTriangleDataFilter: Valid filter execu DataType::float32, DataType::float64, DataType::float64, DataType::float64, DataType::float64}; data.skippedArrayMask = {false, false, false, false, false, false, false, false, false, false}; data.startImportRow = 6; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {636474}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); @@ -119,7 +119,7 @@ TEST_CASE("OrientationAnalysis::ExportGBCDTriangleDataFilter: Valid filter execu DataType::float32, DataType::float64, DataType::float64, DataType::float64, DataType::float64}; data.skippedArrayMask = {false, false, false, false, false, false, false, false, false, false}; data.startImportRow = 5; - data.spaceAsDelimiter = true; + data.delimiters = {' '}; data.tupleDims = {636474}; args.insertOrAssign(k_TextImporterData_Key, std::make_any(data)); diff --git a/src/complex/Parameters/util/TextImporterData.cpp b/src/complex/Parameters/util/TextImporterData.cpp index 4e9599f6fd..d79b4bba37 100644 --- a/src/complex/Parameters/util/TextImporterData.cpp +++ b/src/complex/Parameters/util/TextImporterData.cpp @@ -44,10 +44,7 @@ const std::string k_InputFilePathKey = "Input File Path"; const std::string k_StartImportRowKey = "Start Import Row"; const std::string k_HeaderLineKey = "Header Line"; const std::string k_HeaderModeKey = "Header Mode"; -const std::string k_TabAsDelimiterKey = "Tab As Delimiter"; -const std::string k_SemicolonAsDelimiterKey = "Semicolon As Delimiter"; -const std::string k_CommaAsDelimiterKey = "Comma As Delimiter"; -const std::string k_SpaceAsDelimiterKey = "Space As Delimiter"; +const std::string k_Delimiters = "Delimiters"; const std::string k_ConsecutiveDelimitersKey = "Consecutive Delimiters"; } // namespace @@ -88,10 +85,7 @@ nlohmann::json TextImporterData::writeJson() const json[k_StartImportRowKey] = startImportRow; json[k_HeaderLineKey] = headersLine; json[k_HeaderModeKey] = headerMode; - json[k_TabAsDelimiterKey] = tabAsDelimiter; - json[k_SemicolonAsDelimiterKey] = semicolonAsDelimiter; - json[k_SpaceAsDelimiterKey] = spaceAsDelimiter; - json[k_CommaAsDelimiterKey] = commaAsDelimiter; + json[k_Delimiters] = delimiters; json[k_ConsecutiveDelimitersKey] = consecutiveDelimiters; return json; @@ -207,46 +201,22 @@ Result TextImporterData::ReadJson(const nlohmann::json& json) } data.headerMode = json[k_HeaderModeKey]; - if(!json.contains(k_TabAsDelimiterKey)) + if(!json.contains(k_Delimiters)) { - return MakeErrorResult(-117, fmt::format("TextImporterData: Cannot find the 'Tab As Delimiter' key \"{}\" in the TextImporterData json object.", k_TabAsDelimiterKey)); + return MakeErrorResult(-117, fmt::format("TextImporterData: Cannot find the 'Delimiters' key \"{}\" in the TextImporterData json object.", k_Delimiters)); } - else if(!json[k_TabAsDelimiterKey].is_boolean()) - { - return MakeErrorResult(-118, fmt::format("TextImporterData: 'Tab As Delimiter' value is of type {} and is not a boolean.", json[k_TabAsDelimiterKey].type_name())); - } - data.tabAsDelimiter = json[k_TabAsDelimiterKey]; - - if(!json.contains(k_SemicolonAsDelimiterKey)) - { - return MakeErrorResult(-119, - fmt::format("TextImporterData: Cannot find the 'Semicolon As Delimiter' key \"{}\" in the TextImporterData json object.", k_SemicolonAsDelimiterKey)); - } - else if(!json[k_SemicolonAsDelimiterKey].is_boolean()) - { - return MakeErrorResult(-120, fmt::format("TextImporterData: 'Semicolon As Delimiter' value is of type {} and is not a boolean.", json[k_SemicolonAsDelimiterKey].type_name())); - } - data.semicolonAsDelimiter = json[k_SemicolonAsDelimiterKey]; - if(!json.contains(k_SpaceAsDelimiterKey)) + nlohmann::json dDelimiters = json[k_Delimiters]; + for(usize i = 0; i < dDelimiters.size(); i++) { - return MakeErrorResult(-121, fmt::format("TextImporterData: Cannot find the 'Space As Delimiter' key \"{}\" in the TextImporterData json object.", k_SpaceAsDelimiterKey)); - } - else if(!json[k_SpaceAsDelimiterKey].is_boolean()) - { - return MakeErrorResult(-122, fmt::format("TextImporterData: 'Space As Delimiter' value is of type {} and is not a boolean.", json[k_SpaceAsDelimiterKey].type_name())); - } - data.spaceAsDelimiter = json[k_SpaceAsDelimiterKey]; + auto delimiter = dDelimiters[i]; + if(!delimiter.is_string() || delimiter.get().size() != 1) + { + return MakeErrorResult(-118, fmt::format("TextImporterData: Delimiter at index {} is of type {} and is not a boolean.", std::to_string(i), delimiter.type_name())); + } - if(!json.contains(k_CommaAsDelimiterKey)) - { - return MakeErrorResult(-123, fmt::format("TextImporterData: Cannot find the 'Comma As Delimiter' key \"{}\" in the TextImporterData json object.", k_CommaAsDelimiterKey)); - } - else if(!json[k_CommaAsDelimiterKey].is_boolean()) - { - return MakeErrorResult(-124, fmt::format("TextImporterData: 'Comma As Delimiter' value is of type {} and is not a boolean.", json[k_CommaAsDelimiterKey].type_name())); + data.delimiters.push_back(delimiter.get()[0]); } - data.commaAsDelimiter = json[k_CommaAsDelimiterKey]; if(!json.contains(k_ConsecutiveDelimitersKey)) { diff --git a/src/complex/Parameters/util/TextImporterData.hpp b/src/complex/Parameters/util/TextImporterData.hpp index 095b8ed4a5..e9add2d13c 100644 --- a/src/complex/Parameters/util/TextImporterData.hpp +++ b/src/complex/Parameters/util/TextImporterData.hpp @@ -63,10 +63,7 @@ struct COMPLEX_EXPORT TextImporterData usize headersLine = 1; HeaderMode headerMode = HeaderMode::CUSTOM; std::vector tupleDims = {1}; - bool tabAsDelimiter = false; - bool semicolonAsDelimiter = false; - bool commaAsDelimiter = false; - bool spaceAsDelimiter = false; + std::vector delimiters = {}; bool consecutiveDelimiters = false; }; } // namespace complex diff --git a/src/complex/Utilities/FilterUtilities.cpp b/src/complex/Utilities/FilterUtilities.cpp index 4998129d6a..59e2168f2d 100644 --- a/src/complex/Utilities/FilterUtilities.cpp +++ b/src/complex/Utilities/FilterUtilities.cpp @@ -23,29 +23,4 @@ Result<> CreateOutputDirectories(const fs::path& outputPath) } return {}; } - -// ----------------------------------------------------------------------------- -std::vector CreateDelimitersVector(bool tabAsDelimiter, bool semicolonAsDelimiter, bool commaAsDelimiter, bool spaceAsDelimiter) -{ - std::vector delimiters; - if(tabAsDelimiter) - { - delimiters.push_back('\t'); - } - if(semicolonAsDelimiter) - { - delimiters.push_back(';'); - } - if(commaAsDelimiter) - { - delimiters.push_back(','); - } - if(spaceAsDelimiter) - { - delimiters.push_back(' '); - } - - return delimiters; -} - } // namespace complex diff --git a/wrapping/python/docs/source/API.rst b/wrapping/python/docs/source/API.rst index bd68ae2bd6..3d90eea6c4 100644 --- a/wrapping/python/docs/source/API.rst +++ b/wrapping/python/docs/source/API.rst @@ -406,10 +406,7 @@ General Parameters :ivar input_file_path: "PathLike". The path to the input file on the file system. :ivar start_import_row: Int. What line number does the data start on. 1 Based numbering scheme. - :ivar comma_as_delimiter: Bool. Are the values comma separated. - :ivar semicolon_as_delimiter: Bool. Are the values semicolon separated. - :ivar space_as_delimiter: Bool. Are the values space separated. - :ivar tab_as_delimiter: Bool. Are the values tab separated. + :ivar delimiters: List[string]. List of delimiters that will be used to separate the lines of the file into columns. :ivar consecutive_delimiters: Bool. Should consecutive delimiters be counted as a single delimiter. :ivar custom_headers: List[string]. If the file does not have headers, this is a list of string values, 1 per column of data, that will also become the names of the created :ref:`DataArray`. :ivar data_types: List[cx.DataType]. The DataType, one per column, that indicates the kind of native numerical values (int, float... ) that will be used in the created :ref:`DataArray`. @@ -426,13 +423,7 @@ General Parameters text_importer_data = cx.TextImporterData() text_importer_data.input_file_path = "/tmp/test_csv_data.csv" text_importer_data.start_import_row = 2 - - text_importer_data.comma_as_delimiter = True - text_importer_data.semicolon_as_delimiter = False - text_importer_data.space_as_delimiter = False - text_importer_data.tab_as_delimiter = False - text_importer_data.consecutive_delimiters = False - + text_importer_data.delimiters = [','] text_importer_data.custom_headers = [] text_importer_data.data_types = [cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.int32 ] text_importer_data.skipped_array_mask = [False,False,False,False,False,False,False ] diff --git a/wrapping/python/examples/import_text.py b/wrapping/python/examples/import_text.py index a1c9fd4f0e..de5bd0f9bb 100644 --- a/wrapping/python/examples/import_text.py +++ b/wrapping/python/examples/import_text.py @@ -10,13 +10,7 @@ text_importer_data = cx.TextImporterData() text_importer_data.input_file_path = "wrapping/python/examples/test_csv_data.csv" text_importer_data.start_import_row = 2 - -text_importer_data.comma_as_delimiter = True -text_importer_data.semicolon_as_delimiter = False -text_importer_data.space_as_delimiter = False -text_importer_data.tab_as_delimiter = False -text_importer_data.consecutive_delimiters = False - +text_importer_data.delimiters = [','] text_importer_data.custom_headers = [] text_importer_data.data_types = [cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.int32 ] text_importer_data.skipped_array_mask = [False,False,False,False,False,False,False ]