Last Changes. Fixed some preflight bugs

Signed-off-by: Michael Jackson <[email protected]>

src/Plugins/ComplexCore/docs/ReadCSVFileFilter.md

@@ -14,45 +14,45 @@ This **Filter** reads text data from any text-based file and imports the data in
 
 The user first selects the **Input Text File** path, which then enables the rest of the interface.
 
-![Input Text File Field](Images/Import_Text_1.png)
+![Input Text File Field](Images/Read_CSV_1.png)
 
 If the chosen **Input Text File** already has headers inside the file, the user can select the **Input File Has Headers** checkbox.  This
 enables the **Headers Line Number** spin box where the user can select which line of the file contains the headers.
 
 *NOTE*: The interface only allows importing data starting at the line after the chosen **Headers Line Number**.  So, in the example below, the **Headers Line Number** is set to 1, so **Start Import Line Number** defaults to 2 and has a range of 2-297 (this particular input file has 297 total lines).  The max range of **Headers Line Number** is, of course, set to 296 so that at least 1 line can be imported.
 
-![Input Text File Field](Images/Import_Text_2.png)
+![Input Text File Field](Images/Read_CSV_2.png)
 
 The user can choose how the data is delimited: comma (,), tab, semicolon (;) or space ( ). The user may also elect to ignore consecutive delimiters, which treats consecutive delimiters as one delimiter.
 
-![Input Text File Field](Images/Import_Text_3.png)
+![Input Text File Field](Images/Read_CSV_3.png)
 
 The user can select the number of preview lines available by changing the **Number of Preview Lines** spin box.  The range in the example is set to 1-296 because the import is currently starting at row 2 (from **Start Import Line Number** spin box).
 
-![Input Text File Field](Images/Import_Text_4.png)
+![Input Text File Field](Images/Read_CSV_4.png)
 
 The user can then set the data format for each column.  Selecting one or more columns will enable the **Column Data Type** combo box, where you can choose a data type or decide to skip importing specific columns as well.
 
-![Input Text File Field](Images/Import_Text_5.png)
-![Input Text File Field](Images/Import_Text_6.png)
+![Input Text File Field](Images/Read_CSV_5.png)
+![Input Text File Field](Images/Read_CSV_6.png)
 
 If the **Input File Has Headers** checkbox is OFF, then it is also possible to double-click the headers in the Preview Table to edit them.  These values will be used as the name of the **Data Array** in DREAM3D-NX.
 
 *NOTE:* Editing table headers is only available when the **Input File Has Headers** checkbox is OFF.  If the **Input File Has Headers** checkbox is ON, then the headers will be read from the **Headers Line Number** in the data file, and the table headers will not be editable.
 
-![Input Text File Field](Images/Import_Text_7.png)
+![Input Text File Field](Images/Read_CSV_7.png)
 
 The user can select the tuple dimensions that will be applied to the imported arrays.
 
-![Input Text File Field](Images/Import_Text_8.png)
+![Input Text File Field](Images/Read_CSV_8.png)
 
 The imported arrays can be stored in either an existing attribute matrix or a new attribute matrix can be created.
 
-![Input Text File Field](Images/Import_Text_9.png)
+![Input Text File Field](Images/Read_CSV_9.png)
 
 Afterwards, you end up with a data structure that looks like this:
 
-![Input Text File Field](Images/Import_Text_10.png)
+![Input Text File Field](Images/Read_CSV_10.png)
 
 ## Parameters ##
 
@@ -70,7 +70,7 @@ Not Applicable
 ## Required Objects ##
 
 | Kind | Default Name | Description |
-|------|--------------|------|----------------------|-------------|
+|------|--------------|------|
 | **Attribute Matrix** | None | The existing attribute matrix to store the imported data arrays (only if Existing Attribute Matrix is turned ON) |
 
 ## Created Objects ##

src/Plugins/ComplexCore/src/ComplexCore/Filters/ReadCSVFileFilter.cpp

@@ -560,7 +560,7 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d
     std::string errMsg = fmt::format("Error: The current tuple dimensions ({}) has 0 total tuples.  At least 1 tuple is required.", tupleDimsStr, tupleTotal, totalImportedLines);
     return {MakeErrorResult<OutputActions>(to_underlying(IssueCodes::INCORRECT_TUPLES), errMsg), {}};
   }
-  else if(tupleTotal > totalImportedLines)
+  else if(tupleTotal > totalImportedLines && !useExistingAM)
   {
     std::string tupleDimsStr = tupleDimsToString(readCSVData.tupleDims);
     std::string errMsg = fmt::format("Error: The current tuple dimensions ({}) has {} total tuples, but this is larger than the total number of available lines to import ({}).", tupleDimsStr,
@@ -597,9 +597,11 @@ IFilter::PreflightResult ReadCSVFileFilter::preflightImpl(const DataStructure& d
   {
     const AttributeMatrix& am = dataStructure.getDataRefAs<AttributeMatrix>(groupPath);
     tupleDims = am.getShape();
-    std::string tupleDimsStr = tupleDimsToString(readCSVData.tupleDims);
-    std::string tupleDimsStr2 = tupleDimsToString(tupleDims);
-    std::string msg = fmt::format("The Array Tuple Dimensions ({}) will be ignored and the Existing Attribute Matrix tuple dimensions ({}) will be used instead.", tupleDimsStr, tupleDimsStr2);
+
+    auto totalLinesRead = std::accumulate(tupleDims.begin(), tupleDims.end(), 1UL, std::multiplies<>());
+
+    std::string msg = fmt::format("The Array Tuple Dimensions ({}) will be ignored and the Existing Attribute Matrix tuple dimensions ({}) will be used. The total number of lines read will be {}.",
+                                  fmt::join(readCSVData.tupleDims, "x"), fmt::join(tupleDims, "x"), totalLinesRead);
     resultOutputActions.warnings().push_back(Warning{to_underlying(IssueCodes::IGNORED_TUPLE_DIMS), msg});
   }
 

src/Plugins/ComplexCore/wrapping/python/complexpy.cpp

@@ -438,7 +438,7 @@ PYBIND11_MODULE(complex, mod)
   readCSVData.def_readwrite("input_file_path", &ReadCSVData::inputFilePath);
   readCSVData.def_readwrite("custom_headers", &ReadCSVData::customHeaders);
   readCSVData.def_readwrite("start_import_row", &ReadCSVData::startImportRow);
-  readCSVData.def_readwrite("data_types", &ReadCSVData::dataTypes);
+  readCSVData.def_readwrite("column_data_types", &ReadCSVData::dataTypes);
   readCSVData.def_readwrite("skipped_array_mask", &ReadCSVData::skippedArrayMask);
   readCSVData.def_readwrite("headers_line", &ReadCSVData::headersLine);
   readCSVData.def_readwrite("header_mode", &ReadCSVData::headerMode);

src/complex/Common/TypesUtility.hpp

@@ -260,6 +260,63 @@ inline const std::vector<std::string>& GetAllDataTypesAsStrings()
   return dataTypes;
 }
 
+inline constexpr StringLiteral DataTypeToHumanString(DataType dataType)
+{
+  switch(dataType)
+  {
+  case DataType::int8: {
+    return "signed int 8 bit";
+  }
+  case DataType::uint8: {
+    return "unsigned int 8 bit";
+  }
+  case DataType::int16: {
+    return "signed int 16 bit";
+  }
+  case DataType::uint16: {
+    return "unsigned int 16 bit";
+  }
+  case DataType::int32: {
+    return "signed int 32 bit";
+  }
+  case DataType::uint32: {
+    return "unsigned int 32 bit";
+  }
+  case DataType::int64: {
+    return "signed int 64 bit";
+  }
+  case DataType::uint64: {
+    return "unsigned int 64 bit";
+  }
+  case DataType::float32: {
+    return "float 32";
+  }
+  case DataType::float64: {
+    return "double 64";
+  }
+  case DataType::boolean: {
+    return "boolean";
+  }
+  default:
+    throw std::runtime_error("complex::DataTypeToString: Unknown DataType");
+  }
+}
+
+/**
+ *
+ * @param humanReadable Strings that would be good for a User interface
+ * @return
+ */
+inline const std::vector<std::string>& GetAllDataTypesAsHumanStrings()
+{
+  static const std::vector<std::string> dataTypes = {
+      DataTypeToHumanString(complex::DataType::int8),    DataTypeToHumanString(complex::DataType::uint8),  DataTypeToHumanString(complex::DataType::int16),
+      DataTypeToHumanString(complex::DataType::uint16),  DataTypeToHumanString(complex::DataType::int32),  DataTypeToHumanString(complex::DataType::uint32),
+      DataTypeToHumanString(complex::DataType::int64),   DataTypeToHumanString(complex::DataType::uint64), DataTypeToHumanString(complex::DataType::float32),
+      DataTypeToHumanString(complex::DataType::float64), DataTypeToHumanString(complex::DataType::boolean)};
+  return dataTypes;
+}
+
 /**
  * @brief Returns a DataType for the passed in string representation
  * @param dataTypeString

wrapping/python/docs/source/API.rst

@@ -391,7 +391,7 @@ General Parameters
    column of data is a single array. 
 
    + The file can be comma, space, tab or semicolon separated.
-   + The file optionally can have a line of headers. The user can specify what line the headers are on
+   + The file optionally can have a line of headers. The user can specify what line number the header is located.
    + The import can start at a user specified line number but will continue to the end of the file.
 
    The primary python object that will hold the information to pass to the filter is the ReadCSVData class described below.
@@ -419,16 +419,15 @@ General Parameters
 .. code:: python
 
    data_structure = cx.DataStructure()
-
+   # Example File has 7 columns to import
    read_csv_data = cx.ReadCSVData()
    read_csv_data.input_file_path = "/tmp/test_csv_data.csv"
    read_csv_data.start_import_row = 2
    read_csv_data.delimiters = [',']
    read_csv_data.custom_headers = []
-   read_csv_data.data_types = [cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.int32 ]
+   read_csv_data.column_data_types = [cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.int32 ]
    read_csv_data.skipped_array_mask = [False,False,False,False,False,False,False ]
    read_csv_data.tuple_dims = [37989]
-
    read_csv_data.headers_line = 1
    read_csv_data.header_mode = cx.ReadCSVData.HeaderMode.Line
 

wrapping/python/examples/read_csv_file.py

@@ -6,16 +6,15 @@
 
 # Create the DataStructure object
 data_structure = cx.DataStructure()
-
+# This file has 7 columns to import
 read_csv_data = cx.ReadCSVData()
 read_csv_data.input_file_path = "wrapping/python/examples/test_csv_data.csv"
 read_csv_data.start_import_row = 2
 read_csv_data.delimiters = [',']
 read_csv_data.custom_headers = []
-read_csv_data.data_types = [cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.int32 ]
+read_csv_data.column_data_types = [cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.float32,cx.DataType.int32 ]
 read_csv_data.skipped_array_mask = [False,False,False,False,False,False,False ]
 read_csv_data.tuple_dims = [37989]
-
 read_csv_data.headers_line = 1
 read_csv_data.header_mode = cx.ReadCSVData.HeaderMode.Line