ENH/BUG: Mask Option Type Clean-Up and Out-of-Core Compatibility Upda…

…te (#978)

Any filter that needs or optionally needs a "Mask" array can now use either a bool or uint8 array.

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/AlignSectionsMutualInformation.cpp

@@ -65,7 +65,7 @@ Result<> AlignSectionsMutualInformation::findShifts(std::vector<int64>& xShifts,
 {
   const auto& imageGeom = m_DataStructure.getDataRefAs<ImageGeom>(m_InputValues->ImageGeometryPath);
   const AttributeMatrix* cellData = imageGeom.getCellData();
-  int64 totalPoints = static_cast<int64>(cellData->getNumTuples());
+  auto totalPoints = static_cast<int64>(cellData->getNumTuples());
 
   std::ofstream outFile;
   if(m_InputValues->WriteAlignmentShifts)
@@ -86,6 +86,20 @@ Result<> AlignSectionsMutualInformation::findShifts(std::vector<int64>& xShifts,
     }
   }
 
+  if(m_InputValues->UseMask)
+  {
+    try
+    {
+      m_MaskCompare = InstantiateMaskCompare(m_DataStructure, m_InputValues->MaskArrayPath);
+    } catch(const std::out_of_range& exception)
+    {
+      // This really should NOT be happening as the path was verified during preflight BUT we may be calling this from
+      // somewhere else that is NOT going through the normal nx::core::IFilter API of Preflight and Execute
+      std::string message = fmt::format("Mask Array DataPath does not exist or is not of the correct type (Bool | UInt8) {}", m_InputValues->MaskArrayPath.toString());
+      return MakeErrorResult(-53702, message);
+    }
+  }
+
   SizeVec3 udims = imageGeom.getDimensions();
   int64 dims[3] = {
       static_cast<int64>(udims[0]),
@@ -246,10 +260,9 @@ void AlignSectionsMutualInformation::formFeaturesSections(std::vector<int32>& mi
 
   auto orientationOps = LaueOps::GetAllOrientationOps();
 
-  Float32Array& quats = m_DataStructure.getDataRefAs<Float32Array>(m_InputValues->QuatsArrayPath);
-  BoolArray* goodVoxelsPtr = m_DataStructure.getDataAs<BoolArray>(m_InputValues->MaskArrayPath);
-  Int32Array& m_CellPhases = m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->CellPhasesArrayPath);
-  UInt32Array& m_CrystalStructures = m_DataStructure.getDataRefAs<UInt32Array>(m_InputValues->CrystalStructuresArrayPath);
+  auto& quats = m_DataStructure.getDataRefAs<Float32Array>(m_InputValues->QuatsArrayPath);
+  auto& m_CellPhases = m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->CellPhasesArrayPath);
+  auto& m_CrystalStructures = m_DataStructure.getDataRefAs<UInt32Array>(m_InputValues->CrystalStructuresArrayPath);
 
   size_t initialVoxelsListSize = 1000;
 
@@ -276,7 +289,7 @@ void AlignSectionsMutualInformation::formFeaturesSections(std::vector<int32>& mi
 
       for(int64 point = currentStartPoint; point < endPoint; point++)
       {
-        if((!m_InputValues->UseMask || (goodVoxelsPtr != nullptr && (*goodVoxelsPtr)[point])) && miFeatureIds[point] == 0 && m_CellPhases[point] > 0)
+        if((!m_InputValues->UseMask || (m_MaskCompare != nullptr && m_MaskCompare->isTrue(point))) && miFeatureIds[point] == 0 && m_CellPhases[point] > 0)
         {
           seed = point;
           currentStartPoint = point;

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/AlignSectionsMutualInformation.hpp

@@ -9,10 +9,10 @@
 #include "simplnx/Parameters/FileSystemPathParameter.hpp"
 #include "simplnx/Parameters/NumberParameter.hpp"
 #include "simplnx/Utilities/AlignSections.hpp"
+#include "simplnx/Utilities/DataArrayUtilities.hpp"
 
 namespace nx::core
 {
-
 struct ORIENTATIONANALYSIS_EXPORT AlignSectionsMutualInformationInputValues
 {
   bool WriteAlignmentShifts;
@@ -56,6 +56,8 @@ class ORIENTATIONANALYSIS_EXPORT AlignSectionsMutualInformation : public AlignSe
   const std::atomic_bool& m_ShouldCancel;
   const IFilter::MessageHandler& m_MessageHandler;
   Result<> m_Result = {};
+
+  std::unique_ptr<MaskCompare> m_MaskCompare = nullptr;
 };
 
 } // namespace nx::core

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/BadDataNeighborOrientationCheck.hpp

@@ -6,20 +6,6 @@
 #include "simplnx/DataStructure/DataStructure.hpp"
 #include "simplnx/Filter/IFilter.hpp"
 
-/**
-* This is example code to put in the Execute Method of the filter.
-  BadDataNeighborOrientationCheckInputValues inputValues;
-
-  inputValues.MisorientationTolerance = filterArgs.value<float32>(k_MisorientationTolerance_Key);
-  inputValues.NumberOfNeighbors = filterArgs.value<int32>(k_NumberOfNeighbors_Key);
-  inputValues.QuatsArrayPath = filterArgs.value<DataPath>(k_QuatsArrayPath_Key);
-  inputValues.MaskArrayPath = filterArgs.value<DataPath>(k_MaskArrayPath_Key);
-  inputValues.CellPhasesArrayPath = filterArgs.value<DataPath>(k_CellPhasesArrayPath_Key);
-  inputValues.CrystalStructuresArrayPath = filterArgs.value<DataPath>(k_CrystalStructuresArrayPath_Key);
-
-  return BadDataNeighborOrientationCheck(dataStructure, messageHandler, shouldCancel, &inputValues)();
-*/
-
 namespace nx::core
 {
 

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/WriteStatsGenOdfAngleFile.cpp

@@ -47,10 +47,10 @@ Result<> WriteStatsGenOdfAngleFile::operator()()
   }
 
   const auto& cellPhases = m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->CellPhasesArrayPath);
-  BoolArray* maskPtr = nullptr;
+  std::unique_ptr<MaskCompare> maskPtr = nullptr;
   if(m_InputValues->UseMask)
   {
-    maskPtr = m_DataStructure.getDataAs<BoolArray>(m_InputValues->MaskArrayPath);
+    maskPtr = InstantiateMaskCompare(m_DataStructure, m_InputValues->MaskArrayPath);
   }
 
   // Figure out how many unique phase values we have by looping over all the phase values
@@ -100,14 +100,14 @@ Result<> WriteStatsGenOdfAngleFile::operator()()
 }
 
 // -----------------------------------------------------------------------------
-int WriteStatsGenOdfAngleFile::determineOutputLineCount(const Int32Array& cellPhases, const BoolArray* mask, usize totalPoints, int32 phase) const
+int WriteStatsGenOdfAngleFile::determineOutputLineCount(const Int32Array& cellPhases, const std::unique_ptr<MaskCompare>& mask, usize totalPoints, int32 phase) const
 {
   int32 lineCount = 0;
   for(usize i = 0; i < totalPoints; i++)
   {
     if(cellPhases[i] == phase)
     {
-      if(!m_InputValues->UseMask || (m_InputValues->UseMask && (*mask)[i]))
+      if(!m_InputValues->UseMask || (m_InputValues->UseMask && mask->isTrue(i)))
       {
         lineCount++;
       }
@@ -118,7 +118,7 @@ int WriteStatsGenOdfAngleFile::determineOutputLineCount(const Int32Array& cellPh
 }
 
 // -----------------------------------------------------------------------------
-Result<> WriteStatsGenOdfAngleFile::writeOutputFile(std::ofstream& out, const Int32Array& cellPhases, const BoolArray* mask, int32 lineCount, usize totalPoints, int32 phase) const
+Result<> WriteStatsGenOdfAngleFile::writeOutputFile(std::ofstream& out, const Int32Array& cellPhases, const std::unique_ptr<MaskCompare>& mask, int32 lineCount, usize totalPoints, int32 phase) const
 {
   const auto& eulerAngles = m_DataStructure.getDataRefAs<Float32Array>(m_InputValues->CellEulerAnglesArrayPath);
 
@@ -147,7 +147,7 @@ Result<> WriteStatsGenOdfAngleFile::writeOutputFile(std::ofstream& out, const In
 
     if(cellPhases[i] == phase)
     {
-      if(!m_InputValues->UseMask || (m_InputValues->UseMask && (*mask)[i]))
+      if(!m_InputValues->UseMask || (m_InputValues->UseMask && mask->isTrue(i)))
       {
         writeLine = true;
       }

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/WriteStatsGenOdfAngleFile.hpp

@@ -8,6 +8,8 @@
 #include "simplnx/Filter/IFilter.hpp"
 #include "simplnx/Parameters/ChoicesParameter.hpp"
 #include "simplnx/Parameters/FileSystemPathParameter.hpp"
+#include "simplnx/Utilities/DataArrayUtilities.hpp"
+
 namespace nx::core
 {
 
@@ -44,8 +46,8 @@ class ORIENTATIONANALYSIS_EXPORT WriteStatsGenOdfAngleFile
 
   const std::atomic_bool& getCancel();
 
-  int determineOutputLineCount(const Int32Array& cellPhases, const BoolArray* mask, usize totalPoints, int32 phase) const;
-  Result<> writeOutputFile(std::ofstream& out, const Int32Array& cellPhases, const BoolArray* mask, int32 lineCount, usize totalPoints, int32 phase) const;
+  int determineOutputLineCount(const Int32Array& cellPhases, const std::unique_ptr<MaskCompare>& mask, usize totalPoints, int32 phase) const;
+  Result<> writeOutputFile(std::ofstream& out, const Int32Array& cellPhases, const std::unique_ptr<MaskCompare>& mask, int32 lineCount, usize totalPoints, int32 phase) const;
 
 private:
   DataStructure& m_DataStructure;

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/AlignSectionsMutualInformationFilter.cpp

@@ -66,7 +66,7 @@ Parameters AlignSectionsMutualInformationFilter::parameters() const
   params.insertLinkableParameter(std::make_unique<BoolParameter>(k_UseMask_Key, "Use Mask Array", "Whether to remove some Cells from consideration in the alignment process.", true));
   params.insert(std::make_unique<ArraySelectionParameter>(k_MaskArrayPath_Key, "Cell Mask Array",
                                                           "Specifies if the Cell is to be counted in the algorithm. Only required if Use Mask Array is checked.", DataPath{},
-                                                          ArraySelectionParameter::AllowedTypes{DataType::boolean}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
+                                                          ArraySelectionParameter::AllowedTypes{DataType::boolean, DataType::uint8}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
   params.linkParameters(k_UseMask_Key, k_MaskArrayPath_Key, true);
 
   params.insertSeparator(Parameters::Separator{"Input Cell Data"});
@@ -102,10 +102,7 @@ IFilter::UniquePointer AlignSectionsMutualInformationFilter::clone() const
 IFilter::PreflightResult AlignSectionsMutualInformationFilter::preflightImpl(const DataStructure& dataStructure, const Arguments& filterArgs, const MessageHandler& messageHandler,
                                                                              const std::atomic_bool& shouldCancel) const
 {
-  auto pWriteAlignmentShiftsValue = filterArgs.value<bool>(k_WriteAlignmentShifts_Key);
   auto pAlignmentShiftFileNameValue = filterArgs.value<FileSystemPathParameter::ValueType>(k_AlignmentShiftFileName_Key);
-  auto pMisorientationToleranceValue = filterArgs.value<float32>(k_MisorientationTolerance_Key);
-  auto pUseGoodVoxelsValue = filterArgs.value<bool>(k_UseMask_Key);
   auto imageGeometryPath = filterArgs.value<DataPath>(k_SelectedImageGeometryPath_Key);
   auto pQuatsArrayPathValue = filterArgs.value<DataPath>(k_QuatsArrayPath_Key);
   auto pCellPhasesArrayPathValue = filterArgs.value<DataPath>(k_CellPhasesArrayPath_Key);
@@ -116,7 +113,7 @@ IFilter::PreflightResult AlignSectionsMutualInformationFilter::preflightImpl(con
   nx::core::Result<OutputActions> resultOutputActions;
   std::vector<PreflightValue> preflightUpdatedValues;
 
-  const ImageGeom& imageGeom = dataStructure.getDataRefAs<ImageGeom>(imageGeometryPath);
+  const auto& imageGeom = dataStructure.getDataRefAs<ImageGeom>(imageGeometryPath);
   if(imageGeom.getCellData() == nullptr)
   {
     return {MakeErrorResult<OutputActions>(-3540, fmt::format("Cannot find cell data Attribute Matrix in the selected Image geometry '{}'", imageGeometryPath.toString()))};

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/BadDataNeighborOrientationCheckFilter.cpp

@@ -71,7 +71,7 @@ Parameters BadDataNeighborOrientationCheckFilter::parameters() const
   params.insert(std::make_unique<ArraySelectionParameter>(k_QuatsArrayPath_Key, "Cell Quaternions", "Specifies the orientation of the Cell in quaternion representation", DataPath{},
                                                           ArraySelectionParameter::AllowedTypes{DataType::float32}, ArraySelectionParameter::AllowedComponentShapes{{4}}));
   params.insert(std::make_unique<ArraySelectionParameter>(k_MaskArrayPath_Key, "Cell Mask Array", "Used to define Cells as good or bad", DataPath{},
-                                                          ArraySelectionParameter::AllowedTypes{DataType::boolean}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
+                                                          ArraySelectionParameter::AllowedTypes{DataType::boolean, DataType::uint8}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
   params.insert(std::make_unique<ArraySelectionParameter>(k_CellPhasesArrayPath_Key, "Cell Phases", "Specifies to which Ensemble each Cell belongs", DataPath({"Phases"}),
                                                           ArraySelectionParameter::AllowedTypes{DataType::int32}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
 
@@ -119,15 +119,11 @@ IFilter::PreflightResult BadDataNeighborOrientationCheckFilter::preflightImpl(co
   {
     return {nonstd::make_unexpected(std::vector<Error>{Error{k_MissingInputArray, fmt::format("Could not find mask array at path '{}'", pGoodVoxelsArrayPathValue.toString())}})};
   }
-  auto* goodVoxelsBoolPtr = dataStructure.getDataAs<BoolArray>(pGoodVoxelsArrayPathValue);
-  if(nullptr == goodVoxelsBoolPtr)
-  {
-    return {nonstd::make_unexpected(
-        std::vector<Error>{Error{k_IncorrectInputArray, fmt::format("Mask array at path '{}' is not of the correct type. It must be Bool.", pGoodVoxelsArrayPathValue.toString())}})};
-  }
+
+  auto* goodVoxelsBoolPtr = dataStructure.getDataAs<IDataArray>(pGoodVoxelsArrayPathValue);
   if(goodVoxelsBoolPtr->getNumberOfComponents() != 1)
   {
-    return {nonstd::make_unexpected(std::vector<Error>{Error{k_IncorrectInputArray, "Mask Input Array must be a 1 component Bool array"}})};
+    return {nonstd::make_unexpected(std::vector<Error>{Error{k_IncorrectInputArray, "Mask Input Array must be a 1 component array"}})};
   }
   dataArrayPaths.push_back(pGoodVoxelsArrayPathValue);
 

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/CAxisSegmentFeaturesFilter.cpp

@@ -66,7 +66,7 @@ Parameters CAxisSegmentFeaturesFilter::parameters() const
   params.insertLinkableParameter(
       std::make_unique<BoolParameter>(k_UseMask_Key, "Use Mask Array", "Specifies whether to use a boolean array to exclude some Cells from the Feature identification process", true));
   params.insert(std::make_unique<ArraySelectionParameter>(k_MaskArrayPath_Key, "Cell Mask Array", "Specifies if the Cell is to be counted in the algorithm. Only required if Use Mask Array is checked",
-                                                          DataPath{}, ArraySelectionParameter::AllowedTypes{DataType::boolean}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
+                                                          DataPath{}, ArraySelectionParameter::AllowedTypes{DataType::boolean, DataType::uint8}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
 
   params.insertSeparator(Parameters::Separator{"Input Cell Data"});
   params.insert(std::make_unique<DataObjectNameParameter>(k_FeatureIdsArrayName_Key, "Cell Feature Ids", "Specifies to which Feature each Cell belongs", "FeatureIds"));

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteStatsGenOdfAngleFileFilter.cpp

@@ -70,7 +70,7 @@ Parameters WriteStatsGenOdfAngleFileFilter::parameters() const
   params.insertLinkableParameter(
       std::make_unique<BoolParameter>(k_UseMask_Key, "Only Write Good Elements", "Whether to only write the Euler angles for those elements denoted as true in the supplied mask array", false));
   params.insert(std::make_unique<ArraySelectionParameter>(k_MaskArrayPath_Key, "Mask Array", "Used to define Elements as good or bad. Only required if Only Write Good Elements is checked", DataPath{},
-                                                          ArraySelectionParameter::AllowedTypes{DataType::boolean}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
+                                                          ArraySelectionParameter::AllowedTypes{DataType::boolean, DataType::uint8}, ArraySelectionParameter::AllowedComponentShapes{{1}}));
 
   params.insertSeparator(Parameters::Separator{"Input Cell Data"});
   params.insert(std::make_unique<ArraySelectionParameter>(k_CellEulerAnglesArrayPath_Key, "Euler Angles", "Three angles defining the orientation of the Element in Bunge convention (Z-X-Z)",

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeans.cpp

@@ -1,6 +1,7 @@
 #include "ComputeKMeans.hpp"
 
 #include "simplnx/DataStructure/DataArray.hpp"
+#include "simplnx/Utilities/DataArrayUtilities.hpp"
 #include "simplnx/Utilities/FilterUtilities.hpp"
 #include "simplnx/Utilities/KUtilities.hpp"
 
@@ -14,7 +15,7 @@ template <typename T>
 class ComputeKMeansTemplate
 {
 public:
-  ComputeKMeansTemplate(ComputeKMeans* filter, const IDataArray& inputIDataArray, IDataArray& meansIDataArray, const BoolArray& maskDataArray, usize numClusters, Int32Array& fIds,
+  ComputeKMeansTemplate(ComputeKMeans* filter, const IDataArray& inputIDataArray, IDataArray& meansIDataArray, const std::unique_ptr<MaskCompare>& maskDataArray, usize numClusters, Int32Array& fIds,
                         KUtilities::DistanceMetric distMetric, std::mt19937_64::result_type seed)
   : m_Filter(filter)
   , m_InputArray(dynamic_cast<const DataArrayT&>(inputIDataArray))
@@ -48,7 +49,7 @@ class ComputeKMeansTemplate
     while(clusterChoices < m_NumClusters)
     {
       usize index = std::floor(dist(gen) * static_cast<float64>(rangeMax));
-      if(m_Mask[index])
+      if(m_Mask->isTrue(index))
       {
         clusterIdxs[clusterChoices] = index;
         clusterChoices++;
@@ -103,7 +104,7 @@ class ComputeKMeansTemplate
   ComputeKMeans* m_Filter;
   const DataArrayT& m_InputArray;
   DataArrayT& m_Means;
-  const BoolArray& m_Mask;
+  const std::unique_ptr<MaskCompare>& m_Mask;
   usize m_NumClusters;
   Int32Array& m_FeatureIds;
   KUtilities::DistanceMetric m_DistMetric;
@@ -125,7 +126,7 @@ class ComputeKMeansTemplate
       {
         return;
       }
-      if(m_Mask[i])
+      if(m_Mask->isTrue(i))
       {
         float64 minDist = std::numeric_limits<float64>::max();
         for(int32 j = 0; j < m_NumClusters; j++)
@@ -207,9 +208,22 @@ const std::atomic_bool& ComputeKMeans::getCancel()
 Result<> ComputeKMeans::operator()()
 {
   auto& clusteringArray = m_DataStructure.getDataRefAs<IDataArray>(m_InputValues->ClusteringArrayPath);
+
+  std::unique_ptr<MaskCompare> maskCompare;
+  try
+  {
+    maskCompare = InstantiateMaskCompare(m_DataStructure, m_InputValues->MaskArrayPath);
+  } catch(const std::out_of_range& exception)
+  {
+    // This really should NOT be happening as the path was verified during preflight BUT we may be calling this from
+    // somewhere else that is NOT going through the normal nx::core::IFilter API of Preflight and Execute
+    std::string message = fmt::format("Mask Array DataPath does not exist or is not of the correct type (Bool | UInt8) {}", m_InputValues->MaskArrayPath.toString());
+    return MakeErrorResult(-54060, message);
+  }
+
   RunTemplateClass<ComputeKMeansTemplate, types::NoBooleanType>(clusteringArray.getDataType(), this, clusteringArray, m_DataStructure.getDataRefAs<IDataArray>(m_InputValues->MeansArrayPath),
-                                                                m_DataStructure.getDataRefAs<BoolArray>(m_InputValues->MaskArrayPath), m_InputValues->InitClusters,
-                                                                m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->FeatureIdsArrayPath), m_InputValues->DistanceMetric, m_InputValues->Seed);
+                                                                maskCompare, m_InputValues->InitClusters, m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->FeatureIdsArrayPath),
+                                                                m_InputValues->DistanceMetric, m_InputValues->Seed);
 
   return {};
 }