ENH: Enable Out-of-Core functionality (#703)

* Enable Out-of-Core * Update filters and unit tests to enable out-of-core data. * Update Application to require static methods for creation and destruction. * Update ParallelAlgorithm classes for disabling onout-of-core data. * Add memory utilities for checking available and total memory. * Moved out of core check in ITK filters to execute helper function * Added out-of-core array checks. * Fixed ConvertColorToGrayScale parallelization code for out-of-core datastores. --------- Signed-off-by: Jared Duffey <[email protected]> Signed-off-by: Michael Jackson <[email protected]> Co-authored-by: Jared Duffey <[email protected]> Co-authored-by: Michael Jackson <[email protected]>
BlueQuartzSoftware · Nov 10, 2023 · 63c80dc · 63c80dc
1 parent 202f73f
commit 63c80dc
Show file tree

Hide file tree

Showing 144 changed files with 1,422 additions and 534 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -465,7 +465,9 @@ set(COMPLEX_HDRS
   ${COMPLEX_SOURCE_DIR}/Utilities/FilterUtilities.hpp
   ${COMPLEX_SOURCE_DIR}/Utilities/GeometryUtilities.hpp
   ${COMPLEX_SOURCE_DIR}/Utilities/GeometryHelpers.hpp
+  ${COMPLEX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
   ${COMPLEX_SOURCE_DIR}/Utilities/StringUtilities.hpp
+  ${COMPLEX_SOURCE_DIR}/Utilities/IParallelAlgorithm.hpp
   ${COMPLEX_SOURCE_DIR}/Utilities/ParallelDataAlgorithm.hpp
   ${COMPLEX_SOURCE_DIR}/Utilities/ParallelData2DAlgorithm.hpp
   ${COMPLEX_SOURCE_DIR}/Utilities/ParallelData3DAlgorithm.hpp
@@ -661,6 +663,8 @@ set(COMPLEX_SRCS
   ${COMPLEX_SOURCE_DIR}/Utilities/TooltipRowItem.cpp
   ${COMPLEX_SOURCE_DIR}/Utilities/DataArrayUtilities.cpp
   ${COMPLEX_SOURCE_DIR}/Utilities/DataGroupUtilities.cpp
+  ${COMPLEX_SOURCE_DIR}/Utilities/MemoryUtilities.cpp
+  ${COMPLEX_SOURCE_DIR}/Utilities/IParallelAlgorithm.cpp
   ${COMPLEX_SOURCE_DIR}/Utilities/ParallelDataAlgorithm.cpp
   ${COMPLEX_SOURCE_DIR}/Utilities/ParallelData2DAlgorithm.cpp
   ${COMPLEX_SOURCE_DIR}/Utilities/ParallelData3DAlgorithm.cpp

diff --git a/docs/Porting_Filters.md b/docs/Porting_Filters.md
@@ -78,15 +78,22 @@ Example of getting an array and summing the values using range based for loop.
 
 ```c++
  // Let's sum up all the areas.
-    Float64Array& faceAreas = dataGraph.getDataRefAs<Float64Array>(triangleAreasDataPath);
+    Float64Array& faceAreasArray = dataGraph.getDataRefAs<Float64Array>(triangleAreasDataPath);
+    AbstractFloat64DataStore& faceAreas = faceAreasArray.getDataStoreRef();
     double sumOfAreas = 0.0;
     for(const auto& area : faceAreas)
     {
       sumOfAreas += area;
     }
 ```
 
-## Chaining Together DataPath + String to form new DataPath
+## DataArray Performance
+
+* When iterating over values, either to read or write, use the reference returned by DataArray<T>::getDataStoreRef().
+* When writing values in a multi-threaded function, use the getValue and setValue methods in AbstractDataStore to ensure that values being both read and written at the same time. The [] operators are not capable of protecting against data corruption.
+  * In situation where values are only being read from the array, the [] operators are both safe and faster to use.
+
+## Chaining Together DataPath + String to form new DataPath ##
 
 ```c++
     DataPath triangleAreasDataPath = geometryPath.createChildPath(triangleFaceDataGroupName).createChildPath("Triangle Areas");

diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ComputeFeatureRect.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ComputeFeatureRect.cpp
@@ -36,7 +36,7 @@ Result<> ComputeFeatureRect::operator()()
 {
   const auto& featureIds = m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->FeatureIdsArrayPath);
   auto& corners = m_DataStructure.getDataRefAs<UInt32Array>(m_InputValues->FeatureRectArrayPath);
-  auto& cornersDataStore = corners.getIDataStoreRefAs<UInt32DataStore>();
+  auto& cornersDataStore = corners.getDataStoreRef();
 
   // Create corners array, which stores pixel coordinates for the top-left and bottom-right coordinates of each feature object
   for(usize i = 0; i < corners.getNumberOfTuples(); i++)

diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertColorToGrayScale.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertColorToGrayScale.cpp
@@ -2,6 +2,8 @@
 
 #include "complex/Common/Array.hpp"
 #include "complex/Common/Range.hpp"
+#include "complex/Core/Application.hpp"
+#include "complex/Core/Preferences.hpp"
 #include "complex/DataStructure/DataArray.hpp"
 #include "complex/DataStructure/DataGroup.hpp"
 #include "complex/Utilities/ParallelDataAlgorithm.hpp"
@@ -35,8 +37,8 @@ class LuminosityImpl
     for(size_t i = start; i < end; i++)
     {
       auto temp = static_cast<int32>(
-          roundf((m_ImageData[m_NumComp * i] * m_ColorWeights.getX()) + (m_ImageData[m_NumComp * i + 1] * m_ColorWeights.getY()) + (m_ImageData[m_NumComp * i + 2] * m_ColorWeights.getZ())));
-      m_FlatImageData[i] = static_cast<uint8>(temp);
+          roundf((m_ImageData.at(m_NumComp * i) * m_ColorWeights.getX()) + (m_ImageData.at(m_NumComp * i + 1) * m_ColorWeights.getY()) + (m_ImageData.at(m_NumComp * i + 2) * m_ColorWeights.getZ())));
+      m_FlatImageData.setValue(i, static_cast<uint8>(temp));
     }
   }
 
@@ -72,7 +74,7 @@ class LightnessImpl
     for(size_t i = start; i < end; i++)
     {
       auto minmax = std::minmax_element(m_ImageData.begin() + (i * m_NumComp), m_ImageData.begin() + (i * m_NumComp + 3));
-      m_FlatImageData[i] = static_cast<uint8_t>(roundf(static_cast<float>(static_cast<int16_t>(*(minmax.first)) + static_cast<int16_t>(*(minmax.second))) / 2.0f));
+      m_FlatImageData.setValue(i, static_cast<uint8_t>(roundf(static_cast<float>(static_cast<int16_t>(*(minmax.first)) + static_cast<int16_t>(*(minmax.second))) / 2.0f)));
     }
   }
 
@@ -108,7 +110,7 @@ class SingleChannelImpl
   {
     for(size_t i = start; i < end; i++)
     {
-      m_FlatImageData[i] = static_cast<uint8_t>((m_ImageData[m_NumComp * i + static_cast<size_t>(m_Channel)]));
+      m_FlatImageData.setValue(i, static_cast<uint8_t>((m_ImageData.at(m_NumComp * i + static_cast<size_t>(m_Channel)))));
     }
   }
 
@@ -134,10 +136,11 @@ class ParallelWrapper
   ParallelWrapper& operator=(ParallelWrapper&&) = delete;      // Move Assignment Not Implemented
 
   template <typename T>
-  static void Run(T impl, size_t totalPoints)
+  static void Run(T impl, size_t totalPoints, typename IParallelAlgorithm::AlgorithmArrays algArrays)
   {
     ParallelDataAlgorithm dataAlg;
     dataAlg.setRange(0, totalPoints);
+    dataAlg.requireArraysInMemory(algArrays);
     dataAlg.execute(impl);
   }
 
@@ -185,19 +188,24 @@ Result<> ConvertColorToGrayScale::operator()()
     size_t comp = inputColorData.getNumberOfComponents();
     size_t totalPoints = inputColorData.getNumberOfTuples();
 
+    typename IParallelAlgorithm::AlgorithmArrays algArrays;
+    algArrays.push_back(&inputColorData);
+    algArrays.push_back(&outputGrayData);
+
     switch(convType)
     {
     case ConversionType::Luminosity:
-      ParallelWrapper::Run<LuminosityImpl>(LuminosityImpl(inputColorData, outputGrayData, m_InputValues->ColorWeights, comp), totalPoints);
+      ParallelWrapper::Run<LuminosityImpl>(LuminosityImpl(inputColorData, outputGrayData, m_InputValues->ColorWeights, comp), totalPoints, algArrays);
       break;
     case ConversionType::Average:
-      ParallelWrapper::Run<LuminosityImpl>(LuminosityImpl(inputColorData, outputGrayData, {0.3333f, 0.3333f, 0.3333f}, comp), totalPoints);
+      ParallelWrapper::Run<LuminosityImpl>(LuminosityImpl(inputColorData, outputGrayData, {0.3333f, 0.3333f, 0.3333f}, comp), totalPoints, algArrays);
       break;
-    case ConversionType::Lightness:
-      ParallelWrapper::Run<LightnessImpl>(LightnessImpl(inputColorData, outputGrayData, comp), totalPoints);
+    case ConversionType::Lightness: {
+      ParallelWrapper::Run<LightnessImpl>(LightnessImpl(inputColorData, outputGrayData, comp), totalPoints, algArrays);
       break;
+    }
     case ConversionType::SingleChannel:
-      ParallelWrapper::Run<SingleChannelImpl>(SingleChannelImpl(inputColorData, outputGrayData, comp, m_InputValues->ColorChannel), totalPoints);
+      ParallelWrapper::Run<SingleChannelImpl>(SingleChannelImpl(inputColorData, outputGrayData, comp, m_InputValues->ColorChannel), totalPoints, algArrays);
       break;
     }
   }

diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertData.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertData.cpp
@@ -2,89 +2,128 @@
 
 #include "complex/DataStructure/DataArray.hpp"
 #include "complex/DataStructure/DataGroup.hpp"
+#include "complex/Utilities/ParallelDataAlgorithm.hpp"
 
 using namespace complex;
 
 namespace Detail
 {
 template <typename O, typename D>
-void ConvertData(DataArray<O>& inputArray, DataArray<D>& outputArray)
+class ConvertDataAlg
 {
-  size_t size = inputArray.getSize();
-  for(size_t v = 0; v < size; ++v)
+public:
+  using InputStoreType = AbstractDataStore<O>;
+  using OutputStoreType = AbstractDataStore<D>;
+
+  ConvertDataAlg(InputStoreType& inputStore, OutputStoreType& outputStore)
+  : m_InputStore(inputStore)
+  , m_OutputStore(outputStore)
   {
-    if constexpr(std::is_same<O, D>::value)
-    {
-      // inputArray and destination arrays have the same type
-      outputArray[v] = inputArray[v];
-    }
-    else if constexpr(std::is_same<O, bool>::value)
-    {
-      // inputArray array is a boolean array
-      outputArray[v] = (inputArray[v] ? 1 : 0);
-    }
-    else if constexpr(std::is_same<D, bool>::value)
-    {
-      // Destination array is a boolean array
-      outputArray[v] = (inputArray[v] != 0);
-    }
-    else
+  }
+
+  void convert(size_t start, size_t end) const
+  {
+    // std::cout << "\tConvertDataAlg: " << start << " to " << end << std::endl;
+
+    for(size_t v = start; v < end; ++v)
     {
-      // All other cases
-      outputArray[v] = static_cast<D>(inputArray[v]);
+      // std::cout << "\tConvertDataAlg: index " << v << std::endl;
+
+      if constexpr(std::is_same<O, D>::value)
+      {
+        // inputArray and destination arrays have the same type
+        m_OutputStore.setValue(v, m_InputStore.getValue(v));
+      }
+      else if constexpr(std::is_same<O, bool>::value)
+      {
+        // inputArray array is a boolean array
+        m_OutputStore.setValue(v, (m_InputStore.getValue(v) ? 1 : 0));
+      }
+      else if constexpr(std::is_same<D, bool>::value)
+      {
+        // Destination array is a boolean array
+        m_OutputStore.setValue(v, (m_InputStore.getValue(v) != 0));
+      }
+      else
+      {
+        // All other cases
+        m_OutputStore.setValue(v, static_cast<D>(m_InputStore.getValue(v)));
+      }
     }
   }
-}
+
+  /**
+   * @brief operator () This is called from the TBB stye of code
+   * @param range The range to compute the values
+   */
+  void operator()(const Range& range) const
+  {
+    convert(range.min(), range.max());
+  }
+
+private:
+  InputStoreType& m_InputStore;
+  OutputStoreType& m_OutputStore;
+};
 
 template <typename T>
 Result<> ConvertData(DataStructure& dataStructure, const ConvertDataInputValues* inputValues)
 {
   DataArray<T>& inputArray = dataStructure.getDataRefAs<DataArray<T>>(inputValues->SelectedArrayPath);
+  AbstractDataStore<T>& inputStore = inputArray.getDataStoreRef();
+
+  typename IParallelAlgorithm::AlgorithmArrays algArrays;
+  algArrays.push_back(&inputArray);
+  algArrays.push_back(dataStructure.getDataAs<IDataArray>(inputValues->OutputArrayName));
+
+  ParallelDataAlgorithm dataAlg;
+  dataAlg.setRange(0, inputArray.size());
+  dataAlg.requireArraysInMemory(algArrays);
 
   switch(inputValues->ScalarType)
   {
   case DataType::int8: {
-    ConvertData<T, int8>(inputArray, dataStructure.getDataRefAs<Int8Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, int8>(inputStore, dataStructure.getDataRefAs<Int8Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::uint8: {
-    ConvertData<T, uint8>(inputArray, dataStructure.getDataRefAs<UInt8Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, uint8>(inputStore, dataStructure.getDataRefAs<UInt8Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::int16: {
-    ConvertData<T, int16>(inputArray, dataStructure.getDataRefAs<Int16Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, int16>(inputStore, dataStructure.getDataRefAs<Int16Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::uint16: {
-    ConvertData<T, uint16>(inputArray, dataStructure.getDataRefAs<UInt16Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, uint16>(inputStore, dataStructure.getDataRefAs<UInt16Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::int32: {
-    ConvertData<T, int32>(inputArray, dataStructure.getDataRefAs<Int32Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, int32>(inputStore, dataStructure.getDataRefAs<Int32Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::uint32: {
-    ConvertData<T, uint32>(inputArray, dataStructure.getDataRefAs<UInt32Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, uint32>(inputStore, dataStructure.getDataRefAs<UInt32Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::int64: {
-    ConvertData<T, int64>(inputArray, dataStructure.getDataRefAs<Int64Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, int64>(inputStore, dataStructure.getDataRefAs<Int64Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::uint64: {
-    ConvertData<T, uint64>(inputArray, dataStructure.getDataRefAs<UInt64Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, uint64>(inputStore, dataStructure.getDataRefAs<UInt64Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::float32: {
-    ConvertData<T, float32>(inputArray, dataStructure.getDataRefAs<Float32Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, float32>(inputStore, dataStructure.getDataRefAs<Float32Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::float64: {
-    ConvertData<T, float64>(inputArray, dataStructure.getDataRefAs<Float64Array>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, float64>(inputStore, dataStructure.getDataRefAs<Float64Array>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   case DataType::boolean: {
-    ConvertData<T, bool>(inputArray, dataStructure.getDataRefAs<BoolArray>(inputValues->OutputArrayName));
+    dataAlg.execute(ConvertDataAlg<T, bool>(inputStore, dataStructure.getDataRefAs<BoolArray>(inputValues->OutputArrayName).getDataStoreRef()));
     break;
   }
   default: {