BUG/API: Histogram Sync (BlueQuartzSoftware#1073)

* Add HistogramUtilities File * Integrated Histogram API into Compute Array Statistics and Compute Array Histogram * complete extensive documentation for histogram utilities * Update test case to reflect new histogram that produces results inline with plot.ly histogram * repair broken pipelines after key updates * Added a size() function to IDataStore for std::vector compatibility * Update CalculateBin for special bool case --------- Co-authored-by: Jared Duffey <[email protected]>
imikejackson · Sep 26, 2024 · 2969e25 · 2969e25
1 parent fe66dd1
commit 2969e25
Show file tree

Hide file tree

Showing 21 changed files with 747 additions and 366 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -499,6 +499,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/FilterUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/GeometryUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/GeometryHelpers.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.hpp

diff --git a/src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline b/src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline
@@ -117,7 +117,8 @@
     {
       "args": {
         "new_data_group_path": "DataContainer/Statistics",
-        "histogram_suffix": " Histogram",
+        "histogram_bin_count_suffix": " Histogram Counts",
+        "histogram_bin_range_suffix": " Histogram Bin Ranges",
         "max_range": 1.0,
         "min_range": 0.0,
         "create_new_data_group": true,
@@ -141,11 +142,11 @@
         "file_extension": ".csv",
         "header_option_index": 1,
         "max_val_per_line": 0,
-        "output_dir": "Data/Output/OrientationAnalysis/",
-        "output_path": "Data/Output/OrientationAnalysis/Test/CI_Histogram.csv",
-        "output_style_index": 1,
+        "output_dir": "Data/Output/OrientationAnalysis/Test",
+        "output_style_index": 0,
         "input_data_array_paths": [
-          "DataContainer/Statistics/Confidence Index Histogram"
+          "DataContainer/Statistics/Confidence Index Histogram Counts",
+          "DataContainer/Statistics/Confidence Index Histogram Bin Ranges"
         ]
       },
       "comments": "",

diff --git a/src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline b/src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline
@@ -448,7 +448,8 @@
     {
       "args": {
         "create_new_data_group": true,
-        "histogram_suffix": " Histogram",
+        "histogram_bin_count_suffix": " Histogram Counts",
+        "histogram_bin_range_suffix": " Histogram Bin Ranges",
         "max_range": 1.0,
         "min_range": 0.0,
         "new_data_group_path": "fw-ar-IF1-aptr12-corr/Histograms",
@@ -472,12 +473,12 @@
         "file_extension": ".csv",
         "header_option_index": 1,
         "input_data_array_paths": [
-          "fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram"
+          "fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Counts",
+          "fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges"
         ],
         "max_val_per_line": 0,
-        "output_dir": "",
-        "output_path": "Data/Output/fw-ar-IF1-aptr12-corr/EqDiamHistogram.csv",
-        "output_style_index": 1
+        "output_dir": "Data/Output/fw-ar-IF1-aptr12-corr",
+        "output_style_index": 0
       },
       "comments": "",
       "filter": {

diff --git a/src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline b/src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline
@@ -448,7 +448,8 @@
     {
       "args": {
         "create_new_data_group": true,
-        "histogram_suffix": "Histogram",
+        "histogram_bin_count_suffix": " Histogram Counts",
+        "histogram_bin_range_suffix": " Histogram Bin Ranges",
         "max_range": 1.0,
         "min_range": 0.0,
         "new_data_group_path": "fw-ar-IF1-avtr12-corr/Histograms",
@@ -472,12 +473,12 @@
         "file_extension": ".csv",
         "header_option_index": 1,
         "input_data_array_paths": [
-          "fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiametersHistogram"
+          "fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Counts",
+          "fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges"
         ],
         "max_val_per_line": 0,
-        "output_dir": "",
-        "output_path": "Data/Output/fw-ar-IF1-avtr12-corr/EqDiamHistogram.csv",
-        "output_style_index": 1
+        "output_dir": "Data/Output/fw-ar-IF1-avtr12-corr",
+        "output_style_index": 0
       },
       "comments": "",
       "filter": {

diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp
@@ -1,106 +1,16 @@
 #include "ComputeArrayHistogram.hpp"
 
 #include "SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp"
-
 #include "simplnx/DataStructure/DataArray.hpp"
 #include "simplnx/DataStructure/DataGroup.hpp"
+#include "simplnx/Utilities/HistogramUtilities.hpp"
 #include "simplnx/Utilities/ParallelAlgorithmUtilities.hpp"
 #include "simplnx/Utilities/ParallelTaskAlgorithm.hpp"
 
-#include <chrono>
 #include <tuple>
 
 using namespace nx::core;
 
-namespace
-{
-template <typename Type>
-class GenerateHistogramFromData
-{
-public:
-  GenerateHistogramFromData(ComputeArrayHistogram& filter, const int32 numBins, const IDataArray& inputArray, AbstractDataStore<float64>& histogram, std::atomic<usize>& overflow,
-                            std::tuple<bool, float64, float64>& range, size_t progressIncrement)
-  : m_Filter(filter)
-  , m_NumBins(numBins)
-  , m_InputArray(inputArray)
-  , m_Histogram(histogram)
-  , m_Overflow(overflow)
-  , m_Range(range)
-  , m_ProgressIncrement(progressIncrement)
-  {
-  }
-  ~GenerateHistogramFromData() = default;
-
-  void operator()() const
-  {
-    const auto& inputStore = m_InputArray.template getIDataStoreRefAs<AbstractDataStore<Type>>();
-    auto end = inputStore.getSize();
-
-    // tuple visualization: Histogram = {(bin maximum, count), (bin maximum, count), ... }
-    float64 min = 0.0;
-    float64 max = 0.0;
-    if(std::get<0>(m_Range))
-    {
-      min = std::get<1>(m_Range);
-      max = std::get<2>(m_Range);
-    }
-    else
-    {
-      auto minMax = std::minmax_element(inputStore.begin(), inputStore.end());
-      min = (static_cast<float64>(*minMax.first) - 1);  // ensure upper limit encapsulates max value
-      max = (static_cast<float64>(*minMax.second) + 1); // ensure lower limit encapsulates min value
-    }
-
-    const float64 increment = (max - min) / static_cast<float64>(m_NumBins);
-    if(m_NumBins == 1) // if one bin, just set the first element to total number of points
-    {
-      m_Histogram[0] = max;
-      m_Histogram[1] = end;
-    }
-    else
-    {
-      size_t progressCounter = 0;
-      for(usize i = 0; i < end; i++)
-      {
-        if(progressCounter > m_ProgressIncrement)
-        {
-          m_Filter.updateThreadSafeProgress(progressCounter);
-          progressCounter = 0;
-        }
-        if(m_Filter.getCancel())
-        {
-          return;
-        }
-        const auto bin = std::floor((inputStore[i] - min) / increment);
-        if((bin >= 0) && (bin < m_NumBins))
-        {
-          m_Histogram[bin * 2 + 1]++;
-        }
-        else
-        {
-          m_Overflow++;
-        }
-        progressCounter++;
-      }
-    }
-
-    for(int64 i = 0; i < m_NumBins; i++)
-    {
-      m_Histogram[(i * 2)] = static_cast<float64>(min + (increment * (static_cast<float64>(i) + 1.0))); // load bin maximum into respective position {(x, ), (x , ), ...}
-    }
-  }
-
-private:
-  ComputeArrayHistogram& m_Filter;
-  const int32 m_NumBins = 1;
-  std::tuple<bool, float64, float64>& m_Range;
-  const IDataArray& m_InputArray;
-  AbstractDataStore<float64>& m_Histogram;
-  std::atomic<usize>& m_Overflow;
-  size_t m_ProgressIncrement = 100;
-};
-} // namespace
-
 // -----------------------------------------------------------------------------
 ComputeArrayHistogram::ComputeArrayHistogram(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel,
                                              ComputeArrayHistogramInputValues* inputValues)
@@ -119,22 +29,6 @@ void ComputeArrayHistogram::updateProgress(const std::string& progressMessage)
 {
   m_MessageHandler({IFilter::Message::Type::Info, progressMessage});
 }
-// -----------------------------------------------------------------------------
-void ComputeArrayHistogram::updateThreadSafeProgress(size_t counter)
-{
-  std::lock_guard<std::mutex> guard(m_ProgressMessage_Mutex);
-
-  m_ProgressCounter += counter;
-
-  auto now = std::chrono::steady_clock::now();
-  if(std::chrono::duration_cast<std::chrono::milliseconds>(now - m_InitialTime).count() > 1000) // every second update
-  {
-    auto progressInt = static_cast<size_t>((static_cast<double>(m_ProgressCounter) / static_cast<double>(m_TotalElements)) * 100.0);
-    std::string progressMessage = "Calculating... ";
-    m_MessageHandler(IFilter::ProgressMessage{IFilter::Message::Type::Progress, progressMessage, static_cast<int32_t>(progressInt)});
-    m_InitialTime = std::chrono::steady_clock::now();
-  }
-}
 
 // -----------------------------------------------------------------------------
 const std::atomic_bool& ComputeArrayHistogram::getCancel()
@@ -145,16 +39,9 @@ const std::atomic_bool& ComputeArrayHistogram::getCancel()
 // -----------------------------------------------------------------------------
 Result<> ComputeArrayHistogram::operator()()
 {
-  const auto numBins = m_InputValues->NumberOfBins;
-  const auto selectedArrayPaths = m_InputValues->SelectedArrayPaths;
-
-  for(const auto& arrayPath : selectedArrayPaths)
-  {
-    m_TotalElements += m_DataStructure.getDataAs<IDataArray>(arrayPath)->getSize();
-  }
-  auto progressIncrement = m_TotalElements / 100;
+  const int32 numBins = m_InputValues->NumberOfBins;
+  const std::vector<DataPath> selectedArrayPaths = m_InputValues->SelectedArrayPaths;
 
-  std::tuple<bool, float64, float64> range = std::make_tuple(m_InputValues->UserDefinedRange, m_InputValues->MinRange, m_InputValues->MaxRange); // Custom bool, min, max
   ParallelTaskAlgorithm taskRunner;
 
   std::atomic<usize> overflow = 0;
@@ -165,13 +52,29 @@ Result<> ComputeArrayHistogram::operator()()
     {
       return {};
     }
-    const auto& inputData = m_DataStructure.getDataRefAs<IDataArray>(selectedArrayPaths[i]);
-    auto& histogram = m_DataStructure.getDataAs<DataArray<float64>>(m_InputValues->CreatedHistogramDataPaths.at(i))->getDataStoreRef();
-    ExecuteParallelFunction<GenerateHistogramFromData>(inputData.getDataType(), taskRunner, *this, numBins, inputData, histogram, overflow, range, progressIncrement);
+
+    const auto* inputData = m_DataStructure.getDataAs<IDataArray>(selectedArrayPaths[i]);
+    auto* binRanges = m_DataStructure.getDataAs<IDataArray>(m_InputValues->CreatedBinRangeDataPaths.at(i));
+    auto& counts = m_DataStructure.getDataAs<DataArray<uint64>>(m_InputValues->CreatedHistogramCountsDataPaths.at(i))->getDataStoreRef();
+    Result<> result = {};
+    if(m_InputValues->UserDefinedRange)
+    {
+      ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges,
+                             std::make_pair(m_InputValues->MinRange, m_InputValues->MaxRange), m_ShouldCancel, numBins, counts, overflow);
+    }
+    else
+    {
+      ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges, m_ShouldCancel, numBins, counts, overflow);
+    }
+
+    if(result.invalid())
+    {
+      return result;
+    }
 
     if(overflow > 0)
     {
-      const std::string arrayName = inputData.getName();
+      const std::string arrayName = inputData->getName();
       ComputeArrayHistogram::updateProgress(fmt::format("{} values not categorized into bin for array {}", overflow.load(), arrayName));
     }
   }

diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp
@@ -7,27 +7,23 @@
 #include "simplnx/Filter/IFilter.hpp"
 #include "simplnx/Parameters/MultiArraySelectionParameter.hpp"
 
-#include <chrono>
-#include <mutex>
-
 namespace nx::core
 {
-
 struct SIMPLNXCORE_EXPORT ComputeArrayHistogramInputValues
 {
-  int32 NumberOfBins = 0;
   bool UserDefinedRange = false;
+  int32 NumberOfBins = 0;
   float64 MinRange = 0.0;
   float64 MaxRange = 0.0;
   MultiArraySelectionParameter::ValueType SelectedArrayPaths = {};
-  MultiArraySelectionParameter::ValueType CreatedHistogramDataPaths = {};
+  MultiArraySelectionParameter::ValueType CreatedBinRangeDataPaths = {};
+  MultiArraySelectionParameter::ValueType CreatedHistogramCountsDataPaths = {};
 };
 
 /**
  * @class ComputeArrayHistogram
  * @brief This filter calculates a Histogram according to user specification and stores it accordingly
  */
-
 class SIMPLNXCORE_EXPORT ComputeArrayHistogram
 {
 public:
@@ -42,20 +38,12 @@ class SIMPLNXCORE_EXPORT ComputeArrayHistogram
   Result<> operator()();
 
   void updateProgress(const std::string& progMessage);
-  void updateThreadSafeProgress(size_t counter);
   const std::atomic_bool& getCancel();
 
 private:
   DataStructure& m_DataStructure;
   const ComputeArrayHistogramInputValues* m_InputValues = nullptr;
   const std::atomic_bool& m_ShouldCancel;
   const IFilter::MessageHandler& m_MessageHandler;
-
-  // Threadsafe Progress Message
-  mutable std::mutex m_ProgressMessage_Mutex;
-  size_t m_TotalElements = 0;
-  size_t m_ProgressCounter = 0;
-  std::chrono::steady_clock::time_point m_InitialTime = std::chrono::steady_clock::now();
 };
-
 } // namespace nx::core