Skip to content

Commit

Permalink
BUG/API: Histogram Sync (#1073)
Browse files Browse the repository at this point in the history 
* Add HistogramUtilities File
* Integrated Histogram API into Compute Array Statistics and Compute Array Histogram
* complete extensive documentation for histogram utilities
* Update test case to reflect new histogram that produces results inline with plot.ly histogram
* repair broken pipelines after key updates
* Added a size() function to IDataStore for std::vector compatibility
* Update CalculateBin for special bool case

---------

Co-authored-by: Jared Duffey <[email protected]>
  • Loading branch information
@nyoungbq @JDuffeyBQ
nyoungbq and JDuffeyBQ authored Sep 26, 2024
1 parent 99a8fe1 commit 7b73128
Show file tree
Hide file tree
Showing 21 changed files with 747 additions and 366 deletions.
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,7 @@ set(SIMPLNX_HDRS
${SIMPLNX_SOURCE_DIR}/Utilities/FilterUtilities.hpp
${SIMPLNX_SOURCE_DIR}/Utilities/GeometryUtilities.hpp
${SIMPLNX_SOURCE_DIR}/Utilities/GeometryHelpers.hpp
${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.hpp
Expand Down
11 changes: 6 additions & 5 deletions src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,8 @@
{
"args": {
"new_data_group_path": "DataContainer/Statistics",
"histogram_suffix": " Histogram",
"histogram_bin_count_suffix": " Histogram Counts",
"histogram_bin_range_suffix": " Histogram Bin Ranges",
"max_range": 1.0,
"min_range": 0.0,
"create_new_data_group": true,
Expand All @@ -141,11 +142,11 @@
"file_extension": ".csv",
"header_option_index": 1,
"max_val_per_line": 0,
"output_dir": "Data/Output/OrientationAnalysis/",
"output_path": "Data/Output/OrientationAnalysis/Test/CI_Histogram.csv",
"output_style_index": 1,
"output_dir": "Data/Output/OrientationAnalysis/Test",
"output_style_index": 0,
"input_data_array_paths": [
"DataContainer/Statistics/Confidence Index Histogram"
"DataContainer/Statistics/Confidence Index Histogram Counts",
"DataContainer/Statistics/Confidence Index Histogram Bin Ranges"
]
},
"comments": "",
Expand Down
11 changes: 6 additions & 5 deletions src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,8 @@
{
"args": {
"create_new_data_group": true,
"histogram_suffix": " Histogram",
"histogram_bin_count_suffix": " Histogram Counts",
"histogram_bin_range_suffix": " Histogram Bin Ranges",
"max_range": 1.0,
"min_range": 0.0,
"new_data_group_path": "fw-ar-IF1-aptr12-corr/Histograms",
Expand All @@ -472,12 +473,12 @@
"file_extension": ".csv",
"header_option_index": 1,
"input_data_array_paths": [
"fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram"
"fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Counts",
"fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges"
],
"max_val_per_line": 0,
"output_dir": "",
"output_path": "Data/Output/fw-ar-IF1-aptr12-corr/EqDiamHistogram.csv",
"output_style_index": 1
"output_dir": "Data/Output/fw-ar-IF1-aptr12-corr",
"output_style_index": 0
},
"comments": "",
"filter": {
Expand Down
11 changes: 6 additions & 5 deletions src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,8 @@
{
"args": {
"create_new_data_group": true,
"histogram_suffix": "Histogram",
"histogram_bin_count_suffix": " Histogram Counts",
"histogram_bin_range_suffix": " Histogram Bin Ranges",
"max_range": 1.0,
"min_range": 0.0,
"new_data_group_path": "fw-ar-IF1-avtr12-corr/Histograms",
Expand All @@ -472,12 +473,12 @@
"file_extension": ".csv",
"header_option_index": 1,
"input_data_array_paths": [
"fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiametersHistogram"
"fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Counts",
"fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges"
],
"max_val_per_line": 0,
"output_dir": "",
"output_path": "Data/Output/fw-ar-IF1-avtr12-corr/EqDiamHistogram.csv",
"output_style_index": 1
"output_dir": "Data/Output/fw-ar-IF1-avtr12-corr",
"output_style_index": 0
},
"comments": "",
"filter": {
Expand Down
143 changes: 23 additions & 120 deletions src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,106 +1,16 @@
#include "ComputeArrayHistogram.hpp"

#include "SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp"

#include "simplnx/DataStructure/DataArray.hpp"
#include "simplnx/DataStructure/DataGroup.hpp"
#include "simplnx/Utilities/HistogramUtilities.hpp"
#include "simplnx/Utilities/ParallelAlgorithmUtilities.hpp"
#include "simplnx/Utilities/ParallelTaskAlgorithm.hpp"

#include <chrono>
#include <tuple>

using namespace nx::core;

namespace
{
template <typename Type>
class GenerateHistogramFromData
{
public:
GenerateHistogramFromData(ComputeArrayHistogram& filter, const int32 numBins, const IDataArray& inputArray, AbstractDataStore<float64>& histogram, std::atomic<usize>& overflow,
std::tuple<bool, float64, float64>& range, size_t progressIncrement)
: m_Filter(filter)
, m_NumBins(numBins)
, m_InputArray(inputArray)
, m_Histogram(histogram)
, m_Overflow(overflow)
, m_Range(range)
, m_ProgressIncrement(progressIncrement)
{
}
~GenerateHistogramFromData() = default;

void operator()() const
{
const auto& inputStore = m_InputArray.template getIDataStoreRefAs<AbstractDataStore<Type>>();
auto end = inputStore.getSize();

// tuple visualization: Histogram = {(bin maximum, count), (bin maximum, count), ... }
float64 min = 0.0;
float64 max = 0.0;
if(std::get<0>(m_Range))
{
min = std::get<1>(m_Range);
max = std::get<2>(m_Range);
}
else
{
auto minMax = std::minmax_element(inputStore.begin(), inputStore.end());
min = (static_cast<float64>(*minMax.first) - 1); // ensure upper limit encapsulates max value
max = (static_cast<float64>(*minMax.second) + 1); // ensure lower limit encapsulates min value
}

const float64 increment = (max - min) / static_cast<float64>(m_NumBins);
if(m_NumBins == 1) // if one bin, just set the first element to total number of points
{
m_Histogram[0] = max;
m_Histogram[1] = end;
}
else
{
size_t progressCounter = 0;
for(usize i = 0; i < end; i++)
{
if(progressCounter > m_ProgressIncrement)
{
m_Filter.updateThreadSafeProgress(progressCounter);
progressCounter = 0;
}
if(m_Filter.getCancel())
{
return;
}
const auto bin = std::floor((inputStore[i] - min) / increment);
if((bin >= 0) && (bin < m_NumBins))
{
m_Histogram[bin * 2 + 1]++;
}
else
{
m_Overflow++;
}
progressCounter++;
}
}

for(int64 i = 0; i < m_NumBins; i++)
{
m_Histogram[(i * 2)] = static_cast<float64>(min + (increment * (static_cast<float64>(i) + 1.0))); // load bin maximum into respective position {(x, ), (x , ), ...}
}
}

private:
ComputeArrayHistogram& m_Filter;
const int32 m_NumBins = 1;
std::tuple<bool, float64, float64>& m_Range;
const IDataArray& m_InputArray;
AbstractDataStore<float64>& m_Histogram;
std::atomic<usize>& m_Overflow;
size_t m_ProgressIncrement = 100;
};
} // namespace

// -----------------------------------------------------------------------------
ComputeArrayHistogram::ComputeArrayHistogram(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel,
ComputeArrayHistogramInputValues* inputValues)
Expand All @@ -119,22 +29,6 @@ void ComputeArrayHistogram::updateProgress(const std::string& progressMessage)
{
m_MessageHandler({IFilter::Message::Type::Info, progressMessage});
}
// -----------------------------------------------------------------------------
void ComputeArrayHistogram::updateThreadSafeProgress(size_t counter)
{
std::lock_guard<std::mutex> guard(m_ProgressMessage_Mutex);

m_ProgressCounter += counter;

auto now = std::chrono::steady_clock::now();
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - m_InitialTime).count() > 1000) // every second update
{
auto progressInt = static_cast<size_t>((static_cast<double>(m_ProgressCounter) / static_cast<double>(m_TotalElements)) * 100.0);
std::string progressMessage = "Calculating... ";
m_MessageHandler(IFilter::ProgressMessage{IFilter::Message::Type::Progress, progressMessage, static_cast<int32_t>(progressInt)});
m_InitialTime = std::chrono::steady_clock::now();
}
}

// -----------------------------------------------------------------------------
const std::atomic_bool& ComputeArrayHistogram::getCancel()
Expand All @@ -145,16 +39,9 @@ const std::atomic_bool& ComputeArrayHistogram::getCancel()
// -----------------------------------------------------------------------------
Result<> ComputeArrayHistogram::operator()()
{
const auto numBins = m_InputValues->NumberOfBins;
const auto selectedArrayPaths = m_InputValues->SelectedArrayPaths;

for(const auto& arrayPath : selectedArrayPaths)
{
m_TotalElements += m_DataStructure.getDataAs<IDataArray>(arrayPath)->getSize();
}
auto progressIncrement = m_TotalElements / 100;
const int32 numBins = m_InputValues->NumberOfBins;
const std::vector<DataPath> selectedArrayPaths = m_InputValues->SelectedArrayPaths;

std::tuple<bool, float64, float64> range = std::make_tuple(m_InputValues->UserDefinedRange, m_InputValues->MinRange, m_InputValues->MaxRange); // Custom bool, min, max
ParallelTaskAlgorithm taskRunner;

std::atomic<usize> overflow = 0;
Expand All @@ -165,13 +52,29 @@ Result<> ComputeArrayHistogram::operator()()
{
return {};
}
const auto& inputData = m_DataStructure.getDataRefAs<IDataArray>(selectedArrayPaths[i]);
auto& histogram = m_DataStructure.getDataAs<DataArray<float64>>(m_InputValues->CreatedHistogramDataPaths.at(i))->getDataStoreRef();
ExecuteParallelFunction<GenerateHistogramFromData>(inputData.getDataType(), taskRunner, *this, numBins, inputData, histogram, overflow, range, progressIncrement);

const auto* inputData = m_DataStructure.getDataAs<IDataArray>(selectedArrayPaths[i]);
auto* binRanges = m_DataStructure.getDataAs<IDataArray>(m_InputValues->CreatedBinRangeDataPaths.at(i));
auto& counts = m_DataStructure.getDataAs<DataArray<uint64>>(m_InputValues->CreatedHistogramCountsDataPaths.at(i))->getDataStoreRef();
Result<> result = {};
if(m_InputValues->UserDefinedRange)
{
ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges,
std::make_pair(m_InputValues->MinRange, m_InputValues->MaxRange), m_ShouldCancel, numBins, counts, overflow);
}
else
{
ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges, m_ShouldCancel, numBins, counts, overflow);
}

if(result.invalid())
{
return result;
}

if(overflow > 0)
{
const std::string arrayName = inputData.getName();
const std::string arrayName = inputData->getName();
ComputeArrayHistogram::updateProgress(fmt::format("{} values not categorized into bin for array {}", overflow.load(), arrayName));
}
}
Expand Down
18 changes: 3 additions & 15 deletions src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,23 @@
#include "simplnx/Filter/IFilter.hpp"
#include "simplnx/Parameters/MultiArraySelectionParameter.hpp"

#include <chrono>
#include <mutex>

namespace nx::core
{

struct SIMPLNXCORE_EXPORT ComputeArrayHistogramInputValues
{
int32 NumberOfBins = 0;
bool UserDefinedRange = false;
int32 NumberOfBins = 0;
float64 MinRange = 0.0;
float64 MaxRange = 0.0;
MultiArraySelectionParameter::ValueType SelectedArrayPaths = {};
MultiArraySelectionParameter::ValueType CreatedHistogramDataPaths = {};
MultiArraySelectionParameter::ValueType CreatedBinRangeDataPaths = {};
MultiArraySelectionParameter::ValueType CreatedHistogramCountsDataPaths = {};
};

/**
* @class ComputeArrayHistogram
* @brief This filter calculates a Histogram according to user specification and stores it accordingly
*/

class SIMPLNXCORE_EXPORT ComputeArrayHistogram
{
public:
Expand All @@ -42,20 +38,12 @@ class SIMPLNXCORE_EXPORT ComputeArrayHistogram
Result<> operator()();

void updateProgress(const std::string& progMessage);
void updateThreadSafeProgress(size_t counter);
const std::atomic_bool& getCancel();

private:
DataStructure& m_DataStructure;
const ComputeArrayHistogramInputValues* m_InputValues = nullptr;
const std::atomic_bool& m_ShouldCancel;
const IFilter::MessageHandler& m_MessageHandler;

// Threadsafe Progress Message
mutable std::mutex m_ProgressMessage_Mutex;
size_t m_TotalElements = 0;
size_t m_ProgressCounter = 0;
std::chrono::steady_clock::time_point m_InitialTime = std::chrono::steady_clock::now();
};

} // namespace nx::core
Loading

0 comments on commit 7b73128

Please sign in to comment.