From aa28672451a146201f21c8a3cc2f7cd793041491 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Fri, 23 Aug 2024 11:24:37 -0400 Subject: [PATCH 01/25] Add HistogramUtilities File --- CMakeLists.txt | 1 + src/simplnx/Utilities/HistogramUtilities.hpp | 90 ++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/simplnx/Utilities/HistogramUtilities.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 48fb70d25f..62002e771a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -499,6 +499,7 @@ set(SIMPLNX_HDRS ${SIMPLNX_SOURCE_DIR}/Utilities/FilterUtilities.hpp ${SIMPLNX_SOURCE_DIR}/Utilities/GeometryUtilities.hpp ${SIMPLNX_SOURCE_DIR}/Utilities/GeometryHelpers.hpp + ${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp ${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp ${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.hpp diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp new file mode 100644 index 0000000000..72d075d0ae --- /dev/null +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -0,0 +1,90 @@ +#pragma once + +#include "simplnx/simplnx_export.hpp" + +#include "simplnx/Common/Result.hpp" +#include "simplnx/DataStructure/IDataArray.hpp" + +namespace nx::core::HistogramUtilities +{ +template +SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputStore, const std::pair& rangeMinMax, const std::function& progressReportingFunction, + const std::atomic_bool& shouldCancel, const int32 numBins, AbstractDataStore& histogramStore, std::atomic& overflow, + size_t progressIncrement) +{ + auto end = inputStore.getSize(); + + const Type increment = (rangeMinMax.second - rangeMinMax.first) / static_cast(numBins); + if(numBins == 1) // if one bin, just set the first element to total number of points + { + histogramStore[0] = rangeMinMax.second; + histogramStore[1] = end; + } + else + { + usize progressCounter = 0; + for(usize i = 0; i < end; i++) + { + if(shouldCancel) + { + return MakeErrorResult(-23761, fmt::format("HistogramUtilities::{}: Signal Interrupt Received. {}:{}", __func__, __FILE__, __LINE__)); + } + if(progressCounter > progressIncrement) + { + progressReportingFunction(progressCounter); + progressCounter = 0; + } + const auto bin = std::floor((inputStore[i] - rangeMinMax.first) / increment); + if((bin >= 0) && (bin < numBins)) + { + histogramStore[bin * 2 + 1]++; + } + else + { + overflow++; + } + progressCounter++; + } + } + + for(int64 i = 0; i < numBins; i++) + { + histogramStore[(i * 2)] = static_cast(rangeMinMax.first + (increment * (static_cast(i) + 1.0))); // load bin maximum into respective position {(x, ), (x , ), ...} + } + + if(overflow > 0) + { + return MakeWarningVoidResult(-23762, fmt::format("HistogramUtilities::{}: Overflow detected: overflow count {}. {}:{}", __func__, overflow, __FILE__, __LINE__)); + } + + return {}; +} + +struct SIMPLNX_EXPORT GenerateHistogramFunctor +{ + template + Result<> operator()(const IDataArray* inputArray, ArgsT... args) + { + const auto& inputStore = inputArray->template getIDataStoreRefAs>(); + + auto minMax = std::minmax_element(inputStore.begin(), inputStore.end()); + + GenerateHistogram(inputStore, std::make_pair(static_cast(*minMax.first) - 1, static_cast(*minMax.second) + 1), std::forward(args)...); + } + + template + Result<> operator()(const IDataArray* inputArray, const std::pair& rangeMinMax, ArgsT... args) + { + const auto& inputStore = inputArray->template getIDataStoreRefAs>(); + + // check range ordering : should be min, max + if(rangeMinMax.first > rangeMinMax.second) + { + return MakeErrorResult(-23760, fmt::format("GenerateHistogramFunctor::{}: The range min value is larger than the max value. Min value: {} | Max Value: {}. {}:{}", __func__, rangeMinMax.first, + rangeMinMax.second, __FILE__, __LINE__)); + } + + GenerateHistogram(inputStore, rangeMinMax, std::forward(args)...); + } +}; +} // namespace nx::core::HistogramUtilities From 3c93a93d84373d2ab797d8c13bacb02f7316becc Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Fri, 30 Aug 2024 12:31:52 -0400 Subject: [PATCH 02/25] V1 functional extraction from filter to utilities [compiling] --- .../Algorithms/ComputeArrayHistogram.cpp | 123 +++--------------- src/simplnx/Utilities/HistogramUtilities.hpp | 82 +++++++++--- .../Utilities/ParallelAlgorithmUtilities.hpp | 89 +++++++++++++ 3 files changed, 176 insertions(+), 118 deletions(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp index 94588211b3..d37ff86fca 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp @@ -1,9 +1,9 @@ #include "ComputeArrayHistogram.hpp" #include "SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp" - #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataGroup.hpp" +#include "simplnx/Utilities/HistogramUtilities.hpp" #include "simplnx/Utilities/ParallelAlgorithmUtilities.hpp" #include "simplnx/Utilities/ParallelTaskAlgorithm.hpp" @@ -12,95 +12,6 @@ using namespace nx::core; -namespace -{ -template -class GenerateHistogramFromData -{ -public: - GenerateHistogramFromData(ComputeArrayHistogram& filter, const int32 numBins, const IDataArray& inputArray, AbstractDataStore& histogram, std::atomic& overflow, - std::tuple& range, size_t progressIncrement) - : m_Filter(filter) - , m_NumBins(numBins) - , m_InputArray(inputArray) - , m_Histogram(histogram) - , m_Overflow(overflow) - , m_Range(range) - , m_ProgressIncrement(progressIncrement) - { - } - ~GenerateHistogramFromData() = default; - - void operator()() const - { - const auto& inputStore = m_InputArray.template getIDataStoreRefAs>(); - auto end = inputStore.getSize(); - - // tuple visualization: Histogram = {(bin maximum, count), (bin maximum, count), ... } - float64 min = 0.0; - float64 max = 0.0; - if(std::get<0>(m_Range)) - { - min = std::get<1>(m_Range); - max = std::get<2>(m_Range); - } - else - { - auto minMax = std::minmax_element(inputStore.begin(), inputStore.end()); - min = (static_cast(*minMax.first) - 1); // ensure upper limit encapsulates max value - max = (static_cast(*minMax.second) + 1); // ensure lower limit encapsulates min value - } - - const float64 increment = (max - min) / static_cast(m_NumBins); - if(m_NumBins == 1) // if one bin, just set the first element to total number of points - { - m_Histogram[0] = max; - m_Histogram[1] = end; - } - else - { - size_t progressCounter = 0; - for(usize i = 0; i < end; i++) - { - if(progressCounter > m_ProgressIncrement) - { - m_Filter.updateThreadSafeProgress(progressCounter); - progressCounter = 0; - } - if(m_Filter.getCancel()) - { - return; - } - const auto bin = std::floor((inputStore[i] - min) / increment); - if((bin >= 0) && (bin < m_NumBins)) - { - m_Histogram[bin * 2 + 1]++; - } - else - { - m_Overflow++; - } - progressCounter++; - } - } - - for(int64 i = 0; i < m_NumBins; i++) - { - m_Histogram[(i * 2)] = static_cast(min + (increment * (static_cast(i) + 1.0))); // load bin maximum into respective position {(x, ), (x , ), ...} - } - } - -private: - ComputeArrayHistogram& m_Filter; - const int32 m_NumBins = 1; - std::tuple& m_Range; - const IDataArray& m_InputArray; - AbstractDataStore& m_Histogram; - std::atomic& m_Overflow; - size_t m_ProgressIncrement = 100; -}; -} // namespace - // ----------------------------------------------------------------------------- ComputeArrayHistogram::ComputeArrayHistogram(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, ComputeArrayHistogramInputValues* inputValues) @@ -145,16 +56,9 @@ const std::atomic_bool& ComputeArrayHistogram::getCancel() // ----------------------------------------------------------------------------- Result<> ComputeArrayHistogram::operator()() { - const auto numBins = m_InputValues->NumberOfBins; - const auto selectedArrayPaths = m_InputValues->SelectedArrayPaths; - - for(const auto& arrayPath : selectedArrayPaths) - { - m_TotalElements += m_DataStructure.getDataAs(arrayPath)->getSize(); - } - auto progressIncrement = m_TotalElements / 100; + const int32 numBins = m_InputValues->NumberOfBins; + const std::vector selectedArrayPaths = m_InputValues->SelectedArrayPaths; - std::tuple range = std::make_tuple(m_InputValues->UserDefinedRange, m_InputValues->MinRange, m_InputValues->MaxRange); // Custom bool, min, max ParallelTaskAlgorithm taskRunner; std::atomic overflow = 0; @@ -165,13 +69,28 @@ Result<> ComputeArrayHistogram::operator()() { return {}; } - const auto& inputData = m_DataStructure.getDataRefAs(selectedArrayPaths[i]); + + const auto* inputData = m_DataStructure.getDataAs(selectedArrayPaths[i]); auto& histogram = m_DataStructure.getDataAs>(m_InputValues->CreatedHistogramDataPaths.at(i))->getDataStoreRef(); - ExecuteParallelFunction(inputData.getDataType(), taskRunner, *this, numBins, inputData, histogram, overflow, range, progressIncrement); + Result<> result = {}; + if(m_InputValues->UserDefinedRange) + { + ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, + std::make_pair(m_InputValues->MinRange, m_InputValues->MaxRange), m_ShouldCancel, numBins, histogram, overflow); + } + else + { + ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, m_ShouldCancel, numBins, histogram, overflow); + } + + if(result.invalid()) + { + return result; + } if(overflow > 0) { - const std::string arrayName = inputData.getName(); + const std::string arrayName = inputData->getName(); ComputeArrayHistogram::updateProgress(fmt::format("{} values not categorized into bin for array {}", overflow.load(), arrayName)); } } diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp index 72d075d0ae..9ee3e6668b 100644 --- a/src/simplnx/Utilities/HistogramUtilities.hpp +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -7,10 +7,11 @@ namespace nx::core::HistogramUtilities { +namespace serial +{ template -SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputStore, const std::pair& rangeMinMax, const std::function& progressReportingFunction, - const std::atomic_bool& shouldCancel, const int32 numBins, AbstractDataStore& histogramStore, std::atomic& overflow, - size_t progressIncrement) +SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputStore, const std::pair& rangeMinMax, const std::atomic_bool& shouldCancel, const int32 numBins, + AbstractDataStore& histogramStore, std::atomic& overflow) { auto end = inputStore.getSize(); @@ -22,18 +23,12 @@ SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputSt } else { - usize progressCounter = 0; for(usize i = 0; i < end; i++) { if(shouldCancel) { return MakeErrorResult(-23761, fmt::format("HistogramUtilities::{}: Signal Interrupt Received. {}:{}", __func__, __FILE__, __LINE__)); } - if(progressCounter > progressIncrement) - { - progressReportingFunction(progressCounter); - progressCounter = 0; - } const auto bin = std::floor((inputStore[i] - rangeMinMax.first) / increment); if((bin >= 0) && (bin < numBins)) { @@ -43,13 +38,12 @@ SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputSt { overflow++; } - progressCounter++; } } - for(int64 i = 0; i < numBins; i++) + for(int32 i = 0; i < numBins; i++) { - histogramStore[(i * 2)] = static_cast(rangeMinMax.first + (increment * (static_cast(i) + 1.0))); // load bin maximum into respective position {(x, ), (x , ), ...} + histogramStore[i * 2] = static_cast(rangeMinMax.first + (increment * (i + static_cast(1.0)))); // load bin maximum into respective position {(x, ), (x , ), ...} } if(overflow > 0) @@ -63,17 +57,17 @@ SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputSt struct SIMPLNX_EXPORT GenerateHistogramFunctor { template - Result<> operator()(const IDataArray* inputArray, ArgsT... args) + Result<> operator()(const IDataArray* inputArray, ArgsT&&... args) const { const auto& inputStore = inputArray->template getIDataStoreRefAs>(); auto minMax = std::minmax_element(inputStore.begin(), inputStore.end()); - GenerateHistogram(inputStore, std::make_pair(static_cast(*minMax.first) - 1, static_cast(*minMax.second) + 1), std::forward(args)...); + return GenerateHistogram(inputStore, std::make_pair(*minMax.first - static_cast(1.0), *minMax.second + static_cast(1.0)), std::forward(args)...); } template - Result<> operator()(const IDataArray* inputArray, const std::pair& rangeMinMax, ArgsT... args) + Result<> operator()(const IDataArray* inputArray, std::pair&& rangeMinMax, ArgsT&&... args) const { const auto& inputStore = inputArray->template getIDataStoreRefAs>(); @@ -84,7 +78,63 @@ struct SIMPLNX_EXPORT GenerateHistogramFunctor rangeMinMax.second, __FILE__, __LINE__)); } - GenerateHistogram(inputStore, rangeMinMax, std::forward(args)...); + return GenerateHistogram(inputStore, std::make_pair(static_cast(rangeMinMax.first), static_cast(rangeMinMax.second)), std::forward(args)...); + } +}; +} // namespace serial + +namespace concurrent +{ +template +class SIMPLNX_EXPORT GenerateHistogramImpl +{ +public: + GenerateHistogramImpl(const AbstractDataStore& inputStore, std::pair&& rangeMinMax, const std::atomic_bool& shouldCancel, const int32 numBins, + AbstractDataStore& histogramStore, std::atomic& overflow) + : m_InputStore(inputStore) + , m_ShouldCancel(shouldCancel) + , m_NumBins(numBins) + , m_HistogramStore(histogramStore) + , m_Overflow(overflow) + { + m_Range = std::make_pair(static_cast(rangeMinMax.first), static_cast(rangeMinMax.second)); + } + + GenerateHistogramImpl(const AbstractDataStore& inputStore, const std::atomic_bool& shouldCancel, const int32 numBins, AbstractDataStore& histogramStore, + std::atomic& overflow) + : m_InputStore(inputStore) + , m_ShouldCancel(shouldCancel) + , m_NumBins(numBins) + , m_HistogramStore(histogramStore) + , m_Overflow(overflow) + { + auto minMax = std::minmax_element(m_InputStore.begin(), m_InputStore.end()); + m_Range = std::make_pair(*minMax.first - static_cast(1.0), *minMax.second + static_cast(1.0)); + } + + ~GenerateHistogramImpl() = default; + + void operator()() const + { + serial::GenerateHistogram(m_InputStore, m_Range, m_ShouldCancel, m_NumBins, m_HistogramStore, m_Overflow); + } + +private: + const std::atomic_bool& m_ShouldCancel; + const int32 m_NumBins = 1; + std::pair m_Range = {static_cast(0.0), static_cast(0.0)}; + const AbstractDataStore& m_InputStore; + AbstractDataStore& m_HistogramStore; + std::atomic& m_Overflow; +}; + +struct InstantiateHistogramImplFunctor +{ + template + auto operator()(const IDataArray* iDataArray, ArgsT&&... args) + { + return GenerateHistogramImpl(iDataArray->template getIDataStoreRefAs>(), std::forward(args)...); } }; +} // namespace concurrent } // namespace nx::core::HistogramUtilities diff --git a/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp b/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp index 04877392a0..cfe49c2679 100644 --- a/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp +++ b/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp @@ -87,4 +87,93 @@ auto ExecuteParallelFunction(DataType dataType, ParallelRunnerT&& runner, ArgsT& } } } + +template +auto ExecuteParallelFunctor(FuncT&& func, DataType dataType, ParallelRunnerT&& runner, ArgsT&&... args) +{ + switch(dataType) + { + case DataType::boolean: { + if constexpr(ArrayTypeOptions::UsingBoolean) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::int8: { + if constexpr(ArrayTypeOptions::UsingInt8) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::int16: { + if constexpr(ArrayTypeOptions::UsingInt16) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::int32: { + if constexpr(ArrayTypeOptions::UsingInt32) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::int64: { + if constexpr(ArrayTypeOptions::UsingInt64) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::uint8: { + if constexpr(ArrayTypeOptions::UsingUInt8) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::uint16: { + if constexpr(ArrayTypeOptions::UsingUInt16) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::uint32: { + if constexpr(ArrayTypeOptions::UsingUInt32) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + } + case DataType::uint64: { + if constexpr(ArrayTypeOptions::UsingUInt64) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::float32: { + if constexpr(ArrayTypeOptions::UsingFloat32) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + case DataType::float64: { + if constexpr(ArrayTypeOptions::UsingFloat64) + { + return runner.template execute<>(func.template operator()(std::forward(args)...)); + } + break; + } + default: { + throw std::runtime_error(fmt::format("nx::core::{}: Error: Invalid DataType. {}:{}", __func__, __FILE__, __LINE__)); + } + } + + throw std::runtime_error(fmt::format("nx::core::{}: Error: Valid DataType, but the template declaration has this as a restricted type. {}:{}", __func__, __FILE__, __LINE__)); +} } // namespace nx::core From 394b00914514a0b7b8f0562bb0a85d9dee097454 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Fri, 30 Aug 2024 12:42:22 -0400 Subject: [PATCH 03/25] missed repairs --- src/simplnx/Utilities/HistogramUtilities.hpp | 2 +- src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp index 9ee3e6668b..524f51a33e 100644 --- a/src/simplnx/Utilities/HistogramUtilities.hpp +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -48,7 +48,7 @@ SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputSt if(overflow > 0) { - return MakeWarningVoidResult(-23762, fmt::format("HistogramUtilities::{}: Overflow detected: overflow count {}. {}:{}", __func__, overflow, __FILE__, __LINE__)); + return MakeWarningVoidResult(-23762, fmt::format("HistogramUtilities::{}: Overflow detected: overflow count {}. {}:{}", __func__, overflow.load(), __FILE__, __LINE__)); } return {}; diff --git a/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp b/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp index cfe49c2679..f0e885581c 100644 --- a/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp +++ b/src/simplnx/Utilities/ParallelAlgorithmUtilities.hpp @@ -2,6 +2,8 @@ #include "simplnx/Common/Types.hpp" +#include + #include namespace nx::core From abf8157e12552595cfc3be59172fbc5e9e5f0dc8 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Tue, 3 Sep 2024 17:07:56 -0400 Subject: [PATCH 04/25] Create proper general case in utilities [broken compile] --- src/simplnx/Utilities/HistogramUtilities.hpp | 147 +++++++++++++++---- 1 file changed, 116 insertions(+), 31 deletions(-) diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp index 524f51a33e..c4881b93fc 100644 --- a/src/simplnx/Utilities/HistogramUtilities.hpp +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -9,41 +9,123 @@ namespace nx::core::HistogramUtilities { namespace serial { -template -SIMPLNX_EXPORT Result<> GenerateHistogram(const AbstractDataStore& inputStore, const std::pair& rangeMinMax, const std::atomic_bool& shouldCancel, const int32 numBins, - AbstractDataStore& histogramStore, std::atomic& overflow) +namespace detail { - auto end = inputStore.getSize(); +template +concept HasBracketOperator = requires(T object, usize i) +{ + { + object[i] + } -> std::same_as; +}; + +/** + * @function FillBinRange + * @brief This function fills a container that is STL compatible and has a bracket operator defined with the bin ranges in the following pattern: + * bin_ranges = {minimum, maximum, next maximum, ...} with the format being that the bin's range is defined by bin_ranges[bin_index] <= X < bin_ranges[bin_index + 1] + * @tparam Type this the end type of the function in that the container and data values are of this type + * @tparam Container this is the type of object the ranges are loaded into, !!! It is expected that this class is STL compatible nd has a defined `[]` operator !!! + * @param outputContainer this is the object that the ranges will be loaded into. ASSUMPTION: size is >= numBins + 1 !!! NO Bounds Check!!! + * @param rangeMinMax this is assumed to be the inclusive minimum value and exclusive maximum value for the overall histogram bins. FORMAT: [minimum, maximum) + * @param numBins this is the total number of bin ranges being calculated and by extension the indexing value for the ranges + * @param increment this is the uniform size of the bins + */ +template +SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair& rangeMinMax, const int32 numBins, const Type increment) +{ + // WARNING: No bounds checking for type compatibility, it is expected to be done higher up where the type is not abstracted + // EXPECTED CONTAINER SIZE: numBins + 1 - const Type increment = (rangeMinMax.second - rangeMinMax.first) / static_cast(numBins); if(numBins == 1) // if one bin, just set the first element to total number of points { - histogramStore[0] = rangeMinMax.second; - histogramStore[1] = end; + outputContainer[0] = rangeMinMax.first; + outputContainer[1] = rangeMinMax.second; + return; + } + + // iterate through loading the middle values of the sequence considering `lower bound inclusive, upper bound exclusive` + for(int32 i = 0; i < numBins; i++) + { + outputContainer[i] = static_cast(rangeMinMax.first + (increment * i)); + } + + outputContainer[numBins] = rangeMinMax.second; +} + +/** + * @function FillBinRange + * @brief This overload is provided in the case the bin size is not provided and therefore must be calculated - see above overload for more detail on functionality + * @tparam Type this the end type of the function in that the container and data values are of this type + * @tparam Container this is the type of object the ranges are loaded into, !!! It is expected that this class is STL compatible nd has a defined `[]` operator !!! + * @param outputContainer this is the object that the ranges will be loaded into. ASSUMPTION: size is >= numBins + 1 !!! NO Bounds Check!!! + * @param rangeMinMax this is assumed to be the inclusive minimum value and exclusive maximum value for the overall histogram bins. FORMAT: [minimum, maximum) + * @param numBins this is the total number of bin ranges being calculated and by extension the indexing value for the ranges + */ +template +SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair& rangeMinMax, const int32 numBins) +{ + // DEV NOTE: this function also serves to act as a jumping off point for implementing logarithmic histograms down the line + + // Uniform Bin Sizes + const Type increment = (rangeMinMax.second - rangeMinMax.first) / static_cast(numBins); + + FillBinRanges(outputContainer, rangeMinMax, numBins, increment); +} +} // namespace detail + +template class Container> +SIMPLNX_EXPORT Result<> GenerateHistogram(const Container& inputStore, const std::pair& rangeMinMax, const std::atomic_bool& shouldCancel, const int32 numBins, + Container& binRangesStore, Container& histogramCountsStore, std::atomic& overflow) +{ + usize end = 0; + if constexpr(std::is_same_v, Container>) + { + end = inputStore.size(); + + // just resize outputs to ensure no wasted space and won't be out of bounds + binRangesStore.resize(numBins + 1); + histogramCountsStore.resize(numBins); } - else + if constexpr(std::is_same_v, Container>) { - for(usize i = 0; i < end; i++) + end = inputStore.getSize(); + if(binRangesStore.getSize() < numBins + 1) + { + return MakeErrorResult(-23761, fmt::format("HistogramUtilities::{}: binRangesStore is too small to hold ranges. Needed: {} | Current Size: {}. {}:{}", __func__, numBins + 1, + binRangesStore.getSize(), __FILE__, __LINE__)); + } + if(histogramCountsStore.getSize() < numBins) { - if(shouldCancel) - { - return MakeErrorResult(-23761, fmt::format("HistogramUtilities::{}: Signal Interrupt Received. {}:{}", __func__, __FILE__, __LINE__)); - } - const auto bin = std::floor((inputStore[i] - rangeMinMax.first) / increment); - if((bin >= 0) && (bin < numBins)) - { - histogramStore[bin * 2 + 1]++; - } - else - { - overflow++; - } + return MakeErrorResult(-23762, fmt::format("HistogramUtilities::{}: histogramCountsStore is too small to hold counts. Needed: {} | Current Size: {}. {}:{}", __func__, numBins, + histogramCountsStore.getSize(), __FILE__, __LINE__)); } } - for(int32 i = 0; i < numBins; i++) + const Type increment = (rangeMinMax.second - rangeMinMax.first) / static_cast(numBins); + + // Fill Bins + detail::FillBinRanges(binRangesStore, rangeMinMax, numBins, increment); + + if(shouldCancel) { - histogramStore[i * 2] = static_cast(rangeMinMax.first + (increment * (i + static_cast(1.0)))); // load bin maximum into respective position {(x, ), (x , ), ...} + return MakeErrorResult(-23762, fmt::format("HistogramUtilities::{}: Signal Interrupt Received. {}:{}", __func__, __FILE__, __LINE__)); + } + + for(usize i = 0; i < end; i++) + { + if(shouldCancel) + { + return MakeErrorResult(-23763, fmt::format("HistogramUtilities::{}: Signal Interrupt Received. {}:{}", __func__, __FILE__, __LINE__)); + } + const auto bin = std::floor((inputStore[i] - rangeMinMax.first) / increment); + if((bin >= 0) && (bin < numBins)) + { + histogramCountsStore[bin]++; + } + else + { + overflow++; + } } if(overflow > 0) @@ -63,7 +145,7 @@ struct SIMPLNX_EXPORT GenerateHistogramFunctor auto minMax = std::minmax_element(inputStore.begin(), inputStore.end()); - return GenerateHistogram(inputStore, std::make_pair(*minMax.first - static_cast(1.0), *minMax.second + static_cast(1.0)), std::forward(args)...); + return GenerateHistogram(inputStore, std::make_pair(*minMax.first, *minMax.second + static_cast(1.0)), std::forward(args)...); } template @@ -85,31 +167,33 @@ struct SIMPLNX_EXPORT GenerateHistogramFunctor namespace concurrent { -template +template class SIMPLNX_EXPORT GenerateHistogramImpl { public: GenerateHistogramImpl(const AbstractDataStore& inputStore, std::pair&& rangeMinMax, const std::atomic_bool& shouldCancel, const int32 numBins, - AbstractDataStore& histogramStore, std::atomic& overflow) + AbstractDataStore& binRangesStore, AbstractDataStore& histogramStore, std::atomic& overflow) : m_InputStore(inputStore) , m_ShouldCancel(shouldCancel) , m_NumBins(numBins) + , m_BinRangesStore(binRangesStore) , m_HistogramStore(histogramStore) , m_Overflow(overflow) { m_Range = std::make_pair(static_cast(rangeMinMax.first), static_cast(rangeMinMax.second)); } - GenerateHistogramImpl(const AbstractDataStore& inputStore, const std::atomic_bool& shouldCancel, const int32 numBins, AbstractDataStore& histogramStore, - std::atomic& overflow) + GenerateHistogramImpl(const AbstractDataStore& inputStore, const std::atomic_bool& shouldCancel, const int32 numBins, AbstractDataStore& binRangesStore, + AbstractDataStore& histogramStore, std::atomic& overflow) : m_InputStore(inputStore) , m_ShouldCancel(shouldCancel) , m_NumBins(numBins) + , m_BinRangesStore(binRangesStore) , m_HistogramStore(histogramStore) , m_Overflow(overflow) { auto minMax = std::minmax_element(m_InputStore.begin(), m_InputStore.end()); - m_Range = std::make_pair(*minMax.first - static_cast(1.0), *minMax.second + static_cast(1.0)); + m_Range = std::make_pair(*minMax.first, *minMax.second + static_cast(1.0)); } ~GenerateHistogramImpl() = default; @@ -124,7 +208,8 @@ class SIMPLNX_EXPORT GenerateHistogramImpl const int32 m_NumBins = 1; std::pair m_Range = {static_cast(0.0), static_cast(0.0)}; const AbstractDataStore& m_InputStore; - AbstractDataStore& m_HistogramStore; + AbstractDataStore& m_BinRangesStore; + AbstractDataStore& m_HistogramStore; std::atomic& m_Overflow; }; From e6193dd29f3a95337d86125f2833c41b2362dfa7 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Wed, 4 Sep 2024 11:42:22 -0400 Subject: [PATCH 05/25] complete extensive documentation for histogram utilities [compiling, not functional] --- .../Algorithms/ComputeArrayHistogram.cpp | 26 ++---- .../Algorithms/ComputeArrayHistogram.hpp | 4 +- .../Filters/ComputeArrayHistogramFilter.cpp | 2 +- src/simplnx/Utilities/HistogramUtilities.hpp | 83 ++++++++++++++++--- 4 files changed, 79 insertions(+), 36 deletions(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp index d37ff86fca..e5e65c7096 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp @@ -7,7 +7,6 @@ #include "simplnx/Utilities/ParallelAlgorithmUtilities.hpp" #include "simplnx/Utilities/ParallelTaskAlgorithm.hpp" -#include #include using namespace nx::core; @@ -30,22 +29,6 @@ void ComputeArrayHistogram::updateProgress(const std::string& progressMessage) { m_MessageHandler({IFilter::Message::Type::Info, progressMessage}); } -// ----------------------------------------------------------------------------- -void ComputeArrayHistogram::updateThreadSafeProgress(size_t counter) -{ - std::lock_guard guard(m_ProgressMessage_Mutex); - - m_ProgressCounter += counter; - - auto now = std::chrono::steady_clock::now(); - if(std::chrono::duration_cast(now - m_InitialTime).count() > 1000) // every second update - { - auto progressInt = static_cast((static_cast(m_ProgressCounter) / static_cast(m_TotalElements)) * 100.0); - std::string progressMessage = "Calculating... "; - m_MessageHandler(IFilter::ProgressMessage{IFilter::Message::Type::Progress, progressMessage, static_cast(progressInt)}); - m_InitialTime = std::chrono::steady_clock::now(); - } -} // ----------------------------------------------------------------------------- const std::atomic_bool& ComputeArrayHistogram::getCancel() @@ -71,16 +54,17 @@ Result<> ComputeArrayHistogram::operator()() } const auto* inputData = m_DataStructure.getDataAs(selectedArrayPaths[i]); - auto& histogram = m_DataStructure.getDataAs>(m_InputValues->CreatedHistogramDataPaths.at(i))->getDataStoreRef(); + auto& counts = m_DataStructure.getDataAs>(m_InputValues->CreatedHistogramCountsDataPaths.at(i))->getDataStoreRef(); + auto* binRanges = m_DataStructure.getDataAs(m_InputValues->CreatedBinRangeDataPaths.at(i)); Result<> result = {}; if(m_InputValues->UserDefinedRange) { - ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, - std::make_pair(m_InputValues->MinRange, m_InputValues->MaxRange), m_ShouldCancel, numBins, histogram, overflow); + ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges, + std::make_pair(m_InputValues->MinRange, m_InputValues->MaxRange), m_ShouldCancel, numBins, counts, overflow); } else { - ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, m_ShouldCancel, numBins, histogram, overflow); + ExecuteParallelFunctor(HistogramUtilities::concurrent::InstantiateHistogramImplFunctor{}, inputData->getDataType(), taskRunner, inputData, binRanges, m_ShouldCancel, numBins, counts, overflow); } if(result.invalid()) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp index c1a2890643..fd41690162 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp @@ -20,7 +20,8 @@ struct SIMPLNXCORE_EXPORT ComputeArrayHistogramInputValues float64 MinRange = 0.0; float64 MaxRange = 0.0; MultiArraySelectionParameter::ValueType SelectedArrayPaths = {}; - MultiArraySelectionParameter::ValueType CreatedHistogramDataPaths = {}; + MultiArraySelectionParameter::ValueType CreatedHistogramCountsDataPaths = {}; + MultiArraySelectionParameter::ValueType CreatedBinRangeDataPaths = {}; }; /** @@ -42,7 +43,6 @@ class SIMPLNXCORE_EXPORT ComputeArrayHistogram Result<> operator()(); void updateProgress(const std::string& progMessage); - void updateThreadSafeProgress(size_t counter); const std::atomic_bool& getCancel(); private: diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp index aeccf287f1..001cbfe8d4 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp @@ -158,7 +158,7 @@ Result<> ComputeArrayHistogramFilter::executeImpl(DataStructure& dataStructure, createdDataPaths.push_back(childPath); } - inputValues.CreatedHistogramDataPaths = createdDataPaths; + inputValues.CreatedHistogramCountsDataPaths = createdDataPaths; return ComputeArrayHistogram(dataStructure, messageHandler, shouldCancel, &inputValues)(); } diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp index c4881b93fc..e3d5458a42 100644 --- a/src/simplnx/Utilities/HistogramUtilities.hpp +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -73,9 +73,25 @@ SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair class Container> -SIMPLNX_EXPORT Result<> GenerateHistogram(const Container& inputStore, const std::pair& rangeMinMax, const std::atomic_bool& shouldCancel, const int32 numBins, - Container& binRangesStore, Container& histogramCountsStore, std::atomic& overflow) +SIMPLNX_EXPORT Result<> GenerateHistogram(const Container& inputStore, Container& binRangesStore, const std::pair& rangeMinMax, const std::atomic_bool& shouldCancel, + const int32 numBins, Container& histogramCountsStore, std::atomic& overflow) { usize end = 0; if constexpr(std::is_same_v, Container>) @@ -136,20 +152,26 @@ SIMPLNX_EXPORT Result<> GenerateHistogram(const Container& inputStore, con return {}; } +/** + * @class GenerateHistogramFunctor + * @brief This is a compatibility functor that leverages existing typecasting functions to execute GenerateHistogram() cleanly. In it there are two + * definitions for the `()` operator that allows for implicit calculation of range, predicated whether a range is passed in or not + */ struct SIMPLNX_EXPORT GenerateHistogramFunctor { template - Result<> operator()(const IDataArray* inputArray, ArgsT&&... args) const + Result<> operator()(const IDataArray* inputArray, IDataArray* binRangesArray, ArgsT&&... args) const { const auto& inputStore = inputArray->template getIDataStoreRefAs>(); auto minMax = std::minmax_element(inputStore.begin(), inputStore.end()); - return GenerateHistogram(inputStore, std::make_pair(*minMax.first, *minMax.second + static_cast(1.0)), std::forward(args)...); + return GenerateHistogram(inputStore, binRangesArray->template getIDataStoreRefAs>(), std::make_pair(*minMax.first, *minMax.second + static_cast(1.0)), + std::forward(args)...); } template - Result<> operator()(const IDataArray* inputArray, std::pair&& rangeMinMax, ArgsT&&... args) const + Result<> operator()(const IDataArray* inputArray, IDataArray* binRangesArray, std::pair&& rangeMinMax, ArgsT&&... args) const { const auto& inputStore = inputArray->template getIDataStoreRefAs>(); @@ -160,19 +182,37 @@ struct SIMPLNX_EXPORT GenerateHistogramFunctor rangeMinMax.second, __FILE__, __LINE__)); } - return GenerateHistogram(inputStore, std::make_pair(static_cast(rangeMinMax.first), static_cast(rangeMinMax.second)), std::forward(args)...); + return GenerateHistogram(inputStore, binRangesArray->template getIDataStoreRefAs>(), + std::make_pair(static_cast(rangeMinMax.first), static_cast(rangeMinMax.second)), std::forward(args)...); } }; } // namespace serial namespace concurrent { +/** + * @class GenerateHistogramImpl + * @brief This class is a pseudo-wrapper for the serial::GenerateHistogram, the reason for this class' existence is to hold/define ownership of objects in each thread + * @tparam Type this the end type of the function in that the container and data values are of this type + * @tparam SizeType this is the scalar type of the bin counts container + */ template class SIMPLNX_EXPORT GenerateHistogramImpl { public: - GenerateHistogramImpl(const AbstractDataStore& inputStore, std::pair&& rangeMinMax, const std::atomic_bool& shouldCancel, const int32 numBins, - AbstractDataStore& binRangesStore, AbstractDataStore& histogramStore, std::atomic& overflow) + /** + * @function constructor + * @brief This constructor requires a defined range and creates the object + * @param inputStore this is the AbstractDataStore holding the data that will be binned + * @param binRangesStore this is the AbstractDataStore that the ranges will be loaded into. + * @param rangeMinMax this is assumed to be the inclusive minimum value and exclusive maximum value for the overall histogram bins. FORMAT: [minimum, maximum) + * @param shouldCancel this is an atomic value that will determine whether execution ends early + * @param numBins this is the total number of bin ranges being calculated and by extension the indexing value for the ranges + * @param histogramStore this is the AbstractDataStore that will hold the counts for each bin (variable type sizing) + * @param overflow this is an atomic counter for the number of values that fall outside the bin range + */ + GenerateHistogramImpl(const AbstractDataStore& inputStore, AbstractDataStore& binRangesStore, std::pair&& rangeMinMax, const std::atomic_bool& shouldCancel, + const int32 numBins, AbstractDataStore& histogramStore, std::atomic& overflow) : m_InputStore(inputStore) , m_ShouldCancel(shouldCancel) , m_NumBins(numBins) @@ -183,7 +223,17 @@ class SIMPLNX_EXPORT GenerateHistogramImpl m_Range = std::make_pair(static_cast(rangeMinMax.first), static_cast(rangeMinMax.second)); } - GenerateHistogramImpl(const AbstractDataStore& inputStore, const std::atomic_bool& shouldCancel, const int32 numBins, AbstractDataStore& binRangesStore, + /** + * @function constructor + * @brief This constructor constructs the object then calculates and stores the range implicitly + * @param inputStore this is the AbstractDataStore holding the data that will be binned + * @param binRangesStore this is the AbstractDataStore that the ranges will be loaded into. + * @param shouldCancel this is an atomic value that will determine whether execution ends early + * @param numBins this is the total number of bin ranges being calculated and by extension the indexing value for the ranges + * @param histogramStore this is the AbstractDataStore that will hold the counts for each bin (variable type sizing) + * @param overflow this is an atomic counter for the number of values that fall outside the bin range + */ + GenerateHistogramImpl(const AbstractDataStore& inputStore, AbstractDataStore& binRangesStore, const std::atomic_bool& shouldCancel, const int32 numBins, AbstractDataStore& histogramStore, std::atomic& overflow) : m_InputStore(inputStore) , m_ShouldCancel(shouldCancel) @@ -198,9 +248,13 @@ class SIMPLNX_EXPORT GenerateHistogramImpl ~GenerateHistogramImpl() = default; + /** + * @function operator() + * @brief This function serves as the execute method + */ void operator()() const { - serial::GenerateHistogram(m_InputStore, m_Range, m_ShouldCancel, m_NumBins, m_HistogramStore, m_Overflow); + serial::GenerateHistogram(m_InputStore, m_BinRangesStore, m_Range, m_ShouldCancel, m_NumBins, m_HistogramStore, m_Overflow); } private: @@ -213,12 +267,17 @@ class SIMPLNX_EXPORT GenerateHistogramImpl std::atomic& m_Overflow; }; +/** + * @class InstantiateHistogramImplFunctor + * @brief This is a compatibility functor that leverages existing typecasting functions to create the appropriately typed GenerateHistogramImpl() cleanly. + * Designed for compatibility with the existing parallel execution classes. + */ struct InstantiateHistogramImplFunctor { template - auto operator()(const IDataArray* iDataArray, ArgsT&&... args) + auto operator()(const IDataArray* inputArray, IDataArray* binRangesArray, ArgsT&&... args) { - return GenerateHistogramImpl(iDataArray->template getIDataStoreRefAs>(), std::forward(args)...); + return GenerateHistogramImpl(inputArray->template getIDataStoreRefAs>(), binRangesArray->template getIDataStoreRefAs>(), std::forward(args)...); } }; } // namespace concurrent From 16b9ec05738a76c239fa826c78a9a63bfb6a6dc8 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Wed, 4 Sep 2024 16:46:40 -0400 Subject: [PATCH 06/25] repair compute array histogram [compiling, functional, failed tests] --- .../Algorithms/ComputeArrayHistogram.cpp | 2 +- .../Algorithms/ComputeArrayHistogram.hpp | 16 +----- .../Algorithms/ComputeArrayStatistics.cpp | 1 + .../Filters/ComputeArrayHistogramFilter.cpp | 57 ++++++++++++------- .../Filters/ComputeArrayHistogramFilter.hpp | 3 +- .../test/ComputeArrayHistogramTest.cpp | 2 +- 6 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp index e5e65c7096..3cc8916832 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.cpp @@ -54,8 +54,8 @@ Result<> ComputeArrayHistogram::operator()() } const auto* inputData = m_DataStructure.getDataAs(selectedArrayPaths[i]); - auto& counts = m_DataStructure.getDataAs>(m_InputValues->CreatedHistogramCountsDataPaths.at(i))->getDataStoreRef(); auto* binRanges = m_DataStructure.getDataAs(m_InputValues->CreatedBinRangeDataPaths.at(i)); + auto& counts = m_DataStructure.getDataAs>(m_InputValues->CreatedHistogramCountsDataPaths.at(i))->getDataStoreRef(); Result<> result = {}; if(m_InputValues->UserDefinedRange) { diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp index fd41690162..40ae1f9ddc 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogram.hpp @@ -7,28 +7,23 @@ #include "simplnx/Filter/IFilter.hpp" #include "simplnx/Parameters/MultiArraySelectionParameter.hpp" -#include -#include - namespace nx::core { - struct SIMPLNXCORE_EXPORT ComputeArrayHistogramInputValues { - int32 NumberOfBins = 0; bool UserDefinedRange = false; + int32 NumberOfBins = 0; float64 MinRange = 0.0; float64 MaxRange = 0.0; MultiArraySelectionParameter::ValueType SelectedArrayPaths = {}; - MultiArraySelectionParameter::ValueType CreatedHistogramCountsDataPaths = {}; MultiArraySelectionParameter::ValueType CreatedBinRangeDataPaths = {}; + MultiArraySelectionParameter::ValueType CreatedHistogramCountsDataPaths = {}; }; /** * @class ComputeArrayHistogram * @brief This filter calculates a Histogram according to user specification and stores it accordingly */ - class SIMPLNXCORE_EXPORT ComputeArrayHistogram { public: @@ -50,12 +45,5 @@ class SIMPLNXCORE_EXPORT ComputeArrayHistogram const ComputeArrayHistogramInputValues* m_InputValues = nullptr; const std::atomic_bool& m_ShouldCancel; const IFilter::MessageHandler& m_MessageHandler; - - // Threadsafe Progress Message - mutable std::mutex m_ProgressMessage_Mutex; - size_t m_TotalElements = 0; - size_t m_ProgressCounter = 0; - std::chrono::steady_clock::time_point m_InitialTime = std::chrono::steady_clock::now(); }; - } // namespace nx::core diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp index 09875f9fea..1cfdb9980c 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp @@ -2,6 +2,7 @@ #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/Utilities/DataArrayUtilities.hpp" +#include "simplnx/Utilities/HistogramUtilities.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" #include "simplnx/Utilities/Math/StatisticsCalculations.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp index 001cbfe8d4..24e2a813e3 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp @@ -70,7 +70,10 @@ Parameters ComputeArrayHistogramFilter::parameters() const params.insert(std::make_unique(k_NewDataGroupPath_Key, "New DataGroup Path", "The path to the new DataGroup in which to store the calculated histogram(s)", DataPath{})); params.insert(std::make_unique(k_DataGroupPath_Key, "Output DataGroup Path", "The complete path to the DataGroup in which to store the calculated histogram(s)", DataPath{}, DataGroupSelectionParameter::AllowedTypes{BaseGroup::GroupType::AttributeMatrix, BaseGroup::GroupType::DataGroup})); - params.insert(std::make_unique(k_HistoName_Key, "Suffix for created Histograms", "String appended to the end of the histogram array names", " Histogram")); + params.insert(std::make_unique(k_HistoBinCountName_Key, "Suffix for created Histogram Bin Counts", "String appended to the end of the histogram array names", " Histogram Counts")); + params.insert( + std::make_unique(k_HistoBinRangeName_Key, "Suffix for created Histogram Bin Ranges", "String appended to the end of the histogram array names", " Histogram Bin Ranges")); + // Associate the Linkable Parameter(s) to the children parameters that they control params.linkParameters(k_UserDefinedRange_Key, k_MinRange_Key, true); params.linkParameters(k_UserDefinedRange_Key, k_MaxRange_Key, true); @@ -95,36 +98,43 @@ IFilter::PreflightResult ComputeArrayHistogramFilter::preflightImpl(const DataSt auto pDataGroupNameValue = filterArgs.value(k_DataGroupPath_Key); auto pSelectedArrayPathsValue = filterArgs.value(k_SelectedArrayPaths_Key); auto pNewDataGroupNameValue = filterArgs.value(k_NewDataGroupPath_Key); // sanity check if is Attribute matrix after impending simplnx update - auto pHistogramSuffix = filterArgs.value(k_HistoName_Key); - - PreflightResult preflightResult; + auto pBinCountSuffix = filterArgs.value(k_HistoBinCountName_Key); + auto pBinRangeSuffix = filterArgs.value(k_HistoBinRangeName_Key); nx::core::Result resultOutputActions; - - std::vector preflightUpdatedValues; + ; if(pNewDataGroupValue) { auto createDataGroupAction = std::make_unique(pNewDataGroupNameValue); resultOutputActions.value().appendAction(std::move(createDataGroupAction)); } + DataPath parentPath = {}; + if(pNewDataGroupValue) + { + parentPath = pNewDataGroupNameValue; + } + else + { + parentPath = pDataGroupNameValue; + } for(auto& selectedArrayPath : pSelectedArrayPathsValue) { - const auto& dataArray = dataStructure.getDataAs(selectedArrayPath); - DataPath childPath; - if(pNewDataGroupValue) + const auto* dataArray = dataStructure.getDataAs(selectedArrayPath); { - childPath = pNewDataGroupNameValue.createChildPath((dataArray->getName() + pHistogramSuffix)); + auto createArrayAction = std::make_unique(nx::core::DataType::uint64, std::vector{static_cast(pNumberOfBinsValue)}, std::vector{1}, + parentPath.createChildPath((dataArray->getName() + pBinCountSuffix))); + resultOutputActions.value().appendAction(std::move(createArrayAction)); } - else + { - childPath = pDataGroupNameValue.createChildPath((dataArray->getName() + pHistogramSuffix)); + auto createArrayAction = std::make_unique(dataArray->getDataType(), std::vector{static_cast(pNumberOfBinsValue + 1)}, std::vector{1}, + parentPath.createChildPath((dataArray->getName() + pBinCountSuffix))); + resultOutputActions.value().appendAction(std::move(createArrayAction)); } - auto createArrayAction = std::make_unique(nx::core::DataType::float64, std::vector{static_cast(pNumberOfBinsValue)}, std::vector{2}, childPath); - resultOutputActions.value().appendAction(std::move(createArrayAction)); } - return {std::move(resultOutputActions), std::move(preflightUpdatedValues)}; + return {std::move(resultOutputActions)}; } //------------------------------------------------------------------------------ @@ -139,7 +149,8 @@ Result<> ComputeArrayHistogramFilter::executeImpl(DataStructure& dataStructure, inputValues.MaxRange = filterArgs.value(k_MaxRange_Key); inputValues.SelectedArrayPaths = filterArgs.value(k_SelectedArrayPaths_Key); - auto histogramSuffix = filterArgs.value(k_HistoName_Key); + auto binCountSuffix = filterArgs.value(k_HistoBinCountName_Key); + auto binRangeSuffix = filterArgs.value(k_HistoBinRangeName_Key); DataPath dataGroupPath; if(filterArgs.value(k_CreateNewDataGroup_Key)) @@ -150,15 +161,19 @@ Result<> ComputeArrayHistogramFilter::executeImpl(DataStructure& dataStructure, { dataGroupPath = filterArgs.value(k_DataGroupPath_Key); } - std::vector createdDataPaths; + std::vector createdCountsDataPaths; + std::vector createdRangesDataPaths; for(auto& selectedArrayPath : inputValues.SelectedArrayPaths) // regenerate based on preflight { const auto& dataArray = dataStructure.getDataAs(selectedArrayPath); - auto childPath = dataGroupPath.createChildPath((dataArray->getName() + histogramSuffix)); - createdDataPaths.push_back(childPath); + auto countsPath = dataGroupPath.createChildPath((dataArray->getName() + binCountSuffix)); + createdCountsDataPaths.push_back(countsPath); + auto rangesPath = dataGroupPath.createChildPath((dataArray->getName() + binRangeSuffix)); + createdRangesDataPaths.push_back(rangesPath); } - inputValues.CreatedHistogramCountsDataPaths = createdDataPaths; + inputValues.CreatedHistogramCountsDataPaths = createdCountsDataPaths; + inputValues.CreatedBinRangeDataPaths = createdRangesDataPaths; return ComputeArrayHistogram(dataStructure, messageHandler, shouldCancel, &inputValues)(); } @@ -194,7 +209,7 @@ Result ComputeArrayHistogramFilter::FromSIMPLJson(const nlohmann::jso results.push_back(SIMPLConversion::ConvertParameter(args, json, SIMPL::k_NewDataContainerNameKey, k_NewDataGroupPath_Key)); results.push_back(SIMPLConversion::Convert2Parameters(args, json, SIMPL::k_NewDataContainerNameKey, SIMPL::k_NewAttributeMatrixNameKey, k_DataGroupPath_Key)); - results.push_back(SIMPLConversion::ConvertParameter(args, json, SIMPL::k_NewDataArrayNameKey, k_HistoName_Key)); + results.push_back(SIMPLConversion::ConvertParameter(args, json, SIMPL::k_NewDataArrayNameKey, k_HistoBinCountName_Key)); Result<> conversionResult = MergeResults(std::move(results)); diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp index b19f8f3fdd..36d33c604a 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.hpp @@ -32,7 +32,8 @@ class SIMPLNXCORE_EXPORT ComputeArrayHistogramFilter : public IFilter static inline constexpr StringLiteral k_SelectedArrayPaths_Key = "selected_array_paths"; static inline constexpr StringLiteral k_NewDataGroupPath_Key = "new_data_group_path"; static inline constexpr StringLiteral k_DataGroupPath_Key = "output_data_group_path"; - static inline constexpr StringLiteral k_HistoName_Key = "histogram_suffix"; + static inline constexpr StringLiteral k_HistoBinCountName_Key = "histogram_bin_count_suffix"; + static inline constexpr StringLiteral k_HistoBinRangeName_Key = "histogram_bin_range_suffix"; /** * @brief Reads SIMPL json and converts it simplnx Arguments. diff --git a/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp b/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp index 877686132c..be2063fe11 100644 --- a/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp +++ b/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp @@ -59,7 +59,7 @@ TEST_CASE("SimplnxCore::ComputeArrayHistogram: Valid Filter Execution", "[Simpln args.insertOrAssign(ComputeArrayHistogramFilter::k_CreateNewDataGroup_Key, std::make_any(true)); args.insertOrAssign(ComputeArrayHistogramFilter::k_SelectedArrayPaths_Key, std::make_any(dataPaths)); args.insertOrAssign(ComputeArrayHistogramFilter::k_NewDataGroupPath_Key, std::make_any(dataGPath)); - args.insertOrAssign(ComputeArrayHistogramFilter::k_HistoName_Key, std::make_any("Histogram")); + args.insertOrAssign(ComputeArrayHistogramFilter::k_HistoBinCountName_Key, std::make_any("Histogram")); // Preflight the filter and check result auto preflightResult = filter.preflight(dataStruct, args); From e8d841fdd35e3598bbf92c61bcf3275ee55c02d0 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Thu, 5 Sep 2024 09:59:00 -0400 Subject: [PATCH 07/25] fix copy bug --- .../SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp index 24e2a813e3..ccb2ce08d3 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeArrayHistogramFilter.cpp @@ -53,17 +53,20 @@ std::vector ComputeArrayHistogramFilter::defaultTags() const Parameters ComputeArrayHistogramFilter::parameters() const { Parameters params; + // Create the parameter descriptors that are needed for this filter params.insertSeparator(Parameters::Separator{"Input Parameter(s)"}); params.insert(std::make_unique(k_NumberOfBins_Key, "Number of Bins", "Specifies number of histogram bins (greater than zero)", 1)); params.insertLinkableParameter( std::make_unique(k_UserDefinedRange_Key, "Use Custom Min & Max Range", "Whether the user can set the min and max values to consider for the histogram", false)); - params.insert(std::make_unique(k_MinRange_Key, "Min Value", "Specifies the lower bound of the histogram.", 0.0)); - params.insert(std::make_unique(k_MaxRange_Key, "Max Value", "Specifies the upper bound of the histogram.", 1.0)); + params.insert(std::make_unique(k_MinRange_Key, "Min Value", "Specifies the inclusive lower bound of the histogram.", 0.0)); + params.insert(std::make_unique(k_MaxRange_Key, "Max Value", "Specifies the exclusive upper bound of the histogram.", 1.0)); + params.insertSeparator(Parameters::Separator{"Input Data"}); params.insert(std::make_unique(k_SelectedArrayPaths_Key, "Input Data Arrays", "The list of arrays to calculate histogram(s) for", MultiArraySelectionParameter::ValueType{}, MultiArraySelectionParameter::AllowedTypes{IArray::ArrayType::DataArray}, nx::core::GetAllNumericTypes())); + params.insertSeparator(Parameters::Separator{"Output parameters"}); params.insertLinkableParameter( std::make_unique(k_CreateNewDataGroup_Key, "Create New DataGroup for Histograms", "Whether or not to store the calculated histogram(s) in a new DataGroup", true)); @@ -129,7 +132,7 @@ IFilter::PreflightResult ComputeArrayHistogramFilter::preflightImpl(const DataSt { auto createArrayAction = std::make_unique(dataArray->getDataType(), std::vector{static_cast(pNumberOfBinsValue + 1)}, std::vector{1}, - parentPath.createChildPath((dataArray->getName() + pBinCountSuffix))); + parentPath.createChildPath((dataArray->getName() + pBinRangeSuffix))); resultOutputActions.value().appendAction(std::move(createArrayAction)); } } From c0a3910f9c7d4f74cd90e99849b8890f763700dc Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Thu, 5 Sep 2024 17:05:57 -0400 Subject: [PATCH 08/25] Update test case to reflect new histogram that produces results inline with plot.ly histogram --- .../test/ComputeArrayHistogramTest.cpp | 67 ++++++++++--------- src/simplnx/Utilities/HistogramUtilities.hpp | 2 +- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp b/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp index be2063fe11..7e2f600c66 100644 --- a/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp +++ b/src/Plugins/SimplnxCore/test/ComputeArrayHistogramTest.cpp @@ -14,12 +14,22 @@ using namespace nx::core; namespace { constexpr float64 k_max_difference = 0.0001; +constexpr StringLiteral k_BinRangesSuffix = " Ranges"; +constexpr StringLiteral k_BinCountsSuffix = " Counts"; +constexpr StringLiteral k_Array0Name = "array0"; +constexpr StringLiteral k_Array1Name = "array1"; +constexpr StringLiteral k_Array2Name = "array2"; -void compareHistograms(const DataArray& calulated, const std::array& actual) +template +void compareHistograms(const AbstractDataStore& calculated, const std::array& actual) { - for(int32 i = 0; i < actual.size(); i++) + if(calculated.getSize() != actual.size()) { - float64 diff = std::fabs(calulated[i] - actual[i]); + throw std::runtime_error("Improper sizing of DataStore"); + } + for(int32 i = 0; i < N; i++) + { + T diff = std::fabs(calculated[i] - actual[i]); REQUIRE(diff < ::k_max_difference); } } @@ -44,10 +54,10 @@ TEST_CASE("SimplnxCore::ComputeArrayHistogram: Valid Filter Execution", "[Simpln Arguments args; // load vector with data paths for test - ::fillArray(*DataArray::CreateWithStore>(dataStruct, "array0", {static_cast(4)}, {static_cast(3)}), + ::fillArray(*DataArray::CreateWithStore>(dataStruct, k_Array0Name, {static_cast(4)}, {static_cast(3)}), {0.0, 5.5, 8.5, 9.2, 16.7, 907.3, 5.0, 6.9, 83.7387483, -56.8, 3.7, -4.9}); - ::fillArray(*DataArray::CreateWithStore>(dataStruct, "array1", {static_cast(4)}, {static_cast(3)}), {56, 82, 46, 93, 73, 57, 24, 32, -90, -35, 74, -19}); - ::fillArray(*DataArray::CreateWithStore>(dataStruct, "array2", {static_cast(4)}, {static_cast(3)}), {83, 93, 75, 67, 8977, 56, 48, 92, 57, 34, 34, 34}); + ::fillArray(*DataArray::CreateWithStore>(dataStruct, k_Array1Name, {static_cast(4)}, {static_cast(3)}), {56, 82, 46, 93, 73, 57, 24, 32, -90, -35, 74, -19}); + ::fillArray(*DataArray::CreateWithStore>(dataStruct, k_Array2Name, {static_cast(4)}, {static_cast(3)}), {83, 93, 75, 67, 8977, 56, 48, 92, 57, 34, 34, 34}); std::vector dataPaths = dataStruct.getAllDataPaths(); auto parentPath = dataPaths[0].getParent(); @@ -59,7 +69,8 @@ TEST_CASE("SimplnxCore::ComputeArrayHistogram: Valid Filter Execution", "[Simpln args.insertOrAssign(ComputeArrayHistogramFilter::k_CreateNewDataGroup_Key, std::make_any(true)); args.insertOrAssign(ComputeArrayHistogramFilter::k_SelectedArrayPaths_Key, std::make_any(dataPaths)); args.insertOrAssign(ComputeArrayHistogramFilter::k_NewDataGroupPath_Key, std::make_any(dataGPath)); - args.insertOrAssign(ComputeArrayHistogramFilter::k_HistoBinCountName_Key, std::make_any("Histogram")); + args.insertOrAssign(ComputeArrayHistogramFilter::k_HistoBinRangeName_Key, std::make_any(std::string{::k_BinRangesSuffix})); + args.insertOrAssign(ComputeArrayHistogramFilter::k_HistoBinCountName_Key, std::make_any(std::string{::k_BinCountsSuffix})); // Preflight the filter and check result auto preflightResult = filter.preflight(dataStruct, args); @@ -69,32 +80,28 @@ TEST_CASE("SimplnxCore::ComputeArrayHistogram: Valid Filter Execution", "[Simpln auto executeResult = filter.execute(dataStruct, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); - // load vector with child paths from filter - std::vector createdDataPaths; - for(auto& selectedArrayPath : dataPaths) // regenerate based on preflight { - const auto& dataArray = dataStruct.getDataAs(selectedArrayPath); - auto childPath = dataGPath.createChildPath((dataArray->getName() + "Histogram")); - createdDataPaths.push_back(childPath); + std::array binRangesSet = {-56.8, 184.475, 425.75, 667.025, 908.3}; + std::array binCountsSet = {11, 0, 0, 1}; + const std::string name = k_Array0Name; + + compareHistograms(dataStruct.getDataAs(dataGPath.createChildPath((name + std::string{k_BinRangesSuffix})))->getDataStoreRef(), binRangesSet); + compareHistograms(dataStruct.getDataAs(dataGPath.createChildPath((name + std::string{k_BinCountsSuffix})))->getDataStoreRef(), binCountsSet); } + { + std::array binRangesSet = {-90, -44, 2, 48, 94}; + std::array binCountsSet = {1, 2, 3, 6}; + const std::string name = k_Array1Name; - std::array array0HistogramSet = {183.725, 11, 425.25, 0, 666.775, 0, 908.3, 1}; - std::array array1HistogramSet = {-44.75, 1, 1.5, 2, 47.75, 3, 94, 6}; - std::array array2HistogramSet = {2269.25, 11, 4505.5, 0, 6741.75, 0, 8978, 1}; - for(const auto& child : createdDataPaths) + compareHistograms(dataStruct.getDataAs(dataGPath.createChildPath((name + std::string{k_BinRangesSuffix})))->getDataStoreRef(), binRangesSet); + compareHistograms(dataStruct.getDataAs(dataGPath.createChildPath((name + std::string{k_BinCountsSuffix})))->getDataStoreRef(), binCountsSet); + } { - auto& dataArray = dataStruct.getDataRefAs>(child); - if(dataArray.getName().find("array0") != std::string::npos) - { - compareHistograms(dataArray, array0HistogramSet); - } - else if(dataArray.getName().find("array1") != std::string::npos) - { - compareHistograms(dataArray, array1HistogramSet); - } - else if(dataArray.getName().find("array2") != std::string::npos) - { - compareHistograms(dataArray, array2HistogramSet); - } + std::array binRangesSet = {34, 2270, 4506, 6742, 8978}; + std::array binCountsSet = {11, 0, 0, 1}; + const std::string name = k_Array2Name; + + compareHistograms(dataStruct.getDataAs(dataGPath.createChildPath((name + std::string{k_BinRangesSuffix})))->getDataStoreRef(), binRangesSet); + compareHistograms(dataStruct.getDataAs(dataGPath.createChildPath((name + std::string{k_BinCountsSuffix})))->getDataStoreRef(), binCountsSet); } } diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp index e3d5458a42..96ce68b614 100644 --- a/src/simplnx/Utilities/HistogramUtilities.hpp +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -46,7 +46,7 @@ SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair(rangeMinMax.first + (increment * i)); + outputContainer[i] = rangeMinMax.first + static_cast(increment * i); } outputContainer[numBins] = rangeMinMax.second; From cef0a2109815b615b29a5bf5f45c47de7476c400 Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Fri, 6 Sep 2024 08:14:26 -0400 Subject: [PATCH 09/25] repair broken pipelines after key updates --- .../pipelines/CI_Histogram.d3dpipeline | 11 ++++++----- .../pipelines/aptr12_Analysis.d3dpipeline | 11 ++++++----- .../pipelines/avtr12_Analysis.d3dpipeline | 11 ++++++----- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline b/src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline index 8873b62eaa..47bb27ec36 100644 --- a/src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline +++ b/src/Plugins/OrientationAnalysis/pipelines/CI_Histogram.d3dpipeline @@ -117,7 +117,8 @@ { "args": { "new_data_group_path": "DataContainer/Statistics", - "histogram_suffix": " Histogram", + "histogram_bin_count_suffix": " Histogram Counts", + "histogram_bin_range_suffix": " Histogram Bin Ranges", "max_range": 1.0, "min_range": 0.0, "create_new_data_group": true, @@ -141,11 +142,11 @@ "file_extension": ".csv", "header_option_index": 1, "max_val_per_line": 0, - "output_dir": "Data/Output/OrientationAnalysis/", - "output_path": "Data/Output/OrientationAnalysis/Test/CI_Histogram.csv", - "output_style_index": 1, + "output_dir": "Data/Output/OrientationAnalysis/Test", + "output_style_index": 0, "input_data_array_paths": [ - "DataContainer/Statistics/Confidence Index Histogram" + "DataContainer/Statistics/Confidence Index Histogram Counts", + "DataContainer/Statistics/Confidence Index Histogram Bin Ranges" ] }, "comments": "", diff --git a/src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline b/src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline index b526795d5c..29a75c7f7a 100644 --- a/src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline +++ b/src/Plugins/OrientationAnalysis/pipelines/aptr12_Analysis.d3dpipeline @@ -448,7 +448,8 @@ { "args": { "create_new_data_group": true, - "histogram_suffix": " Histogram", + "histogram_bin_count_suffix": " Histogram Counts", + "histogram_bin_range_suffix": " Histogram Bin Ranges", "max_range": 1.0, "min_range": 0.0, "new_data_group_path": "fw-ar-IF1-aptr12-corr/Histograms", @@ -472,12 +473,12 @@ "file_extension": ".csv", "header_option_index": 1, "input_data_array_paths": [ - "fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram" + "fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Counts", + "fw-ar-IF1-aptr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges" ], "max_val_per_line": 0, - "output_dir": "", - "output_path": "Data/Output/fw-ar-IF1-aptr12-corr/EqDiamHistogram.csv", - "output_style_index": 1 + "output_dir": "Data/Output/fw-ar-IF1-aptr12-corr", + "output_style_index": 0 }, "comments": "", "filter": { diff --git a/src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline b/src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline index df912c6f3d..0022ca5a01 100644 --- a/src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline +++ b/src/Plugins/OrientationAnalysis/pipelines/avtr12_Analysis.d3dpipeline @@ -448,7 +448,8 @@ { "args": { "create_new_data_group": true, - "histogram_suffix": "Histogram", + "histogram_bin_count_suffix": " Histogram Counts", + "histogram_bin_range_suffix": " Histogram Bin Ranges", "max_range": 1.0, "min_range": 0.0, "new_data_group_path": "fw-ar-IF1-avtr12-corr/Histograms", @@ -472,12 +473,12 @@ "file_extension": ".csv", "header_option_index": 1, "input_data_array_paths": [ - "fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiametersHistogram" + "fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Counts", + "fw-ar-IF1-avtr12-corr/Histograms/EquivalentDiameters Histogram Bin Ranges" ], "max_val_per_line": 0, - "output_dir": "", - "output_path": "Data/Output/fw-ar-IF1-avtr12-corr/EqDiamHistogram.csv", - "output_style_index": 1 + "output_dir": "Data/Output/fw-ar-IF1-avtr12-corr", + "output_style_index": 0 }, "comments": "", "filter": { From 2a6064a784912cdb06b2431b8e38ec1091e2a65f Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Fri, 6 Sep 2024 16:59:02 -0400 Subject: [PATCH 10/25] Relax Container Type restriction in histogram utilities --- src/simplnx/Utilities/HistogramUtilities.hpp | 23 +++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp index 96ce68b614..55573e562f 100644 --- a/src/simplnx/Utilities/HistogramUtilities.hpp +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -79,32 +79,39 @@ SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair class Container> -SIMPLNX_EXPORT Result<> GenerateHistogram(const Container& inputStore, Container& binRangesStore, const std::pair& rangeMinMax, const std::atomic_bool& shouldCancel, - const int32 numBins, Container& histogramCountsStore, std::atomic& overflow) +template class InputContainer, template class OutputContainer> +SIMPLNX_EXPORT Result<> GenerateHistogram(const InputContainer& inputStore, OutputContainer& binRangesStore, const std::pair& rangeMinMax, const std::atomic_bool& shouldCancel, + const int32 numBins, OutputContainer& histogramCountsStore, std::atomic& overflow) { usize end = 0; - if constexpr(std::is_same_v, Container>) + if constexpr(std::is_same_v, InputContainer>) { end = inputStore.size(); - + } + if constexpr(std::is_same_v, OutputContainer>) + { // just resize outputs to ensure no wasted space and won't be out of bounds binRangesStore.resize(numBins + 1); histogramCountsStore.resize(numBins); } - if constexpr(std::is_same_v, Container>) + if constexpr(std::is_same_v, InputContainer>) { end = inputStore.getSize(); + } + if constexpr(std::is_same_v, OutputContainer>) + { if(binRangesStore.getSize() < numBins + 1) { return MakeErrorResult(-23761, fmt::format("HistogramUtilities::{}: binRangesStore is too small to hold ranges. Needed: {} | Current Size: {}. {}:{}", __func__, numBins + 1, From 5b5851965cab11bd22be2c414e19b28be4e9f8ce Mon Sep 17 00:00:00 2001 From: nyoungbq Date: Tue, 10 Sep 2024 16:49:21 -0400 Subject: [PATCH 11/25] Initial compute Array statistics parallel switch [broken] --- .../Algorithms/ComputeArrayStatistics.cpp | 60 ++++++++----- .../Algorithms/ComputeArrayStatistics.hpp | 3 +- src/simplnx/Utilities/HistogramUtilities.hpp | 25 +++--- .../Utilities/Math/StatisticsCalculations.hpp | 89 +++++-------------- 4 files changed, 72 insertions(+), 105 deletions(-) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp index 1cfdb9980c..966fe864fe 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp @@ -2,8 +2,8 @@ #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/Utilities/DataArrayUtilities.hpp" -#include "simplnx/Utilities/HistogramUtilities.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" +#include "simplnx/Utilities/HistogramUtilities.hpp" #include "simplnx/Utilities/Math/StatisticsCalculations.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" @@ -601,7 +601,13 @@ void FindStatisticsImpl(const ContainerType& data, std::vector& arrays, auto* array7Ptr = dynamic_cast(arrays[8]); if(array7Ptr == nullptr) { - throw std::invalid_argument("findStatisticsImpl() could not dynamic_cast 'Histogram' array to needed type. Check input array selection."); + throw std::invalid_argument("findStatisticsImpl() could not dynamic_cast 'Histogram Bin Counts' array to needed type. Check input array selection."); + } + + auto* array12Ptr = dynamic_cast*>(arrays[12]); + if(array12Ptr == nullptr) + { + throw std::invalid_argument("findStatisticsImpl() could not dynamic_cast 'Histogram Bin Ranges' array to needed type. Check input array selection."); } auto* array10Ptr = dynamic_cast(arrays[10]); @@ -610,17 +616,20 @@ void FindStatisticsImpl(const ContainerType& data, std::vector& arrays, throw std::invalid_argument("findStatisticsImpl() could not dynamic_cast 'Most Populated Bin' array to needed type. Check input array selection."); } - auto* arr10DataStorePtr = array10Ptr->getDataStore(); - if(auto* arr7DataStorePtr = array7Ptr->getDataStore(); arr7DataStorePtr != nullptr) - { - std::vector values = StatisticsCalculations::findHistogram(data, inputValues->MinRange, inputValues->MaxRange, inputValues->UseFullRange, inputValues->NumBins); - arr7DataStorePtr->setTuple(0, values); + auto& binCountsStore = array7Ptr->getDataStoreRef(); + auto& binRangesStore = array12Ptr->getDataStoreRef(); + auto& mostPopBinStore = array10Ptr->getDataStoreRef(); - auto maxElementIt = std::max_element(values.begin(), values.end()); - uint64 index = std::distance(values.begin(), maxElementIt); - arr10DataStorePtr->setComponent(0, 0, index); - arr10DataStorePtr->setComponent(0, 1, values[index]); - } + auto range = StatisticsCalculations::findHistogramRange(data, static_cast(inputValues->MinRange), static_cast(inputValues->MaxRange), inputValues->UseFullRange); + + std::atomic_bool neverCancel{false}; + std::atomic overflow{0}; + Result<> result = HistogramUtilities::serial::GenerateHistogram(data, binRangesStore, range, neverCancel, inputValues->NumBins, binCountsStore, overflow); + + auto maxElementIt = std::max_element(binCountsStore.begin(), binCountsStore.end()); + uint64 index = std::distance(binCountsStore.begin(), maxElementIt); + mostPopBinStore.setComponent(0, 0, index); + mostPopBinStore.setComponent(0, 1, binCountsStore[index]); if(inputValues->FindModalBinRanges) { @@ -630,7 +639,7 @@ void FindStatisticsImpl(const ContainerType& data, std::vector& arrays, throw std::invalid_argument("findStatisticsImpl() could not dynamic_cast 'Mode' array to needed type. Check input array selection."); } - auto* array11Ptr = dynamic_cast*>(arrays[11]); + auto* array11Ptr = dynamic_cast*>(arrays[11]); if(array11Ptr == nullptr) { throw std::invalid_argument("findStatisticsImpl() could not dynamic_cast 'Modal Bin Ranges' array to needed type. Check input array selection."); @@ -638,9 +647,9 @@ void FindStatisticsImpl(const ContainerType& data, std::vector& arrays, for(const T& mode : array5Ptr->at(0)) { - std::pair range = StatisticsCalculations::findModalBinRange(data, inputValues->MinRange, inputValues->MaxRange, inputValues->UseFullRange, inputValues->NumBins, mode); - array11Ptr->addEntry(0, range.first); - array11Ptr->addEntry(0, range.second); + std::pair modalRange = StatisticsCalculations::findModalBinRange(data, binRangesStore, mode); + array11Ptr->addEntry(0, modalRange.first); + array11Ptr->addEntry(0, modalRange.second); } } } @@ -856,7 +865,7 @@ struct ComputeArrayStatisticsFunctor { return MakeErrorResult(-563502, "ComputeArrayStatisticsFunctor could not dynamic_cast 'Feature-Has-Data' array to needed type. Check input array selection."); } - arrayPtr->fill(0); + arrayPtr->fill(false); } if(inputValues->FindLength) { @@ -924,13 +933,21 @@ struct ComputeArrayStatisticsFunctor if(inputValues->FindHistogram) { { - auto* arrayPtr = dataStructure.getDataAs(inputValues->HistogramArrayName); + auto* arrayPtr = dataStructure.getDataAs(inputValues->BinCountsArrayName); if(arrayPtr == nullptr) { - return MakeErrorResult(-563511, "ComputeArrayStatisticsFunctor could not dynamic_cast 'Histogram' array to needed type. Check input array selection."); + return MakeErrorResult(-563511, "ComputeArrayStatisticsFunctor could not dynamic_cast 'Histogram Bin Counts' array to needed type. Check input array selection."); } arrayPtr->fill(0); } + { + auto* arrayPtr = dataStructure.getDataAs(inputValues->BinRangesArrayName); + if(arrayPtr == nullptr) + { + return MakeErrorResult(-563514, "ComputeArrayStatisticsFunctor could not dynamic_cast 'Histogram Bin Ranges' array to needed type. Check input array selection."); + } + arrayPtr->fill(static_cast(0.0)); + } { auto* arrayPtr = dataStructure.getDataAs(inputValues->MostPopulatedBinArrayName); if(arrayPtr == nullptr) @@ -997,7 +1014,7 @@ Result<> ComputeArrayStatistics::operator()() return {}; } - std::vector arrays(12, nullptr); + std::vector arrays(13, nullptr); if(m_InputValues->FindLength) { @@ -1033,7 +1050,8 @@ Result<> ComputeArrayStatistics::operator()() } if(m_InputValues->FindHistogram) { - arrays[8] = m_DataStructure.getDataAs(m_InputValues->HistogramArrayName); + arrays[8] = m_DataStructure.getDataAs(m_InputValues->BinCountsArrayName); + arrays[12] = m_DataStructure.getDataAs(m_InputValues->BinRangesArrayName); arrays[10] = m_DataStructure.getDataAs(m_InputValues->MostPopulatedBinArrayName); } if(m_InputValues->FindModalBinRanges) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.hpp index 06265f8692..3802632fe7 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.hpp @@ -38,7 +38,8 @@ struct SIMPLNXCORE_EXPORT ComputeArrayStatisticsInputValues DataPath FeatureIdsArrayPath; DataPath MaskArrayPath; DataPath DestinationAttributeMatrix; - DataPath HistogramArrayName; + DataPath BinCountsArrayName; + DataPath BinRangesArrayName; DataPath MostPopulatedBinArrayName; DataPath ModalBinArrayName; DataPath FeatureHasDataArrayName; diff --git a/src/simplnx/Utilities/HistogramUtilities.hpp b/src/simplnx/Utilities/HistogramUtilities.hpp index 55573e562f..9797e37b27 100644 --- a/src/simplnx/Utilities/HistogramUtilities.hpp +++ b/src/simplnx/Utilities/HistogramUtilities.hpp @@ -9,16 +9,6 @@ namespace nx::core::HistogramUtilities { namespace serial { -namespace detail -{ -template -concept HasBracketOperator = requires(T object, usize i) -{ - { - object[i] - } -> std::same_as; -}; - /** * @function FillBinRange * @brief This function fills a container that is STL compatible and has a bracket operator defined with the bin ranges in the following pattern: @@ -30,7 +20,7 @@ concept HasBracketOperator = requires(T object, usize i) * @param numBins this is the total number of bin ranges being calculated and by extension the indexing value for the ranges * @param increment this is the uniform size of the bins */ -template +template SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair& rangeMinMax, const int32 numBins, const Type increment) { // WARNING: No bounds checking for type compatibility, it is expected to be done higher up where the type is not abstracted @@ -61,7 +51,7 @@ SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair +template SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair& rangeMinMax, const int32 numBins) { // DEV NOTE: this function also serves to act as a jumping off point for implementing logarithmic histograms down the line @@ -71,7 +61,12 @@ SIMPLNX_EXPORT void FillBinRanges(Container& outputContainer, const std::pair +SIMPLNX_EXPORT Type CalculateBin(Type value, Type min, Type increment) +{ + return std::floor((value - min) / increment); +} /** * @function GenerateHistogram @@ -127,7 +122,7 @@ SIMPLNX_EXPORT Result<> GenerateHistogram(const InputContainer& inputStore const Type increment = (rangeMinMax.second - rangeMinMax.first) / static_cast(numBins); // Fill Bins - detail::FillBinRanges(binRangesStore, rangeMinMax, numBins, increment); + FillBinRanges(binRangesStore, rangeMinMax, numBins, increment); if(shouldCancel) { @@ -140,7 +135,7 @@ SIMPLNX_EXPORT Result<> GenerateHistogram(const InputContainer& inputStore { return MakeErrorResult(-23763, fmt::format("HistogramUtilities::{}: Signal Interrupt Received. {}:{}", __func__, __FILE__, __LINE__)); } - const auto bin = std::floor((inputStore[i] - rangeMinMax.first) / increment); + const auto bin = CalculateBin(inputStore[i], rangeMinMax.first, increment); if((bin >= 0) && (bin < numBins)) { histogramCountsStore[bin]++; diff --git a/src/simplnx/Utilities/Math/StatisticsCalculations.hpp b/src/simplnx/Utilities/Math/StatisticsCalculations.hpp index 29a306fdab..a5996ff4a2 100644 --- a/src/simplnx/Utilities/Math/StatisticsCalculations.hpp +++ b/src/simplnx/Utilities/Math/StatisticsCalculations.hpp @@ -236,104 +236,57 @@ size_t findNumUniqueValues(const C& source) // ----------------------------------------------------------------------------- template