diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertColorToGrayScale.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertColorToGrayScale.cpp index 468100ed89..c2b4de3ba9 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertColorToGrayScale.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertColorToGrayScale.cpp @@ -134,11 +134,11 @@ class ParallelWrapper ParallelWrapper& operator=(ParallelWrapper&&) = delete; // Move Assignment Not Implemented template - static void Run(T impl, size_t totalPoints) + static void Run(T impl, size_t totalPoints, typename IParallelAlgorithm::AlgorithmArrays algArrays) { ParallelDataAlgorithm dataAlg; dataAlg.setRange(0, totalPoints); - dataAlg.setParallelizationEnabled(false); + dataAlg.requireArraysInMemory(algArrays); dataAlg.execute(impl); } @@ -186,19 +186,23 @@ Result<> ConvertColorToGrayScale::operator()() size_t comp = inputColorData.getNumberOfComponents(); size_t totalPoints = inputColorData.getNumberOfTuples(); + typename IParallelAlgorithm::AlgorithmArrays algArrays; + algArrays.push_back(&inputColorData); + algArrays.push_back(&outputGrayData); + switch(convType) { case ConversionType::Luminosity: - ParallelWrapper::Run(LuminosityImpl(inputColorData, outputGrayData, m_InputValues->ColorWeights, comp), totalPoints); + ParallelWrapper::Run(LuminosityImpl(inputColorData, outputGrayData, m_InputValues->ColorWeights, comp), totalPoints, algArrays); break; case ConversionType::Average: - ParallelWrapper::Run(LuminosityImpl(inputColorData, outputGrayData, {0.3333f, 0.3333f, 0.3333f}, comp), totalPoints); + ParallelWrapper::Run(LuminosityImpl(inputColorData, outputGrayData, {0.3333f, 0.3333f, 0.3333f}, comp), totalPoints, algArrays); break; case ConversionType::Lightness: - ParallelWrapper::Run(LightnessImpl(inputColorData, outputGrayData, comp), totalPoints); + ParallelWrapper::Run(LightnessImpl(inputColorData, outputGrayData, comp), totalPoints, algArrays); break; case ConversionType::SingleChannel: - ParallelWrapper::Run(SingleChannelImpl(inputColorData, outputGrayData, comp, m_InputValues->ColorChannel), totalPoints); + ParallelWrapper::Run(SingleChannelImpl(inputColorData, outputGrayData, comp, m_InputValues->ColorChannel), totalPoints, algArrays); break; } } diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertData.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertData.cpp index 4aa948905f..b648e63320 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertData.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertData.cpp @@ -72,9 +72,13 @@ Result<> ConvertData(DataStructure& dataStructure, const ConvertDataInputValues* DataArray& inputArray = dataStructure.getDataRefAs>(inputValues->SelectedArrayPath); AbstractDataStore& inputStore = inputArray.getDataStoreRef(); + typename IParallelAlgorithm::AlgorithmArrays algArrays; + algArrays.push_back(&inputArray); + algArrays.push_back(dataStructure.getDataAs(inputValues->OutputArrayName)); + ParallelDataAlgorithm dataAlg; dataAlg.setRange(0, inputArray.size()); - dataAlg.setParallelizationEnabled(false); + dataAlg.requireArraysInMemory(algArrays); switch(inputValues->ScalarType) { diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/FindArrayStatisticsFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/FindArrayStatisticsFilter.cpp index 1c41beb566..a7f8837612 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/FindArrayStatisticsFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/FindArrayStatisticsFilter.cpp @@ -404,12 +404,8 @@ IFilter::PreflightResult FindArrayStatisticsFilter::preflightImpl(const DataStru Result<> FindArrayStatisticsFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { - std::cout << "Values: 0" << std::endl; - FindArrayStatisticsInputValues inputValues; - std::cout << "Values: 1" << std::endl; - inputValues.FindHistogram = filterArgs.value(k_FindHistogram_Key); inputValues.MinRange = filterArgs.value(k_MinRange_Key); inputValues.MaxRange = filterArgs.value(k_MaxRange_Key); @@ -447,8 +443,6 @@ Result<> FindArrayStatisticsFilter::executeImpl(DataStructure& dataStructure, co inputValues.StandardizedArrayName = inputValues.SelectedArrayPath.getParent().createChildPath(filterArgs.value(k_StandardizedArrayName_Key)); inputValues.NumUniqueValuesName = inputValues.DestinationAttributeMatrix.createChildPath(filterArgs.value(k_NumUniqueValues_Key)); - std::cout << "Values: 2" << std::endl; - return FindArrayStatistics(dataStructure, messageHandler, shouldCancel, &inputValues)(); } } // namespace complex diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportHDF5Dataset.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportHDF5Dataset.cpp index c792e1532c..871a9e77e4 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportHDF5Dataset.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/ImportHDF5Dataset.cpp @@ -43,16 +43,33 @@ std::vector createDimensionVector(const std::string& cDimsStr) } template -Result<> fillDataArray(DataStructure& dataStructure, const DataPath& dataArrayPath, const complex::HDF5::DatasetReader& datasetReader) +Result<> fillDataStore(DataArray& dataArray, const DataPath& dataArrayPath, const complex::HDF5::DatasetReader& datasetReader) { - auto& dataArray = dataStructure.getDataRefAs>(dataArrayPath); - auto& absDataStore = dataArray.getDataStoreRef(); + DataStore& dataStore = dataArray.getIDataStoreRefAs>(); + if(!datasetReader.readIntoSpan(dataStore.createSpan())) + { + return {MakeErrorResult(-21002, fmt::format("Error reading dataset '{}' with '{}' total elements into data store for data array '{}' with '{}' total elements ('{}' tuples and '{}' components)", + dataArrayPath.getTargetName(), datasetReader.getNumElements(), dataArrayPath.toString(), dataArray.getSize(), dataArray.getNumberOfTuples(), + dataArray.getNumberOfComponents()))}; + } + + return {}; +} + +template +Result<> fillOocDataStore(DataArray& dataArray, const DataPath& dataArrayPath, const complex::HDF5::DatasetReader& datasetReader) +{ + if(Memory::GetTotalMemory() <= dataArray.getSize() * sizeof(T)) + { + return MakeErrorResult(-21004, fmt::format("Error reading dataset '{}' with '{}' total elements. Not enough memory to import data.", dataArray.getName(), datasetReader.getNumElements())); + } + auto& absDataStore = dataArray.getDataStoreRef(); std::vector data(absDataStore.getSize()); nonstd::span span{data.data(), data.size()}; if(!datasetReader.readIntoSpan(span)) { - return {MakeErrorResult(-21002, fmt::format("Error reading dataset '{}' with '{}' total elements into data store for data array '{}' with '{}' total elements ('{}' tuples and '{}' components)", + return {MakeErrorResult(-21003, fmt::format("Error reading dataset '{}' with '{}' total elements into data store for data array '{}' with '{}' total elements ('{}' tuples and '{}' components)", dataArrayPath.getTargetName(), datasetReader.getNumElements(), dataArrayPath.toString(), dataArray.getSize(), dataArray.getNumberOfTuples(), dataArray.getNumberOfComponents()))}; } @@ -60,6 +77,20 @@ Result<> fillDataArray(DataStructure& dataStructure, const DataPath& dataArrayPa return {}; } + +template +Result<> fillDataArray(DataStructure& dataStructure, const DataPath& dataArrayPath, const complex::HDF5::DatasetReader& datasetReader) +{ + auto& dataArray = dataStructure.getDataRefAs>(dataArrayPath); + if(dataArray.getDataFormat().empty()) + { + return fillDataStore(dataArray, dataArrayPath, datasetReader); + } + else + { + return fillOocDataStore(dataArray, dataArrayPath, datasetReader); + } +} } // namespace namespace complex diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/FindKernelAvgMisorientations.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/FindKernelAvgMisorientations.cpp index 37d35dafa9..c0a41c7762 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/FindKernelAvgMisorientations.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/FindKernelAvgMisorientations.cpp @@ -208,8 +208,16 @@ Result<> FindKernelAvgMisorientations::operator()() // set up threadsafe messenger m_TotalElements = udims[2] * udims[1] * udims[0]; + typename IParallelAlgorithm::AlgorithmArrays algArrays; + algArrays.push_back(m_DataStructure.getDataAs(m_InputValues->CellPhasesArrayPath)); + algArrays.push_back(m_DataStructure.getDataAs(m_InputValues->CrystalStructuresArrayPath)); + algArrays.push_back(m_DataStructure.getDataAs(m_InputValues->FeatureIdsArrayPath)); + algArrays.push_back(m_DataStructure.getDataAs(m_InputValues->KernelAverageMisorientationsArrayName)); + algArrays.push_back(m_DataStructure.getDataAs(m_InputValues->QuatsArrayPath)); + ParallelData3DAlgorithm parallelAlgorithm; parallelAlgorithm.setRange(Range3D(0, udims[0], 0, udims[1], 0, udims[2])); + parallelAlgorithm.requireArraysInMemory(algArrays); parallelAlgorithm.execute(FindKernelAvgMisorientationsImpl(this, m_DataStructure, m_InputValues, m_ShouldCancel)); return {}; diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateFaceIPFColoring.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateFaceIPFColoring.cpp index a947c7daef..6bbfed727f 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateFaceIPFColoring.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateFaceIPFColoring.cpp @@ -183,9 +183,17 @@ Result<> GenerateFaceIPFColoring::operator()() auto& faceIpfColors = m_DataStructure.getDataRefAs(faceIpfColorsArrayPath); int64 numTriangles = faceLabels.getNumberOfTuples(); + typename IParallelAlgorithm::AlgorithmArrays algArrays; + algArrays.push_back(&faceLabels); + algArrays.push_back(&faceNormals); + algArrays.push_back(&eulerAngles); + algArrays.push_back(&phases); + algArrays.push_back(&crystalStructures); + algArrays.push_back(&faceIpfColors); + ParallelDataAlgorithm parallelTask; parallelTask.setRange(0, numTriangles); - parallelTask.setParallelizationEnabled(false); + parallelTask.requireArraysInMemory(algArrays); parallelTask.execute(CalculateFaceIPFColorsImpl(faceLabels, phases, faceNormals, eulerAngles, crystalStructures, faceIpfColors)); return {}; diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateGBCDPoleFigure.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateGBCDPoleFigure.cpp index f4f8a919a6..4101b2e067 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateGBCDPoleFigure.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateGBCDPoleFigure.cpp @@ -324,9 +324,14 @@ Result<> GenerateGBCDPoleFigure::operator()() m_MessageHandler({IFilter::Message::Type::Info, fmt::format("Generating Intensity Plot for phase {}", m_InputValues->PhaseOfInterest)}); + typename IParallelAlgorithm::AlgorithmArrays algArrays; + algArrays.push_back(&poleFigure); + algArrays.push_back(&gbcd); + ParallelData2DAlgorithm dataAlg; dataAlg.setRange(0, xPoints, 0, yPoints); - dataAlg.setParallelizationEnabled(false); + dataAlg.requireArraysInMemory(algArrays); + dataAlg.execute( GenerateGBCDPoleFigureImpl(poleFigure, {xPoints, yPoints}, orientOps, gbcdDeltas, gbcdLimits, gbcdSizes, gbcd, m_InputValues->PhaseOfInterest, m_InputValues->MisorientationRotation)); diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateIPFColors.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateIPFColors.cpp index ce948fad1f..b32a496241 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateIPFColors.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/GenerateIPFColors.cpp @@ -159,15 +159,24 @@ Result<> GenerateIPFColors::operator()() MatrixMath::Normalize3x1(normRefDir[0], normRefDir[1], normRefDir[2]); + typename IParallelAlgorithm::AlgorithmArrays algArrays; + algArrays.push_back(&eulers); + algArrays.push_back(&phases); + algArrays.push_back(&crystalStructures); + algArrays.push_back(&ipfColors); + complex::IDataArray* goodVoxelsArray = nullptr; if(m_InputValues->useGoodVoxels) { goodVoxelsArray = m_DataStructure.getDataAs(m_InputValues->goodVoxelsArrayPath); + algArrays.push_back(goodVoxelsArray); } + // Allow data-based parallelization ParallelDataAlgorithm dataAlg; dataAlg.setRange(0, totalPoints); - dataAlg.setParallelizationEnabled(false); + dataAlg.requireArraysInMemory(algArrays); + dataAlg.execute(GenerateIPFColorsImpl(this, normRefDir, eulers, phases, crystalStructures, numPhases, goodVoxelsArray, ipfColors)); if(m_PhaseWarningCount > 0) diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/GenerateFZQuaternions.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/GenerateFZQuaternions.cpp index b9a58dec79..f4a25ec904 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/GenerateFZQuaternions.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/GenerateFZQuaternions.cpp @@ -264,9 +264,21 @@ Result<> GenerateFZQuaternions::executeImpl(DataStructure& dataStructure, const std::atomic_int32_t warningCount = 0; int32_t numPhases = static_cast(xtalArray.getNumberOfTuples()); + typename IParallelAlgorithm::AlgorithmArrays algArrays; + algArrays.push_back(&phaseArray); + algArrays.push_back(&quatArray); + algArrays.push_back(&xtalArray); + algArrays.push_back(&fzQuatArray); + + if(pUseGoodVoxelsValue) + { + algArrays.push_back(maskArray); + } + // Parallel algorithm to find duplicate nodes ParallelDataAlgorithm dataAlg; dataAlg.setRange(0ULL, static_cast(quatArray.getNumberOfTuples())); + dataAlg.requireArraysInMemory(algArrays); if(pUseGoodVoxelsValue) { diff --git a/src/Plugins/OrientationAnalysis/test/CMakeLists.txt b/src/Plugins/OrientationAnalysis/test/CMakeLists.txt index 0846dffd0f..b9ea548532 100644 --- a/src/Plugins/OrientationAnalysis/test/CMakeLists.txt +++ b/src/Plugins/OrientationAnalysis/test/CMakeLists.txt @@ -12,6 +12,7 @@ set(${PLUGIN_NAME}UnitTest_SRCS CAxisSegmentFeaturesTest.cpp ConvertHexGridToSquareGridTest.cpp ConvertOrientationsTest.cpp + ConvertQuaternionTest.cpp CreateEnsembleInfoTest.cpp EBSDSegmentFeaturesFilterTest.cpp EbsdToH5EbsdTest.cpp diff --git a/src/complex/Common/Uuid.hpp b/src/complex/Common/Uuid.hpp index 85e36f7b3d..f8a3a0d149 100644 --- a/src/complex/Common/Uuid.hpp +++ b/src/complex/Common/Uuid.hpp @@ -171,10 +171,10 @@ struct COMPLEX_EXPORT Uuid static inline Uuid GenerateV4() { - static std::random_device rd; - static std::mt19937 gen(rd()); - static std::uniform_int_distribution<> dis(0, 15); - static std::uniform_int_distribution<> dis2(8, 11); + static thread_local std::random_device rd; + static thread_local std::mt19937 gen(rd()); + static thread_local std::uniform_int_distribution<> dis(0, 15); + static thread_local std::uniform_int_distribution<> dis2(8, 11); std::stringstream ss; int i; diff --git a/src/complex/Core/Application.cpp b/src/complex/Core/Application.cpp index 920e551b7f..c21bb2246e 100644 --- a/src/complex/Core/Application.cpp +++ b/src/complex/Core/Application.cpp @@ -283,7 +283,7 @@ void Application::loadPlugin(const std::filesystem::path& path, bool verbose) fmt::print("Loading Plugin: {}\n", path.string()); } auto pluginLoader = std::make_shared(path); - if(!getFilterList()->addPlugin(pluginLoader)) + if(getFilterList()->addPlugin(pluginLoader).invalid()) { return; } diff --git a/src/complex/Filter/FilterList.cpp b/src/complex/Filter/FilterList.cpp index 65921a87db..fcaee24875 100644 --- a/src/complex/Filter/FilterList.cpp +++ b/src/complex/Filter/FilterList.cpp @@ -91,27 +91,26 @@ AbstractPlugin* FilterList::getPlugin(const FilterHandle& handle) const return nullptr; } -bool FilterList::addPlugin(const std::shared_ptr& loader) +Result<> FilterList::addPlugin(const std::shared_ptr& loader) { if(!loader->isLoaded()) { - return false; + return MakeErrorResult(-444, "Plugin was not loaded"); } AbstractPlugin* plugin = loader->getPlugin(); Uuid pluginUuid = plugin->getId(); if(m_PluginMap.count(pluginUuid) > 0) { - // throw std::runtime_error(fmt::format("Attempted to add plugin '{}' with uuid '{}', but plugin '{}' already exists with that uuid", plugin->getName(), pluginUuid.str(), - // m_PluginMap[pluginUuid]->getPlugin()->getName())); - return false; + return MakeErrorResult(-445, fmt::format("Attempted to add plugin '{}' with uuid '{}', but plugin '{}' already exists with that uuid", plugin->getName(), pluginUuid.str(), + m_PluginMap[pluginUuid]->getPlugin()->getName())); } auto pluginHandles = plugin->getFilterHandles(); m_FilterHandles.merge(pluginHandles); m_PluginMap[pluginUuid] = loader; - return true; + return {}; } -bool FilterList::addPlugin(const std::string& path) +Result<> FilterList::addPlugin(const std::string& path) { return addPlugin(std::dynamic_pointer_cast(std::make_shared(path))); } diff --git a/src/complex/Filter/FilterList.hpp b/src/complex/Filter/FilterList.hpp index c7e8a4c20b..5e2834290f 100644 --- a/src/complex/Filter/FilterList.hpp +++ b/src/complex/Filter/FilterList.hpp @@ -127,7 +127,7 @@ class COMPLEX_EXPORT FilterList * @param loader * @return bool */ - bool addPlugin(const std::shared_ptr& loader); + Result<> addPlugin(const std::shared_ptr& loader); /** * @brief Attempts to add the plugin at the specified filepath. Returns true @@ -135,7 +135,7 @@ class COMPLEX_EXPORT FilterList * @param path * @return bool */ - bool addPlugin(const std::string& path); + Result<> addPlugin(const std::string& path); /** * @brief Removes the plugin with the given uuid. diff --git a/src/complex/Utilities/IParallelAlgorithm.cpp b/src/complex/Utilities/IParallelAlgorithm.cpp index 1c54f63267..a964c76041 100644 --- a/src/complex/Utilities/IParallelAlgorithm.cpp +++ b/src/complex/Utilities/IParallelAlgorithm.cpp @@ -26,10 +26,20 @@ void IParallelAlgorithm::setParallelizationEnabled(bool doParallel) m_RunParallel = doParallel; } // ----------------------------------------------------------------------------- -void IParallelAlgorithm::requireArraysInMemory(const std::vector& arrays) +void IParallelAlgorithm::requireArraysInMemory(const AlgorithmArrays& arrays) { + if(arrays.size() == 0) + { + return; + } + for(const auto* array : arrays) { + if(array == nullptr) + { + continue; + } + if(array->getIDataStoreRef().getDataFormat().empty() == false) { setParallelizationEnabled(false); diff --git a/src/complex/Utilities/IParallelAlgorithm.hpp b/src/complex/Utilities/IParallelAlgorithm.hpp index fdb3c8f3c9..c4bde3c8bd 100644 --- a/src/complex/Utilities/IParallelAlgorithm.hpp +++ b/src/complex/Utilities/IParallelAlgorithm.hpp @@ -11,6 +11,8 @@ namespace complex class COMPLEX_EXPORT IParallelAlgorithm { public: + using AlgorithmArrays = std::vector; + IParallelAlgorithm(const IParallelAlgorithm&) = default; IParallelAlgorithm(IParallelAlgorithm&&) noexcept = default; IParallelAlgorithm& operator=(const IParallelAlgorithm&) = default; @@ -28,7 +30,7 @@ class COMPLEX_EXPORT IParallelAlgorithm */ void setParallelizationEnabled(bool doParallel); - void requireArraysInMemory(const std::vector& arrays); + void requireArraysInMemory(const AlgorithmArrays& arrays); protected: IParallelAlgorithm();