diff --git a/CMakeLists.txt b/CMakeLists.txt index 35cb04f916..221218bc73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -548,6 +548,7 @@ set(SIMPLNX_HDRS ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/Writers/ObjectWriter.hpp ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/Text/CsvParser.hpp + ${SIMPLNX_SOURCE_DIR}/Utilities/MD5.hpp ) set(SIMPLNX_GENERATED_HEADERS @@ -743,6 +744,7 @@ set(SIMPLNX_SRCS ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/Writers/ObjectWriter.cpp ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/Text/CsvParser.cpp + ${SIMPLNX_SOURCE_DIR}/Utilities/MD5.cpp ) # Add Core FilterParameters diff --git a/src/Plugins/ITKImageProcessing/test/CMakeLists.txt b/src/Plugins/ITKImageProcessing/test/CMakeLists.txt index 89af617cb7..c498452f8a 100644 --- a/src/Plugins/ITKImageProcessing/test/CMakeLists.txt +++ b/src/Plugins/ITKImageProcessing/test/CMakeLists.txt @@ -98,8 +98,6 @@ target_sources(${PLUGIN_NAME}UnitTest PRIVATE ${${PLUGIN_NAME}_SOURCE_DIR}/test/ITKTestBase.hpp ${${PLUGIN_NAME}_SOURCE_DIR}/test/ITKTestBase.cpp - ${${PLUGIN_NAME}_SOURCE_DIR}/test/MD5.hpp - ${${PLUGIN_NAME}_SOURCE_DIR}/test/MD5.cpp ) # ----------------------------------------------------------------------------- diff --git a/src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp b/src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp index 74adcb583f..b448ea2c7a 100644 --- a/src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp +++ b/src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp @@ -2,7 +2,6 @@ #include "ITKImageProcessing/Filters/ITKImageReaderFilter.hpp" #include "ITKImageProcessing/Filters/ITKImageWriterFilter.hpp" -#include "MD5.hpp" #include #include @@ -12,6 +11,7 @@ #include #include "simplnx/Common/Types.hpp" +#include "simplnx/Utilities/MD5.hpp" #include diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.cpp index 3c8063e1a5..efe418a18c 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.cpp @@ -247,12 +247,16 @@ Result<> MultiWriteOutStl(const fs::path& path, const IGeometry::MeshIndexType n return result; } +/** + * @brief This class provides an interface to write the STL Files in parallel + */ class MultiWriteStlFileImpl { public: - MultiWriteStlFileImpl(const fs::path& path, const IGeometry::MeshIndexType numTriangles, const std::string& header, const IGeometry::MeshIndexArrayType& triangles, const Float32Array& vertices, - const Int32Array& featureIds, const int32 featureId) - : m_Path(path) + MultiWriteStlFileImpl(WriteStlFile* filter, const fs::path path, const IGeometry::MeshIndexType numTriangles, const std::string header, const IGeometry::MeshIndexArrayType& triangles, + const Float32Array& vertices, const Int32Array& featureIds, const int32 featureId) + : m_Filter(filter) + , m_Path(path) , m_NumTriangles(numTriangles) , m_Header(header) , m_Triangles(triangles) @@ -265,17 +269,15 @@ class MultiWriteStlFileImpl void operator()() const { - Result<> result; - // Create output file writer in binary write out mode to ensure cross-compatibility FILE* filePtr = fopen(m_Path.string().c_str(), "wb"); if(filePtr == nullptr) { fclose(filePtr); - std::cout << fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", m_Path.string(), m_Path.filename().string()) << "\n"; + m_Filter->sendThreadSafeProgressMessage( + {MakeWarningVoidResult(-27876, fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", m_Path.string(), m_Path.filename().string()))}); return; - // return {MakeWarningVoidResult(-27876, fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", m_Path.string(), m_Path.filename().string()))}; } int32 triCount = 0; @@ -283,9 +285,9 @@ class MultiWriteStlFileImpl { // Scope header output processing to keep overhead low and increase readability if(m_Header.size() >= 80) { - result = MakeWarningVoidResult(-27874, - fmt::format("Warning: Writing STL File '{}'. Header was over the 80 characters supported by STL. Length of header: {}. Only the first 80 bytes will be written.", - m_Path.filename().string(), m_Header.length())); + m_Filter->sendThreadSafeProgressMessage(MakeWarningVoidResult( + -27874, fmt::format("Warning: Writing STL File '{}'. Header was over the 80 characters supported by STL. Length of header: {}. Only the first 80 bytes will be written.", + m_Path.filename().string(), m_Header.length()))); } std::array stlFileHeader = {}; @@ -370,12 +372,9 @@ class MultiWriteStlFileImpl if(totalWritten != 50) { fclose(filePtr); - std::cout << fmt::format("Error Writing STL File '{}': Not enough bytes written for triangle {}. Only {} bytes written of 50 bytes", m_Path.filename().string(), triCount, totalWritten) - << "\n"; + m_Filter->sendThreadSafeProgressMessage({MakeWarningVoidResult( + -27873, fmt::format("Error Writing STL File '{}': Not enough bytes written for triangle {}. Only {} bytes written of 50 bytes", m_Path.filename().string(), triCount, totalWritten))}); break; - // return {MakeWarningVoidResult( - // -27873, fmt::format("Error Writing STL File '{}': Not enough bytes written for triangle {}. Only {} bytes written of 50 bytes", path.filename().string(), triCount, - // totalWritten))}; } triCount++; } @@ -386,6 +385,7 @@ class MultiWriteStlFileImpl } private: + WriteStlFile* m_Filter = nullptr; const fs::path m_Path; const IGeometry::MeshIndexType m_NumTriangles; const std::string m_Header; @@ -468,6 +468,10 @@ Result<> WriteStlFile::operator()() } } + // The writing of the files can happen in parallel as much as the Operating System will allow + ParallelTaskAlgorithm taskRunner; + taskRunner.setParallelizationEnabled(true); + // Store a list of Atomic Files, so we can clean up or finish depending on the outcome of all the writes std::vector> fileList; @@ -489,20 +493,18 @@ Result<> WriteStlFile::operator()() { return ConvertResult(std::move(fileList[fileIndex])); } - m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", featureId)); - - auto result = ::MultiWriteOutStl(fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId)}, triangles, vertices, - featureIds, featureId); - // if valid Loop over all the triangles for this spin - if(result.invalid()) + taskRunner.execute(MultiWriteStlFileImpl(this, fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId)}, triangles, + vertices, featureIds, featureId)); + fileIndex++; + if(m_HasErrors) { - return result; + break; } - - fileIndex++; } + taskRunner.wait(); } + if(groupingType == GroupingType::FeaturesAndPhases) { const auto& featureIds = m_DataStructure.getDataRefAs(m_InputValues->FeatureIdsPath); @@ -527,20 +529,17 @@ Result<> WriteStlFile::operator()() { return ConvertResult(std::move(fileList[fileIndex])); } - - m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", featureId)); - - auto result = - ::MultiWriteOutStl(fileList[fileIndex].value().tempFilePath(), nTriangles, - {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId) + " Phase " + StringUtilities::number(value)}, triangles, vertices, featureIds, featureId); - // if valid loop over all the triangles for this spin - if(result.invalid()) + m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Phase {} and Feature Id {}", value, featureId)); + taskRunner.execute(MultiWriteStlFileImpl(this, fileList[fileIndex].value().tempFilePath(), nTriangles, + {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId) + " Phase " + StringUtilities::number(value)}, triangles, vertices, featureIds, + featureId)); + fileIndex++; + if(m_HasErrors) { - return result; + break; } - - fileIndex++; } + taskRunner.wait(); } // Group Triangles by Part Number which is a single component Int32 Array @@ -552,10 +551,6 @@ Result<> WriteStlFile::operator()() std::unordered_set uniquePartNumbers(partNumbers.cbegin(), partNumbers.cend()); fileList.reserve(uniquePartNumbers.size()); // Reserved enough file names - // The writing of the files can happen in parallel as much as the Operating System will allow - ParallelTaskAlgorithm taskRunner; - taskRunner.setParallelizationEnabled(true); - // Loop over each Part Number and write a file usize fileIndex = 0; for(const auto currentPartNumber : uniquePartNumbers) @@ -566,26 +561,40 @@ Result<> WriteStlFile::operator()() { return ConvertResult(std::move(fileList[fileIndex])); } - m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Part Number {}", currentPartNumber)); - - taskRunner.execute(MultiWriteStlFileImpl(fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Part Number " + StringUtilities::number(currentPartNumber)}, triangles, - vertices, partNumbers, currentPartNumber)); + taskRunner.execute(MultiWriteStlFileImpl(this, fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Part Number " + StringUtilities::number(currentPartNumber)}, + triangles, vertices, partNumbers, currentPartNumber)); fileIndex++; + if(m_HasErrors) + { + break; + } } taskRunner.wait(); } + // Commit all the temp files for(auto& atomicFile : fileList) { Result<> commitResult = atomicFile.value().commit(); if(commitResult.invalid()) { - return commitResult; + m_Result = MergeResults(m_Result, commitResult); } } - return {}; + return m_Result; +} + +// ----------------------------------------------------------------------------- +void WriteStlFile::sendThreadSafeProgressMessage(Result<> result) +{ + std::lock_guard guard(m_ProgressMessage_Mutex); + if(result.invalid()) + { + m_HasErrors = true; + m_Result = MergeResults(m_Result, result); + } } // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast, cppcoreguidelines-pro-bounds-pointer-arithmetic) diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.hpp index f55ead9c3d..b99aec26c2 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.hpp @@ -51,11 +51,17 @@ class SIMPLNXCORE_EXPORT WriteStlFile const std::atomic_bool& getCancel(); + void sendThreadSafeProgressMessage(Result<> result); + private: DataStructure& m_DataStructure; const WriteStlFileInputValues* m_InputValues = nullptr; const std::atomic_bool& m_ShouldCancel; const IFilter::MessageHandler& m_MessageHandler; + mutable std::mutex m_ProgressMessage_Mutex; + + mutable bool m_HasErrors = false; + Result<> m_Result; }; } // namespace nx::core diff --git a/src/Plugins/SimplnxCore/test/WriteStlFileTest.cpp b/src/Plugins/SimplnxCore/test/WriteStlFileTest.cpp index 6a68c31990..80f81e6895 100644 --- a/src/Plugins/SimplnxCore/test/WriteStlFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/WriteStlFileTest.cpp @@ -1,5 +1,6 @@ #include +#include "SimplnxCore/Filters/CombineStlFilesFilter.hpp" #include "SimplnxCore/Filters/WriteStlFileFilter.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" @@ -10,13 +11,16 @@ #include "simplnx/UnitTest/UnitTestCommon.hpp" #include - namespace fs = std::filesystem; -using namespace nx::core; +using namespace nx::core; +using namespace nx::core::Constants; namespace { const std::string k_ExemplarDir = fmt::format("{}/6_6_write_stl_file_test", unit_test::k_TestFilesDir); +const DataPath k_ComputedTriangleDataContainerName({"ComputedTriangleDataContainer"}); +const DataPath k_ExemplarTriangleDataContainerName({k_TriangleDataContainerName}); +const std::string k_PartNumberName = "Part Number"; std::vector readIn(fs::path filePath) { @@ -116,3 +120,72 @@ TEST_CASE("SimplnxCore::WriteStlFileFilter: Single File Valid", "[SimplnxCore][W ::CompareSingleResult(); } + +TEST_CASE("SimplnxCore::WriteStlFileFilter:Part_Number", "[SimplnxCore][WriteStlFileFilter]") +{ + const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "6_6_write_stl_file_test.tar.gz", "6_6_write_stl_file_test"); + + const nx::core::UnitTest::TestFileSentinel testDataSentinel2(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "6_6_combine_stl_files_v2.tar.gz", + "6_6_combine_stl_files.dream3d"); + DataStructure dataStructure; + + { + CombineStlFilesFilter filter; + Arguments args; + std::string inputStlDir = fmt::format("{}/6_6_combine_stl_files_v2/STL_Models", unit_test::k_TestFilesDir.view()); + + // Create default Parameters for the filter. + args.insertOrAssign(CombineStlFilesFilter::k_StlFilesPath_Key, std::make_any(fs::path(inputStlDir))); + args.insertOrAssign(CombineStlFilesFilter::k_TriangleGeometryPath_Key, std::make_any(k_ComputedTriangleDataContainerName)); + args.insertOrAssign(CombineStlFilesFilter::k_FaceAttributeMatrixName_Key, std::make_any(k_FaceData)); + args.insertOrAssign(CombineStlFilesFilter::k_FaceNormalsArrayName_Key, std::make_any("Face Normals")); + args.insertOrAssign(CombineStlFilesFilter::k_VertexAttributeMatrixName_Key, std::make_any(k_VertexData)); + args.insertOrAssign(CombineStlFilesFilter::k_LabelFaces_Key, std::make_any(true)); + args.insertOrAssign(CombineStlFilesFilter::k_FaceLabelName_Key, std::make_any(k_PartNumberName)); + args.insertOrAssign(CombineStlFilesFilter::k_LabelVertices_Key, std::make_any(true)); + args.insertOrAssign(CombineStlFilesFilter::k_VertexLabelName_Key, std::make_any(k_PartNumberName)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions) + + // Execute the filter and check the result + auto executeResult = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result) + } + + { + // Instantiate the filter, a DataStructure object and an Arguments Object + WriteStlFileFilter filter; + // auto exemplarFilePath = fs::path(fmt::format("{}/exemplar.dream3d", k_ExemplarDir)); + // DataStructure dataStructure = UnitTest::LoadDataStructure(exemplarFilePath); + Arguments args; + + // Create default Parameters for the filter. + args.insertOrAssign(WriteStlFileFilter::k_GroupingType_Key, std::make_any(3)); + args.insertOrAssign(WriteStlFileFilter::k_OutputStlDirectory_Key, std::make_any(fs::path(std::string(unit_test::k_BinaryTestOutputDir)))); + args.insertOrAssign(WriteStlFileFilter::k_OutputStlPrefix_Key, std::make_any("Part_Number_")); + args.insertOrAssign(WriteStlFileFilter::k_TriangleGeomPath_Key, std::make_any(k_ComputedTriangleDataContainerName)); + args.insertOrAssign(WriteStlFileFilter::k_PartNumberPath_Key, std::make_any(k_ComputedTriangleDataContainerName.createChildPath(k_FaceData).createChildPath(k_PartNumberName))); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions) + + // Execute the filter and check the result + auto executeResult = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result) + } + + fs::path writtenFilePath = fs::path(std::string(unit_test::k_BinaryTestOutputDir) + "/Part_Number_1.stl"); + REQUIRE(fs::exists(writtenFilePath)); + auto fileContents = readIn(writtenFilePath); + std::string md5Hash = nx::core::UnitTest::ComputeMD5Hash(fileContents); + REQUIRE(md5Hash == "a0383b898d0668d70f08e135e5064efb"); + + writtenFilePath = fs::path(std::string(unit_test::k_BinaryTestOutputDir) + "/Part_Number_2.stl"); + REQUIRE(fs::exists(writtenFilePath)); + fileContents = readIn(writtenFilePath); + md5Hash = nx::core::UnitTest::ComputeMD5Hash(fileContents); + REQUIRE(md5Hash == "d45a0d99495df506384fdbbb46a79f5c"); +} diff --git a/src/Plugins/ITKImageProcessing/test/MD5.cpp b/src/simplnx/Utilities/MD5.cpp similarity index 100% rename from src/Plugins/ITKImageProcessing/test/MD5.cpp rename to src/simplnx/Utilities/MD5.cpp diff --git a/src/Plugins/ITKImageProcessing/test/MD5.hpp b/src/simplnx/Utilities/MD5.hpp similarity index 100% rename from src/Plugins/ITKImageProcessing/test/MD5.hpp rename to src/simplnx/Utilities/MD5.hpp diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp index 7de7ebcc22..27b03cecd1 100644 --- a/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp +++ b/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp @@ -19,6 +19,7 @@ #include "simplnx/Parameters/BoolParameter.hpp" #include "simplnx/Parameters/GeometrySelectionParameter.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" +#include "simplnx/Utilities/MD5.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" #include "simplnx/Utilities/Parsing/HDF5/Writers/FileWriter.hpp" @@ -210,6 +211,17 @@ namespace UnitTest { inline constexpr float EPSILON = 0.0001; +template +std::string ComputeMD5Hash(const std::vector& outputDataArray) +{ + const T* dataPtr = outputDataArray.data(); + usize arraySize = outputDataArray.size(); + MD5 md5; + md5.update(reinterpret_cast(dataPtr), arraySize * sizeof(T)); + md5.finalize(); + return md5.hexdigest(); +} + /** * @brief This class will decompress a tar.gz file using the locally installed copy of cmake and when * then class goes out of scope the extracted contents will be deleted from disk.