Add unit test for Part Number code path for STL Writer

- Move MD5 class into simplnx/Utilities since it can be used else where in the code

Signed-off-by: Michael Jackson <[email protected]>

CMakeLists.txt

@@ -548,6 +548,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/Writers/ObjectWriter.hpp
 
   ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/Text/CsvParser.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MD5.hpp
 )
 
 set(SIMPLNX_GENERATED_HEADERS
@@ -743,6 +744,7 @@ set(SIMPLNX_SRCS
   ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/Writers/ObjectWriter.cpp
 
   ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/Text/CsvParser.cpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MD5.cpp
 )
 
 # Add Core FilterParameters

src/Plugins/ITKImageProcessing/test/CMakeLists.txt

@@ -98,8 +98,6 @@ target_sources(${PLUGIN_NAME}UnitTest
   PRIVATE
     ${${PLUGIN_NAME}_SOURCE_DIR}/test/ITKTestBase.hpp
     ${${PLUGIN_NAME}_SOURCE_DIR}/test/ITKTestBase.cpp
-    ${${PLUGIN_NAME}_SOURCE_DIR}/test/MD5.hpp
-    ${${PLUGIN_NAME}_SOURCE_DIR}/test/MD5.cpp
 )
 
 # -----------------------------------------------------------------------------

src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp

@@ -2,7 +2,6 @@
 
 #include "ITKImageProcessing/Filters/ITKImageReaderFilter.hpp"
 #include "ITKImageProcessing/Filters/ITKImageWriterFilter.hpp"
-#include "MD5.hpp"
 
 #include <itkImportImageFilter.h>
 #include <itkNumericTraits.h>
@@ -12,6 +11,7 @@
 #include <itkTestingHashImageFilter.h>
 
 #include "simplnx/Common/Types.hpp"
+#include "simplnx/Utilities/MD5.hpp"
 
 #include <fmt/format.h>
 

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.cpp

@@ -247,12 +247,16 @@ Result<> MultiWriteOutStl(const fs::path& path, const IGeometry::MeshIndexType n
   return result;
 }
 
+/**
+ * @brief This class provides an interface to write the STL Files in parallel
+ */
 class MultiWriteStlFileImpl
 {
 public:
-  MultiWriteStlFileImpl(const fs::path& path, const IGeometry::MeshIndexType numTriangles, const std::string& header, const IGeometry::MeshIndexArrayType& triangles, const Float32Array& vertices,
-                        const Int32Array& featureIds, const int32 featureId)
-  : m_Path(path)
+  MultiWriteStlFileImpl(WriteStlFile* filter, const fs::path path, const IGeometry::MeshIndexType numTriangles, const std::string header, const IGeometry::MeshIndexArrayType& triangles,
+                        const Float32Array& vertices, const Int32Array& featureIds, const int32 featureId)
+  : m_Filter(filter)
+  , m_Path(path)
   , m_NumTriangles(numTriangles)
   , m_Header(header)
   , m_Triangles(triangles)
@@ -265,27 +269,25 @@ class MultiWriteStlFileImpl
 
   void operator()() const
   {
-    Result<> result;
-
     // Create output file writer in binary write out mode to ensure cross-compatibility
     FILE* filePtr = fopen(m_Path.string().c_str(), "wb");
 
     if(filePtr == nullptr)
     {
       fclose(filePtr);
-      std::cout << fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", m_Path.string(), m_Path.filename().string()) << "\n";
+      m_Filter->sendThreadSafeProgressMessage(
+          {MakeWarningVoidResult(-27876, fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", m_Path.string(), m_Path.filename().string()))});
       return;
-      // return {MakeWarningVoidResult(-27876, fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", m_Path.string(), m_Path.filename().string()))};
     }
 
     int32 triCount = 0;
 
     { // Scope header output processing to keep overhead low and increase readability
       if(m_Header.size() >= 80)
       {
-        result = MakeWarningVoidResult(-27874,
-                                       fmt::format("Warning: Writing STL File '{}'. Header was over the 80 characters supported by STL. Length of header: {}. Only the first 80 bytes will be written.",
-                                                   m_Path.filename().string(), m_Header.length()));
+        m_Filter->sendThreadSafeProgressMessage(MakeWarningVoidResult(
+            -27874, fmt::format("Warning: Writing STL File '{}'. Header was over the 80 characters supported by STL. Length of header: {}. Only the first 80 bytes will be written.",
+                                m_Path.filename().string(), m_Header.length())));
       }
 
       std::array<char, 80> stlFileHeader = {};
@@ -370,12 +372,9 @@ class MultiWriteStlFileImpl
       if(totalWritten != 50)
       {
         fclose(filePtr);
-        std::cout << fmt::format("Error Writing STL File '{}': Not enough bytes written for triangle {}. Only {} bytes written of 50 bytes", m_Path.filename().string(), triCount, totalWritten)
-                  << "\n";
+        m_Filter->sendThreadSafeProgressMessage({MakeWarningVoidResult(
+            -27873, fmt::format("Error Writing STL File '{}': Not enough bytes written for triangle {}. Only {} bytes written of 50 bytes", m_Path.filename().string(), triCount, totalWritten))});
         break;
-        //        return {MakeWarningVoidResult(
-        //            -27873, fmt::format("Error Writing STL File '{}': Not enough bytes written for triangle {}. Only {} bytes written of 50 bytes", path.filename().string(), triCount,
-        //            totalWritten))};
       }
       triCount++;
     }
@@ -386,6 +385,7 @@ class MultiWriteStlFileImpl
   }
 
 private:
+  WriteStlFile* m_Filter = nullptr;
   const fs::path m_Path;
   const IGeometry::MeshIndexType m_NumTriangles;
   const std::string m_Header;
@@ -468,6 +468,10 @@ Result<> WriteStlFile::operator()()
     }
   }
 
+  // The writing of the files can happen in parallel as much as the Operating System will allow
+  ParallelTaskAlgorithm taskRunner;
+  taskRunner.setParallelizationEnabled(true);
+
   // Store a list of Atomic Files, so we can clean up or finish depending on the outcome of all the writes
   std::vector<Result<AtomicFile>> fileList;
 
@@ -489,20 +493,18 @@ Result<> WriteStlFile::operator()()
       {
         return ConvertResult(std::move(fileList[fileIndex]));
       }
-
       m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", featureId));
-
-      auto result = ::MultiWriteOutStl(fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId)}, triangles, vertices,
-                                       featureIds, featureId);
-      // if valid Loop over all the triangles for this spin
-      if(result.invalid())
+      taskRunner.execute(MultiWriteStlFileImpl(this, fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId)}, triangles,
+                                               vertices, featureIds, featureId));
+      fileIndex++;
+      if(m_HasErrors)
       {
-        return result;
+        break;
       }
-
-      fileIndex++;
     }
+    taskRunner.wait();
   }
+
   if(groupingType == GroupingType::FeaturesAndPhases)
   {
     const auto& featureIds = m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->FeatureIdsPath);
@@ -527,20 +529,17 @@ Result<> WriteStlFile::operator()()
       {
         return ConvertResult(std::move(fileList[fileIndex]));
       }
-
-      m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", featureId));
-
-      auto result =
-          ::MultiWriteOutStl(fileList[fileIndex].value().tempFilePath(), nTriangles,
-                             {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId) + " Phase " + StringUtilities::number(value)}, triangles, vertices, featureIds, featureId);
-      // if valid loop over all the triangles for this spin
-      if(result.invalid())
+      m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Phase {} and Feature Id {}", value, featureId));
+      taskRunner.execute(MultiWriteStlFileImpl(this, fileList[fileIndex].value().tempFilePath(), nTriangles,
+                                               {"DREAM3D Generated For Feature ID " + StringUtilities::number(featureId) + " Phase " + StringUtilities::number(value)}, triangles, vertices, featureIds,
+                                               featureId));
+      fileIndex++;
+      if(m_HasErrors)
       {
-        return result;
+        break;
       }
-
-      fileIndex++;
     }
+    taskRunner.wait();
   }
 
   // Group Triangles by Part Number which is a single component Int32 Array
@@ -552,10 +551,6 @@ Result<> WriteStlFile::operator()()
     std::unordered_set<int32> uniquePartNumbers(partNumbers.cbegin(), partNumbers.cend());
     fileList.reserve(uniquePartNumbers.size()); // Reserved enough file names
 
-    // The writing of the files can happen in parallel as much as the Operating System will allow
-    ParallelTaskAlgorithm taskRunner;
-    taskRunner.setParallelizationEnabled(true);
-
     // Loop over each Part Number and write a file
     usize fileIndex = 0;
     for(const auto currentPartNumber : uniquePartNumbers)
@@ -566,26 +561,40 @@ Result<> WriteStlFile::operator()()
       {
         return ConvertResult(std::move(fileList[fileIndex]));
       }
-
       m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Part Number {}", currentPartNumber));
-
-      taskRunner.execute(MultiWriteStlFileImpl(fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Part Number " + StringUtilities::number(currentPartNumber)}, triangles,
-                                               vertices, partNumbers, currentPartNumber));
+      taskRunner.execute(MultiWriteStlFileImpl(this, fileList[fileIndex].value().tempFilePath(), nTriangles, {"DREAM3D Generated For Part Number " + StringUtilities::number(currentPartNumber)},
+                                               triangles, vertices, partNumbers, currentPartNumber));
       fileIndex++;
+      if(m_HasErrors)
+      {
+        break;
+      }
     }
     taskRunner.wait();
   }
 
+  // Commit all the temp files
   for(auto& atomicFile : fileList)
   {
     Result<> commitResult = atomicFile.value().commit();
     if(commitResult.invalid())
     {
-      return commitResult;
+      m_Result = MergeResults(m_Result, commitResult);
     }
   }
 
-  return {};
+  return m_Result;
+}
+
+// -----------------------------------------------------------------------------
+void WriteStlFile::sendThreadSafeProgressMessage(Result<> result)
+{
+  std::lock_guard<std::mutex> guard(m_ProgressMessage_Mutex);
+  if(result.invalid())
+  {
+    m_HasErrors = true;
+    m_Result = MergeResults(m_Result, result);
+  }
 }
 
 // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast, cppcoreguidelines-pro-bounds-pointer-arithmetic)

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteStlFile.hpp

@@ -51,11 +51,17 @@ class SIMPLNXCORE_EXPORT WriteStlFile
 
   const std::atomic_bool& getCancel();
 
+  void sendThreadSafeProgressMessage(Result<> result);
+
 private:
   DataStructure& m_DataStructure;
   const WriteStlFileInputValues* m_InputValues = nullptr;
   const std::atomic_bool& m_ShouldCancel;
   const IFilter::MessageHandler& m_MessageHandler;
+  mutable std::mutex m_ProgressMessage_Mutex;
+
+  mutable bool m_HasErrors = false;
+  Result<> m_Result;
 };
 
 } // namespace nx::core

src/Plugins/SimplnxCore/test/WriteStlFileTest.cpp

@@ -1,5 +1,6 @@
 #include <catch2/catch.hpp>
 
+#include "SimplnxCore/Filters/CombineStlFilesFilter.hpp"
 #include "SimplnxCore/Filters/WriteStlFileFilter.hpp"
 #include "SimplnxCore/SimplnxCore_test_dirs.hpp"
 
@@ -10,13 +11,16 @@
 #include "simplnx/UnitTest/UnitTestCommon.hpp"
 
 #include <filesystem>
-
 namespace fs = std::filesystem;
-using namespace nx::core;
 
+using namespace nx::core;
+using namespace nx::core::Constants;
 namespace
 {
 const std::string k_ExemplarDir = fmt::format("{}/6_6_write_stl_file_test", unit_test::k_TestFilesDir);
+const DataPath k_ComputedTriangleDataContainerName({"ComputedTriangleDataContainer"});
+const DataPath k_ExemplarTriangleDataContainerName({k_TriangleDataContainerName});
+const std::string k_PartNumberName = "Part Number";
 
 std::vector<char> readIn(fs::path filePath)
 {
@@ -116,3 +120,72 @@ TEST_CASE("SimplnxCore::WriteStlFileFilter: Single File Valid", "[SimplnxCore][W
 
   ::CompareSingleResult();
 }
+
+TEST_CASE("SimplnxCore::WriteStlFileFilter:Part_Number", "[SimplnxCore][WriteStlFileFilter]")
+{
+  const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "6_6_write_stl_file_test.tar.gz", "6_6_write_stl_file_test");
+
+  const nx::core::UnitTest::TestFileSentinel testDataSentinel2(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "6_6_combine_stl_files_v2.tar.gz",
+                                                               "6_6_combine_stl_files.dream3d");
+  DataStructure dataStructure;
+
+  {
+    CombineStlFilesFilter filter;
+    Arguments args;
+    std::string inputStlDir = fmt::format("{}/6_6_combine_stl_files_v2/STL_Models", unit_test::k_TestFilesDir.view());
+
+    // Create default Parameters for the filter.
+    args.insertOrAssign(CombineStlFilesFilter::k_StlFilesPath_Key, std::make_any<FileSystemPathParameter::ValueType>(fs::path(inputStlDir)));
+    args.insertOrAssign(CombineStlFilesFilter::k_TriangleGeometryPath_Key, std::make_any<DataPath>(k_ComputedTriangleDataContainerName));
+    args.insertOrAssign(CombineStlFilesFilter::k_FaceAttributeMatrixName_Key, std::make_any<std::string>(k_FaceData));
+    args.insertOrAssign(CombineStlFilesFilter::k_FaceNormalsArrayName_Key, std::make_any<std::string>("Face Normals"));
+    args.insertOrAssign(CombineStlFilesFilter::k_VertexAttributeMatrixName_Key, std::make_any<std::string>(k_VertexData));
+    args.insertOrAssign(CombineStlFilesFilter::k_LabelFaces_Key, std::make_any<bool>(true));
+    args.insertOrAssign(CombineStlFilesFilter::k_FaceLabelName_Key, std::make_any<std::string>(k_PartNumberName));
+    args.insertOrAssign(CombineStlFilesFilter::k_LabelVertices_Key, std::make_any<bool>(true));
+    args.insertOrAssign(CombineStlFilesFilter::k_VertexLabelName_Key, std::make_any<std::string>(k_PartNumberName));
+
+    // Preflight the filter and check result
+    auto preflightResult = filter.preflight(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions)
+
+    // Execute the filter and check the result
+    auto executeResult = filter.execute(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result)
+  }
+
+  {
+    // Instantiate the filter, a DataStructure object and an Arguments Object
+    WriteStlFileFilter filter;
+    // auto exemplarFilePath = fs::path(fmt::format("{}/exemplar.dream3d", k_ExemplarDir));
+    // DataStructure dataStructure = UnitTest::LoadDataStructure(exemplarFilePath);
+    Arguments args;
+
+    // Create default Parameters for the filter.
+    args.insertOrAssign(WriteStlFileFilter::k_GroupingType_Key, std::make_any<ChoicesParameter::ValueType>(3));
+    args.insertOrAssign(WriteStlFileFilter::k_OutputStlDirectory_Key, std::make_any<FileSystemPathParameter::ValueType>(fs::path(std::string(unit_test::k_BinaryTestOutputDir))));
+    args.insertOrAssign(WriteStlFileFilter::k_OutputStlPrefix_Key, std::make_any<StringParameter::ValueType>("Part_Number_"));
+    args.insertOrAssign(WriteStlFileFilter::k_TriangleGeomPath_Key, std::make_any<DataPath>(k_ComputedTriangleDataContainerName));
+    args.insertOrAssign(WriteStlFileFilter::k_PartNumberPath_Key, std::make_any<DataPath>(k_ComputedTriangleDataContainerName.createChildPath(k_FaceData).createChildPath(k_PartNumberName)));
+
+    // Preflight the filter and check result
+    auto preflightResult = filter.preflight(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions)
+
+    // Execute the filter and check the result
+    auto executeResult = filter.execute(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result)
+  }
+
+  fs::path writtenFilePath = fs::path(std::string(unit_test::k_BinaryTestOutputDir) + "/Part_Number_1.stl");
+  REQUIRE(fs::exists(writtenFilePath));
+  auto fileContents = readIn(writtenFilePath);
+  std::string md5Hash = nx::core::UnitTest::ComputeMD5Hash(fileContents);
+  REQUIRE(md5Hash == "a0383b898d0668d70f08e135e5064efb");
+
+  writtenFilePath = fs::path(std::string(unit_test::k_BinaryTestOutputDir) + "/Part_Number_2.stl");
+  REQUIRE(fs::exists(writtenFilePath));
+  fileContents = readIn(writtenFilePath);
+  md5Hash = nx::core::UnitTest::ComputeMD5Hash(fileContents);
+  REQUIRE(md5Hash == "d45a0d99495df506384fdbbb46a79f5c");
+}

test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp

@@ -19,6 +19,7 @@
 #include "simplnx/Parameters/BoolParameter.hpp"
 #include "simplnx/Parameters/GeometrySelectionParameter.hpp"
 #include "simplnx/Utilities/FilterUtilities.hpp"
+#include "simplnx/Utilities/MD5.hpp"
 #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp"
 #include "simplnx/Utilities/Parsing/HDF5/Writers/FileWriter.hpp"
 
@@ -210,6 +211,17 @@ namespace UnitTest
 {
 inline constexpr float EPSILON = 0.0001;
 
+template <class T>
+std::string ComputeMD5Hash(const std::vector<T>& outputDataArray)
+{
+  const T* dataPtr = outputDataArray.data();
+  usize arraySize = outputDataArray.size();
+  MD5 md5;
+  md5.update(reinterpret_cast<const uint8*>(dataPtr), arraySize * sizeof(T));
+  md5.finalize();
+  return md5.hexdigest();
+}
+
 /**
  * @brief This class will decompress a tar.gz file using the locally installed copy of cmake and when
  * then class goes out of scope the extracted contents will be deleted from disk.