ENH: Write Temp Files for All Writers (#790)

Creates a new temporary file for all output from writers with the new AtomicFile class

Co-authored-by: Michael Jackson <[email protected]>

CMakeLists.txt

@@ -286,6 +286,7 @@ set(COMPLEX_SOURCE_DIR ${complex_SOURCE_DIR}/src/complex)
 set(COMPLEX_HDRS
   ${COMPLEX_SOURCE_DIR}/Common/Any.hpp
   ${COMPLEX_SOURCE_DIR}/Common/Array.hpp
+  ${COMPLEX_SOURCE_DIR}/Common/AtomicFile.hpp
   ${COMPLEX_SOURCE_DIR}/Common/Bit.hpp
   ${COMPLEX_SOURCE_DIR}/Common/BoundingBox.hpp
   ${COMPLEX_SOURCE_DIR}/Common/ComplexConstants.hpp
@@ -536,6 +537,7 @@ set(COMPLEX_GENERATED_HEADERS
 )
 
 set(COMPLEX_SRCS
+  ${COMPLEX_SOURCE_DIR}/Common/AtomicFile.cpp
   ${COMPLEX_SOURCE_DIR}/Common/RgbColor.cpp
   ${COMPLEX_SOURCE_DIR}/Common/Range.cpp
   ${COMPLEX_SOURCE_DIR}/Common/Range2D.cpp

src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteAbaqusHexahedron.cpp

@@ -1,5 +1,6 @@
 #include "WriteAbaqusHexahedron.hpp"
 
+#include "complex/Common/AtomicFile.hpp"
 #include "complex/DataStructure/DataArray.hpp"
 #include "complex/DataStructure/DataGroup.hpp"
 #include "complex/DataStructure/Geometry/ImageGeom.hpp"
@@ -316,15 +317,11 @@ int32 writeSects(const std::string& file, const Int32Array& featureIds, int32 ho
   return err;
 }
 
-void deleteFile(const std::vector<std::string>& fileNames)
+void deleteFile(const std::vector<std::unique_ptr<AtomicFile>>& fileList)
 {
-  for(const auto& fileName : fileNames)
+  for(const auto& atomicFile : fileList)
   {
-    auto path = fs::path(fileName);
-    if(fs::exists(path))
-    {
-      fs::remove(path);
-    }
+    atomicFile->removeTempFile();
   }
 }
 } // namespace
@@ -366,72 +363,77 @@ Result<> WriteAbaqusHexahedron::operator()()
   usize totalPoints = imageGeom.getNumberOfCells();
 
   // Create file names
-  std::vector<std::string> fileNames = {};
-  fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_nodes.inp");
-  fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elems.inp");
-  fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_sects.inp");
-  fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elset.inp");
-  fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + ".inp");
-
-  int32 err = writeNodes(this, fileNames[0], cDims.data(), origin.data(), spacing.data(), getCancel()); // Nodes file
+  std::vector<std::unique_ptr<AtomicFile>> fileList = {};
+  fileList.push_back(std::make_unique<AtomicFile>(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_nodes.inp"));
+  fileList.push_back(std::make_unique<AtomicFile>(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elems.inp"));
+  fileList.push_back(std::make_unique<AtomicFile>(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_sects.inp"));
+  fileList.push_back(std::make_unique<AtomicFile>(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elset.inp"));
+  fileList.push_back(std::make_unique<AtomicFile>(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + ".inp"));
+
+  int32 err = writeNodes(this, fileList[0]->tempFilePath().string(), cDims.data(), origin.data(), spacing.data(), getCancel()); // Nodes file
   if(err < 0)
   {
-    return MakeErrorResult(-1113, fmt::format("Error writing output nodes file '{}'", fileNames[0]));
+    return MakeErrorResult(-1113, fmt::format("Error writing output nodes file '{}'", fileList[0]->tempFilePath().string()));
   }
   if(getCancel()) // Filter has been cancelled
   {
-    deleteFile(fileNames); // delete files
+    deleteFile(fileList); // delete files
     return {};
   }
   m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 1/5) Complete");
 
-  err = writeElems(this, fileNames[1], cDims.data(), pDims, getCancel()); // Elements file
+  err = writeElems(this, fileList[1]->tempFilePath().string(), cDims.data(), pDims, getCancel()); // Elements file
   if(err < 0)
   {
-    return MakeErrorResult(-1114, fmt::format("Error writing output elems file '{}'", fileNames[1]));
+    return MakeErrorResult(-1114, fmt::format("Error writing output elems file '{}'", fileList[1]->tempFilePath().string()));
   }
   if(getCancel()) // Filter has been cancelled
   {
-    deleteFile(fileNames); // delete files
+    deleteFile(fileList); // delete files
     return {};
   }
   m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 2/5) Complete");
 
-  err = writeSects(fileNames[2], featureIds, m_InputValues->HourglassStiffness); // Sections file
+  err = writeSects(fileList[2]->tempFilePath().string(), featureIds, m_InputValues->HourglassStiffness); // Sections file
   if(err < 0)
   {
-    return MakeErrorResult(-1115, fmt::format("Error writing output sects file '{}'", fileNames[2]));
+    return MakeErrorResult(-1115, fmt::format("Error writing output sects file '{}'", fileList[2]->tempFilePath().string()));
   }
   if(getCancel()) // Filter has been cancelled
   {
-    deleteFile(fileNames); // delete files
+    deleteFile(fileList); // delete files
     return {};
   }
   m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 3/5) Complete");
 
-  err = writeElset(this, fileNames[3], totalPoints, featureIds, getCancel()); // Element set file
+  err = writeElset(this, fileList[3]->tempFilePath().string(), totalPoints, featureIds, getCancel()); // Element set file
   if(err < 0)
   {
-    return MakeErrorResult(-1116, fmt::format("Error writing output elset file '{}'", fileNames[3]));
+    return MakeErrorResult(-1116, fmt::format("Error writing output elset file '{}'", fileList[3]->tempFilePath().string()));
   }
   if(getCancel()) // Filter has been cancelled
   {
-    deleteFile(fileNames); // delete files
+    deleteFile(fileList); // delete files
     return {};
   }
   m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 4/5) Complete");
 
-  err = writeMaster(fileNames[4], m_InputValues->JobName, m_InputValues->FilePrefix); // Master file
+  err = writeMaster(fileList[4]->tempFilePath().string(), m_InputValues->JobName, m_InputValues->FilePrefix); // Master file
   if(err < 0)
   {
-    return MakeErrorResult(-1117, fmt::format("Error writing output master file '{}'", fileNames[4]));
+    return MakeErrorResult(-1117, fmt::format("Error writing output master file '{}'", fileList[4]->tempFilePath().string()));
   }
   if(getCancel()) // Filter has been cancelled
   {
-    deleteFile(fileNames); // delete files
+    deleteFile(fileList); // delete files
     return {};
   }
   m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 5/5) Complete");
 
+  for(auto& file : fileList)
+  {
+    file->commit();
+  }
+
   return {};
 }

src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteStlFile.cpp

@@ -1,45 +1,12 @@
 #include "WriteStlFile.hpp"
 
+#include "complex/Common/AtomicFile.hpp"
 #include "complex/DataStructure/Geometry/TriangleGeom.hpp"
 #include "complex/Utilities/FilterUtilities.hpp"
 #include "complex/Utilities/StringUtilities.hpp"
 
 using namespace complex;
 
-namespace
-{
-int32_t writeHeader(FILE* f, const std::string& header, int32_t triCount)
-{
-  if(nullptr == f)
-  {
-    return -1;
-  }
-
-  char h[80];
-  size_t headLength = 80;
-  if(header.length() < 80)
-  {
-    headLength = static_cast<size_t>(header.length());
-  }
-
-  std::string c_str = header;
-  ::memset(h, 0, 80);
-  ::memcpy(h, c_str.data(), headLength);
-  // Return the number of bytes written - which should be 80
-  fwrite(h, 1, 80, f);
-  fwrite(&triCount, 1, 4, f);
-  return 0;
-}
-
-void writeNumTrianglesToFile(const std::string& filename, int32_t triCount)
-{
-  FILE* out = fopen(filename.c_str(), "r+b");
-  fseek(out, 80L, SEEK_SET);
-  fwrite(reinterpret_cast<char*>(&triCount), 1, 4, out);
-  fclose(out);
-}
-} // namespace
-
 // -----------------------------------------------------------------------------
 WriteStlFile::WriteStlFile(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, WriteStlFileInputValues* inputValues)
 : m_DataStructure(dataStructure)
@@ -110,8 +77,8 @@ Result<> WriteStlFile::operator()()
 
   int32_t triCount = 0;
 
-  // Loop over the unique Spins
-  for(const auto& [spin, value] : uniqueGrainIdToPhase)
+  // Loop over the unique feature Ids
+  for(const auto& [featureId, value] : uniqueGrainIdToPhase)
   {
     // Generate the output file name
     std::string filename = m_InputValues->OutputStlDirectory.string() + "/" + m_InputValues->OutputStlPrefix;
@@ -120,18 +87,22 @@ Result<> WriteStlFile::operator()()
       filename += "Ensemble_" + StringUtilities::number(value) + "_";
     }
 
-    filename += "Feature_" + StringUtilities::number(spin) + ".stl";
-    FILE* f = fopen(filename.c_str(), "wb");
+    filename += "Feature_" + StringUtilities::number(featureId) + ".stl";
+
+    AtomicFile atomicFile(filename, true);
+
+    FILE* f = fopen(atomicFile.tempFilePath().string().c_str(), "wb");
 
     if(f == nullptr)
     {
       fclose(f);
-      return {MakeWarningVoidResult(-27875, fmt::format("Error Writing STL File. Unable to create file at path {}.", filename))};
+      atomicFile.setAutoCommit(false); // Set this to false otherwise
+      return {MakeWarningVoidResult(-27875, fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", atomicFile.tempFilePath().string(), filename))};
     }
 
-    m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", spin));
+    m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", featureId));
 
-    std::string header = "DREAM3D Generated For Feature ID " + StringUtilities::number(spin);
+    std::string header = "DREAM3D Generated For Feature ID " + StringUtilities::number(featureId);
     if(m_InputValues->GroupByFeature)
     {
       header += " Phase " + StringUtilities::number(value);
@@ -140,10 +111,24 @@ Result<> WriteStlFile::operator()()
     if(header.size() >= 80)
     {
       fclose(f);
-      return {MakeWarningVoidResult(-27874, fmt::format("Error Writing STL File. Header was over the 80 characters supported by STL. Length of header: {}.", header.length()))};
+      atomicFile.setAutoCommit(false); // Set this to false otherwise
+      atomicFile.removeTempFile();     // Remove the temp file
+      return {MakeWarningVoidResult(-27874, fmt::format("Error Writing STL File '{}'. Header was over the 80 characters supported by STL. Length of header: {}.", filename, header.length()))};
     }
 
-    writeHeader(f, header, 0);
+    char h[80];
+    size_t headLength = 80;
+    if(header.length() < 80)
+    {
+      headLength = static_cast<size_t>(header.length());
+    }
+
+    std::string c_str = header;
+    ::memset(h, 0, 80);
+    ::memcpy(h, c_str.data(), headLength);
+    // Return the number of bytes written - which should be 80
+    fwrite(h, 1, 80, f);
+    fwrite(&triCount, 1, 4, f);
     triCount = 0; // Reset this to Zero. Increment for every triangle written
 
     // Loop over all the triangles for this spin
@@ -158,11 +143,11 @@ Result<> WriteStlFile::operator()()
       vert1[1] = static_cast<float>(vertices[nId0 * 3 + 1]);
       vert1[2] = static_cast<float>(vertices[nId0 * 3 + 2]);
 
-      if(featureIds[t * 2] == spin)
+      if(featureIds[t * 2] == featureId)
       {
         // winding = 0; // 0 = Write it using forward spin
       }
-      else if(featureIds[t * 2 + 1] == spin)
+      else if(featureIds[t * 2 + 1] == featureId)
       {
         // winding = 1; // Write it using backward spin
         // Switch the 2 node indices
@@ -205,12 +190,17 @@ Result<> WriteStlFile::operator()()
       if(totalWritten != 50)
       {
         fclose(f);
-        return {MakeWarningVoidResult(-27873, fmt::format("Error Writing STL File. Not enough elements written for Feature Id {}. Wrote {} of 50.", spin, totalWritten))};
+        atomicFile.setAutoCommit(false); // Set this to false otherwise
+        atomicFile.removeTempFile();     // Remove the temp file
+        return {MakeWarningVoidResult(-27873,
+                                      fmt::format("Error Writing STL File '{}'. Not enough elements written for Feature Id {}. Wrote {} of 50. No file written.", filename, featureId, totalWritten))};
       }
       triCount++;
     }
+
+    fseek(f, 80L, SEEK_SET);
+    fwrite(reinterpret_cast<char*>(&triCount), 1, 4, f);
     fclose(f);
-    writeNumTrianglesToFile(filename, triCount);
   }
 
   return {};

src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteVtkRectilinearGrid.cpp

@@ -241,14 +241,6 @@ const std::atomic_bool& WriteVtkRectilinearGrid::getCancel()
 // -----------------------------------------------------------------------------
 Result<> WriteVtkRectilinearGrid::operator()()
 {
-  // Make sure any directory path is also available as the user may have just typed
-  // in a path without actually creating the full path
-  Result<> createDirectoriesResult = CreateOutputDirectories(m_InputValues->OutputFile.parent_path());
-  if(createDirectoriesResult.invalid())
-  {
-    return createDirectoriesResult;
-  }
-
   const auto& imageGeom = m_DataStructure.getDataRefAs<ImageGeom>(m_InputValues->ImageGeometryPath);
   SizeVec3 dims = imageGeom.getDimensions();
   FloatVec3 res = imageGeom.getSpacing();

src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteASCIIDataFilter.cpp

@@ -1,5 +1,6 @@
 #include "WriteASCIIDataFilter.hpp"
 
+#include "complex/Common/AtomicFile.hpp"
 #include "complex/Common/TypeTraits.hpp"
 #include "complex/Parameters/ChoicesParameter.hpp"
 #include "complex/Parameters/FileSystemPathParameter.hpp"
@@ -173,7 +174,8 @@ Result<> WriteASCIIDataFilter::executeImpl(DataStructure& dataStructure, const A
 
   if(static_cast<WriteASCIIDataFilter::OutputStyle>(fileType) == WriteASCIIDataFilter::OutputStyle::SingleFile)
   {
-    auto outputPath = filterArgs.value<FileSystemPathParameter::ValueType>(k_OutputPath_Key);
+    AtomicFile atomicFile(filterArgs.value<FileSystemPathParameter::ValueType>(k_OutputPath_Key), false);
+    auto outputPath = atomicFile.tempFilePath();
     // Make sure any directory path is also available as the user may have just typed
     // in a path without actually creating the full path
     Result<> createDirectoriesResult = complex::CreateOutputDirectories(outputPath.parent_path());
@@ -182,14 +184,18 @@ Result<> WriteASCIIDataFilter::executeImpl(DataStructure& dataStructure, const A
       return createDirectoriesResult;
     }
 
-    // Create the output file
-    std::ofstream outStrm(outputPath, std::ios_base::out | std::ios_base::binary);
-    if(!outStrm.is_open())
+    // Scope file writer in code block to get around file lock on windows (enforce destructor order)
     {
-      return MakeErrorResult(-11021, fmt::format("Unable to create output file {}", outputPath.string()));
-    }
+      // Create the output file
+      std::ofstream outStrm(outputPath, std::ios_base::out | std::ios_base::binary);
+      if(!outStrm.is_open())
+      {
+        return MakeErrorResult(-11021, fmt::format("Unable to create output file {}", outputPath.string()));
+      }
 
-    OStreamUtilities::PrintDataSetsToSingleFile(outStrm, selectedDataArrayPaths, dataStructure, messageHandler, shouldCancel, delimiter, includeIndex, includeHeaders);
+      OStreamUtilities::PrintDataSetsToSingleFile(outStrm, selectedDataArrayPaths, dataStructure, messageHandler, shouldCancel, delimiter, includeIndex, includeHeaders);
+    }
+    atomicFile.setAutoCommit(true);
   }
 
   if(static_cast<WriteASCIIDataFilter::OutputStyle>(fileType) == WriteASCIIDataFilter::OutputStyle::MultipleFiles)

src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoRectilinearCoordinateFilter.cpp

@@ -2,6 +2,7 @@
 
 #include "ComplexCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.hpp"
 
+#include "complex/Common/AtomicFile.hpp"
 #include "complex/DataStructure/DataPath.hpp"
 #include "complex/Filter/Actions/EmptyAction.hpp"
 #include "complex/Parameters/ArraySelectionParameter.hpp"
@@ -96,13 +97,17 @@ Result<> WriteAvizoRectilinearCoordinateFilter::executeImpl(DataStructure& dataS
 {
   AvizoWriterInputValues inputValues;
 
-  inputValues.OutputFile = filterArgs.value<FileSystemPathParameter::ValueType>(k_OutputFile_Key);
+  AtomicFile atomicFile(filterArgs.value<FileSystemPathParameter::ValueType>(k_OutputFile_Key), true);
+
+  inputValues.OutputFile = atomicFile.tempFilePath();
   inputValues.WriteBinaryFile = filterArgs.value<bool>(k_WriteBinaryFile_Key);
   inputValues.GeometryPath = filterArgs.value<DataPath>(k_GeometryPath_Key);
   inputValues.FeatureIdsArrayPath = filterArgs.value<DataPath>(k_FeatureIdsArrayPath_Key);
   inputValues.Units = filterArgs.value<StringParameter::ValueType>(k_Units_Key);
 
-  return WriteAvizoRectilinearCoordinate(dataStructure, messageHandler, shouldCancel, &inputValues)();
+  auto result = WriteAvizoRectilinearCoordinate(dataStructure, messageHandler, shouldCancel, &inputValues)();
+  atomicFile.setAutoCommit(result.valid());
+  return result;
 }
 
 namespace