From 5b9d046ba29adf3a68c9a3849bbb4d5f483e0a53 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Tue, 19 Dec 2023 07:05:34 -0500 Subject: [PATCH] ENH: Write Temp Files for All Writers (#790) Creates a new temporary file for all output from writers with the new AtomicFile class Co-authored-by: Michael Jackson --- CMakeLists.txt | 2 + .../Algorithms/WriteAbaqusHexahedron.cpp | 60 +-- .../Filters/Algorithms/WriteStlFile.cpp | 84 ++-- .../Algorithms/WriteVtkRectilinearGrid.cpp | 8 - .../Filters/WriteASCIIDataFilter.cpp | 20 +- .../WriteAvizoRectilinearCoordinateFilter.cpp | 9 +- .../WriteAvizoUniformCoordinateFilter.cpp | 9 +- .../Filters/WriteDREAM3DFilter.cpp | 19 +- .../Filters/WriteFeatureDataCSVFilter.cpp | 21 +- .../Filters/WriteLosAlamosFFTFilter.cpp | 9 +- .../Filters/WriteVtkRectilinearGridFilter.cpp | 15 +- .../Filters/ITKImageWriter.cpp | 103 ++-- .../Algorithms/ConvertHexGridToSquareGrid.cpp | 53 ++- .../Filters/Algorithms/EbsdToH5Ebsd.cpp | 443 +++++++++--------- .../Filters/WriteGBCDGMTFileFilter.cpp | 10 +- .../Filters/WriteGBCDTriangleDataFilter.cpp | 9 +- .../Filters/WriteINLFileFilter.cpp | 9 +- src/complex/Common/AtomicFile.cpp | 97 ++++ src/complex/Common/AtomicFile.hpp | 46 ++ src/complex/Utilities/OStreamUtilities.cpp | 64 +-- 20 files changed, 690 insertions(+), 400 deletions(-) create mode 100644 src/complex/Common/AtomicFile.cpp create mode 100644 src/complex/Common/AtomicFile.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c3d9175e2..ae7ca2fcdb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,6 +286,7 @@ set(COMPLEX_SOURCE_DIR ${complex_SOURCE_DIR}/src/complex) set(COMPLEX_HDRS ${COMPLEX_SOURCE_DIR}/Common/Any.hpp ${COMPLEX_SOURCE_DIR}/Common/Array.hpp + ${COMPLEX_SOURCE_DIR}/Common/AtomicFile.hpp ${COMPLEX_SOURCE_DIR}/Common/Bit.hpp ${COMPLEX_SOURCE_DIR}/Common/BoundingBox.hpp ${COMPLEX_SOURCE_DIR}/Common/ComplexConstants.hpp @@ -536,6 +537,7 @@ set(COMPLEX_GENERATED_HEADERS ) set(COMPLEX_SRCS + ${COMPLEX_SOURCE_DIR}/Common/AtomicFile.cpp ${COMPLEX_SOURCE_DIR}/Common/RgbColor.cpp ${COMPLEX_SOURCE_DIR}/Common/Range.cpp ${COMPLEX_SOURCE_DIR}/Common/Range2D.cpp diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteAbaqusHexahedron.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteAbaqusHexahedron.cpp index 0f5e24ddbc..e6674585f5 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteAbaqusHexahedron.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteAbaqusHexahedron.cpp @@ -1,5 +1,6 @@ #include "WriteAbaqusHexahedron.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataArray.hpp" #include "complex/DataStructure/DataGroup.hpp" #include "complex/DataStructure/Geometry/ImageGeom.hpp" @@ -316,15 +317,11 @@ int32 writeSects(const std::string& file, const Int32Array& featureIds, int32 ho return err; } -void deleteFile(const std::vector& fileNames) +void deleteFile(const std::vector>& fileList) { - for(const auto& fileName : fileNames) + for(const auto& atomicFile : fileList) { - auto path = fs::path(fileName); - if(fs::exists(path)) - { - fs::remove(path); - } + atomicFile->removeTempFile(); } } } // namespace @@ -366,72 +363,77 @@ Result<> WriteAbaqusHexahedron::operator()() usize totalPoints = imageGeom.getNumberOfCells(); // Create file names - std::vector fileNames = {}; - fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_nodes.inp"); - fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elems.inp"); - fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_sects.inp"); - fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elset.inp"); - fileNames.push_back(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + ".inp"); - - int32 err = writeNodes(this, fileNames[0], cDims.data(), origin.data(), spacing.data(), getCancel()); // Nodes file + std::vector> fileList = {}; + fileList.push_back(std::make_unique(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_nodes.inp")); + fileList.push_back(std::make_unique(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elems.inp")); + fileList.push_back(std::make_unique(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_sects.inp")); + fileList.push_back(std::make_unique(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + "_elset.inp")); + fileList.push_back(std::make_unique(m_InputValues->OutputPath.string() + "/" + m_InputValues->FilePrefix + ".inp")); + + int32 err = writeNodes(this, fileList[0]->tempFilePath().string(), cDims.data(), origin.data(), spacing.data(), getCancel()); // Nodes file if(err < 0) { - return MakeErrorResult(-1113, fmt::format("Error writing output nodes file '{}'", fileNames[0])); + return MakeErrorResult(-1113, fmt::format("Error writing output nodes file '{}'", fileList[0]->tempFilePath().string())); } if(getCancel()) // Filter has been cancelled { - deleteFile(fileNames); // delete files + deleteFile(fileList); // delete files return {}; } m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 1/5) Complete"); - err = writeElems(this, fileNames[1], cDims.data(), pDims, getCancel()); // Elements file + err = writeElems(this, fileList[1]->tempFilePath().string(), cDims.data(), pDims, getCancel()); // Elements file if(err < 0) { - return MakeErrorResult(-1114, fmt::format("Error writing output elems file '{}'", fileNames[1])); + return MakeErrorResult(-1114, fmt::format("Error writing output elems file '{}'", fileList[1]->tempFilePath().string())); } if(getCancel()) // Filter has been cancelled { - deleteFile(fileNames); // delete files + deleteFile(fileList); // delete files return {}; } m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 2/5) Complete"); - err = writeSects(fileNames[2], featureIds, m_InputValues->HourglassStiffness); // Sections file + err = writeSects(fileList[2]->tempFilePath().string(), featureIds, m_InputValues->HourglassStiffness); // Sections file if(err < 0) { - return MakeErrorResult(-1115, fmt::format("Error writing output sects file '{}'", fileNames[2])); + return MakeErrorResult(-1115, fmt::format("Error writing output sects file '{}'", fileList[2]->tempFilePath().string())); } if(getCancel()) // Filter has been cancelled { - deleteFile(fileNames); // delete files + deleteFile(fileList); // delete files return {}; } m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 3/5) Complete"); - err = writeElset(this, fileNames[3], totalPoints, featureIds, getCancel()); // Element set file + err = writeElset(this, fileList[3]->tempFilePath().string(), totalPoints, featureIds, getCancel()); // Element set file if(err < 0) { - return MakeErrorResult(-1116, fmt::format("Error writing output elset file '{}'", fileNames[3])); + return MakeErrorResult(-1116, fmt::format("Error writing output elset file '{}'", fileList[3]->tempFilePath().string())); } if(getCancel()) // Filter has been cancelled { - deleteFile(fileNames); // delete files + deleteFile(fileList); // delete files return {}; } m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 4/5) Complete"); - err = writeMaster(fileNames[4], m_InputValues->JobName, m_InputValues->FilePrefix); // Master file + err = writeMaster(fileList[4]->tempFilePath().string(), m_InputValues->JobName, m_InputValues->FilePrefix); // Master file if(err < 0) { - return MakeErrorResult(-1117, fmt::format("Error writing output master file '{}'", fileNames[4])); + return MakeErrorResult(-1117, fmt::format("Error writing output master file '{}'", fileList[4]->tempFilePath().string())); } if(getCancel()) // Filter has been cancelled { - deleteFile(fileNames); // delete files + deleteFile(fileList); // delete files return {}; } m_MessageHandler(IFilter::Message::Type::Info, "Writing Sections (File 5/5) Complete"); + for(auto& file : fileList) + { + file->commit(); + } + return {}; } diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteStlFile.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteStlFile.cpp index 79e01d647d..b00aa33c4f 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteStlFile.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteStlFile.cpp @@ -1,45 +1,12 @@ #include "WriteStlFile.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/Geometry/TriangleGeom.hpp" #include "complex/Utilities/FilterUtilities.hpp" #include "complex/Utilities/StringUtilities.hpp" using namespace complex; -namespace -{ -int32_t writeHeader(FILE* f, const std::string& header, int32_t triCount) -{ - if(nullptr == f) - { - return -1; - } - - char h[80]; - size_t headLength = 80; - if(header.length() < 80) - { - headLength = static_cast(header.length()); - } - - std::string c_str = header; - ::memset(h, 0, 80); - ::memcpy(h, c_str.data(), headLength); - // Return the number of bytes written - which should be 80 - fwrite(h, 1, 80, f); - fwrite(&triCount, 1, 4, f); - return 0; -} - -void writeNumTrianglesToFile(const std::string& filename, int32_t triCount) -{ - FILE* out = fopen(filename.c_str(), "r+b"); - fseek(out, 80L, SEEK_SET); - fwrite(reinterpret_cast(&triCount), 1, 4, out); - fclose(out); -} -} // namespace - // ----------------------------------------------------------------------------- WriteStlFile::WriteStlFile(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, WriteStlFileInputValues* inputValues) : m_DataStructure(dataStructure) @@ -110,8 +77,8 @@ Result<> WriteStlFile::operator()() int32_t triCount = 0; - // Loop over the unique Spins - for(const auto& [spin, value] : uniqueGrainIdToPhase) + // Loop over the unique feature Ids + for(const auto& [featureId, value] : uniqueGrainIdToPhase) { // Generate the output file name std::string filename = m_InputValues->OutputStlDirectory.string() + "/" + m_InputValues->OutputStlPrefix; @@ -120,18 +87,22 @@ Result<> WriteStlFile::operator()() filename += "Ensemble_" + StringUtilities::number(value) + "_"; } - filename += "Feature_" + StringUtilities::number(spin) + ".stl"; - FILE* f = fopen(filename.c_str(), "wb"); + filename += "Feature_" + StringUtilities::number(featureId) + ".stl"; + + AtomicFile atomicFile(filename, true); + + FILE* f = fopen(atomicFile.tempFilePath().string().c_str(), "wb"); if(f == nullptr) { fclose(f); - return {MakeWarningVoidResult(-27875, fmt::format("Error Writing STL File. Unable to create file at path {}.", filename))}; + atomicFile.setAutoCommit(false); // Set this to false otherwise + return {MakeWarningVoidResult(-27875, fmt::format("Error Opening STL File. Unable to create temp file at path '{}' for original file '{}'", atomicFile.tempFilePath().string(), filename))}; } - m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", spin)); + m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Writing STL for Feature Id {}", featureId)); - std::string header = "DREAM3D Generated For Feature ID " + StringUtilities::number(spin); + std::string header = "DREAM3D Generated For Feature ID " + StringUtilities::number(featureId); if(m_InputValues->GroupByFeature) { header += " Phase " + StringUtilities::number(value); @@ -140,10 +111,24 @@ Result<> WriteStlFile::operator()() if(header.size() >= 80) { fclose(f); - return {MakeWarningVoidResult(-27874, fmt::format("Error Writing STL File. Header was over the 80 characters supported by STL. Length of header: {}.", header.length()))}; + atomicFile.setAutoCommit(false); // Set this to false otherwise + atomicFile.removeTempFile(); // Remove the temp file + return {MakeWarningVoidResult(-27874, fmt::format("Error Writing STL File '{}'. Header was over the 80 characters supported by STL. Length of header: {}.", filename, header.length()))}; } - writeHeader(f, header, 0); + char h[80]; + size_t headLength = 80; + if(header.length() < 80) + { + headLength = static_cast(header.length()); + } + + std::string c_str = header; + ::memset(h, 0, 80); + ::memcpy(h, c_str.data(), headLength); + // Return the number of bytes written - which should be 80 + fwrite(h, 1, 80, f); + fwrite(&triCount, 1, 4, f); triCount = 0; // Reset this to Zero. Increment for every triangle written // Loop over all the triangles for this spin @@ -158,11 +143,11 @@ Result<> WriteStlFile::operator()() vert1[1] = static_cast(vertices[nId0 * 3 + 1]); vert1[2] = static_cast(vertices[nId0 * 3 + 2]); - if(featureIds[t * 2] == spin) + if(featureIds[t * 2] == featureId) { // winding = 0; // 0 = Write it using forward spin } - else if(featureIds[t * 2 + 1] == spin) + else if(featureIds[t * 2 + 1] == featureId) { // winding = 1; // Write it using backward spin // Switch the 2 node indices @@ -205,12 +190,17 @@ Result<> WriteStlFile::operator()() if(totalWritten != 50) { fclose(f); - return {MakeWarningVoidResult(-27873, fmt::format("Error Writing STL File. Not enough elements written for Feature Id {}. Wrote {} of 50.", spin, totalWritten))}; + atomicFile.setAutoCommit(false); // Set this to false otherwise + atomicFile.removeTempFile(); // Remove the temp file + return {MakeWarningVoidResult(-27873, + fmt::format("Error Writing STL File '{}'. Not enough elements written for Feature Id {}. Wrote {} of 50. No file written.", filename, featureId, totalWritten))}; } triCount++; } + + fseek(f, 80L, SEEK_SET); + fwrite(reinterpret_cast(&triCount), 1, 4, f); fclose(f); - writeNumTrianglesToFile(filename, triCount); } return {}; diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteVtkRectilinearGrid.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteVtkRectilinearGrid.cpp index 8442b1ad48..d761fb0fb7 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteVtkRectilinearGrid.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/WriteVtkRectilinearGrid.cpp @@ -241,14 +241,6 @@ const std::atomic_bool& WriteVtkRectilinearGrid::getCancel() // ----------------------------------------------------------------------------- Result<> WriteVtkRectilinearGrid::operator()() { - // Make sure any directory path is also available as the user may have just typed - // in a path without actually creating the full path - Result<> createDirectoriesResult = CreateOutputDirectories(m_InputValues->OutputFile.parent_path()); - if(createDirectoriesResult.invalid()) - { - return createDirectoriesResult; - } - const auto& imageGeom = m_DataStructure.getDataRefAs(m_InputValues->ImageGeometryPath); SizeVec3 dims = imageGeom.getDimensions(); FloatVec3 res = imageGeom.getSpacing(); diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteASCIIDataFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteASCIIDataFilter.cpp index 939448441b..2252742266 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteASCIIDataFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteASCIIDataFilter.cpp @@ -1,5 +1,6 @@ #include "WriteASCIIDataFilter.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/Common/TypeTraits.hpp" #include "complex/Parameters/ChoicesParameter.hpp" #include "complex/Parameters/FileSystemPathParameter.hpp" @@ -173,7 +174,8 @@ Result<> WriteASCIIDataFilter::executeImpl(DataStructure& dataStructure, const A if(static_cast(fileType) == WriteASCIIDataFilter::OutputStyle::SingleFile) { - auto outputPath = filterArgs.value(k_OutputPath_Key); + AtomicFile atomicFile(filterArgs.value(k_OutputPath_Key), false); + auto outputPath = atomicFile.tempFilePath(); // Make sure any directory path is also available as the user may have just typed // in a path without actually creating the full path Result<> createDirectoriesResult = complex::CreateOutputDirectories(outputPath.parent_path()); @@ -182,14 +184,18 @@ Result<> WriteASCIIDataFilter::executeImpl(DataStructure& dataStructure, const A return createDirectoriesResult; } - // Create the output file - std::ofstream outStrm(outputPath, std::ios_base::out | std::ios_base::binary); - if(!outStrm.is_open()) + // Scope file writer in code block to get around file lock on windows (enforce destructor order) { - return MakeErrorResult(-11021, fmt::format("Unable to create output file {}", outputPath.string())); - } + // Create the output file + std::ofstream outStrm(outputPath, std::ios_base::out | std::ios_base::binary); + if(!outStrm.is_open()) + { + return MakeErrorResult(-11021, fmt::format("Unable to create output file {}", outputPath.string())); + } - OStreamUtilities::PrintDataSetsToSingleFile(outStrm, selectedDataArrayPaths, dataStructure, messageHandler, shouldCancel, delimiter, includeIndex, includeHeaders); + OStreamUtilities::PrintDataSetsToSingleFile(outStrm, selectedDataArrayPaths, dataStructure, messageHandler, shouldCancel, delimiter, includeIndex, includeHeaders); + } + atomicFile.setAutoCommit(true); } if(static_cast(fileType) == WriteASCIIDataFilter::OutputStyle::MultipleFiles) diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoRectilinearCoordinateFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoRectilinearCoordinateFilter.cpp index 76a94b0225..c730b93d4b 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoRectilinearCoordinateFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoRectilinearCoordinateFilter.cpp @@ -2,6 +2,7 @@ #include "ComplexCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/Filter/Actions/EmptyAction.hpp" #include "complex/Parameters/ArraySelectionParameter.hpp" @@ -96,13 +97,17 @@ Result<> WriteAvizoRectilinearCoordinateFilter::executeImpl(DataStructure& dataS { AvizoWriterInputValues inputValues; - inputValues.OutputFile = filterArgs.value(k_OutputFile_Key); + AtomicFile atomicFile(filterArgs.value(k_OutputFile_Key), true); + + inputValues.OutputFile = atomicFile.tempFilePath(); inputValues.WriteBinaryFile = filterArgs.value(k_WriteBinaryFile_Key); inputValues.GeometryPath = filterArgs.value(k_GeometryPath_Key); inputValues.FeatureIdsArrayPath = filterArgs.value(k_FeatureIdsArrayPath_Key); inputValues.Units = filterArgs.value(k_Units_Key); - return WriteAvizoRectilinearCoordinate(dataStructure, messageHandler, shouldCancel, &inputValues)(); + auto result = WriteAvizoRectilinearCoordinate(dataStructure, messageHandler, shouldCancel, &inputValues)(); + atomicFile.setAutoCommit(result.valid()); + return result; } namespace diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoUniformCoordinateFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoUniformCoordinateFilter.cpp index 744aca6979..736f748e2c 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoUniformCoordinateFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteAvizoUniformCoordinateFilter.cpp @@ -2,6 +2,7 @@ #include "ComplexCore/Filters/Algorithms/WriteAvizoUniformCoordinate.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/Filter/Actions/EmptyAction.hpp" #include "complex/Parameters/ArraySelectionParameter.hpp" @@ -96,13 +97,17 @@ Result<> WriteAvizoUniformCoordinateFilter::executeImpl(DataStructure& dataStruc { AvizoWriterInputValues inputValues; - inputValues.OutputFile = filterArgs.value(k_OutputFile_Key); + AtomicFile atomicFile(filterArgs.value(k_OutputFile_Key), true); + + inputValues.OutputFile = atomicFile.tempFilePath(); inputValues.WriteBinaryFile = filterArgs.value(k_WriteBinaryFile_Key); inputValues.GeometryPath = filterArgs.value(k_GeometryPath_Key); inputValues.FeatureIdsArrayPath = filterArgs.value(k_FeatureIdsArrayPath_Key); inputValues.Units = filterArgs.value(k_Units_Key); - return WriteAvizoUniformCoordinate(dataStructure, messageHandler, shouldCancel, &inputValues)(); + auto result = WriteAvizoUniformCoordinate(dataStructure, messageHandler, shouldCancel, &inputValues)(); + atomicFile.setAutoCommit(result.valid()); + return result; } namespace diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteDREAM3DFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteDREAM3DFilter.cpp index 89d4ae3d3b..96f8a5cbb0 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteDREAM3DFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteDREAM3DFilter.cpp @@ -1,15 +1,17 @@ #include "WriteDREAM3DFilter.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataGroup.hpp" #include "complex/Parameters/BoolParameter.hpp" #include "complex/Parameters/FileSystemPathParameter.hpp" #include "complex/Pipeline/Pipeline.hpp" #include "complex/Pipeline/PipelineFilter.hpp" #include "complex/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" - +#include "complex/Utilities/Parsing/HDF5/Writers/FileWriter.hpp" #include "complex/Utilities/SIMPLConversion.hpp" -#include "complex/Utilities/Parsing/HDF5/Writers/FileWriter.hpp" +#include +namespace fs = std::filesystem; namespace { @@ -82,7 +84,9 @@ IFilter::PreflightResult WriteDREAM3DFilter::preflightImpl(const DataStructure& Result<> WriteDREAM3DFilter::executeImpl(DataStructure& dataStructure, const Arguments& args, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { - auto exportFilePath = args.value(k_ExportFilePath); + AtomicFile atomicFile(args.value(k_ExportFilePath), false); + + auto exportFilePath = atomicFile.tempFilePath(); auto writeXdmf = args.value(k_WriteXdmf); Pipeline pipeline; @@ -99,6 +103,15 @@ Result<> WriteDREAM3DFilter::executeImpl(DataStructure& dataStructure, const Arg } auto results = DREAM3D::WriteFile(exportFilePath, dataStructure, pipeline, writeXdmf); + if(results.valid()) + { + atomicFile.commit(); + if(writeXdmf) + { + fs::path xdmfFilePath = exportFilePath.replace_extension(".xdmf"); + fs::rename(xdmfFilePath, args.value(k_ExportFilePath).replace_extension(".xdmf")); + } + } return results; } diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteFeatureDataCSVFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteFeatureDataCSVFilter.cpp index bd9f9bae92..4ee04f6ab8 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteFeatureDataCSVFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteFeatureDataCSVFilter.cpp @@ -1,5 +1,6 @@ #include "WriteFeatureDataCSVFilter.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/Common/TypeTraits.hpp" #include "complex/DataStructure/AttributeMatrix.hpp" #include "complex/DataStructure/DataPath.hpp" @@ -94,7 +95,9 @@ IFilter::PreflightResult WriteFeatureDataCSVFilter::preflightImpl(const DataStru Result<> WriteFeatureDataCSVFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { - auto pOutputFilePath = filterArgs.value(k_FeatureDataFile_Key); + AtomicFile atomicFile(filterArgs.value(k_FeatureDataFile_Key)); + + auto pOutputFilePath = atomicFile.tempFilePath(); auto pWriteNeighborListDataValue = filterArgs.value(k_WriteNeighborListData_Key); auto pWriteNumFeaturesLineValue = filterArgs.value(k_WriteNumFeaturesLine_Key); auto pDelimiterChoiceIntValue = filterArgs.value(k_DelimiterChoiceInt_Key); @@ -140,15 +143,19 @@ Result<> WriteFeatureDataCSVFilter::executeImpl(DataStructure& dataStructure, co } } - std::ofstream fout(pOutputFilePath.string(), std::ofstream::out | std::ios_base::binary); // test name resolution and create file - if(!fout.is_open()) + // Scope file writer in code block to get around file lock on windows (enforce destructor order) { - return MakeErrorResult(-64640, fmt::format("Error opening path {}", pOutputFilePath.string())); - } + std::ofstream fout(pOutputFilePath.string(), std::ofstream::out | std::ios_base::binary); // test name resolution and create file + if(!fout.is_open()) + { + return MakeErrorResult(-64640, fmt::format("Error opening path {}", pOutputFilePath.string())); + } - // call ostream function - OStreamUtilities::PrintDataSetsToSingleFile(fout, arrayPaths, dataStructure, messageHandler, shouldCancel, delimiter, true, true, false, "Feature_ID", neighborPaths, pWriteNumFeaturesLineValue); + // call ostream function + OStreamUtilities::PrintDataSetsToSingleFile(fout, arrayPaths, dataStructure, messageHandler, shouldCancel, delimiter, true, true, false, "Feature_ID", neighborPaths, pWriteNumFeaturesLineValue); + } + atomicFile.commit(); return {}; } diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteLosAlamosFFTFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteLosAlamosFFTFilter.cpp index 311b576f93..10736f372d 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteLosAlamosFFTFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteLosAlamosFFTFilter.cpp @@ -2,6 +2,7 @@ #include "ComplexCore/Filters/Algorithms/WriteLosAlamosFFT.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/Filter/Actions/EmptyAction.hpp" #include "complex/Parameters/ArraySelectionParameter.hpp" @@ -107,15 +108,19 @@ IFilter::PreflightResult WriteLosAlamosFFTFilter::preflightImpl(const DataStruct Result<> WriteLosAlamosFFTFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { + AtomicFile atomicFile(filterArgs.value(k_OutputFile_Key), true); + WriteLosAlamosFFTInputValues inputValues; - inputValues.OutputFile = filterArgs.value(k_OutputFile_Key); + inputValues.OutputFile = atomicFile.tempFilePath(); inputValues.FeatureIdsArrayPath = filterArgs.value(k_FeatureIdsArrayPath_Key); inputValues.CellEulerAnglesArrayPath = filterArgs.value(k_CellEulerAnglesArrayPath_Key); inputValues.CellPhasesArrayPath = filterArgs.value(k_CellPhasesArrayPath_Key); inputValues.ImageGeomPath = filterArgs.value(k_ImageGeomPath); - return WriteLosAlamosFFT(dataStructure, messageHandler, shouldCancel, &inputValues)(); + auto result = WriteLosAlamosFFT(dataStructure, messageHandler, shouldCancel, &inputValues)(); + atomicFile.setAutoCommit(result.valid()); + return result; } namespace diff --git a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteVtkRectilinearGridFilter.cpp b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteVtkRectilinearGridFilter.cpp index 0390b042eb..deea8b06ff 100644 --- a/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteVtkRectilinearGridFilter.cpp +++ b/src/Plugins/ComplexCore/src/ComplexCore/Filters/WriteVtkRectilinearGridFilter.cpp @@ -2,6 +2,7 @@ #include "ComplexCore/Filters/Algorithms/WriteVtkRectilinearGrid.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/DataStructure/Geometry/ImageGeom.hpp" #include "complex/Parameters/BoolParameter.hpp" @@ -112,14 +113,24 @@ IFilter::PreflightResult WriteVtkRectilinearGridFilter::preflightImpl(const Data Result<> WriteVtkRectilinearGridFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { + AtomicFile atomicFile(filterArgs.value(k_OutputFile_Key), false); + + auto dirResult = atomicFile.getResult(); + if(dirResult.invalid()) + { + return dirResult; + } + WriteVtkRectilinearGridInputValues inputValues; - inputValues.OutputFile = filterArgs.value(k_OutputFile_Key); + inputValues.OutputFile = atomicFile.tempFilePath(); inputValues.WriteBinaryFile = filterArgs.value(k_WriteBinaryFile_Key); inputValues.ImageGeometryPath = filterArgs.value(k_ImageGeometryPath_Key); inputValues.SelectedDataArrayPaths = filterArgs.value(k_SelectedDataArrayPaths_Key); - return WriteVtkRectilinearGrid(dataStructure, messageHandler, shouldCancel, &inputValues)(); + auto result = WriteVtkRectilinearGrid(dataStructure, messageHandler, shouldCancel, &inputValues)(); + atomicFile.setAutoCommit(result.valid()); + return result; } namespace diff --git a/src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Filters/ITKImageWriter.cpp b/src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Filters/ITKImageWriter.cpp index 7121d7c99d..518ae53a59 100644 --- a/src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Filters/ITKImageWriter.cpp +++ b/src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Filters/ITKImageWriter.cpp @@ -1,5 +1,6 @@ #include "ITKImageWriter.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/DataStructure/DataStore.hpp" #include "complex/DataStructure/Geometry/ImageGeom.hpp" @@ -47,25 +48,39 @@ bool Is2DFormat(const fs::path& fileName) } template -void WriteAsOneFile(itk::Image& image, const fs::path& filePath /*, const IFilter::MessageHandler& messanger*/) +Result<> WriteAsOneFile(itk::Image& image, const fs::path& filePath /*, const IFilter::MessageHandler& messanger*/) { - using ImageType = itk::Image; - using FileWriterType = itk::ImageFileWriter; - auto writer = FileWriterType::New(); + fs::path tempPath(fmt::format("{}/{}", fs::temp_directory_path().string(), filePath.filename().string())); + try + { + using ImageType = itk::Image; + using FileWriterType = itk::ImageFileWriter; + auto writer = FileWriterType::New(); - // messanger(fmt::format("Saving {}", fileName)); + // messanger(fmt::format("Saving {}", fileName)); + + writer->SetInput(&image); + writer->SetFileName(tempPath.string()); + writer->UseCompressionOn(); + writer->Update(); + } catch(const itk::ExceptionObject& err) + { + // Handle errors from the writer deleting the directory + fs::remove(tempPath); + + return MakeErrorResult(-21011, fmt::format("ITK exception was thrown while writing output file: {}", err.GetDescription())); + } - writer->SetInput(&image); - writer->SetFileName(filePath.string()); - writer->UseCompressionOn(); - writer->Update(); + fs::rename(tempPath, filePath); + return {}; } template -void WriteAs2DStack(itk::Image& image, uint32 z_size, const fs::path& filePath, uint64 indexOffset) +Result<> WriteAs2DStack(itk::Image& image, uint32 z_size, const fs::path& filePath, uint64 indexOffset) { + // write to temp directory auto namesGenerator = itk::NumericSeriesFileNames::New(); - fs::path parentPath = filePath.parent_path(); + fs::path parentPath = fs::temp_directory_path(); fs::path fileName = filePath.stem(); fs::path extension = filePath.extension(); std::string format = fmt::format("{}/{}%03d{}", parentPath.string(), fileName.string(), extension.string()); @@ -73,14 +88,37 @@ void WriteAs2DStack(itk::Image& image, uint32 z_size, const namesGenerator->SetIncrementIndex(1); namesGenerator->SetStartIndex(indexOffset); namesGenerator->SetEndIndex(z_size - 1); - using InputImageType = itk::Image; - using OutputImageType = itk::Image; - using SeriesWriterType = itk::ImageSeriesWriter; - auto writer = SeriesWriterType::New(); - writer->SetInput(&image); - writer->SetFileNames(namesGenerator->GetFileNames()); - writer->UseCompressionOn(); - writer->Update(); + + // generate all the files in that new directory + try + { + using InputImageType = itk::Image; + using OutputImageType = itk::Image; + using SeriesWriterType = itk::ImageSeriesWriter; + auto writer = SeriesWriterType::New(); + writer->SetInput(&image); + writer->SetFileNames(namesGenerator->GetFileNames()); + writer->UseCompressionOn(); + writer->Update(); + } catch(const itk::ExceptionObject& err) + { + // Handle errors from the writer deleting the directory + for(const auto& name : namesGenerator->GetFileNames()) + { + fs::remove(name); + } + + return MakeErrorResult(-21011, fmt::format("ITK exception was thrown while writing output file: {}", err.GetDescription())); + } + + // Move all the files from the new directory to the users actual directory + for(const auto& name : namesGenerator->GetFileNames()) + { + fs::path tempFile(name); + fs::rename(tempFile, {fmt::format("{}/{}", filePath.parent_path().string(), tempFile.filename().string())}); + } + + return {}; } template @@ -90,28 +128,21 @@ Result<> WriteImage(IDataStore& dataStore, const ITK::ImageGeomData& imageGeom, auto& typedDataStore = dynamic_cast>&>(dataStore); - try + typename itk::Image::Pointer image = ITK::WrapDataStoreInImage(typedDataStore, imageGeom); + if(Is2DFormat(filePath) && Dimensions == 3) { - typename itk::Image::Pointer image = ITK::WrapDataStoreInImage(typedDataStore, imageGeom); - if(Is2DFormat(filePath) && Dimensions == 3) - { - typename ImageType::SizeType size = image->GetLargestPossibleRegion().GetSize(); - if(size[2] < 2) - { - return MakeErrorResult(-21012, "Image is 2D, not 3D."); - } - WriteAs2DStack(*image, size[2], filePath, indexOffset); - } - else + typename ImageType::SizeType size = image->GetLargestPossibleRegion().GetSize(); + if(size[2] < 2) { - WriteAsOneFile(*image, filePath); + return MakeErrorResult(-21012, "Image is 2D, not 3D."); } - } catch(const itk::ExceptionObject& err) + + return WriteAs2DStack(*image, size[2], filePath, indexOffset); + } + else { - return MakeErrorResult(-21011, fmt::format("ITK exception was thrown while writing output file: {}", err.GetDescription())); + return WriteAsOneFile(*image, filePath); } - - return {}; } template diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ConvertHexGridToSquareGrid.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ConvertHexGridToSquareGrid.cpp index 8a525dd042..2c1bb52d7c 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ConvertHexGridToSquareGrid.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ConvertHexGridToSquareGrid.cpp @@ -1,5 +1,6 @@ #include "ConvertHexGridToSquareGrid.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataArray.hpp" #include "complex/DataStructure/DataGroup.hpp" #include "complex/Parameters/ChoicesParameter.hpp" @@ -30,7 +31,16 @@ class Converter , m_SqrYStep(spacingXY.at(1)) { } - ~Converter() noexcept = default; + ~Converter() noexcept + { + if(m_Valid) + { + for(const auto& atomicFile : m_AtomicFiles) + { + atomicFile->commit(); + } + } + } Converter(const Converter&) = delete; // Copy Constructor Default Implemented Converter(Converter&&) = delete; // Move Constructor Not Implemented @@ -49,40 +59,58 @@ class Converter int32 err = reader.readFile(); if(err < 0 && err != -600) { + m_Valid = false; return MakeErrorResult(reader.getErrorCode(), reader.getErrorMessage()); } if(err == -600) { + m_Valid = false; return MakeWarningVoidResult(reader.getErrorCode(), reader.getErrorMessage()); } if(reader.getGrid().find(EbsdLib::Ang::SquareGrid) == 0) { + m_Valid = false; return MakeErrorResult(-55000, fmt::format("Ang file is already a square grid: {}", inputPath.string())); } std::string origHeader = reader.getOriginalHeader(); if(origHeader.empty()) { + m_Valid = false; return MakeErrorResult(-55001, fmt::format("Header for input hex grid file was empty: {}", inputPath.string())); } std::istringstream headerStream(origHeader, std::ios_base::in | std::ios_base::binary); - fs::path outPath = fs::absolute(m_OutputPath) / (m_FilePrefix + inputPath.filename().string()); + m_AtomicFiles.emplace_back(std::make_unique((fs::absolute(m_OutputPath) / (m_FilePrefix + inputPath.filename().string())), false)); + fs::path outPath = m_AtomicFiles[m_Index]->tempFilePath(); + + // Ensure the output path exists by creating it if necessary + if(!fs::exists(m_OutputPath)) + { + auto result = m_AtomicFiles[m_Index]->getResult(); + if(result.invalid()) + { + m_Valid = false; + return result; + } + } if(outPath == inputPath) { + m_Valid = false; return MakeErrorResult(-201, "New ang file is the same as the old ang file. Overwriting is NOT allowed"); } std::ofstream outFile(outPath, std::ios_base::out | std::ios_base::binary); - // Ensure the output path exists by creating it if necessary - if(!fs::exists(outPath.parent_path())) + if(!fs::exists(outPath)) { - return MakeErrorResult(-77750, fmt::format("The parent path was not created and does not exist: {}", outPath.parent_path().string())); + m_Valid = false; + return MakeErrorResult(-77750, fmt::format("The parent path was not created and does not exist: {}", outPath.string())); } if(!outFile.is_open()) { + m_Valid = false; return MakeErrorResult(-200, fmt::format("Ang square output file could not be opened for writing: {}", outPath.string())); } @@ -132,6 +160,7 @@ class Converter { if(m_ShouldCancel) { + m_Valid = false; return {}; } for(int32 i = 0; i < m_SqrNumCols; i++) @@ -188,6 +217,8 @@ class Converter << "\n"; } } + + m_Index++; } return {}; @@ -197,6 +228,9 @@ class Converter const std::atomic_bool& m_ShouldCancel; const fs::path& m_OutputPath; const std::string& m_FilePrefix; + std::vector> m_AtomicFiles = {}; + usize m_Index = 0; + bool m_Valid = true; const float32 m_SqrXStep; const float32 m_SqrYStep; @@ -341,13 +375,20 @@ Result<> ConvertHexGridToSquareGrid::operator()() int32 progress; int64 z = m_InputValues->InputFileListInfo.startIndex; + usize index = 0; auto result = Result<>{}; ::Converter converter(getCancel(), m_InputValues->OutputPath, m_InputValues->OutputFilePrefix, m_InputValues->XYSpacing); for(const auto& filepath : fileList) { m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Now Processing: {}", filepath)); - result = MergeResults(converter(fs::path(filepath)), result); + result = MergeResults(converter(filepath), result); + if(result.invalid()) + { + return result; + } + + index++; { z++; diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/EbsdToH5Ebsd.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/EbsdToH5Ebsd.cpp index 2ac6cfc8de..5aefbb3951 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/EbsdToH5Ebsd.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/EbsdToH5Ebsd.cpp @@ -1,11 +1,11 @@ #include "EbsdToH5Ebsd.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataArray.hpp" #include "complex/DataStructure/DataGroup.hpp" #include "complex/Utilities/FilterUtilities.hpp" #include "EbsdLib/Core/EbsdLibConstants.h" -#include "EbsdLib/Core/EbsdMacros.h" #include "EbsdLib/IO/HKL/CtfFields.h" #include "EbsdLib/IO/HKL/H5CtfImporter.h" #include "EbsdLib/IO/TSL/AngFields.h" @@ -51,265 +51,278 @@ Result<> EbsdToH5Ebsd::operator()() } } - // Make sure any directory path is also available as the user may have just typed - // in a path without actually creating the full path - Result<> createDirectoriesResult = complex::CreateOutputDirectories(absPath.parent_path()); - if(createDirectoriesResult.invalid()) - { - return createDirectoriesResult; - } - - // Create output H5Ebsd File - hid_t fileId = H5Support::H5Utilities::createFile(absPath.string()); - if(fileId < 0) - { - return MakeErrorResult(-99501, fmt::format("The output HDF5 file could not be created. Check permissions or if the file is in use by another program")); - } - - H5Support::H5ScopedFileSentinel sentinel(fileId, true); - - herr_t err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::ZResolution, m_InputValues->ZSpacing); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Z Spacing Scalar to the HDF5 File"); - return MakeErrorResult(-99502, ss); - } - - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::StackingOrder, static_cast(m_InputValues->StackingOrder)); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Stacking Order Scalar to the HDF5 File"); - return MakeErrorResult(-99503, ss); - } - - err = H5Support::H5Lite::writeStringAttribute(fileId, EbsdLib::H5Ebsd::StackingOrder, "Name", EbsdToH5EbsdInputConstants::k_StackingChoices[m_InputValues->StackingOrder]); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Stacking Order Name Attribute to the HDF5 File"); - return MakeErrorResult(-99504, ss); - } + AtomicFile atomicFile(absPath.string(), false); - auto eulerTransformation = EbsdToH5EbsdInputConstants::k_NoEulerTransform; - auto sampleTransformation = EbsdToH5EbsdInputConstants::k_NoSampleTransform; - switch(m_InputValues->ReferenceFrame) + auto dirResult = atomicFile.getResult(); + if(dirResult.invalid()) { - case EbsdToH5EbsdInputConstants::k_Edax: - eulerTransformation = EbsdToH5EbsdInputConstants::k_EdaxEulerTransform; - sampleTransformation = EbsdToH5EbsdInputConstants::k_EdaxSampleTransform; - break; - case EbsdToH5EbsdInputConstants::k_Oxford: - eulerTransformation = EbsdToH5EbsdInputConstants::k_OxfordEulerTransform; - sampleTransformation = EbsdToH5EbsdInputConstants::k_OxfordSampleTransform; - break; - case EbsdToH5EbsdInputConstants::k_Hedm: - eulerTransformation = EbsdToH5EbsdInputConstants::k_HedmEulerTransform; - sampleTransformation = EbsdToH5EbsdInputConstants::k_HedmSampleTransform; - break; - default: - break; + return dirResult; } - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::SampleTransformationAngle, sampleTransformation[EbsdToH5EbsdInputConstants::k_AngleIndex]); - if(err < 0) + // Scope file writer in code block to get around file lock on windows (enforce destructor order) { - std::string ss = fmt::format("Could not write the Sample Transformation Angle to the HDF5 File"); - return MakeErrorResult(-99505, ss); - } - - int32_t rank = 1; - hsize_t dims[3] = {3, 0, 0}; - err = H5Support::H5Lite::writePointerDataset(fileId, EbsdLib::H5Ebsd::SampleTransformationAxis, rank, dims, sampleTransformation.data()); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Sample Transformation Axis to the HDF5 File"); - return MakeErrorResult(-99506, ss); - } - - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::EulerTransformationAngle, eulerTransformation[EbsdToH5EbsdInputConstants::k_AngleIndex]); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Euler Transformation Angle to the HDF5 File"); - return MakeErrorResult(-99507, ss); - } + // Create output H5Ebsd File + hid_t fileId = H5Support::H5Utilities::createFile(absPath.string()); + if(fileId < 0) + { + return MakeErrorResult(-99501, fmt::format("The output HDF5 file could not be created. Check permissions or if the file is in use by another program")); + } - err = H5Support::H5Lite::writePointerDataset(fileId, EbsdLib::H5Ebsd::EulerTransformationAxis, rank, dims, eulerTransformation.data()); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Euler Transformation Axis to the HDF5 File"); - return MakeErrorResult(-99508, ss); - } + // Use a file sentinel to ensure the file is closed before coming out of this scoped block of code + auto fileSentinel = H5Support::H5ScopedFileSentinel(fileId, true); - // Now generate all the file names the user is asking for and populate the table - std::vector fileList = m_InputValues->InputFileListInfo.generate(); - if(fileList.empty()) - { - return MakeErrorResult(-99509, fmt::format("Generated File List was empty. Parent path to files is '{}'", m_InputValues->InputFileListInfo.inputPath)); - } - EbsdImporter::Pointer fileImporter; + fileId = H5Support::H5Utilities::createFile(atomicFile.tempFilePath().string()); + if(fileId < 0) + { + return MakeErrorResult(-99501, fmt::format("The output HDF5 file could not be created. Check permissions or if the file is in use by another program")); + } - int32 zStartIndex = m_InputValues->InputFileListInfo.startIndex; + H5Support::H5ScopedFileSentinel sentinel(fileId, true); - // Write the Manufacturer of the OIM file here - // This list will grow to be the number of EBSD file formats we support - auto firstFilePath = fs::path(fileList[0]); - std::string ext = firstFilePath.extension().string(); - ext.erase(0, 1); // Remove the '.' from the string - if(ext == EbsdLib::Ang::FileExt) - { - err = H5Support::H5Lite::writeStringDataset(fileId, EbsdLib::H5Ebsd::Manufacturer, EbsdLib::Ang::Manufacturer); + herr_t err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::ZResolution, m_InputValues->ZSpacing); if(err < 0) { + std::string ss = fmt::format("Could not write the Z Spacing Scalar to the HDF5 File"); + return MakeErrorResult(-99502, ss); + } - std::string ss = fmt::format("Could not write the Manufacturer Data to the HDF5 File"); - return MakeErrorResult(-99509, ss); + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::StackingOrder, static_cast(m_InputValues->StackingOrder)); + if(err < 0) + { + std::string ss = fmt::format("Could not write the Stacking Order Scalar to the HDF5 File"); + return MakeErrorResult(-99503, ss); } - fileImporter = H5AngImporter::New(); - } - else if(ext == EbsdLib::Ctf::FileExt) - { - err = H5Support::H5Lite::writeStringDataset(fileId, EbsdLib::H5Ebsd::Manufacturer, EbsdLib::Ctf::Manufacturer); + + err = H5Support::H5Lite::writeStringAttribute(fileId, EbsdLib::H5Ebsd::StackingOrder, "Name", EbsdToH5EbsdInputConstants::k_StackingChoices[m_InputValues->StackingOrder]); if(err < 0) { - std::string ss = fmt::format("Could not write the Manufacturer Data to the HDF5 File"); - return MakeErrorResult(-99510, ss); + std::string ss = fmt::format("Could not write the Stacking Order Name Attribute to the HDF5 File"); + return MakeErrorResult(-99504, ss); } - fileImporter = H5CtfImporter::New(); - CtfReader ctfReader; - ctfReader.setFileName(fileList.front()); - err = ctfReader.readHeaderOnly(); - if(ctfReader.getZCells() > 1 && fileList.size() == 1) + + auto eulerTransformation = EbsdToH5EbsdInputConstants::k_NoEulerTransform; + auto sampleTransformation = EbsdToH5EbsdInputConstants::k_NoSampleTransform; + switch(m_InputValues->ReferenceFrame) { - zStartIndex = 0; + case EbsdToH5EbsdInputConstants::k_Edax: + eulerTransformation = EbsdToH5EbsdInputConstants::k_EdaxEulerTransform; + sampleTransformation = EbsdToH5EbsdInputConstants::k_EdaxSampleTransform; + break; + case EbsdToH5EbsdInputConstants::k_Oxford: + eulerTransformation = EbsdToH5EbsdInputConstants::k_OxfordEulerTransform; + sampleTransformation = EbsdToH5EbsdInputConstants::k_OxfordSampleTransform; + break; + case EbsdToH5EbsdInputConstants::k_Hedm: + eulerTransformation = EbsdToH5EbsdInputConstants::k_HedmEulerTransform; + sampleTransformation = EbsdToH5EbsdInputConstants::k_HedmSampleTransform; + break; + default: + break; } + + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::SampleTransformationAngle, sampleTransformation[EbsdToH5EbsdInputConstants::k_AngleIndex]); if(err < 0) { - std::string ss = fmt::format("Error reading CTF file header"); - return MakeErrorResult(-99511, ss); + std::string ss = fmt::format("Could not write the Sample Transformation Angle to the HDF5 File"); + return MakeErrorResult(-99505, ss); } - } - else - { - std::string ss = fmt::format("The file extension was not detected correctly"); - return MakeErrorResult(-99512, ss); - } - std::vector indices; - // Loop on Each EBSD File - int64_t z = zStartIndex; - int64_t xDim = 0, yDim = 0; - float xRes = 0.0f, yRes = 0.0f; - /* There is a frailness about the z index and the file list. The programmer - * using this code MUST ensure that the list of files that is sent into this - * class is in the appropriate order to match up with the z index (slice index) - * otherwise the import will have subtle errors. The programmer is urged NOT to - * simply gather a list from the file system as those lists are sorted in such - * a way that if the number of digits appearing in the filename are NOT the same - * then the list will be wrong, ie, this example: - * - * slice_1.ang - * slice_2.ang - * .... - * slice_10.ang - * - * Most, if not ALL C++ libraries when asked for that list will return the list - * sorted like the following: - * - * slice_1.ang - * slice_10.ang - * slice_2.ang - * - * which is going to cause problems because the data is going to be placed - * into the HDF5 file at the wrong index. YOU HAVE BEEN WARNED. - */ - int64_t biggestXDim = 0; - int64_t biggestYDim = 0; - int32_t totalSlicesImported = 0; - for(const auto& ebsdFName : fileList) - { - m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Converting File: '{}'", ebsdFName)); + int32_t rank = 1; + hsize_t dims[3] = {3, 0, 0}; + err = H5Support::H5Lite::writePointerDataset(fileId, EbsdLib::H5Ebsd::SampleTransformationAxis, rank, dims, sampleTransformation.data()); + if(err < 0) + { + std::string ss = fmt::format("Could not write the Sample Transformation Axis to the HDF5 File"); + return MakeErrorResult(-99506, ss); + } - err = fileImporter->importFile(fileId, z, ebsdFName); + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::EulerTransformationAngle, eulerTransformation[EbsdToH5EbsdInputConstants::k_AngleIndex]); if(err < 0) { - return MakeErrorResult(err, std::string(fileImporter->getPipelineMessage())); + std::string ss = fmt::format("Could not write the Euler Transformation Angle to the HDF5 File"); + return MakeErrorResult(-99507, ss); } - totalSlicesImported = totalSlicesImported + fileImporter->numberOfSlicesImported(); - fileImporter->getDims(xDim, yDim); - fileImporter->getSpacing(xRes, yRes); - if(xDim > biggestXDim) + err = H5Support::H5Lite::writePointerDataset(fileId, EbsdLib::H5Ebsd::EulerTransformationAxis, rank, dims, eulerTransformation.data()); + if(err < 0) { - biggestXDim = xDim; + std::string ss = fmt::format("Could not write the Euler Transformation Axis to the HDF5 File"); + return MakeErrorResult(-99508, ss); } - if(yDim > biggestYDim) + + // Now generate all the file names the user is asking for and populate the table + std::vector fileList = m_InputValues->InputFileListInfo.generate(); + if(fileList.empty()) { - biggestYDim = yDim; + return MakeErrorResult(-99509, fmt::format("Generated File List was empty. Parent path to files is '{}'", m_InputValues->InputFileListInfo.inputPath)); } + EbsdImporter::Pointer fileImporter; + + int32 zStartIndex = m_InputValues->InputFileListInfo.startIndex; - indices.push_back(static_cast(z)); - ++z; - if(getCancel()) + // Write the Manufacturer of the OIM file here + // This list will grow to be the number of EBSD file formats we support + auto firstFilePath = fs::path(fileList[0]); + std::string ext = firstFilePath.extension().string(); + ext.erase(0, 1); // Remove the '.' from the string + if(ext == EbsdLib::Ang::FileExt) { - return {}; + err = H5Support::H5Lite::writeStringDataset(fileId, EbsdLib::H5Ebsd::Manufacturer, EbsdLib::Ang::Manufacturer); + if(err < 0) + { + + std::string ss = fmt::format("Could not write the Manufacturer Data to the HDF5 File"); + return MakeErrorResult(-99509, ss); + } + fileImporter = H5AngImporter::New(); + } + else if(ext == EbsdLib::Ctf::FileExt) + { + err = H5Support::H5Lite::writeStringDataset(fileId, EbsdLib::H5Ebsd::Manufacturer, EbsdLib::Ctf::Manufacturer); + if(err < 0) + { + std::string ss = fmt::format("Could not write the Manufacturer Data to the HDF5 File"); + return MakeErrorResult(-99510, ss); + } + fileImporter = H5CtfImporter::New(); + CtfReader ctfReader; + ctfReader.setFileName(fileList.front()); + err = ctfReader.readHeaderOnly(); + if(ctfReader.getZCells() > 1 && fileList.size() == 1) + { + zStartIndex = 0; + } + if(err < 0) + { + std::string ss = fmt::format("Error reading CTF file header"); + return MakeErrorResult(-99511, ss); + } + } + else + { + std::string ss = fmt::format("The file extension was not detected correctly"); + return MakeErrorResult(-99512, ss); } - } - // Write Z index start, Z index end and Z Spacing to the HDF5 file - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::ZStartIndex, zStartIndex); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Z Start Index Scalar to the HDF5 File"); - return MakeErrorResult(-99514, ss); - } + std::vector indices; + // Loop on Each EBSD File + int64_t z = zStartIndex; + int64_t xDim = 0, yDim = 0; + float xRes = 0.0f, yRes = 0.0f; + /* There is a frailness about the z index and the file list. The programmer + * using this code MUST ensure that the list of files that is sent into this + * class is in the appropriate order to match up with the z index (slice index) + * otherwise the import will have subtle errors. The programmer is urged NOT to + * simply gather a list from the file system as those lists are sorted in such + * a way that if the number of digits appearing in the filename are NOT the same + * then the list will be wrong, ie, this example: + * + * slice_1.ang + * slice_2.ang + * .... + * slice_10.ang + * + * Most, if not ALL C++ libraries when asked for that list will return the list + * sorted like the following: + * + * slice_1.ang + * slice_10.ang + * slice_2.ang + * + * which is going to cause problems because the data is going to be placed + * into the HDF5 file at the wrong index. YOU HAVE BEEN WARNED. + */ + int64_t biggestXDim = 0; + int64_t biggestYDim = 0; + int32_t totalSlicesImported = 0; + for(const auto& ebsdFName : fileList) + { + m_MessageHandler(IFilter::Message::Type::Info, fmt::format("Converting File: '{}'", ebsdFName)); + + err = fileImporter->importFile(fileId, z, ebsdFName); + if(err < 0) + { + return MakeErrorResult(err, std::string(fileImporter->getPipelineMessage())); + } + totalSlicesImported = totalSlicesImported + fileImporter->numberOfSlicesImported(); + + fileImporter->getDims(xDim, yDim); + fileImporter->getSpacing(xRes, yRes); + if(xDim > biggestXDim) + { + biggestXDim = xDim; + } + if(yDim > biggestYDim) + { + biggestYDim = yDim; + } + + indices.push_back(static_cast(z)); + ++z; + if(getCancel()) + { + return {}; + } + } - auto zEndIndex = zStartIndex + totalSlicesImported - 1; - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::ZEndIndex, zEndIndex); - if(err < 0) - { - std::string ss = fmt::format("Could not write the Z End Index Scalar to the HDF5 File"); - return MakeErrorResult(-99515, ss); - } + // Write Z index start, Z index end and Z Spacing to the HDF5 file + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::ZStartIndex, zStartIndex); + if(err < 0) + { + std::string ss = fmt::format("Could not write the Z Start Index Scalar to the HDF5 File"); + return MakeErrorResult(-99514, ss); + } - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::XPoints, biggestXDim); - if(err < 0) - { - std::string ss = fmt::format("Could not write the XPoints Scalar to HDF5 file"); - return MakeErrorResult(-99516, ss); - } + auto zEndIndex = zStartIndex + totalSlicesImported - 1; + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::ZEndIndex, zEndIndex); + if(err < 0) + { + std::string ss = fmt::format("Could not write the Z End Index Scalar to the HDF5 File"); + return MakeErrorResult(-99515, ss); + } - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::YPoints, biggestYDim); - if(err < 0) - { - std::string ss = fmt::format("Could not write the YPoints Scalar to HDF5 file"); - return MakeErrorResult(-99517, ss); - } + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::XPoints, biggestXDim); + if(err < 0) + { + std::string ss = fmt::format("Could not write the XPoints Scalar to HDF5 file"); + return MakeErrorResult(-99516, ss); + } - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::XResolution, xRes); - if(err < 0) - { - std::string ss = fmt::format("Could not write the XResolution Scalar to HDF5 file"); - return MakeErrorResult(-99518, ss); - } + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::YPoints, biggestYDim); + if(err < 0) + { + std::string ss = fmt::format("Could not write the YPoints Scalar to HDF5 file"); + return MakeErrorResult(-99517, ss); + } - err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::YResolution, yRes); - if(err < 0) - { - std::string ss = fmt::format("Could not write the YResolution Scalar to HDF5 file"); - return MakeErrorResult(-99519, ss); - } + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::XResolution, xRes); + if(err < 0) + { + std::string ss = fmt::format("Could not write the XResolution Scalar to HDF5 file"); + return MakeErrorResult(-99518, ss); + } - if(!getCancel()) - { - // Write an Index data set which contains all the z index values which - // should help speed up the reading side of this file - std::vector dimsL = {static_cast(indices.size())}; - err = H5Support::H5Lite::writeVectorDataset(fileId, std::string(EbsdLib::H5Ebsd::Index), dimsL, indices); + err = H5Support::H5Lite::writeScalarDataset(fileId, EbsdLib::H5Ebsd::YResolution, yRes); if(err < 0) { - std::string ss = fmt::format("Error writing index dataset to H5Ebsd"); - return MakeErrorResult(-99520, ss); + std::string ss = fmt::format("Could not write the YResolution Scalar to HDF5 file"); + return MakeErrorResult(-99519, ss); + } + + if(!getCancel()) + { + // Write an Index data set which contains all the z index values which + // should help speed up the reading side of this file + std::vector dimsL = {static_cast(indices.size())}; + err = H5Support::H5Lite::writeVectorDataset(fileId, std::string(EbsdLib::H5Ebsd::Index), dimsL, indices); + if(err < 0) + { + std::string ss = fmt::format("Error writing index dataset to H5Ebsd"); + return MakeErrorResult(-99520, ss); + } } } + atomicFile.commit(); return {}; } diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDGMTFileFilter.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDGMTFileFilter.cpp index c586b368f6..c3e545ff78 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDGMTFileFilter.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDGMTFileFilter.cpp @@ -1,5 +1,6 @@ #include "WriteGBCDGMTFileFilter.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataArray.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/Parameters/ArraySelectionParameter.hpp" @@ -123,14 +124,19 @@ IFilter::PreflightResult WriteGBCDGMTFileFilter::preflightImpl(const DataStructu Result<> WriteGBCDGMTFileFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { + AtomicFile atomicFile(filterArgs.value(k_OutputFile_Key), true); + WriteGBCDGMTFileInputValues inputValues; + inputValues.PhaseOfInterest = filterArgs.value(k_PhaseOfInterest_Key); inputValues.MisorientationRotation = filterArgs.value(k_MisorientationRotation_Key); - inputValues.OutputFile = filterArgs.value(k_OutputFile_Key); + inputValues.OutputFile = atomicFile.tempFilePath(); inputValues.GBCDArrayPath = filterArgs.value(k_GBCDArrayPath_Key); inputValues.CrystalStructuresArrayPath = filterArgs.value(k_CrystalStructuresArrayPath_Key); - return WriteGBCDGMTFile(dataStructure, messageHandler, shouldCancel, &inputValues)(); + auto result = WriteGBCDGMTFile(dataStructure, messageHandler, shouldCancel, &inputValues)(); + atomicFile.setAutoCommit(result.valid()); + return result; } namespace diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDTriangleDataFilter.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDTriangleDataFilter.cpp index 9af6a84b79..53633918a9 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDTriangleDataFilter.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteGBCDTriangleDataFilter.cpp @@ -2,6 +2,7 @@ #include "OrientationAnalysis/Filters/Algorithms/WriteGBCDTriangleData.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/Filter/Actions/EmptyAction.hpp" #include "complex/Parameters/ArraySelectionParameter.hpp" @@ -109,15 +110,19 @@ IFilter::PreflightResult WriteGBCDTriangleDataFilter::preflightImpl(const DataSt Result<> WriteGBCDTriangleDataFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { + AtomicFile atomicFile(filterArgs.value(k_OutputFile_Key), true); + WriteGBCDTriangleDataInputValues inputValues; - inputValues.OutputFile = filterArgs.value(k_OutputFile_Key); + inputValues.OutputFile = atomicFile.tempFilePath(); inputValues.SurfaceMeshFaceLabelsArrayPath = filterArgs.value(k_SurfaceMeshFaceLabelsArrayPath_Key); inputValues.SurfaceMeshFaceNormalsArrayPath = filterArgs.value(k_SurfaceMeshFaceNormalsArrayPath_Key); inputValues.SurfaceMeshFaceAreasArrayPath = filterArgs.value(k_SurfaceMeshFaceAreasArrayPath_Key); inputValues.FeatureEulerAnglesArrayPath = filterArgs.value(k_FeatureEulerAnglesArrayPath_Key); - return WriteGBCDTriangleData(dataStructure, messageHandler, shouldCancel, &inputValues)(); + auto result = WriteGBCDTriangleData(dataStructure, messageHandler, shouldCancel, &inputValues)(); + atomicFile.setAutoCommit(result.valid()); + return result; } namespace diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteINLFileFilter.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteINLFileFilter.cpp index 15558b3152..4b4dd11939 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteINLFileFilter.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/WriteINLFileFilter.cpp @@ -2,6 +2,7 @@ #include "OrientationAnalysis/Filters/Algorithms/WriteINLFile.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/DataStructure/DataPath.hpp" #include "complex/Filter/Actions/EmptyAction.hpp" #include "complex/Parameters/ArraySelectionParameter.hpp" @@ -110,9 +111,11 @@ IFilter::PreflightResult WriteINLFileFilter::preflightImpl(const DataStructure& Result<> WriteINLFileFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { + AtomicFile atomicFile(filterArgs.value(k_OutputFile_Key), true); + WriteINLFileInputValues inputValues; - inputValues.OutputFile = filterArgs.value(k_OutputFile_Key); + inputValues.OutputFile = atomicFile.tempFilePath(); inputValues.ImageGeomPath = filterArgs.value(k_ImageGeomPath_Key); inputValues.FeatureIdsArrayPath = filterArgs.value(k_FeatureIdsArrayPath_Key); inputValues.CellPhasesArrayPath = filterArgs.value(k_CellPhasesArrayPath_Key); @@ -121,7 +124,9 @@ Result<> WriteINLFileFilter::executeImpl(DataStructure& dataStructure, const Arg inputValues.MaterialNameArrayPath = filterArgs.value(k_MaterialNameArrayPath_Key); inputValues.NumFeaturesArrayPath = filterArgs.value(k_NumFeaturesArrayPath_Key); - return WriteINLFile(dataStructure, messageHandler, shouldCancel, &inputValues)(); + auto result = WriteINLFile(dataStructure, messageHandler, shouldCancel, &inputValues)(); + atomicFile.setAutoCommit(result.valid()); + return result; } namespace diff --git a/src/complex/Common/AtomicFile.cpp b/src/complex/Common/AtomicFile.cpp new file mode 100644 index 0000000000..a81207cfb1 --- /dev/null +++ b/src/complex/Common/AtomicFile.cpp @@ -0,0 +1,97 @@ +#include "AtomicFile.hpp" + +#include "complex/Utilities/FilterUtilities.hpp" + +#include + +#include + +using namespace complex; + +namespace +{ +constexpr std::array chars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', + 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; +std::string randomDirName() +{ + std::mt19937_64 gen(static_cast(std::chrono::steady_clock::now().time_since_epoch().count())); + std::uniform_int_distribution dist(0, chars.size() - 1); + + std::string randomDir = ""; + for(uint32 i = 0; i < 24; i++) + { + randomDir += chars[dist(gen)]; + } + return randomDir; +} +} // namespace + +AtomicFile::AtomicFile(const std::string& filename, bool autoCommit) +: m_FilePath(fs::path(filename)) +, m_AutoCommit(autoCommit) +{ + m_TempFilePath = fs::path(fmt::format("{}/{}/{}", m_FilePath.parent_path().string(), ::randomDirName(), m_FilePath.filename().string())); + m_Result = createOutputDirectories(); +} + +AtomicFile::AtomicFile(fs::path&& filepath, bool autoCommit) +: m_FilePath(std::move(filepath)) +, m_AutoCommit(autoCommit) +{ + m_TempFilePath = fs::path(fmt::format("{}/{}/{}", m_FilePath.parent_path().string(), ::randomDirName(), m_FilePath.filename().string())); + m_Result = createOutputDirectories(); +} + +AtomicFile::~AtomicFile() +{ + if(m_AutoCommit) + { + commit(); + } + if(fs::exists(m_TempFilePath) || fs::exists(m_TempFilePath.parent_path())) + { + removeTempFile(); + } +} + +fs::path AtomicFile::tempFilePath() const +{ + return m_TempFilePath; +} + +void AtomicFile::commit() const +{ + if(!fs::exists(m_TempFilePath)) + { + throw std::runtime_error(m_TempFilePath.string() + " does not exist"); + } + + fs::rename(m_TempFilePath, m_FilePath); +} + +void AtomicFile::setAutoCommit(bool value) +{ + m_AutoCommit = value; +} + +bool AtomicFile::getAutoCommit() const +{ + return m_AutoCommit; +} + +void AtomicFile::removeTempFile() const +{ + fs::remove_all(m_TempFilePath.parent_path()); +} + +Result<> AtomicFile::getResult() const +{ + return m_Result; +} + +Result<> AtomicFile::createOutputDirectories() +{ + // Make sure any directory path is also available as the user may have just typed + // in a path without actually creating the full path + return CreateOutputDirectories(m_TempFilePath.parent_path()); +} diff --git a/src/complex/Common/AtomicFile.hpp b/src/complex/Common/AtomicFile.hpp new file mode 100644 index 0000000000..f6d68c257e --- /dev/null +++ b/src/complex/Common/AtomicFile.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include "complex/complex_export.hpp" + +#include "complex/Common/Result.hpp" + +#include +#include + +namespace complex +{ +namespace fs = std::filesystem; + +/** + * @class AtomicFile + * @brief The AtomicFile class accepts a filepath which it stores and + * used to create a temporary file. This temporary can be written to in place + * of the original file path. Upon commit() the temporary file will be moved to + * the end location passed in originally. By enabling autoCommit the temporary file + * will be swapped upon object destruction. The temporary file will always be deleted + * upon object destruction. + */ +class COMPLEX_EXPORT AtomicFile +{ +public: + explicit AtomicFile(const std::string& filename, bool autoCommit = false); + explicit AtomicFile(fs::path&& filename, bool autoCommit = false); + + ~AtomicFile(); + + fs::path tempFilePath() const; + void commit() const; + void setAutoCommit(bool value); + bool getAutoCommit() const; + void removeTempFile() const; + Result<> getResult() const; + +private: + fs::path m_FilePath; + fs::path m_TempFilePath; + bool m_AutoCommit = false; + Result<> m_Result = {}; + + Result<> createOutputDirectories(); +}; +} // namespace complex diff --git a/src/complex/Utilities/OStreamUtilities.cpp b/src/complex/Utilities/OStreamUtilities.cpp index 2583f54510..8fa55f8e05 100644 --- a/src/complex/Utilities/OStreamUtilities.cpp +++ b/src/complex/Utilities/OStreamUtilities.cpp @@ -1,5 +1,6 @@ #include "OStreamUtilities.hpp" +#include "complex/Common/AtomicFile.hpp" #include "complex/Utilities/FilterUtilities.hpp" #include @@ -370,50 +371,57 @@ void PrintDataSetsToMultipleFiles(const std::vector& objectPaths, Data for(const auto& dataPath : objectPaths) { - auto outputFilePath = fmt::format("{}/{}{}", directoryPath, dataPath.getTargetName(), fileExtension); - mesgHandler(IFilter::Message::Type::Info, fmt::format("Writing IArray ({}) to output file {}", dataPath.getTargetName(), outputFilePath)); + AtomicFile atomicFile(fmt::format("{}/{}{}", directoryPath, dataPath.getTargetName(), fileExtension), false); - std::ofstream outStrm(outputFilePath, std::ios_base::out | std::ios_base::binary); + auto outputFilePath = atomicFile.tempFilePath().string(); + mesgHandler(IFilter::Message::Type::Info, fmt::format("Writing IArray ({}) to output file {}", dataPath.getTargetName(), outputFilePath)); - std::pair result = {0, "PrintDataSetsToMultipleFiles default failure. If you are seeing this error something bad has happened."}; - auto* dataArray = dataStructure.getDataAs(dataPath); - if(dataArray != nullptr) + // Scope file writer in code block to get around file lock on windows (enforce destructor order) { - if(exportToBinary) + std::ofstream outStrm(outputFilePath, std::ios_base::out | std::ios_base::binary); + + std::pair result = {0, "PrintDataSetsToMultipleFiles default failure. If you are seeing this error something bad has happened."}; + auto* dataArray = dataStructure.getDataAs(dataPath); + if(dataArray != nullptr) { - result = dataArray->getIDataStore()->writeBinaryFile(outputFilePath); + if(exportToBinary) + { + result = dataArray->getIDataStore()->writeBinaryFile(outputFilePath); + } + else + { + ExecuteDataFunction(PrintDataArray{}, dataArray->getDataType(), outStrm, dataArray, mesgHandler, shouldCancel, delimiter, componentsPerLine); + } } - else + auto* stringArray = dataStructure.getDataAs(dataPath); + if(stringArray != nullptr) { - ExecuteDataFunction(PrintDataArray{}, dataArray->getDataType(), outStrm, dataArray, mesgHandler, shouldCancel, delimiter, componentsPerLine); + if(exportToBinary) + { + throw std::runtime_error(fmt::format("{}({}): Function {}: Error. Cannot print a StringArray to binary: '{}'", "PrintDataSetsToMultipleFiles", __FILE__, __LINE__, dataPath.getTargetName())); + } + PrintStringArray(outStrm, *stringArray, mesgHandler, shouldCancel, delimiter, componentsPerLine); } - } - auto* stringArray = dataStructure.getDataAs(dataPath); - if(stringArray != nullptr) - { - if(exportToBinary) + auto* neighborList = dataStructure.getDataAs(dataPath); + if(neighborList != nullptr) { - throw std::runtime_error(fmt::format("{}({}): Function {}: Error. Cannot print a StringArray to binary: '{}'", "PrintDataSetsToMultipleFiles", __FILE__, __LINE__, dataPath.getTargetName())); + if(exportToBinary) + { + throw std::runtime_error( + fmt::format("{}({}): Function {}: Error. Cannot print a NeighborList to binary: '{}'", "PrintDataSetsToMultipleFiles", __FILE__, __LINE__, dataPath.getTargetName())); + } + ExecuteNeighborFunction(PrintNeighborList{}, neighborList->getDataType(), outStrm, neighborList, mesgHandler, shouldCancel, delimiter, includeIndex, includeHeaders); } - PrintStringArray(outStrm, *stringArray, mesgHandler, shouldCancel, delimiter, componentsPerLine); - } - auto* neighborList = dataStructure.getDataAs(dataPath); - if(neighborList != nullptr) - { - if(exportToBinary) + if(result.first < 0) { - throw std::runtime_error(fmt::format("{}({}): Function {}: Error. Cannot print a NeighborList to binary: '{}'", "PrintDataSetsToMultipleFiles", __FILE__, __LINE__, dataPath.getTargetName())); + mesgHandler(IFilter::Message::Type::Error, result.second); } - ExecuteNeighborFunction(PrintNeighborList{}, neighborList->getDataType(), outStrm, neighborList, mesgHandler, shouldCancel, delimiter, includeIndex, includeHeaders); - } - if(result.first < 0) - { - mesgHandler(IFilter::Message::Type::Error, result.second); } if(shouldCancel) { return; } + atomicFile.commit(); } };