From 8b74807760c5157c9c8809c627767b09b821fa1a Mon Sep 17 00:00:00 2001 From: Jessica Marquis Date: Wed, 27 Mar 2024 12:26:11 -0400 Subject: [PATCH] FILTER : ExtractPipelineToFile added (#897) This filter extracts the JSON formatted pipeline from either a DREAM3D version 6 or DREAM3D-NX version 7 .dream3d file and saves that pipeline into a file on the local drive. --- src/Plugins/SimplnxCore/CMakeLists.txt | 1 + .../docs/ExtractPipelineToFileFilter.md | 15 ++ .../ExtractPipelineToFile.d3dpipeline | 21 +++ .../Filters/ExtractPipelineToFileFilter.cpp | 151 ++++++++++++++++++ .../Filters/ExtractPipelineToFileFilter.hpp | 107 +++++++++++++ src/Plugins/SimplnxCore/test/CMakeLists.txt | 3 +- .../test/ExtractPipelineToFileTest.cpp | 139 ++++++++++++++++ src/simplnx/Pipeline/Pipeline.hpp | 1 + 8 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 src/Plugins/SimplnxCore/docs/ExtractPipelineToFileFilter.md create mode 100644 src/Plugins/SimplnxCore/pipelines/ExtractPipelineToFile.d3dpipeline create mode 100644 src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.cpp create mode 100644 src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.hpp create mode 100644 src/Plugins/SimplnxCore/test/ExtractPipelineToFileTest.cpp diff --git a/src/Plugins/SimplnxCore/CMakeLists.txt b/src/Plugins/SimplnxCore/CMakeLists.txt index 0e7e4ea6f3..9eadba5950 100644 --- a/src/Plugins/SimplnxCore/CMakeLists.txt +++ b/src/Plugins/SimplnxCore/CMakeLists.txt @@ -43,6 +43,7 @@ set(FilterList ExecuteProcessFilter ExtractComponentAsArrayFilter ExtractInternalSurfacesFromTriangleGeometry + ExtractPipelineToFileFilter ExtractVertexGeometryFilter FeatureFaceCurvatureFilter FillBadDataFilter diff --git a/src/Plugins/SimplnxCore/docs/ExtractPipelineToFileFilter.md b/src/Plugins/SimplnxCore/docs/ExtractPipelineToFileFilter.md new file mode 100644 index 0000000000..068eb9d176 --- /dev/null +++ b/src/Plugins/SimplnxCore/docs/ExtractPipelineToFileFilter.md @@ -0,0 +1,15 @@ +# Extract DREAM.3D Pipeline To File + +## Description + +This **Filter** reads the pipeline from an hdf5 file with the .dream3d extension and writes it back out to a json formatted pipeline file with the appropriate extension based on whether the pipeline is a DREAM.3D version 6 (.json) or DREAM3D-NX version 7 (.d3dpipeline) formatted pipeline. + +% Auto generated parameter table will be inserted here + +## Example Pipelines + +ExtractPipelineToFile + +## License & Copyright + +Please see the description file distributed with this plugin. diff --git a/src/Plugins/SimplnxCore/pipelines/ExtractPipelineToFile.d3dpipeline b/src/Plugins/SimplnxCore/pipelines/ExtractPipelineToFile.d3dpipeline new file mode 100644 index 0000000000..267807ca89 --- /dev/null +++ b/src/Plugins/SimplnxCore/pipelines/ExtractPipelineToFile.d3dpipeline @@ -0,0 +1,21 @@ +{ + "isDisabled": false, + "name": "ExtractPipelineToFile.d3dpipeline", + "pinnedParams": [], + "pipeline": [ + { + "args": { + "input_file_path": "Data/Output/Reconstruction/SmallIN100_Final.dream3d", + "output_file_path": "Data/Output/Reconstruction/SmallIN100_Final_Pipeline.d3dpipeline" + }, + "comments": "", + "filter": { + "name": "nx::core::ExtractPipelineToFileFilter", + "uuid": "27203c99-9975-4528-88cc-42b270165d79" + }, + "isDisabled": false + } + ], + "version": 1, + "workflowParams": [] +} diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.cpp new file mode 100644 index 0000000000..519d084388 --- /dev/null +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.cpp @@ -0,0 +1,151 @@ +#include "ExtractPipelineToFileFilter.hpp" + +#include "simplnx/Common/AtomicFile.hpp" +#include "simplnx/Common/StringLiteral.hpp" +#include "simplnx/Parameters/FileSystemPathParameter.hpp" +#include "simplnx/Parameters/StringParameter.hpp" +#include "simplnx/Pipeline/Pipeline.hpp" +#include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" + +#include "simplnx/Utilities/SIMPLConversion.hpp" + +#include + +namespace +{ +constexpr nx::core::StringLiteral k_ImportedPipeline = "Imported Pipeline"; +} // namespace + +namespace nx::core +{ +//------------------------------------------------------------------------------ +std::string ExtractPipelineToFileFilter::name() const +{ + return FilterTraits::name; +} + +//------------------------------------------------------------------------------ +std::string ExtractPipelineToFileFilter::className() const +{ + return FilterTraits::className; +} + +//------------------------------------------------------------------------------ +Uuid ExtractPipelineToFileFilter::uuid() const +{ + return FilterTraits::uuid; +} + +//------------------------------------------------------------------------------ +std::string ExtractPipelineToFileFilter::humanName() const +{ + return "Extract DREAM.3D Pipeline To File"; +} + +//------------------------------------------------------------------------------ +std::vector ExtractPipelineToFileFilter::defaultTags() const +{ + return {className(), "IO", "Input", "Read", "Import", "Output", "Write", "Export", "Pipeline", "JSON"}; +} + +//------------------------------------------------------------------------------ +Parameters ExtractPipelineToFileFilter::parameters() const +{ + Parameters params; + params.insertSeparator(Parameters::Separator{"Input Parameters"}); + params.insert(std::make_unique(k_ImportFileData, "Input DREAM3D File Path", "The file path to the .dream3d that holds the pipeline to be extracted.", + FileSystemPathParameter::ValueType{}, FileSystemPathParameter::ExtensionsType{".dream3d"}, FileSystemPathParameter::PathType::InputFile)); + params.insertSeparator(Parameters::Separator{"Output Parameters"}); + params.insert(std::make_unique(k_OutputFile, "Output File Path", "The file path in which to save the extracted pipeline", FileSystemPathParameter::ValueType{}, + FileSystemPathParameter::ExtensionsType{Pipeline::k_Extension, Pipeline::k_SIMPLExtension}, FileSystemPathParameter::PathType::OutputFile)); + return params; +} + +//------------------------------------------------------------------------------ +IFilter::UniquePointer ExtractPipelineToFileFilter::clone() const +{ + return std::make_unique(); +} + +//------------------------------------------------------------------------------ +IFilter::PreflightResult ExtractPipelineToFileFilter::preflightImpl(const DataStructure& dataStructure, const Arguments& args, const MessageHandler& messageHandler, + const std::atomic_bool& shouldCancel) const +{ + const auto importFile = args.value(k_ImportFileData); + auto outputFile = args.value(k_OutputFile); + + Result pipelineResult = DREAM3D::ImportPipelineJsonFromFile(importFile); + if(pipelineResult.invalid()) + { + return {ConvertInvalidResult(std::move(pipelineResult))}; + } + + Result results; + + const nlohmann::json pipelineJson = pipelineResult.value(); + const bool isLegacy = pipelineJson.contains(nx::core::Pipeline::k_SIMPLPipelineBuilderKey); + + fs::path finalOutputPath = outputFile; + std::string extension = isLegacy ? Pipeline::k_SIMPLExtension : Pipeline::k_Extension; + if(!finalOutputPath.has_extension()) + { + finalOutputPath.concat(extension); + results.warnings().push_back(Warning{ + -2580, fmt::format("Output file '{}' is missing an extension. A {} extension will be added to the provided output file so that the extracted pipeline will be written to the file at path '{}'", + outputFile.string(), extension, finalOutputPath.string())}); + } + if(finalOutputPath.extension().string() != extension) + { + finalOutputPath.replace_extension(extension); + results.warnings().push_back( + Warning{-2581, fmt::format("Output file '{}' has the incorrect extension. A {} extension will be used instead so that the extracted pipeline will be written to the file at path '{}'", + outputFile.string(), extension, finalOutputPath.string())}); + } + + return {results}; +} + +//------------------------------------------------------------------------------ +Result<> ExtractPipelineToFileFilter::executeImpl(DataStructure& dataStructure, const Arguments& args, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, + const std::atomic_bool& shouldCancel) const +{ + const auto importFile = args.value(k_ImportFileData); + auto outputFile = args.value(k_OutputFile); + + Result pipelineResult = DREAM3D::ImportPipelineJsonFromFile(importFile); + if(pipelineResult.invalid()) + { + return ConvertResult(std::move(pipelineResult)); + } + const nlohmann::json pipelineJson = pipelineResult.value(); + const bool isLegacy = pipelineJson.contains(nx::core::Pipeline::k_SIMPLPipelineBuilderKey); + + std::string extension = isLegacy ? Pipeline::k_SIMPLExtension : Pipeline::k_Extension; + if(!outputFile.has_extension()) + { + outputFile.concat(extension); + } + if(outputFile.extension().string() != extension) + { + outputFile.replace_extension(extension); + } + AtomicFile atomicFile(outputFile.string(), false); + { + const fs::path exportFilePath = atomicFile.tempFilePath(); + std::ofstream fOut(exportFilePath.string(), std::ofstream::out); // test name resolution and create file + if(!fOut.is_open()) + { + return MakeErrorResult(-2582, fmt::format("Error opening output path {}", exportFilePath.string())); + } + + fOut << pipelineJson.dump(2); + } + atomicFile.commit(); + return {}; +} + +Result ExtractPipelineToFileFilter::FromSIMPLJson(const nlohmann::json& json) +{ + return {}; +} +} // namespace nx::core diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.hpp new file mode 100644 index 0000000000..9a5723ecf6 --- /dev/null +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ExtractPipelineToFileFilter.hpp @@ -0,0 +1,107 @@ +#pragma once + +#include "SimplnxCore/SimplnxCore_export.hpp" + +#include "simplnx/Filter/Arguments.hpp" +#include "simplnx/Filter/FilterTraits.hpp" +#include "simplnx/Filter/IFilter.hpp" +#include "simplnx/Filter/Parameters.hpp" + +namespace nx::core +{ +/** + * @class ExtractPipelineToFileFilter + * @brief The ExtractPipelineToFileFilter is an IFilter class designed to extract the pipeline data + * from a target DREAM.3D file and export it out to it's own file. + */ +class SIMPLNXCORE_EXPORT ExtractPipelineToFileFilter : public IFilter +{ +public: + ExtractPipelineToFileFilter() = default; + ~ExtractPipelineToFileFilter() noexcept override = default; + + ExtractPipelineToFileFilter(const ExtractPipelineToFileFilter&) = delete; + ExtractPipelineToFileFilter(ExtractPipelineToFileFilter&&) noexcept = delete; + + ExtractPipelineToFileFilter& operator=(const ExtractPipelineToFileFilter&) = delete; + ExtractPipelineToFileFilter& operator=(ExtractPipelineToFileFilter&&) noexcept = delete; + + // Parameter Keys + static inline constexpr StringLiteral k_ImportFileData = "input_file_path"; + static inline constexpr StringLiteral k_OutputFile = "output_file_path"; + + /** + * @brief Reads SIMPL json and converts it simplnx Arguments. + * @param json + * @return Result + */ + static Result FromSIMPLJson(const nlohmann::json& json); + + /** + * @brief Returns the name of the filter class. + * @return std::string + */ + std::string name() const override; + + /** + * @brief Returns the C++ classname of this filter. + * @return std::string + */ + std::string className() const override; + + /** + * @brief Returns the ExtractPipelineToFileFilter class's UUID. + * @return Uuid + */ + Uuid uuid() const override; + + /** + * @brief Returns the human readable name of the filter. + * @return std::string + */ + std::string humanName() const override; + + /** + * @brief Returns the default tags for this filter. + * @return + */ + std::vector defaultTags() const override; + + /** + * @brief Returns a collection of the filter's parameters (i.e. its inputs) + * @return Parameters + */ + Parameters parameters() const override; + + /** + * @brief Returns a copy of the filter as a std::unique_ptr. + * @return UniquePointer + */ + UniquePointer clone() const override; + +protected: + /** + * @brief Classes that implement IFilter must provide this function for preflight. + * Runs after the filter runs the checks in its parameters. + * @param dataStructure + * @param args + * @param messageHandler + * @return Result + */ + PreflightResult preflightImpl(const DataStructure& dataStructure, const Arguments& args, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const override; + + /** + * @brief Classes that implement IFilter must provide this function for execute. + * Runs after the filter applies the OutputActions from preflight. + * @param dataStructure + * @param args + * @param pipelineNode + * @param messageHandler + * @return Result<> + */ + Result<> executeImpl(DataStructure& dataStructure, const Arguments& args, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, + const std::atomic_bool& shouldCancel) const override; +}; +} // namespace nx::core + +SIMPLNX_DEF_FILTER_TRAITS(nx::core, ExtractPipelineToFileFilter, "27203c99-9975-4528-88cc-42b270165d79"); diff --git a/src/Plugins/SimplnxCore/test/CMakeLists.txt b/src/Plugins/SimplnxCore/test/CMakeLists.txt index 095a01b877..87fde40928 100644 --- a/src/Plugins/SimplnxCore/test/CMakeLists.txt +++ b/src/Plugins/SimplnxCore/test/CMakeLists.txt @@ -43,6 +43,7 @@ set(${PLUGIN_NAME}UnitTest_SRCS ExecuteProcessTest.cpp ExtractComponentAsArrayTest.cpp ExtractInternalSurfacesFromTriangleGeometryTest.cpp + ExtractPipelineToFileTest.cpp ExtractVertexGeometryTest.cpp FeatureFaceCurvatureTest.cpp FillBadDataTest.cpp @@ -69,7 +70,7 @@ set(${PLUGIN_NAME}UnitTest_SRCS FindVolFractionsTest.cpp GenerateColorTableTest.cpp GenerateVectorColorsTest.cpp - GeneratePythonSkeletonTest.cpp + GeneratePythonSkeletonTest.cpp IdentifySampleTest.cpp FlyingEdges3DTest.cpp ImageGeomTest.cpp diff --git a/src/Plugins/SimplnxCore/test/ExtractPipelineToFileTest.cpp b/src/Plugins/SimplnxCore/test/ExtractPipelineToFileTest.cpp new file mode 100644 index 0000000000..4f72942a59 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/ExtractPipelineToFileTest.cpp @@ -0,0 +1,139 @@ +#include "SimplnxCore/Filters/ExtractPipelineToFileFilter.hpp" +#include "SimplnxCore/SimplnxCore_test_dirs.hpp" + +#include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/Parameters/FileSystemPathParameter.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" + +#include + +#include +#include + +namespace fs = std::filesystem; +using namespace nx::core; +using namespace nx::core::Constants; + +namespace +{ +fs::path k_OutputFileName(fmt::format("{}/Small_IN100_Pipeline", unit_test::k_BinaryTestOutputDir)); +fs::path k_JsonOutputFile(k_OutputFileName.string() + Pipeline::k_SIMPLExtension.str()); +fs::path k_NXOutputFile(k_OutputFileName.string() + Pipeline::k_Extension.str()); +} // namespace + +TEST_CASE("SimplnxCore::ExtractPipelineToFileFilter : Valid Execution", "[SimplnxCore][ExtractPipelineToFileFilter]") +{ + // Instantiate the filter, a DataStructure object and an Arguments Object + DataStructure dataStructure; + Arguments args; + ExtractPipelineToFileFilter filter; + + const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "Small_IN100_dream3d.tar.gz", "Small_IN100.dream3d"); + auto exemplarDream3dFile = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir)); + + // Create default Parameters for the filter. + args.insertOrAssign(ExtractPipelineToFileFilter::k_ImportFileData, std::make_any(exemplarDream3dFile)); + args.insertOrAssign(ExtractPipelineToFileFilter::k_OutputFile, std::make_any(k_JsonOutputFile)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions) + + // Execute the filter and check the result + auto executeResult = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result) + + REQUIRE(fs::exists(k_JsonOutputFile)); + + fs::remove(k_JsonOutputFile); +} + +TEST_CASE("SimplnxCore::ExtractPipelineToFileFilter : Valid Execution - incorrect output extension", "[SimplnxCore][ExtractPipelineToFileFilter]") +{ + // Instantiate the filter, a DataStructure object and an Arguments Object + DataStructure dataStructure; + Arguments args; + ExtractPipelineToFileFilter filter; + + const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "Small_IN100_dream3d.tar.gz", "Small_IN100.dream3d"); + auto exemplarDream3dFile = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir)); + + // Create default Parameters for the filter. + args.insertOrAssign(ExtractPipelineToFileFilter::k_ImportFileData, std::make_any(exemplarDream3dFile)); + args.insertOrAssign(ExtractPipelineToFileFilter::k_OutputFile, std::make_any(k_NXOutputFile)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions) + + // Execute the filter and check the result + auto executeResult = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result) + + REQUIRE(preflightResult.outputActions.warnings().front().code == -2581); + REQUIRE(fs::exists(k_JsonOutputFile)); + + fs::remove(k_JsonOutputFile); +} + +TEST_CASE("SimplnxCore::ExtractPipelineToFileFilter : Invalid Execution - missing output extension", "[SimplnxCore][ExtractPipelineToFileFilter]") +{ + // Instantiate the filter, a DataStructure object and an Arguments Object + DataStructure dataStructure; + Arguments args; + ExtractPipelineToFileFilter filter; + + const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "Small_IN100_dream3d.tar.gz", "Small_IN100.dream3d"); + auto exemplarDream3dFile = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir)); + + // Create default Parameters for the filter. + args.insertOrAssign(ExtractPipelineToFileFilter::k_ImportFileData, std::make_any(exemplarDream3dFile)); + args.insertOrAssign(ExtractPipelineToFileFilter::k_OutputFile, std::make_any(k_OutputFileName)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(preflightResult.outputActions) + + // Execute the filter and check the result + auto executeResult = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(executeResult.result) + + REQUIRE(!fs::exists(k_JsonOutputFile)); +} + +TEST_CASE("SimplnxCore::ExtractPipelineToFileFilter : Invalid Execution - invalid input file format", "[SimplnxCore][ExtractPipelineToFileFilter]") +{ + // Instantiate the filter, a DataStructure object and an Arguments Object + DataStructure dataStructure; + Arguments args; + ExtractPipelineToFileFilter filter; + + auto testInvalidInputFile = fs::path(fmt::format("{}/TestDataStructureNoPipeline.dream3d", unit_test::k_BinaryTestOutputDir)); + { + DataStructure testDataStructure = UnitTest::CreateDataStructure(); + auto fileWriterResult = nx::core::HDF5::FileWriter::CreateFile(testInvalidInputFile); + REQUIRE(fileWriterResult.valid()); + nx::core::HDF5::FileWriter fileWriter = std::move(fileWriterResult.value()); + auto writeResult = HDF5::DataStructureWriter::WriteFile(dataStructure, fileWriter); + REQUIRE(writeResult.valid()); + } + + fs::path outputFilePath(fmt::format("{}/TestDataStructureNoPipeline{}", unit_test::k_BinaryTestOutputDir, Pipeline::k_SIMPLExtension.str())); + + // Create default Parameters for the filter. + args.insertOrAssign(ExtractPipelineToFileFilter::k_ImportFileData, std::make_any(testInvalidInputFile)); + args.insertOrAssign(ExtractPipelineToFileFilter::k_OutputFile, std::make_any(outputFilePath)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(preflightResult.outputActions) + + // Execute the filter and check the result + auto executeResult = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(executeResult.result) + + REQUIRE(!fs::exists(k_JsonOutputFile)); + + fs::remove(testInvalidInputFile); +} diff --git a/src/simplnx/Pipeline/Pipeline.hpp b/src/simplnx/Pipeline/Pipeline.hpp index 0ab8288ad9..2765d797cf 100644 --- a/src/simplnx/Pipeline/Pipeline.hpp +++ b/src/simplnx/Pipeline/Pipeline.hpp @@ -33,6 +33,7 @@ class SIMPLNX_EXPORT Pipeline : public AbstractPipelineNode, protected PipelineN using const_iterator = collection_type::const_iterator; static constexpr StringLiteral k_Extension = ".d3dpipeline"; + static constexpr StringLiteral k_SIMPLExtension = ".json"; static constexpr StringLiteral k_SIMPLPipelineBuilderKey = "PipelineBuilder"; /**