From 40512e34ec32fd5628c15e6eabd0cd25d7b55eac Mon Sep 17 00:00:00 2001 From: Joey Kleingers Date: Fri, 21 Jun 2024 09:47:37 -0400 Subject: [PATCH] ENH: Update Python Plugin Generation File Structure (#989) Signed-off-by: Joey Kleingers --- src/Plugins/SimplnxCore/CMakeLists.txt | 10 +++ .../Filters/CreatePythonSkeletonFilter.cpp | 56 +++++++++++--- .../utils/PythonPluginMetaTemplate.py | 38 +++++++++ .../utils/PythonPluginPyProjectTemplate.py | 17 ++++ .../utils/PythonPluginSourceTemplate.in.hpp | 18 +++++ .../utils/PythonPluginTemplateFile.hpp | 77 +++++++++++++++++-- .../test/CreatePythonSkeletonTest.cpp | 11 +-- .../{src/ExamplePlugin => }/environment.yml | 0 8 files changed, 208 insertions(+), 19 deletions(-) create mode 100644 src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginMetaTemplate.py create mode 100644 src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginPyProjectTemplate.py rename wrapping/python/plugins/ExamplePlugin/{src/ExamplePlugin => }/environment.yml (100%) diff --git a/src/Plugins/SimplnxCore/CMakeLists.txt b/src/Plugins/SimplnxCore/CMakeLists.txt index 46a7905760..9a135bad87 100644 --- a/src/Plugins/SimplnxCore/CMakeLists.txt +++ b/src/Plugins/SimplnxCore/CMakeLists.txt @@ -373,6 +373,16 @@ string(HEX ${PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE} PYTHON_PLUGIN_ENVIRONMENT_TEMPL string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE ${PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE}) string(APPEND PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE "0x00") +file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginPyProjectTemplate.py" PYTHON_PLUGIN_PYPROJECT_TEMPLATE) +string(HEX ${PYTHON_PLUGIN_PYPROJECT_TEMPLATE} PYTHON_PLUGIN_PYPROJECT_TEMPLATE) +string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_PYPROJECT_TEMPLATE ${PYTHON_PLUGIN_PYPROJECT_TEMPLATE}) +string(APPEND PYTHON_PLUGIN_PYPROJECT_TEMPLATE "0x00") + +file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginMetaTemplate.py" PYTHON_PLUGIN_META_TEMPLATE) +string(HEX ${PYTHON_PLUGIN_META_TEMPLATE} PYTHON_PLUGIN_META_TEMPLATE) +string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_META_TEMPLATE ${PYTHON_PLUGIN_META_TEMPLATE}) +string(APPEND PYTHON_PLUGIN_META_TEMPLATE "0x00") + cmpConfigureFileWithMD5Check(CONFIGURED_TEMPLATE_PATH "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.in.hpp" GENERATED_FILE_PATH "${${PLUGIN_NAME}_BINARY_DIR}/generated/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.hpp") diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/CreatePythonSkeletonFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/CreatePythonSkeletonFilter.cpp index 5921b9ff87..59adf0166d 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/CreatePythonSkeletonFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/CreatePythonSkeletonFilter.cpp @@ -100,39 +100,77 @@ IFilter::PreflightResult CreatePythonSkeletonFilter::preflightImpl(const DataStr if(useExistingPlugin) { pluginPath = pluginInputDir.string(); + pluginName = pluginInputDir.filename().string(); } - std::string fullPath = fmt::format("{}{}{}{}Plugin.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator}); + + std::string fullPath = fmt::format("{}{}{}{}meta.yaml", pluginPath, std::string{fs::path::preferred_separator}, "conda", std::string{fs::path::preferred_separator}); + if(std::filesystem::exists({fullPath})) + { + fullPath = std::string("[REPLACE]: ").append(fullPath); + } + else + { + fullPath = std::string("[New]: ").append(fullPath); + } + preflightUpdatedValue << fullPath << '\n'; + + fullPath = fmt::format("{}{}{}{}environment.yml", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator}); + if(std::filesystem::exists({fullPath})) + { + fullPath = std::string("[REPLACE]: ").append(fullPath); + } + else + { + fullPath = std::string("[New]: ").append(fullPath); + } + preflightUpdatedValue << fullPath << '\n'; + + fullPath = fmt::format("{}{}{}{}pyproject.toml", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator}); + if(std::filesystem::exists({fullPath})) + { + fullPath = std::string("[REPLACE]: ").append(fullPath); + } + else + { + fullPath = std::string("[New]: ").append(fullPath); + } + preflightUpdatedValue << fullPath << '\n'; + + fullPath = fmt::format("{}{}{}{}{}{}__init__.py", pluginPath, std::string{fs::path::preferred_separator}, "src", std::string{fs::path::preferred_separator}, pluginName, + std::string{fs::path::preferred_separator}); if(std::filesystem::exists({fullPath})) { - fullPath = "[REPLACE]: " + fullPath; + fullPath = std::string("[REPLACE]: ").append(fullPath); } else { - fullPath = "[New]: " + fullPath; + fullPath = std::string("[New]: ").append(fullPath); } preflightUpdatedValue << fullPath << '\n'; - fullPath = fmt::format("{}{}{}{}__init__.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator}); + fullPath = fmt::format("{}{}{}{}{}{}Plugin.py", pluginPath, std::string{fs::path::preferred_separator}, "src", std::string{fs::path::preferred_separator}, pluginName, + std::string{fs::path::preferred_separator}); if(std::filesystem::exists({fullPath})) { - fullPath = "[REPLACE]: " + fullPath; + fullPath = std::string("[REPLACE]: ").append(fullPath); } else { - fullPath = "[New]: " + fullPath; + fullPath = std::string("[New]: ").append(fullPath); } preflightUpdatedValue << fullPath << '\n'; for(const auto& filterName : filterList) { - fullPath = fmt::format("{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, filterName); + fullPath = fmt::format("{}{}{}{}{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, "src", std::string{fs::path::preferred_separator}, pluginName, + std::string{fs::path::preferred_separator}, filterName); if(std::filesystem::exists({fullPath})) { - fullPath = "[REPLACE]: " + fullPath; + fullPath = std::string("[REPLACE]: ").append(fullPath); } else { - fullPath = "[New]: " + fullPath; + fullPath = std::string("[New]: ").append(fullPath); } preflightUpdatedValue << fullPath << '\n'; } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginMetaTemplate.py b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginMetaTemplate.py new file mode 100644 index 0000000000..80b5e78493 --- /dev/null +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginMetaTemplate.py @@ -0,0 +1,38 @@ +{% set name = "simplnx.#PLUGIN_NAME#" %} +{% set version = "1.0.0" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + - path: ../ + folder: #PLUGIN_NAME# + +build: + noarch: python + script: {{ PYTHON }} -m pip install ./#PLUGIN_NAME# -vv --no-deps --no-build-isolation + number: 0 + +requirements: + build: + - python >=3.10 + - pytest + - pip + - hatchling + run: + - python >=3.10 + +test: + imports: + - #PLUGIN_NAME# + commands: + - pip check + requires: + - pip + +about: + home: https://www.dream3d.io/ + license: BSD + summary: DREAM3D-NX consists of data analysis tools (Filters) that allow for the construction of customized workflows (Pipelines) to analyze data. + diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginPyProjectTemplate.py b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginPyProjectTemplate.py new file mode 100644 index 0000000000..6e725338ae --- /dev/null +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginPyProjectTemplate.py @@ -0,0 +1,17 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/#PLUGIN_NAME#"] + +[project] +name = "simplnx.#PLUGIN_NAME#" +version = "1.0.0" +requires-python = ">=3.10" +dependencies = [ + "numpy" +] + +[project.entry-points."simplnx.plugins"] +plugin = "#PLUGIN_NAME#" \ No newline at end of file diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp index d092a5c34a..e66468bd57 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp @@ -46,4 +46,22 @@ inline const std::string PluginEnvironmentPythonFile() return {k_PluginEnvironmentPythonFileCharArray}; } +// clang-format off +static const char k_PluginPyProjectPythonFileCharArray[] = {@PYTHON_PLUGIN_PYPROJECT_TEMPLATE@}; +// clang-format on + +inline const std::string PluginPyProjectPythonFile() +{ + return {k_PluginPyProjectPythonFileCharArray}; +} + +// clang-format off +static const char k_PluginMetaPythonFileCharArray[] = {@PYTHON_PLUGIN_META_TEMPLATE@}; +// clang-format on + +inline const std::string PluginMetaPythonFile() +{ + return {k_PluginMetaPythonFileCharArray}; +} + }; // namespace nx::core \ No newline at end of file diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp index f4f619a8a8..a039db556b 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp @@ -163,7 +163,7 @@ inline Result<> InsertFilterNameInPluginFiles(const std::filesystem::path& plugi outFile << "# FILTER_START: " << filterName << "\n" << "try:\n" << " from " << pluginName << "." << filterName << " import " << filterName << "\n" - << " _filters_.append(" << filterName << ")\n" + << " _filters.append(" << filterName << ")\n" << "except ImportError:\n" << " pass\n" << "# FILTER_END: " << filterName << "\n\n"; @@ -330,14 +330,26 @@ inline std::string GeneratePythonPlugin(const std::string& pluginName, const std inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirectory, const std::string& pluginName, const std::string& pluginShortName, const std::string& pluginDescription, const std::string& pluginFilterList) { - auto pluginRootPath = outputDirectory / pluginName; + auto pluginSrcPath = pluginRootPath / "src" / pluginName; auto result = nx::core::CreateOutputDirectories(pluginRootPath); if(result.invalid()) { return result; } - auto outputPath = pluginRootPath / "Plugin.py"; + auto pluginCondaPath = pluginRootPath / "conda"; + result = nx::core::CreateOutputDirectories(pluginCondaPath); + if(result.invalid()) + { + return result; + } + auto pluginDocsPath = pluginRootPath / "docs"; + result = nx::core::CreateOutputDirectories(pluginDocsPath); + if(result.invalid()) + { + return result; + } + auto outputPath = pluginSrcPath / "Plugin.py"; { auto atomicFileResult = AtomicFile::Create(outputPath); if(atomicFileResult.invalid()) @@ -365,7 +377,7 @@ inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirect } // Write the __init__.py file - outputPath = pluginRootPath / "__init__.py"; + outputPath = pluginSrcPath / "__init__.py"; { auto atomicFileResult = AtomicFile::Create(outputPath); if(atomicFileResult.invalid()) @@ -447,11 +459,66 @@ inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirect return commitResult; } } + + // Write the pyproject.toml file + outputPath = pluginRootPath / "pyproject.toml"; + { + auto atomicFileResult = AtomicFile::Create(outputPath); + if(atomicFileResult.invalid()) + { + return ConvertResult(std::move(atomicFileResult)); + } + AtomicFile pyProjTempFile = std::move(atomicFileResult.value()); + { + // Scope this so that the file closes first before we then 'commit' with the atomic file + std::ofstream fout(pyProjTempFile.tempFilePath(), std::ios_base::out | std::ios_base::binary); + if(!fout.is_open()) + { + return MakeErrorResult(-74100, fmt::format("Error creating and opening output file at path: {}", pyProjTempFile.tempFilePath().string())); + } + std::string content = PluginPyProjectPythonFile(); + content = StringUtilities::replace(content, "#PLUGIN_NAME#", pluginName); + fout << content; + } + Result<> commitResult = pyProjTempFile.commit(); + if(commitResult.invalid()) + { + return commitResult; + } + } + + // Write the meta.yaml file + outputPath = pluginCondaPath / "meta.yaml"; + { + auto atomicFileResult = AtomicFile::Create(outputPath); + if(atomicFileResult.invalid()) + { + return ConvertResult(std::move(atomicFileResult)); + } + AtomicFile metaTempFile = std::move(atomicFileResult.value()); + { + // Scope this so that the file closes first before we then 'commit' with the atomic file + std::ofstream fout(metaTempFile.tempFilePath(), std::ios_base::out | std::ios_base::binary); + if(!fout.is_open()) + { + return MakeErrorResult(-74100, fmt::format("Error creating and opening output file at path: {}", metaTempFile.tempFilePath().string())); + } + std::string content = PluginMetaPythonFile(); + content = StringUtilities::replace(content, "#PLUGIN_NAME#", pluginName); + fout << content; + } + Result<> commitResult = metaTempFile.commit(); + if(commitResult.invalid()) + { + return commitResult; + } + } + // Now loop over each Filter and generate the skeleton files auto filterList = StringUtilities::split(pluginFilterList, ','); for(const auto& name : filterList) { - WritePythonFilterToFile(pluginRootPath, name, name, Uuid::GenerateV4().str()); + WritePythonFilterToFile(pluginSrcPath, name, name, Uuid::GenerateV4().str()); } return {}; diff --git a/src/Plugins/SimplnxCore/test/CreatePythonSkeletonTest.cpp b/src/Plugins/SimplnxCore/test/CreatePythonSkeletonTest.cpp index 76717f04a7..e9c09e5436 100644 --- a/src/Plugins/SimplnxCore/test/CreatePythonSkeletonTest.cpp +++ b/src/Plugins/SimplnxCore/test/CreatePythonSkeletonTest.cpp @@ -5,6 +5,7 @@ // #include "simplnx/Parameters/ArrayCreationParameter.hpp" #include "simplnx/Parameters/FileSystemPathParameter.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" namespace fs = std::filesystem; using namespace nx::core; @@ -36,17 +37,17 @@ TEST_CASE("SimplnxCore::GeneratePythonSkeleton") // Preflight the filter and check result auto preflightResult = filter.preflight(dataStructure, args); - REQUIRE(preflightResult.outputActions.valid()); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); // Execute the filter and check the result auto executeResult = filter.execute(dataStructure, args); - REQUIRE(executeResult.result.valid()); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); } // Generate filters into an existing plugin { const std::string k_FilterNames = "ThirdFilter,FourthFilter"; - const std::filesystem::path k_InputDirectory = k_OutputDirectory / k_PluginName; + const std::filesystem::path k_InputDirectory = k_OutputDirectory / k_PluginName / "src" / k_PluginName; // Create default Parameters for the filter. args.insertOrAssign(CreatePythonSkeletonFilter::k_UseExistingPlugin_Key, true); @@ -55,11 +56,11 @@ TEST_CASE("SimplnxCore::GeneratePythonSkeleton") // Preflight the filter and check result auto preflightResult = filter.preflight(dataStructure, args); - REQUIRE(preflightResult.outputActions.valid()); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); // Execute the filter and check the result auto executeResult = filter.execute(dataStructure, args); - REQUIRE(executeResult.result.valid()); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); } } } diff --git a/wrapping/python/plugins/ExamplePlugin/src/ExamplePlugin/environment.yml b/wrapping/python/plugins/ExamplePlugin/environment.yml similarity index 100% rename from wrapping/python/plugins/ExamplePlugin/src/ExamplePlugin/environment.yml rename to wrapping/python/plugins/ExamplePlugin/environment.yml