ENH: Update Python Plugin Generation File Structure (#989)

Signed-off-by: Joey Kleingers <[email protected]>

src/Plugins/SimplnxCore/CMakeLists.txt

@@ -373,6 +373,16 @@ string(HEX ${PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE} PYTHON_PLUGIN_ENVIRONMENT_TEMPL
 string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE ${PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE})
 string(APPEND PYTHON_PLUGIN_ENVIRONMENT_TEMPLATE "0x00")
 
+file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginPyProjectTemplate.py" PYTHON_PLUGIN_PYPROJECT_TEMPLATE)
+string(HEX ${PYTHON_PLUGIN_PYPROJECT_TEMPLATE} PYTHON_PLUGIN_PYPROJECT_TEMPLATE)
+string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_PYPROJECT_TEMPLATE ${PYTHON_PLUGIN_PYPROJECT_TEMPLATE})
+string(APPEND PYTHON_PLUGIN_PYPROJECT_TEMPLATE "0x00")
+
+file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginMetaTemplate.py" PYTHON_PLUGIN_META_TEMPLATE)
+string(HEX ${PYTHON_PLUGIN_META_TEMPLATE} PYTHON_PLUGIN_META_TEMPLATE)
+string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_META_TEMPLATE ${PYTHON_PLUGIN_META_TEMPLATE})
+string(APPEND PYTHON_PLUGIN_META_TEMPLATE "0x00")
+
 cmpConfigureFileWithMD5Check(CONFIGURED_TEMPLATE_PATH "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.in.hpp"
                                GENERATED_FILE_PATH "${${PLUGIN_NAME}_BINARY_DIR}/generated/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.hpp")
 

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/CreatePythonSkeletonFilter.cpp

@@ -100,39 +100,77 @@ IFilter::PreflightResult CreatePythonSkeletonFilter::preflightImpl(const DataStr
   if(useExistingPlugin)
   {
     pluginPath = pluginInputDir.string();
+    pluginName = pluginInputDir.filename().string();
   }
-  std::string fullPath = fmt::format("{}{}{}{}Plugin.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator});
+
+  std::string fullPath = fmt::format("{}{}{}{}meta.yaml", pluginPath, std::string{fs::path::preferred_separator}, "conda", std::string{fs::path::preferred_separator});
+  if(std::filesystem::exists({fullPath}))
+  {
+    fullPath = std::string("[REPLACE]: ").append(fullPath);
+  }
+  else
+  {
+    fullPath = std::string("[New]: ").append(fullPath);
+  }
+  preflightUpdatedValue << fullPath << '\n';
+
+  fullPath = fmt::format("{}{}{}{}environment.yml", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator});
+  if(std::filesystem::exists({fullPath}))
+  {
+    fullPath = std::string("[REPLACE]: ").append(fullPath);
+  }
+  else
+  {
+    fullPath = std::string("[New]: ").append(fullPath);
+  }
+  preflightUpdatedValue << fullPath << '\n';
+
+  fullPath = fmt::format("{}{}{}{}pyproject.toml", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator});
+  if(std::filesystem::exists({fullPath}))
+  {
+    fullPath = std::string("[REPLACE]: ").append(fullPath);
+  }
+  else
+  {
+    fullPath = std::string("[New]: ").append(fullPath);
+  }
+  preflightUpdatedValue << fullPath << '\n';
+
+  fullPath = fmt::format("{}{}{}{}{}{}__init__.py", pluginPath, std::string{fs::path::preferred_separator}, "src", std::string{fs::path::preferred_separator}, pluginName,
+                         std::string{fs::path::preferred_separator});
   if(std::filesystem::exists({fullPath}))
   {
-    fullPath = "[REPLACE]: " + fullPath;
+    fullPath = std::string("[REPLACE]: ").append(fullPath);
   }
   else
   {
-    fullPath = "[New]: " + fullPath;
+    fullPath = std::string("[New]: ").append(fullPath);
   }
   preflightUpdatedValue << fullPath << '\n';
 
-  fullPath = fmt::format("{}{}{}{}__init__.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator});
+  fullPath = fmt::format("{}{}{}{}{}{}Plugin.py", pluginPath, std::string{fs::path::preferred_separator}, "src", std::string{fs::path::preferred_separator}, pluginName,
+                         std::string{fs::path::preferred_separator});
   if(std::filesystem::exists({fullPath}))
   {
-    fullPath = "[REPLACE]: " + fullPath;
+    fullPath = std::string("[REPLACE]: ").append(fullPath);
   }
   else
   {
-    fullPath = "[New]: " + fullPath;
+    fullPath = std::string("[New]: ").append(fullPath);
   }
   preflightUpdatedValue << fullPath << '\n';
 
   for(const auto& filterName : filterList)
   {
-    fullPath = fmt::format("{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, filterName);
+    fullPath = fmt::format("{}{}{}{}{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, "src", std::string{fs::path::preferred_separator}, pluginName,
+                           std::string{fs::path::preferred_separator}, filterName);
     if(std::filesystem::exists({fullPath}))
     {
-      fullPath = "[REPLACE]: " + fullPath;
+      fullPath = std::string("[REPLACE]: ").append(fullPath);
     }
     else
     {
-      fullPath = "[New]: " + fullPath;
+      fullPath = std::string("[New]: ").append(fullPath);
     }
     preflightUpdatedValue << fullPath << '\n';
   }

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginMetaTemplate.py

@@ -0,0 +1,38 @@
+{% set name = "simplnx.#PLUGIN_NAME#" %}
+{% set version = "1.0.0" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  - path: ../
+    folder: #PLUGIN_NAME#
+
+build:
+  noarch: python
+  script: {{ PYTHON }} -m pip install ./#PLUGIN_NAME# -vv --no-deps --no-build-isolation
+  number: 0
+
+requirements:
+  build:
+    - python >=3.10
+    - pytest
+    - pip
+    - hatchling
+  run:
+    - python >=3.10
+
+test:
+  imports:
+    - #PLUGIN_NAME#
+  commands:
+    - pip check
+  requires:
+    - pip
+
+about:
+  home: https://www.dream3d.io/
+  license: BSD
+  summary: DREAM3D-NX consists of data analysis tools (Filters) that allow for the construction of customized workflows (Pipelines) to analyze data.
+

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginPyProjectTemplate.py

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/#PLUGIN_NAME#"]
+
+[project]
+name = "simplnx.#PLUGIN_NAME#"
+version = "1.0.0"
+requires-python = ">=3.10"
+dependencies = [
+  "numpy"
+]
+
+[project.entry-points."simplnx.plugins"]
+plugin = "#PLUGIN_NAME#"

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp

@@ -46,4 +46,22 @@ inline const std::string PluginEnvironmentPythonFile()
   return {k_PluginEnvironmentPythonFileCharArray};
 }
 
+// clang-format off
+static const char k_PluginPyProjectPythonFileCharArray[] = {@PYTHON_PLUGIN_PYPROJECT_TEMPLATE@};
+// clang-format on
+
+inline const std::string PluginPyProjectPythonFile()
+{
+  return {k_PluginPyProjectPythonFileCharArray};
+}
+
+// clang-format off
+static const char k_PluginMetaPythonFileCharArray[] = {@PYTHON_PLUGIN_META_TEMPLATE@};
+// clang-format on
+
+inline const std::string PluginMetaPythonFile()
+{
+  return {k_PluginMetaPythonFileCharArray};
+}
+
 }; // namespace nx::core

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp

@@ -163,7 +163,7 @@ inline Result<> InsertFilterNameInPluginFiles(const std::filesystem::path& plugi
         outFile << "# FILTER_START: " << filterName << "\n"
                 << "try:\n"
                 << "  from " << pluginName << "." << filterName << " import " << filterName << "\n"
-                << "  _filters_.append(" << filterName << ")\n"
+                << "  _filters.append(" << filterName << ")\n"
                 << "except ImportError:\n"
                 << "  pass\n"
                 << "# FILTER_END: " << filterName << "\n\n";
@@ -330,14 +330,26 @@ inline std::string GeneratePythonPlugin(const std::string& pluginName, const std
 inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirectory, const std::string& pluginName, const std::string& pluginShortName, const std::string& pluginDescription,
                                        const std::string& pluginFilterList)
 {
-
   auto pluginRootPath = outputDirectory / pluginName;
+  auto pluginSrcPath = pluginRootPath / "src" / pluginName;
   auto result = nx::core::CreateOutputDirectories(pluginRootPath);
   if(result.invalid())
   {
     return result;
   }
-  auto outputPath = pluginRootPath / "Plugin.py";
+  auto pluginCondaPath = pluginRootPath / "conda";
+  result = nx::core::CreateOutputDirectories(pluginCondaPath);
+  if(result.invalid())
+  {
+    return result;
+  }
+  auto pluginDocsPath = pluginRootPath / "docs";
+  result = nx::core::CreateOutputDirectories(pluginDocsPath);
+  if(result.invalid())
+  {
+    return result;
+  }
+  auto outputPath = pluginSrcPath / "Plugin.py";
   {
     auto atomicFileResult = AtomicFile::Create(outputPath);
     if(atomicFileResult.invalid())
@@ -365,7 +377,7 @@ inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirect
   }
 
   // Write the __init__.py file
-  outputPath = pluginRootPath / "__init__.py";
+  outputPath = pluginSrcPath / "__init__.py";
   {
     auto atomicFileResult = AtomicFile::Create(outputPath);
     if(atomicFileResult.invalid())
@@ -447,11 +459,66 @@ inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirect
       return commitResult;
     }
   }
+
+  // Write the pyproject.toml file
+  outputPath = pluginRootPath / "pyproject.toml";
+  {
+    auto atomicFileResult = AtomicFile::Create(outputPath);
+    if(atomicFileResult.invalid())
+    {
+      return ConvertResult(std::move(atomicFileResult));
+    }
+    AtomicFile pyProjTempFile = std::move(atomicFileResult.value());
+    {
+      // Scope this so that the file closes first before we then 'commit' with the atomic file
+      std::ofstream fout(pyProjTempFile.tempFilePath(), std::ios_base::out | std::ios_base::binary);
+      if(!fout.is_open())
+      {
+        return MakeErrorResult(-74100, fmt::format("Error creating and opening output file at path: {}", pyProjTempFile.tempFilePath().string()));
+      }
+      std::string content = PluginPyProjectPythonFile();
+      content = StringUtilities::replace(content, "#PLUGIN_NAME#", pluginName);
+      fout << content;
+    }
+    Result<> commitResult = pyProjTempFile.commit();
+    if(commitResult.invalid())
+    {
+      return commitResult;
+    }
+  }
+
+  // Write the meta.yaml file
+  outputPath = pluginCondaPath / "meta.yaml";
+  {
+    auto atomicFileResult = AtomicFile::Create(outputPath);
+    if(atomicFileResult.invalid())
+    {
+      return ConvertResult(std::move(atomicFileResult));
+    }
+    AtomicFile metaTempFile = std::move(atomicFileResult.value());
+    {
+      // Scope this so that the file closes first before we then 'commit' with the atomic file
+      std::ofstream fout(metaTempFile.tempFilePath(), std::ios_base::out | std::ios_base::binary);
+      if(!fout.is_open())
+      {
+        return MakeErrorResult(-74100, fmt::format("Error creating and opening output file at path: {}", metaTempFile.tempFilePath().string()));
+      }
+      std::string content = PluginMetaPythonFile();
+      content = StringUtilities::replace(content, "#PLUGIN_NAME#", pluginName);
+      fout << content;
+    }
+    Result<> commitResult = metaTempFile.commit();
+    if(commitResult.invalid())
+    {
+      return commitResult;
+    }
+  }
+
   // Now loop over each Filter and generate the skeleton files
   auto filterList = StringUtilities::split(pluginFilterList, ',');
   for(const auto& name : filterList)
   {
-    WritePythonFilterToFile(pluginRootPath, name, name, Uuid::GenerateV4().str());
+    WritePythonFilterToFile(pluginSrcPath, name, name, Uuid::GenerateV4().str());
   }
 
   return {};

src/Plugins/SimplnxCore/test/CreatePythonSkeletonTest.cpp

@@ -5,6 +5,7 @@
 //
 #include "simplnx/Parameters/ArrayCreationParameter.hpp"
 #include "simplnx/Parameters/FileSystemPathParameter.hpp"
+#include "simplnx/UnitTest/UnitTestCommon.hpp"
 
 namespace fs = std::filesystem;
 using namespace nx::core;
@@ -36,17 +37,17 @@ TEST_CASE("SimplnxCore::GeneratePythonSkeleton")
 
       // Preflight the filter and check result
       auto preflightResult = filter.preflight(dataStructure, args);
-      REQUIRE(preflightResult.outputActions.valid());
+      SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions);
 
       // Execute the filter and check the result
       auto executeResult = filter.execute(dataStructure, args);
-      REQUIRE(executeResult.result.valid());
+      SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
     }
 
     // Generate filters into an existing plugin
     {
       const std::string k_FilterNames = "ThirdFilter,FourthFilter";
-      const std::filesystem::path k_InputDirectory = k_OutputDirectory / k_PluginName;
+      const std::filesystem::path k_InputDirectory = k_OutputDirectory / k_PluginName / "src" / k_PluginName;
 
       // Create default Parameters for the filter.
       args.insertOrAssign(CreatePythonSkeletonFilter::k_UseExistingPlugin_Key, true);
@@ -55,11 +56,11 @@ TEST_CASE("SimplnxCore::GeneratePythonSkeleton")
 
       // Preflight the filter and check result
       auto preflightResult = filter.preflight(dataStructure, args);
-      REQUIRE(preflightResult.outputActions.valid());
+      SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions);
 
       // Execute the filter and check the result
       auto executeResult = filter.execute(dataStructure, args);
-      REQUIRE(executeResult.result.valid());
+      SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
     }
   }
 }