Allow discovery and loading of pip installed python plugins

.gitignore

@@ -29,3 +29,4 @@ test_data/
 CMakeUserPresets.json
 Workspace
 __pycache__/
+dist/

src/nxrunner/src/nxrunner.cpp

@@ -577,6 +577,7 @@ int main(int argc, char* argv[])
     auto manualImportFinder = nx::python::ManualImportFinderHolder::Create();
     manualImportFinder.addToMetaPath();
     nx::python::LoadPythonPlugins(pythonPlugins, outputCallback, pluginLoadErrorCallback, pythonErrorCallback);
+    nx::python::LoadInstalledPythonPlugins(outputCallback, pluginLoadErrorCallback, pythonErrorCallback);
   } catch(const std::exception& exception)
   {
     std::cout << "Aborting python plugin loading due to exception: \n";

wrapping/python/CxPybind/CxPybind/CxPybind.hpp

@@ -696,7 +696,11 @@ inline void Internals::loadPythonPlugin(py::module_& mod)
 
   auto pluginLoader = std::make_shared<InMemoryPluginLoader>(plugin);
 
-  m_App->getFilterList()->addPlugin(std::dynamic_pointer_cast<IPluginLoader>(pluginLoader));
+  Result<> result = m_App->getFilterList()->addPlugin(std::dynamic_pointer_cast<IPluginLoader>(pluginLoader));
+  if(result.invalid())
+  {
+    throw std::runtime_error(result.errors().at(0).message);
+  }
 
   m_PythonPlugins.insert({plugin->getId(), plugin});
 }

wrapping/python/NxPythonEmbed/NxPythonEmbed/NxPythonEmbed.hpp

@@ -304,4 +304,126 @@ inline std::vector<std::string> LoadPythonPlugins(const std::set<std::string>& p
   }
   return loadedPythonPlugins;
 }
+
+inline py::module_ GetImportlibMetadata()
+{
+  auto sys = py::module_::import("sys");
+  if(sys.attr("version_info").attr("__lt__")(py::make_tuple(3, 10)))
+  {
+    return py::module_::import("importlib_metadata");
+  }
+  else
+  {
+    return py::module_::import("importlib.metadata");
+  }
+}
+
+class PluginEntrypoint
+{
+public:
+  PluginEntrypoint(py::object object)
+  : m_Entrypoint(object)
+  {
+  }
+
+  py::object& getPyObject()
+  {
+    return m_Entrypoint;
+  }
+
+  const py::object& getPyObject() const
+  {
+    return m_Entrypoint;
+  }
+
+  std::string getValue() const
+  {
+    return m_Entrypoint.attr("value").cast<std::string>();
+  }
+
+  py::object load()
+  {
+    return m_Entrypoint.attr("load")();
+  }
+
+private:
+  py::object m_Entrypoint;
+};
+
+inline std::vector<PluginEntrypoint> GetInstalledPythonPlugins()
+{
+  using namespace pybind11::literals;
+  py::module_ importlibMetadata = GetImportlibMetadata();
+  auto discoveredPlugins = importlibMetadata.attr("entry_points")("group"_a = "simplnx.plugins");
+  std::vector<PluginEntrypoint> plugins;
+  for(py::handle pluginEntryPoint : discoveredPlugins)
+  {
+    plugins.emplace_back(py::reinterpret_borrow<py::object>(pluginEntryPoint));
+  }
+  return plugins;
+}
+
+inline std::vector<std::string> LoadInstalledPythonPlugins(OutputCallback outputCallback = {}, PluginLoadErrorCallback pluginLoadErrorCallback = {}, PythonErrorCallback pythonErrorCallback = {})
+{
+  std::vector<std::string> loadedPythonPlugins;
+
+  try
+  {
+    auto simplnxModule = py::module_::import(SIMPLNX_PYTHON_MODULE);
+    std::vector<PluginEntrypoint> discoveredPlugins = GetInstalledPythonPlugins();
+    if(outputCallback)
+    {
+      std::set<std::string> pluginNames;
+      for(auto& pluginEntryPoint : discoveredPlugins)
+      {
+        pluginNames.insert(pluginEntryPoint.getValue());
+      }
+      outputCallback(fmt::format("Loading installed Python plugins: {}", pluginNames));
+    }
+    for(auto& pluginEntryPoint : discoveredPlugins)
+    {
+      std::string pluginName = pluginEntryPoint.getValue();
+      if(outputCallback)
+      {
+        outputCallback(fmt::format("Attempting to load installed Python plugin: '{}'", pluginName));
+      }
+      try
+      {
+        py::object mod = pluginEntryPoint.load();
+        simplnxModule.attr("load_python_plugin")(mod);
+        if(outputCallback)
+        {
+          outputCallback(fmt::format("Successfully loaded installed Python plugin '{}'", pluginName));
+        }
+        loadedPythonPlugins.push_back(pluginName);
+      } catch(const py::error_already_set& exception)
+      {
+        if(pluginLoadErrorCallback)
+        {
+          pluginLoadErrorCallback(PluginLoadErrorInfo{ExceptionType::Python, pluginName, exception.what()});
+        }
+      } catch(const std::exception& exception)
+      {
+        if(pluginLoadErrorCallback)
+        {
+          pluginLoadErrorCallback(PluginLoadErrorInfo{ExceptionType::Cpp, pluginName, exception.what()});
+        }
+      }
+    }
+  } catch(const py::error_already_set& exception)
+  {
+    if(pythonErrorCallback)
+    {
+      pythonErrorCallback(PythonErrorInfo{ExceptionType::Python, exception.what()});
+    }
+  } catch(const std::exception& exception)
+  {
+    if(pythonErrorCallback)
+    {
+      pythonErrorCallback(PythonErrorInfo{ExceptionType::Cpp, exception.what()});
+    }
+  }
+
+  return loadedPythonPlugins;
+}
 } // namespace nx::python

wrapping/python/plugins/DataAnalysisToolkit/conda/meta.yaml

@@ -0,0 +1,42 @@
+{% set name = "simplnx.DataAnalysisToolkit" %}
+{% set version = "1.0.0" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  - path: ../
+    folder: DataAnalysisToolkit
+
+build:
+  noarch: python
+  script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation
+  number: 0
+
+requirements:
+  build:
+    - python >=3.10
+    - pytest
+    - pip
+  run:
+    - python >=3.10
+    - matplotlib
+    - numpy
+    - opencv
+    - scipy
+    - h5py
+    - pillow
+
+test:
+  imports:
+    - DataAnalysisToolkit
+  commands:
+    - pip check
+  requires:
+    - pip
+
+about:
+  home: https://www.dream3d.io/
+  license: BSD
+  summary: DREAM3D-NX consists of data analysis tools (Filters) that allow for the construction of customized workflows (Pipelines) to analyze data.

wrapping/python/plugins/DataAnalysisToolkit/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/DataAnalysisToolkit"]
+
+[project]
+name = "simplnx.DataAnalysisToolkit"
+version = "1.0.0"
+requires-python = ">=3.10"
+dependencies = [
+  "matplotlib",
+  "numpy",
+  "opencv",
+  "scipy",
+  "h5py",
+  "pillow",
+]
+
+[project.entry-points."simplnx.plugins"]
+plugin = "DataAnalysisToolkit"

wrapping/python/plugins/ExamplePlugin/conda/meta.yaml

@@ -0,0 +1,36 @@
+{% set name = "simplnx.ExamplePlugin" %}
+{% set version = "1.0.0" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  - path: ../
+    folder: ExamplePlugin
+
+build:
+  noarch: python
+  script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation
+  number: 0
+
+requirements:
+  build:
+    - python >=3.10
+    - pytest
+    - pip
+  run:
+    - python >=3.10
+
+test:
+  imports:
+    - ExamplePlugin
+  commands:
+    - pip check
+  requires:
+    - pip
+
+about:
+  home: https://www.dream3d.io/
+  license: BSD
+  summary: DREAM3D-NX consists of data analysis tools (Filters) that allow for the construction of customized workflows (Pipelines) to analyze data.

wrapping/python/plugins/ExamplePlugin/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ExamplePlugin"]
+
+[project]
+name = "simplnx.ExamplePlugin"
+version = "1.0.0"
+requires-python = ">=3.10"
+
+[project.entry-points."simplnx.plugins"]
+plugin = "ExamplePlugin"