PYTHON: Add API to DataStructure, Document API, add example uses. (#792)

Signed-off-by: Michael Jackson <[email protected]>

.github/workflows/linux.yml

@@ -43,7 +43,7 @@ jobs:
           sudo apt-get -y install ninja-build
       - name: Install Sphinx
         run: |
-          sudo pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme
+          sudo pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
       - name: Setup NuGet Credentials
         shell: bash
         run: |

.github/workflows/macos.yml

@@ -32,7 +32,7 @@ jobs:
           brew install ninja
       - name: Install Sphinx
         run: |
-          sudo pip3 install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme
+          sudo pip3 install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
       - name: Setup NuGet Credentials
         shell: bash
         run: |

.github/workflows/windows.yml

@@ -49,7 +49,7 @@ jobs:
           -source "https://nuget.pkg.github.com/BlueQuartzSoftware/index.json"
       - name: Install Sphinx
         run: |
-          pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme
+          pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
       - name: Configure
         run: |
           cmake --preset ci-windows-${{matrix.toolset}} ${{github.workspace}} -T ${{matrix.toolset}}

CMakeLists.txt

@@ -63,7 +63,7 @@ option(COMPLEX_ENABLE_INSTALL "Enables COMPLEX install rules" ON)
 file(TO_CMAKE_PATH "${CMAKE_COMMAND}" CMAKE_COMMAND_NORM)
 
 project(complex
-  VERSION 1.2.0
+  VERSION 1.2.1
   DESCRIPTION "SIMPL Redesign"
   HOMEPAGE_URL "https://github.com/BlueQuartzSoftware/complex"
   LANGUAGES CXX

conda/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "complex" %}
-{% set version = "1.2.0" %}
+{% set version = "1.2.1" %}
 
 package:
   name: {{ name|lower }}

conda/recipe.yaml

@@ -1,5 +1,5 @@
 context:
-  version: "1.2.0"
+  version: "1.2.1"
   name: complex
 
 package:

src/Plugins/ComplexCore/wrapping/python/complexpy.cpp

@@ -10,7 +10,6 @@
 #include "ComplexCore/ComplexCoreFilterBinding.hpp"
 
 #include <ComplexCore/ComplexCorePlugin.hpp>
-
 #include <complex/DataStructure/AttributeMatrix.hpp>
 #include <complex/DataStructure/DataArray.hpp>
 #include <complex/DataStructure/DataGroup.hpp>
@@ -80,6 +79,7 @@
 #include <complex/Pipeline/AbstractPipelineNode.hpp>
 #include <complex/Pipeline/Pipeline.hpp>
 #include <complex/Pipeline/PipelineFilter.hpp>
+#include <complex/Utilities/DataGroupUtilities.hpp>
 
 #include <fmt/ranges.h>
 
@@ -505,12 +505,118 @@ PYBIND11_MODULE(complex, mod)
 
   dataObject.def_property_readonly("id", &DataObject::getId);
   dataObject.def_property_readonly("name", &DataObject::getName);
+  dataObject.def_property_readonly("type", &DataObject::getDataObjectType);
 
   dataStructure.def(py::init<>());
   dataStructure.def("__getitem__", py::overload_cast<const DataPath&>(&DataStructure::getSharedData));
+  dataStructure.def("__getitem__", [](DataStructure& self, const std::string& path) {
+    auto pathConversionResult = DataPath::FromString(path);
+    if(!pathConversionResult)
+    {
+      return std::shared_ptr<DataObject>(nullptr);
+    }
+    return self.getSharedData(pathConversionResult.value());
+  });
   dataStructure.def_property_readonly("size", &DataStructure::getSize);
   dataStructure.def("__len__", &DataStructure::getSize);
   dataStructure.def("remove", py::overload_cast<const DataPath&>(&DataStructure::removeData));
+  dataStructure.def("remove", [](DataStructure& self, const std::string& path) {
+    auto pathConversionResult = DataPath::FromString(path);
+    if(!pathConversionResult)
+    {
+      return false;
+    }
+    return self.removeData(pathConversionResult.value());
+  });
+  dataStructure.def("hierarchy_to_str", [](DataStructure& self) {
+    std::stringstream ss;
+    self.exportHierarchyAsText(ss);
+    return ss.str();
+  });
+  dataStructure.def("hierarchy_to_graphviz", [](DataStructure& self) {
+    std::stringstream ss;
+    self.exportHierarchyAsGraphViz(ss);
+    return ss.str();
+  });
+  dataStructure.def("get_children", [](DataStructure& self, complex::DataPath& parentPath) {
+    if(parentPath.empty())
+    {
+      std::vector<DataPath> outputPaths;
+      for(const auto* object : self.getTopLevelData())
+      {
+        auto topLevelPath = DataPath::FromString(object->getDataPaths()[0].getTargetName()).value();
+        outputPaths.push_back(topLevelPath);
+      }
+      return outputPaths;
+    }
+    else
+    {
+      auto result = complex::GetAllChildDataPaths(self, parentPath);
+      if(result)
+      {
+        return result.value();
+      }
+      return std::vector<DataPath>{};
+    }
+  });
+  dataStructure.def("get_children", [](DataStructure& self, const std::string& parentPath) {
+    if(parentPath.empty())
+    {
+      std::vector<DataPath> outputPaths;
+      for(const auto* object : self.getTopLevelData())
+      {
+        auto topLevelPath = DataPath::FromString(object->getDataPaths()[0].getTargetName()).value();
+        outputPaths.push_back(topLevelPath);
+      }
+      return outputPaths;
+    }
+    else
+    {
+      auto pathConversionResult = DataPath::FromString(parentPath);
+      if(!pathConversionResult)
+      {
+        return std::vector<DataPath>{};
+      }
+
+      auto result = complex::GetAllChildDataPaths(self, pathConversionResult.value());
+      if(result)
+      {
+        return result.value();
+      }
+      return std::vector<DataPath>{};
+    }
+  });
+
+  auto dataObjectType = py::enum_<DataObject::Type>(dataObject, "DataObjectType");
+  dataObjectType.value("DataObject", DataObject::Type::DataObject);
+  dataObjectType.value("DynamicListArray", DataObject::Type::DynamicListArray);
+  dataObjectType.value("ScalarData", DataObject::Type::ScalarData);
+  dataObjectType.value("BaseGroup", DataObject::Type::BaseGroup);
+  dataObjectType.value("AttributeMatrix", DataObject::Type::AttributeMatrix);
+  dataObjectType.value("DataGroup", DataObject::Type::DataGroup);
+  dataObjectType.value("IDataArray", DataObject::Type::IDataArray);
+  dataObjectType.value("DataArray", DataObject::Type::DataArray);
+  dataObjectType.value("IGeometry", DataObject::Type::IGeometry);
+  dataObjectType.value("IGridGeometry", DataObject::Type::IGridGeometry);
+  dataObjectType.value("RectGridGeom", DataObject::Type::RectGridGeom);
+  dataObjectType.value("ImageGeom", DataObject::Type::ImageGeom);
+  dataObjectType.value("INodeGeometry0D", DataObject::Type::INodeGeometry0D);
+  dataObjectType.value("VertexGeom", DataObject::Type::VertexGeom);
+  dataObjectType.value("INodeGeometry1D", DataObject::Type::INodeGeometry1D);
+  dataObjectType.value("EdgeGeom", DataObject::Type::EdgeGeom);
+  dataObjectType.value("INodeGeometry2D", DataObject::Type::INodeGeometry2D);
+  dataObjectType.value("QuadGeom", DataObject::Type::QuadGeom);
+  dataObjectType.value("TriangleGeom", DataObject::Type::TriangleGeom);
+  dataObjectType.value("INodeGeometry3D", DataObject::Type::INodeGeometry3D);
+  dataObjectType.value("HexahedralGeom", DataObject::Type::HexahedralGeom);
+  dataObjectType.value("TetrahedralGeom", DataObject::Type::TetrahedralGeom);
+  dataObjectType.value("INeighborList", DataObject::Type::INeighborList);
+  dataObjectType.value("NeighborList", DataObject::Type::NeighborList);
+  dataObjectType.value("StringArray", DataObject::Type::StringArray);
+  dataObjectType.value("AbstractMontage", DataObject::Type::AbstractMontage);
+  dataObjectType.value("GridMontage", DataObject::Type::GridMontage);
+  dataObjectType.value("Unknown", DataObject::Type::Unknown);
+  dataObjectType.value("Any", DataObject::Type::Any);
 
   py::class_<BaseGroup, DataObject, std::shared_ptr<BaseGroup>> baseGroup(mod, "BaseGroup");
   baseGroup.def("contains", py::overload_cast<const std::string&>(&BaseGroup::contains, py::const_));
@@ -593,6 +699,7 @@ PYBIND11_MODULE(complex, mod)
   iDataArray.def_property_readonly("store", py::overload_cast<>(&IDataArray::getIDataStore));
   iDataArray.def_property_readonly("tdims", &IDataArray::getTupleShape);
   iDataArray.def_property_readonly("cdims", &IDataArray::getComponentShape);
+  iDataArray.def_property_readonly("data_type", &IDataArray::getDataType);
 
   auto dataArrayInt8 = COMPLEX_PY_BIND_DATA_ARRAY(mod, Int8Array);
   auto dataArrayUInt8 = COMPLEX_PY_BIND_DATA_ARRAY(mod, UInt8Array);

src/complex/DataStructure/DataStructure.cpp

@@ -908,15 +908,14 @@ void DataStructure::exportHierarchyAsText(std::ostream& outputStream) const
   {
     auto topLevelPath = DataPath::FromString(object->getDataPaths()[0].getTargetName()).value();
     outputStream << k_Delimiter << topLevelPath.getTargetName() << "\n";
-    auto optionalChildPaths = GetAllChildDataPaths(*this, topLevelPath);
+    auto optionalDataPaths = GetAllChildDataPaths(*this, topLevelPath);
 
-    if(optionalChildPaths.has_value() && !optionalChildPaths.value().empty())
+    if(optionalDataPaths.has_value() && !optionalDataPaths.value().empty())
     {
       // Begin recursion
-      recurseHierarchyToText(outputStream, optionalChildPaths.value(), "");
+      recurseHierarchyToText(outputStream, optionalDataPaths.value(), "");
     }
   }
-  outputStream << '\n'; // for readability
 }
 
 void DataStructure::recurseHierarchyToGraphViz(std::ostream& outputStream, const std::vector<DataPath> paths, const std::string& parent) const
@@ -929,12 +928,15 @@ void DataStructure::recurseHierarchyToGraphViz(std::ostream& outputStream, const
 
     // pull child paths or skip to next iteration
     auto optionalChildPaths = GetAllChildDataPaths(*this, path);
-    if(optionalChildPaths.has_value() && !optionalChildPaths.value().empty())
+    if(!optionalChildPaths.has_value() || optionalChildPaths.value().empty())
     {
-      // Begin recursion
-      recurseHierarchyToGraphViz(outputStream, optionalChildPaths.value(), path.getTargetName());
+      continue;
     }
+
+    // recurse
+    recurseHierarchyToGraphViz(outputStream, optionalChildPaths.value(), path.getTargetName());
   }
+  // outputStream << "\n"; // for readability
 }
 
 void DataStructure::recurseHierarchyToText(std::ostream& outputStream, const std::vector<DataPath> paths, std::string indent) const

test/DataStructTest.cpp

@@ -17,6 +17,7 @@
 #include "complex/UnitTest/UnitTestCommon.hpp"
 #include "complex/Utilities/DataArrayUtilities.hpp"
 #include "complex/Utilities/DataGroupUtilities.hpp"
+#include "complex/unit_test/complex_test_dirs.hpp"
 
 #include <catch2/catch.hpp>
 
@@ -42,6 +43,26 @@ constexpr StringLiteral k_SharedPolyhedrons = "SharedPolyhedronList";
 constexpr StringLiteral k_HexGeo = "Hex Geometry";
 } // namespace
 
+// This test will ensure we don't run into runtime exceptions trying to run the functions
+TEST_CASE("ComplexCore::exportHierarchyAsGraphViz")
+{
+  DataStructure dataStructure = UnitTest::CreateComplexMultiLevelDataGraph();
+  auto outputPath = fs::path(fmt::format("{}/exportHierarchyAsGraphViz_test.dot", unit_test::k_BinaryTestOutputDir));
+  std::cout << outputPath << std::endl;
+  std::ofstream output(outputPath, std::ios_base::trunc);
+  dataStructure.exportHierarchyAsGraphViz(output);
+}
+
+// This test will ensure we don't run into runtime exceptions trying to run the functions
+TEST_CASE("ComplexCore::exportHierarchyAsText")
+{
+  DataStructure dataStructure = UnitTest::CreateComplexMultiLevelDataGraph();
+  auto outputPath = fs::path(fmt::format("{}/exportHierarchyAsText_test.txt", unit_test::k_BinaryTestOutputDir));
+  std::cout << outputPath << std::endl;
+  std::ofstream output(outputPath, std::ios_base::trunc);
+  dataStructure.exportHierarchyAsText(output);
+}
+
 DataStructure createTestDataStructure()
 {
   DataStructure dataStruct;

wrapping/python/ReadMe.md

@@ -1,5 +1,20 @@
 # Complex Python Information
 
+## Checklist when updating Python Bindings
+
+- Update Version number.
+
+  - If you add API then update the third number
+  - If you break API (any where in complex), update the second number
+
+- Document **ALL** new API in the appropriate documentation file(s)
+- Create a ReleaseNotes_1XX.rst file with the appropriate highlights from the release
+- Create example python code for any new API
+- Update example python codes for any changed API
+- Add unit test for any NEW API
+- Update Unit test for changed API
+- Tag the repository with the correct version in the correct semantic form
+
 ## Creating the Python Bindings
 
 ### MacOS: Use Mamba
@@ -32,22 +47,27 @@ Create the package from the `complex` sources
 
 ```shell
     [user@host] $ cd complex/conda
-    (nx-build) [user@host] $ conda mambabuild --python 3.8 .; conda mambabuild --python 3.9 .; conda mambabuild --python 3.10 . 
+    (nx-build) [user@host] $ conda mambabuild --python 3.8 .
+    (nx-build) [user@host] $ conda mambabuild --python 3.9 .
+    (nx-build) [user@host] $ conda mambabuild --python 3.10 . 
 
 ```
 
 ### Windows/Linux
 
 ```shell
+    [user@host] $ conda create on nx-build python=3.10 mamba boa
     [user@host] $ cd complex/conda
     [user@host] $ conda build . 
 ```
 
 For faster environment solves mamba can also be used.
 
 ```shell
-conda install boa
-conda mambabuild --python 3.8|3.9|3.10 .
+    [user@host] $ conda install boa
+    [user@host] $ conda mambabuild --python 3.8 .
+    [user@host] $ conda mambabuild --python 3.9 .
+    [user@host] $ conda mambabuild --python 3.10 .
 ```
 
 ### Uploading to Anaconda.org
@@ -56,15 +76,15 @@ Open a "base" anaconda prompt.
 
 ```shell
     [user@host] $ anaconda login
-    [user@host] $ anaconda upload -c bluequartzsoftware [path/to/tar.bz]
+    [user@host] $ anaconda upload --user bluequartzsoftware [path/to/tar.bz]
 ```
 
 ## Using the Python Bindings
 
 ```shell
 conda config --add channels conda-forge
 conda config --set channel_priority strict
-conda create -n cxpython python=3.8
+conda create -n cxpython python=3.10
 conda activate cxpython
 conda install -c bluequartzsoftware complex
 ```

wrapping/python/ToDo.md

@@ -0,0 +1,10 @@
+# ToDo list for python bindings
+
+- Create each kind of geometry using the python bindings
+
+  - combine that with numpy to generate some nodes for a node geometry
+
+- Document all wrapped methods in the complexpy.cpp file.
+- Example of using MatPlotLib to generate a plot and save the whole plot as an image file
+- Example of using Pandas DataFrame
+- Example of looping over X number of EBSD files and generate a Pole figure for each file

wrapping/python/docs/index_template.rst

@@ -5,7 +5,7 @@
 DREAM3D-NX Python Docs
 =======================
 
-Latest Version: 1.2.0
+Latest Version: 1.2.1
 ---------------------
 
 The *complex* library can be installed through an Anaconda packages from the *BlueQuartzSoftware* channel. This can be achieved
@@ -32,6 +32,7 @@ by creating a new virtual environment
    Reference_Frame_Notes
    ReleaseNotes_110
    ReleaseNotes_120
+   ReleaseNotes_121
 @PLUGIN_LIST@
 
 Indices and tables

wrapping/python/docs/source/API.rst

@@ -352,7 +352,7 @@ General Parameters
   .. code:: python
 
     generated_file_list_value = cx.GeneratedFileListParameter.ValueType()
-    generated_file_list_value.input_path = "/Users/mjackson/DREAM3DNXData/Data/Porosity_Image"
+    generated_file_list_value.input_path = "DREAM3DNXData/Data/Porosity_Image"
     generated_file_list_value.ordering = cx.GeneratedFileListParameter.Ordering.LowToHigh
 
     generated_file_list_value.file_prefix = "slice-"