Shell commands and function callbacks (#1193)

* WIP: Shell commands and function callbacks (#1175)

Work from @siramok.

* Initial callback and shell command implementations

* Initial work to combine commands and callbacks

* Move callback registration and execution into Command

* Catch the case where neither action is defined

* Allow defining multiline shell commands and callbacks

* Minor formatting tweaks

* Initial implementation of triggers with callbacks

* Improved organization, more error catching

* Make sure that triggers either have a condition or callback

* Make void callbacks take conduit node parameters

* Initial attempt at exposing callbacks through ascent-jupyter-bridge

* Let void callbacks return arbitrary data via conduit nodes

* Disallow anonymous callbacks, add some tests

* Add tests for shell commands and callbacks

* Refactor callback API into the main API, adjusts tests to reflect the change

* Minor cleanups that I missed

* Fix minor bug with the trigger + callback test

* alt download link for zlib

* typo

* Fix several bugs in the tests

* Implement all suggestions

---------

Co-authored-by: Andres Sewell <[email protected]>
Co-authored-by: Andres Sewell <[email protected]>

src/libs/ascent/CMakeLists.txt

@@ -163,6 +163,7 @@ set(ascent_headers
     runtimes/flow_filters/ascent_runtime_htg_filters.hpp
     runtimes/flow_filters/ascent_runtime_trigger_filters.hpp
     runtimes/flow_filters/ascent_runtime_query_filters.hpp
+    runtimes/flow_filters/ascent_runtime_command_filters.hpp
     runtimes/flow_filters/ascent_runtime_vtkh_utils.hpp
     runtimes/flow_filters/ascent_runtime_utils.hpp
     # utils
@@ -228,6 +229,7 @@ set(ascent_sources
     runtimes/flow_filters/ascent_runtime_htg_filters.cpp
     runtimes/flow_filters/ascent_runtime_trigger_filters.cpp
     runtimes/flow_filters/ascent_runtime_query_filters.cpp
+    runtimes/flow_filters/ascent_runtime_command_filters.cpp
     runtimes/flow_filters/ascent_runtime_utils.cpp
     # utils
     utils/ascent_actions_utils.cpp

src/libs/ascent/ascent.cpp

@@ -858,6 +858,83 @@ about(conduit::Node &n)
 
 }
 
+// Callback maps
+static std::map<std::string, void (*)(conduit::Node &, conduit::Node &)> m_void_callback_map;
+static std::map<std::string, bool (*)(void)> m_bool_callback_map;
+
+//-----------------------------------------------------------------------------
+void
+register_callback(const std::string &callback_name,
+                  void (*callback_function)(conduit::Node &, conduit::Node &))
+{
+    if (callback_name == "")
+    {
+        ASCENT_ERROR("cannot register an anonymous void callback");
+    }
+    else if (m_void_callback_map.count(callback_name) != 0)
+    {
+        ASCENT_ERROR("cannot register more than one void callback under the name '" << callback_name << "'");
+    }
+    else if (m_bool_callback_map.count(callback_name) != 0)
+    {
+        ASCENT_ERROR("cannot register both a void and bool callback under the same name '" << callback_name << "'");
+    }
+    m_void_callback_map.insert(std::make_pair(callback_name, callback_function));
+}
+
+//-----------------------------------------------------------------------------
+void
+register_callback(const std::string &callback_name,
+                  bool (*callback_function)(void))
+{
+    if (callback_name == "")
+    {
+        ASCENT_ERROR("cannot register an anonymous bool callback");
+    }
+    else if (m_bool_callback_map.count(callback_name) != 0)
+    {
+        ASCENT_ERROR("cannot register more than one bool callback under the name '" << callback_name << "'");
+    }
+    else if (m_void_callback_map.count(callback_name) != 0)
+    {
+        ASCENT_ERROR("cannot register both a void and bool callback under the same name '" << callback_name << "'");
+    }
+    m_bool_callback_map.insert(std::make_pair(callback_name, callback_function));
+}
+
+//-----------------------------------------------------------------------------
+void
+execute_callback(std::string callback_name,
+                 conduit::Node &params,
+                 conduit::Node &output)
+{
+    if (m_void_callback_map.count(callback_name) != 1)
+    {
+        ASCENT_ERROR("requested void callback '" << callback_name << "' was never registered");
+    }
+    auto callback_function = m_void_callback_map.at(callback_name);
+    return callback_function(params, output);
+}
+
+//-----------------------------------------------------------------------------
+bool
+execute_callback(std::string callback_name)
+{
+    if (m_bool_callback_map.count(callback_name) != 1)
+    {
+        ASCENT_ERROR("requested bool callback '" << callback_name << "' was never registered");
+    }
+    auto callback_function = m_bool_callback_map.at(callback_name);
+    return callback_function();
+}
+
+//-----------------------------------------------------------------------------
+void
+reset_callbacks()
+{
+    m_void_callback_map.clear();
+    m_bool_callback_map.clear();
+}
 
 //-----------------------------------------------------------------------------
 // -- end ascent:: --

src/libs/ascent/ascent.hpp

@@ -66,13 +66,30 @@ class ASCENT_API Ascent
     conduit::Node  m_info;
 };
 
-
 //-----------------------------------------------------------------------------
 std::string ASCENT_API about();
 
 //-----------------------------------------------------------------------------
 void        ASCENT_API about(conduit::Node &node);
 
+//-----------------------------------------------------------------------------
+void        ASCENT_API register_callback(const std::string &callback_name,
+                                         void (*callback_function)(conduit::Node &, conduit::Node &));
+//-----------------------------------------------------------------------------
+void        ASCENT_API register_callback(const std::string &callback_name,
+                                         bool (*callback_function)(void));
+
+//-----------------------------------------------------------------------------
+void        ASCENT_API execute_callback(std::string callback_name,
+                                        conduit::Node &params,
+                                        conduit::Node &output);
+
+//-----------------------------------------------------------------------------
+bool        ASCENT_API execute_callback(std::string callback_name);
+
+//-----------------------------------------------------------------------------
+void        ASCENT_API reset_callbacks();
+
 };
 //-----------------------------------------------------------------------------
 // -- end ascent:: --

src/libs/ascent/python/ascent_module/ascent_mpi_python.cpp

@@ -151,6 +151,29 @@ struct PyAscent_MPI_Ascent
     Ascent *ascent; // NoteIterator is light weight, we can deal with copies
 };
 
+//---------------------------------------------------------------------------//
+// Helper that promotes ascent error to python error
+//---------------------------------------------------------------------------//
+static void
+PyAscent_MPI_Ascent_Error_To_PyErr(const conduit::Error &e)
+{
+    std::ostringstream oss;
+    oss << "Ascent Error: " << e.message();
+    PyErr_SetString(PyExc_RuntimeError,
+                    oss.str().c_str());
+}
+
+//---------------------------------------------------------------------------//
+// Helper that promotes ascent error to python error
+//---------------------------------------------------------------------------//
+static void
+PyAscent_MPI_Cpp_Error_To_PyErr(const char *msg)
+{
+    std::ostringstream oss;
+    oss << "Ascent Error: " << msg;
+    PyErr_SetString(PyExc_RuntimeError,
+                    oss.str().c_str());
+}
 
 //---------------------------------------------------------------------------//
 static PyObject *
@@ -465,6 +488,72 @@ PyAscent_MPI_about()
     return (PyObject*)py_node_res;
 }
 
+//---------------------------------------------------------------------------//
+// ascent::execute_callback
+//---------------------------------------------------------------------------//
+static PyObject *
+PyAscent_MPI_execute_callback(PyObject *self,
+                              PyObject *args)
+{
+    char *callback_name;
+    PyObject *py_params = NULL;
+    PyObject *py_output = NULL;
+
+    if (!PyArg_ParseTuple(args,
+                          "sOO",
+                          &callback_name,
+                          &py_params,
+                          &py_output))
+    {
+        return NULL;
+    }
+
+    try
+    {
+        if(py_params != NULL && py_output != NULL)
+        {
+            if(!PyConduit_Node_Check(py_params))
+            {
+                PyErr_SetString(PyExc_TypeError,
+                                "Ascent::execute_callback 'params' argument must be a "
+                                "conduit::Node");
+                return NULL;
+            }
+            else if (!PyConduit_Node_Check(py_output))
+            {
+                PyErr_SetString(PyExc_TypeError,
+                                "Ascent::execute_callback 'output' argument must be a "
+                                "conduit::Node");
+                return NULL;
+            }
+            std::string callback_name_string = callback_name;
+            Node *params = PyConduit_Node_Get_Node_Ptr(py_params);
+            Node *output = PyConduit_Node_Get_Node_Ptr(py_output);
+            ascent::execute_callback(callback_name, *params, *output);
+            Py_RETURN_NONE;
+        }
+    }
+    catch(conduit::Error e)
+    {
+        PyAscent_MPI_Ascent_Error_To_PyErr(e);
+        return NULL;
+    }
+    // also try to bottle other errors, to prevent python
+    // from crashing due to uncaught exception
+    catch(std::exception &e)
+    {
+        PyAscent_MPI_Cpp_Error_To_PyErr(e.what());
+        return NULL;
+    }
+    catch(...)
+    {
+        PyAscent_MPI_Cpp_Error_To_PyErr("unknown cpp exception thrown");
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
 //---------------------------------------------------------------------------//
 // Python Module Method Defs
 //---------------------------------------------------------------------------//
@@ -476,6 +565,11 @@ static PyMethodDef ascent_mpi_python_funcs[] =
       METH_NOARGS,
       NULL},
     //-----------------------------------------------------------------------//
+    {"execute_callback",
+     (PyCFunction)PyAscent_MPI_execute_callback,
+      METH_VARARGS,
+      NULL},
+    //-----------------------------------------------------------------------//
     // end ascent methods table
     //-----------------------------------------------------------------------//
     {NULL, NULL, METH_VARARGS, NULL}

src/libs/ascent/python/ascent_module/ascent_python.cpp

@@ -510,7 +510,7 @@ static PyMethodDef PyAscent_Ascent_METHODS[] = {
      METH_VARARGS | METH_KEYWORDS,
      "{todo}"},
     //-----------------------------------------------------------------------//
-     {"execute",
+    {"execute",
      (PyCFunction)PyAscent_Ascent_execute,
      METH_VARARGS | METH_KEYWORDS,
      "{todo}"},
@@ -596,6 +596,72 @@ PyAscent_about()
     return (PyObject*)py_node_res;
 }
 
+//---------------------------------------------------------------------------//
+// ascent::execute_callback
+//---------------------------------------------------------------------------//
+static PyObject *
+PyAscent_execute_callback(PyObject *self,
+                          PyObject *args)
+{
+    char *callback_name;
+    PyObject *py_params = NULL;
+    PyObject *py_output = NULL;
+
+    if (!PyArg_ParseTuple(args,
+                          "sOO",
+                          &callback_name,
+                          &py_params,
+                          &py_output))
+    {
+        return NULL;
+    }
+
+    try
+    {
+        if(py_params != NULL && py_output != NULL)
+        {
+            if(!PyConduit_Node_Check(py_params))
+            {
+                PyErr_SetString(PyExc_TypeError,
+                                "Ascent::execute_callback 'params' argument must be a "
+                                "conduit::Node");
+                return NULL;
+            }
+            else if (!PyConduit_Node_Check(py_output))
+            {
+                PyErr_SetString(PyExc_TypeError,
+                                "Ascent::execute_callback 'output' argument must be a "
+                                "conduit::Node");
+                return NULL;
+            }
+            std::string callback_name_string = callback_name;
+            Node *params = PyConduit_Node_Get_Node_Ptr(py_params);
+            Node *output = PyConduit_Node_Get_Node_Ptr(py_output);
+            ascent::execute_callback(callback_name, *params, *output);
+            Py_RETURN_NONE;
+        }
+    }
+    catch(conduit::Error e)
+    {
+        PyAscent_Ascent_Error_To_PyErr(e);
+        return NULL;
+    }
+    // also try to bottle other errors, to prevent python
+    // from crashing due to uncaught exception
+    catch(std::exception &e)
+    {
+        PyAscent_Cpp_Error_To_PyErr(e.what());
+        return NULL;
+    }
+    catch(...)
+    {
+        PyAscent_Cpp_Error_To_PyErr("unknown cpp exception thrown");
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
 //---------------------------------------------------------------------------//
 // Python Module Method Defs
 //---------------------------------------------------------------------------//
@@ -607,6 +673,11 @@ static PyMethodDef ascent_python_funcs[] =
       METH_NOARGS,
       NULL},
     //-----------------------------------------------------------------------//
+    {"execute_callback",
+     (PyCFunction)PyAscent_execute_callback,
+      METH_VARARGS,
+      NULL},
+    //-----------------------------------------------------------------------//
     // end ascent methods table
     //-----------------------------------------------------------------------//
     {NULL, NULL, METH_VARARGS, NULL}

src/libs/ascent/python/ascent_module/py_src/ascent.py

@@ -24,6 +24,14 @@ def about():
         raise ImportError('failed to import ascent_python, was Ascent built with serial support ENABLE_SERIAL=ON ?')
     return None
 
+def execute_callback(callback_name, params, output):
+    try:
+        from .ascent_python import execute_callback as ascent_execute_callback
+        return ascent_execute_callback(callback_name, params, output)
+    except ImportError:
+        raise ImportError('failed to import ascent_python, was Ascent built with serial support ENABLE_SERIAL=ON ?')
+    return None
+
 def Ascent():
     try:
         from .ascent_python import Ascent as ascent_obj

src/libs/ascent/python/ascent_module/py_src/mpi/ascent_mpi.py

@@ -21,24 +21,31 @@ def about():
         from .ascent_mpi_python import about as ascent_about
         return ascent_about()
     except ImportError:
-        raise ImportError('failed to import ascent_mpi_python, was Ascent built with mpi support?')
+        raise ImportError('failed to import ascent_mpi_python, was Ascent built with MPI support ENABLE_MPI=ON ?')
     return None
 
+def execute_callback(callback_name, params, output):
+    try:
+        from .ascent_mpi_python import execute_callback as ascent_execute_callback
+        return ascent_execute_callback(callback_name, params, output)
+    except ImportError:
+        raise ImportError('failed to import ascent_mpi_python, was Ascent built with MPI support ENABLE_MPI=ON ?')
+    return None
 
 def Ascent():
     try:
         from .ascent_mpi_python import Ascent as ascent_obj
         return ascent_obj()
     except ImportError:
-        raise ImportError('failed to import ascent_mpi_python, was Ascent built with mpi support?')
+        raise ImportError('failed to import ascent_mpi_python, was Ascent built with MPI support ENABLE_MPI=ON ?')
     return None
 
 def jupyter_bridge():
     try:
         from ..bridge_kernel.server import jupyter_extract
         return jupyter_extract()
     except ImportError:
-        raise ImportError('failed to import ascent_mpi_python, was Ascent built with mpi support?')
+        raise ImportError('failed to import ascent_mpi_python, was Ascent built with MPI support ENABLE_MPI=ON ?')
     return None
 
 

src/libs/ascent/runtimes/ascent_empty_runtime.cpp

@@ -99,7 +99,6 @@ EmptyRuntime::Info()
     return m_info;
 }
 
-
 //-----------------------------------------------------------------------------
 void
 EmptyRuntime::Cleanup()