merian-nodes: Attempt to recover from errors in process

LDAP · Dec 2, 2024 · 640dbe8 · 640dbe8
1 parent 20fe1ba
commit 640dbe8
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 4 deletions.
diff --git a/include/merian-nodes/graph/graph.hpp b/include/merian-nodes/graph/graph.hpp
@@ -3,6 +3,7 @@
 #include "errors.hpp"
 #include "graph_run.hpp"
 #include "merian/utils/chrono.hpp"
+#include "merian/utils/vector.hpp"
 #include "node.hpp"
 #include "resource.hpp"
 
@@ -54,8 +55,10 @@ struct NodeData {
 
     // User disabled
     bool disable{};
-    // Disabled because a input is not connected;
+    // Errors during build
     std::vector<std::string> errors{};
+    // Errors during run - triggers rebuild and gets build error
+    std::vector<std::string> run_errors{};
 
     // Cache input connectors (node->describe_inputs())
     // (on start_nodes added and checked for name conflicts)
@@ -587,7 +590,20 @@ class Graph : public std::enable_shared_from_this<Graph<ITERATIONS_IN_FLIGHT>> {
                 if (debug_utils)
                     debug_utils->cmd_begin_label(cmd, registry.node_name(node));
 
-                run_node(run, cmd, node, data, profiler);
+                try {
+                    run_node(run, cmd, node, data, profiler);
+                } catch (const graph_errors::node_error& e) {
+                    data.run_errors.emplace_back(fmt::format("node error: {}", e.what()));
+                } catch (const ShaderCompiler::compilation_failed& e) {
+                    data.run_errors.emplace_back(fmt::format("compilation failed: {}", e.what()));
+                }
+                if (!data.run_errors.empty()) {
+                    SPDLOG_ERROR("executing node '{}' failed:\n - {}", data.identifier,
+                                 fmt::join(data.run_errors, "\n   - "));
+
+                    request_reconnect();
+                    SPDLOG_ERROR("emergency reconnect.");
+                }
 
                 if (debug_utils)
                     debug_utils->cmd_end_label(cmd);
@@ -1595,6 +1611,11 @@ class Graph : public std::enable_shared_from_this<Graph<ITERATIONS_IN_FLIGHT>> {
                 to_erase.push_back(node);
                 continue;
             }
+            if (!data.run_errors.empty()) {
+                SPDLOG_DEBUG("node {} ({}) has run errors, converting to build errors.");
+                move_all(data.errors, data.run_errors);
+                data.run_errors.clear();
+            }
             if (!data.errors.empty()) {
                 SPDLOG_DEBUG("node {} ({}) is erroneous, skipping...", data.identifier,
                              registry.node_name(node));

diff --git a/include/merian-nodes/graph/node.hpp b/include/merian-nodes/graph/node.hpp
@@ -35,8 +35,8 @@ class Node : public std::enable_shared_from_this<Node> {
     //
     // Note that input and output names must be unique.
     //
-    // If you throw node_error or compilation_failed the graph will disable the node for this connect attempt and
-    // set the error state for this node. Your inputs will then be invisible.
+    // If you throw node_error or compilation_failed the graph will disable the node for this
+    // connect attempt and set the error state for this node. Your inputs will then be invisible.
     [[nodiscard]]
     virtual std::vector<InputConnectorHandle> describe_inputs() {
         return {};
@@ -101,6 +101,9 @@ class Node : public std::enable_shared_from_this<Node> {
     //
     // You can provide data that that is required for the current run by setting the io map
     // in_flight_data. The pointer is persisted and supplied again after (graph ring size - 1) runs.
+    //
+    // You can throw node_error and compilation_failed here. The graph then attemps to finish the
+    // run and rebuild, however this is not supported and not recommened.
     virtual void process([[maybe_unused]] GraphRun& run,
                          [[maybe_unused]] const vk::CommandBuffer& cmd,
                          [[maybe_unused]] const DescriptorSetHandle& descriptor_set,