Communicate remote errors to UI

Viewer/ecflowUI/src/VAvisoAttr.cpp

@@ -42,7 +42,7 @@ QString VAvisoAttrType::toolTip(QStringList d) const {
         t += "<b>Revision:</b> " + d[RevisionIndex] + "<br>";
         t += "<b>Auth:</b> " + d[AuthIndex];
         if (auto& reason = d[ReasonIndex]; !reason.isEmpty()) {
-            t += "<br><b>Reason:</b> " + d[ReasonIndex];
+            t += "<br><b>Reason:</b> <span style=\"color:red\">" + d[ReasonIndex] + "</span>";
         }
     }
     return t;

Viewer/ecflowUI/src/VIcon.cpp

@@ -139,6 +139,12 @@ class VCheckpointErrorIcon : public VIcon {
     bool show(VNode*) override;
 };
 
+class VRemoteErrorIcon : public VIcon {
+public:
+    explicit VRemoteErrorIcon(const std::string& name) : VIcon(name) {}
+    bool show(VNode*) override;
+};
+
 //==========================================================
 //
 // Create VIcon instances
@@ -162,6 +168,7 @@ static VRestoredIcon restoredIcon("restored");
 static VSlowJobCreationIcon slowJobCreationIcon("slow_job");
 static VNoLogIcon noLog("no_log");
 static VCheckpointErrorIcon noCheckptIcon("checkpt_err");
+static VRemoteErrorIcon remoteErrorIcon("remote_err");
 
 //==========================================================
 //
@@ -557,3 +564,11 @@ bool VCheckpointErrorIcon::show(VNode* n) {
 
     return n->isFlagSet(ecf::Flag::CHECKPT_ERROR);
 }
+
+bool VRemoteErrorIcon::show(VNode* n) {
+    if (!n || n->isServer()) {
+        return false;
+    }
+
+    return n->isFlagSet(ecf::Flag::REMOTE_ERROR);
+}

Viewer/ecflowUI/src/VMirrorAttr.cpp

@@ -39,7 +39,7 @@ QString VMirrorAttrType::toolTip(QStringList d) const {
         t += "<b>SSL:</b> " + d[SslIndex] + "<br>";
         t += "<b>Auth:</b> " + d[AuthIndex];
         if (const auto& reason = d[ReasonIndex]; !reason.isEmpty()) {
-            t += "<br><b>Reason:</b> " + d[ReasonIndex];
+            t += "<br><b>Reason:</b> <span style=\"color:red\">" + d[ReasonIndex] + "</span>";
         }
     }
     return t;

Viewer/ecflowUI/src/viewer.qrc

@@ -14,6 +14,7 @@
     <file alias="icon_archived.svg">../images/icon_archived.svg</file>
     <file alias="icon_calendar.svg">../images/icon_calendar.svg</file>
     <file alias="icon_checkpt_err.svg">../images/icon_checkpt_err.svg</file>
+    <file alias="icon_remote_err.svg">../images/icon_no_access.svg</file>
     <file alias="icon_clock.svg">../images/icon_clock.svg</file>     
     <file alias="icon_clock_free.svg">../images/icon_clock_free.svg</file>
     <file alias="icon_complete.svg">../images/icon_complete.svg</file>

libs/core/src/ecflow/core/Ecf.cpp

@@ -13,8 +13,8 @@
 bool Ecf::server_                            = false;
 bool Ecf::debug_equality_                    = false;
 unsigned int Ecf::debug_level_               = 0;
-unsigned int Ecf::state_change_no_           = 0;
-unsigned int Ecf::modify_change_no_          = 0;
+thread_local Ecf::atomic_counter_t Ecf::state_change_no_  = 0;
+thread_local Ecf::atomic_counter_t Ecf::modify_change_no_ = 0;
 bool DebugEquality::ignore_server_variables_ = false;
 
 const char* Ecf::SERVER_NAME() {
@@ -59,7 +59,7 @@ const std::string& Ecf::CHECK_CMD() {
     return CHECK_CMD;
 }
 
-// -remote has been removed form firefox, since version 39
+//-remote has been removed from firefox, since version 39
 //-remote openfile(file)                    -> -file <file>
 //-remote openurl(url)                      -> -url <url>
 //-remote openurl(url,new-window)           -> -new-window <url>
@@ -80,15 +80,14 @@ const std::string& Ecf::URL() {
     return URL;
 }
 
-unsigned int Ecf::incr_state_change_no() {
+Ecf::counter_t Ecf::incr_state_change_no() {
     if (server_) {
         return ++state_change_no_;
     }
     return state_change_no_;
 }
 
-unsigned int Ecf::incr_modify_change_no() {
-
+Ecf::counter_t Ecf::incr_modify_change_no() {
     if (server_) {
         return ++modify_change_no_;
     }

libs/core/src/ecflow/core/Ecf.hpp

@@ -15,35 +15,46 @@
 /// \brief Provides globals used by server for determining change
 ///
 
+#include <atomic>
 #include <string>
 
-// class Ecf: This class is used in the server to determine incremental changes
-//            to the data model. Each Node/attribute stores a state change no
-//            When ever there is a change, we increment local state change
-//            number with this global.
-// When making large scale changes, ie nodes added or deleted we use modify change no
-// Note: The client will need to at some point copy over the full defs
-//       at this point the state change no add modify number is also copied.
-//       The client passes these two number back to server, the server then
-//       uses these two numbers to determine what's changed.
-//
+/**
+ * This class holds *global data*, and is used in the server to determine incremental changes to the data model.
+ *
+ * This data is composed of two parts:
+ *  - `state_change_no_`: leads the number of node state changes on the currently loaded defs
+ *  - `modify_change_no_`: leads the number of structural changes to the currently loaded defs
+ *
+ * Each Node/Attribute stores a local `state_change_no_`, and whenever there is a change, this local number is
+ * assigned based on the incremented value of the `global state_change_no_`.
+ *
+ * When making large scale changes (e.g. adding or deleting nodes), we increment the global `modify_change_no_`.
+ *
+ * The synchronization strategy is such that when the client eventually copies over the full defs, the state_change_no_
+ * and modify_change_no_ are also copied. In future synchronization attempts, the client includes these two numbers in
+ * the sync request, and thus allows the server to determine what has changed.
+ */
+
 class Ecf {
 public:
+    using counter_t        = unsigned int;
+    using atomic_counter_t = std::atomic<counter_t>;
+
     // Disable default construction
     Ecf() = delete;
     // Disable copy (and move) semantics
     Ecf(const Ecf&)                  = delete;
     const Ecf& operator=(const Ecf&) = delete;
 
     /// Increment and then return state change no
-    static unsigned int incr_state_change_no();
-    static unsigned int state_change_no() { return state_change_no_; }
-    static void set_state_change_no(unsigned int x) { state_change_no_ = x; }
+    static counter_t incr_state_change_no();
+    static counter_t state_change_no() { return state_change_no_; }
+    static void set_state_change_no(counter_t x) { state_change_no_ = x; }
 
     /// The modify_change_no_ is used for node addition and deletion and re-ordering
-    static unsigned int incr_modify_change_no();
-    static unsigned int modify_change_no() { return modify_change_no_; }
-    static void set_modify_change_no(unsigned int x) { modify_change_no_ = x; }
+    static counter_t incr_modify_change_no();
+    static counter_t modify_change_no() { return modify_change_no_; }
+    static void set_modify_change_no(counter_t x) { modify_change_no_ = x; }
 
     /// Returns true if we are on the server side.
     /// Only in server side do we increment state/modify numbers
@@ -80,8 +91,8 @@ class Ecf {
     static bool server_;
     static bool debug_equality_;
     static unsigned int debug_level_;
-    static unsigned int state_change_no_;
-    static unsigned int modify_change_no_;
+    static thread_local atomic_counter_t state_change_no_;
+    static thread_local atomic_counter_t modify_change_no_;
 };
 
 /// Make sure the Ecf number don't change

libs/core/src/ecflow/core/Serialization.hpp

@@ -65,11 +65,14 @@ void save_as_string(std::string& outbound_data, const T& t) {
     }
     outbound_data = archive_stream.str();
 
-    //std::cout << "*** save_as_string: " << outbound_data << "\n";
+    std::cout << "*** save_as_string: " << outbound_data << "\n";
 }
 
 template <typename T>
 void restore_from_string(const std::string& archive_data, T& restored) {
+
+    std::cout << "*** restore_from_string: " << archive_data << "\n";
+
     std::istringstream archive_stream(archive_data);
     cereal::JSONInputArchive iarchive(archive_stream); // Create an input archive
     iarchive(restored);                                // Read the data from the archive

libs/node/src/ecflow/node/AvisoAttr.cpp

@@ -16,6 +16,7 @@
 #include "ecflow/core/Message.hpp"
 #include "ecflow/core/exceptions/Exceptions.hpp"
 #include "ecflow/node/Node.hpp"
+#include "ecflow/node/Operations.hpp"
 
 namespace ecf {
 
@@ -48,6 +49,13 @@ AvisoAttr::AvisoAttr(Node* parent,
     }
 }
 
+AvisoAttr AvisoAttr::make_detached() const {
+    AvisoAttr detached   = *this;
+    detached.parent_     = nullptr;
+    detached.controller_ = nullptr;
+    return detached;
+}
+
 void AvisoAttr::set_listener(std::string_view listener) {
     state_change_no_ = Ecf::incr_state_change_no();
 
@@ -108,12 +116,29 @@ bool AvisoAttr::isFree() const {
         return a.configuration.revision() < b.configuration.revision();
     });
 
-    // (b) update the revision, in the listener configuration
-    this->revision_ = max->configuration.revision();
-    ALOG(D, "AvisoAttr: " << aviso_path << " updated revision to " << this->revision_);
     state_change_no_ = Ecf::incr_state_change_no();
 
-    return true;
+    // (b) update the revision, in the listener configuration
+    if (max->notification.success()) {
+        ALOG(D, "AvisoAttr::isFree: " << aviso_path << " updated revision to " << this->revision_);
+        this->revision_ = max->configuration.revision();
+        parent_->flag().clear(Flag::REMOTE_ERROR);
+        parent_->flag().set_state_change_no(state_change_no_);
+        reason_ = "";
+
+        for (auto* parent = parent_; parent; parent = parent->parent()) {
+            parent->set_state_change_no(state_change_no_);
+        }
+        return max->notification.match().has_value();
+    }
+    else {
+        parent_->flag().set(Flag::REMOTE_ERROR);
+        parent_->flag().set_state_change_no(state_change_no_);
+        reason_ = max->notification.reason();
+
+        ecf::visit_parents(*parent_, [n = this->state_change_no_](Node& node) { node.set_state_change_no(n); });
+        return false;
+    }
 }
 
 void AvisoAttr::start() const {
@@ -157,17 +182,21 @@ void AvisoAttr::start_controller(const std::string& aviso_path,
                                  std::uint32_t polling,
                                  const std::string& aviso_auth) const {
 
-    // Controller -- start up the Aviso controller, and subscribe the Aviso listener
-    controller_ = std::make_shared<controller_t>();
-    controller_->subscribe(ecf::service::aviso::AvisoRequest::make_listen_start(
-        aviso_path, aviso_listener, aviso_url, aviso_schema, polling, revision_, aviso_auth));
-    // Controller -- effectively start the Aviso listener
-    // n.b. this must be done after subscribing in the controller, so that the polling interval is set
-    controller_->start();
+    if (!controller_) {
+        // Controller -- start up the Aviso controller, and subscribe the Aviso listener
+        controller_ = std::make_shared<controller_t>();
+        controller_->subscribe(ecf::service::aviso::AvisoRequest::make_listen_start(
+            aviso_path, aviso_listener, aviso_url, aviso_schema, polling, revision_, aviso_auth));
+        // Controller -- effectively start the Aviso listener
+        // n.b. this must be done after subscribing in the controller, so that the polling interval is set
+        controller_->start();
+    }
 }
 
 void AvisoAttr::stop_controller(const std::string& aviso_path) const {
     if (controller_ != nullptr) {
+        ALOG(D, "AvisoAttr: finishing polling for Aviso attribute (" << parent_path_ << ":" << name_ << ")");
+
         controller_->unsubscribe(ecf::service::aviso::AvisoRequest::make_listen_finish(aviso_path));
 
         // Controller -- shutdown up the Aviso controller
@@ -178,7 +207,6 @@ void AvisoAttr::stop_controller(const std::string& aviso_path) const {
 
 void AvisoAttr::finish() const {
     using namespace ecf;
-    LOG(Log::DBG, Message("AvisoAttr: unsubscribe Aviso attribute (name: ", name_, ", listener: ", listener_, ")"));
 
     std::string aviso_path = path();
     stop_controller(aviso_path);

libs/node/src/ecflow/node/AvisoAttr.hpp

@@ -78,6 +78,8 @@ class AvisoAttr {
 
     AvisoAttr& operator=(const AvisoAttr& rhs) = default;
 
+    [[nodiscard]] AvisoAttr make_detached() const;
+
     [[nodiscard]] inline Node* parent() const { return parent_; }
     [[nodiscard]] inline const std::string& name() const { return name_; }
     [[nodiscard]] inline const std::string& listener() const { return listener_; }
@@ -124,7 +126,7 @@ class AvisoAttr {
     polling_t polling_;
 
     auth_t auth_;
-    reason_t reason_;
+    mutable reason_t reason_{};
 
     // The following are mutable as they are modified by the const method isFree()
     mutable revision_t revision_;