Enable REST/UDP proxies to use HTTP connection

Also, fix an eventual race condition accessing:
 - Ecf::state_change_no_ and Ecf::state_modify_no_
Apart from making these global (instead of thread_local),
the Http library is restricted to the use of a single thread

Re ECFLOW-1957

libs/CMakeLists.txt

@@ -479,6 +479,7 @@ set(srcs
   # Server -- Headers
   server/src/ecflow/server/BaseServer.hpp
   server/src/ecflow/server/CheckPtSaver.hpp
+  server/src/ecflow/server/HttpServer.hpp
   server/src/ecflow/server/NodeTreeTraverser.hpp
   server/src/ecflow/server/PeriodicScheduler.hpp
   server/src/ecflow/server/Server.hpp
@@ -491,6 +492,7 @@ set(srcs
   # Server -- Sources
   server/src/ecflow/server/BaseServer.cpp
   server/src/ecflow/server/CheckPtSaver.cpp
+  server/src/ecflow/server/HttpServer.cpp
   server/src/ecflow/server/NodeTreeTraverser.cpp
   server/src/ecflow/server/PeriodicScheduler.cpp
   server/src/ecflow/server/Server.cpp
@@ -569,8 +571,10 @@ ecbuild_add_library(
     $<$<BOOL:${OPENSSL_FOUND}>:OpenSSL::SSL>
     $<$<NOT:$<BOOL:${APPLE}>>:crypt>
     Threads::Threads
-  DEFINITIONS
+    $<$<BOOL:${ZLIB_FOUND}>:ZLIB::ZLIB>
+  PUBLIC_DEFINITIONS
     CMAKE
+    $<$<BOOL:${ZLIB_FOUND}>:ECF_HTTP_COMPRESSION>
 )
 target_clangformat(ecflow_all)
 

libs/base/src/ecflow/base/HttpClient.cpp

@@ -18,8 +18,7 @@
 #include "ecflow/core/Converter.hpp"
 
 HttpClient::HttpClient(Cmd_ptr cmd_ptr, const std::string& host, const std::string& port, int timeout)
-    : stopped_(false),
-      host_(host),
+    : host_(host),
       port_(port),
       client_(host, ecf::convert_to<int>(port)) {
 
@@ -34,16 +33,29 @@ void HttpClient::run() {
     std::string outbound;
     ecf::save_as_string(outbound, outbound_request_);
 
-    auto result   = client_.Post("/v1/ecflow", outbound, "application/json");
-    auto response = result.value();
-
-    ecf::restore_from_string(response.body, inbound_response_);
-};
+    auto result = client_.Post("/v1/ecflow", outbound, "application/json");
+    if (result) {
+        auto response = result.value();
+        ecf::restore_from_string(response.body, inbound_response_);
+    }
+    else {
+        status_ = result.error();
+        reason_ = httplib::to_string(status_);
+    }
+}
 
 bool HttpClient::handle_server_response(ServerReply& server_reply, bool debug) const {
     if (debug) {
         std::cout << "  Client::handle_server_response" << std::endl;
     }
     server_reply.set_host_port(host_, port_); // client context, needed by some commands, ie. SServerLoadCmd
-    return inbound_response_.handle_server_response(server_reply, outbound_request_.get_cmd(), debug);
+
+    if (status_ == httplib::Error::Success) {
+        return inbound_response_.handle_server_response(server_reply, outbound_request_.get_cmd(), debug);
+    }
+    else {
+        std::stringstream ss;
+        ss << "HttpClient::handle_server_response: Error: " << status_ << " " << reason_;
+        throw std::runtime_error(ss.str());
+    }
 }

libs/base/src/ecflow/base/HttpClient.hpp

@@ -11,18 +11,18 @@
 #ifndef ecflow_base_HttpClient_HPP
 #define ecflow_base_HttpClient_HPP
 
+#define CPPHTTPLIB_THREAD_POOL_COUNT 1
+#define CPPHTTPLIB_OPENSSL_SUPPORT 1
+#define CPPHTTPLIB_ZLIB_SUPPORT 1
+
 #include <httplib.h>
 
 #include "ecflow/base/ClientToServerRequest.hpp"
 #include "ecflow/base/Connection.hpp"
 #include "ecflow/base/ServerToClientResponse.hpp"
 
 ///
-/// \brief  This class acts as the client part. ( in client/server architecture)
-///
-/// \note The plug command can move a node to another server hence the server
-/// itself will NEED to ACT as a client. This is why client lives in Base and
-/// not the Client project
+/// \brief  This class acts as an HTTP client
 ///
 
 class HttpClient {
@@ -38,10 +38,14 @@ class HttpClient {
     bool handle_server_response(ServerReply&, bool debug) const;
 
 private:
-    bool stopped_;
     std::string host_; /// the servers name
     std::string port_; /// the port on the server
     httplib::Client client_;
+
+    httplib::Response response_;
+    httplib::Error status_ = httplib::Error::Success;
+    std::string reason_    = "";
+
     ClientToServerRequest outbound_request_;  /// The request we will send to the server
     ServerToClientResponse inbound_response_; /// The response we get back from the server
 };

libs/client/src/ecflow/client/ClientEnvironment.cpp

@@ -245,6 +245,12 @@ void ClientEnvironment::read_environment_variables() {
 
     ecf::environment::get(ecf::environment::ECF_TRYNO, task_try_num_);
 
+    std::string host_protocol;
+    ecf::environment::get(ecf::environment::ECF_HOST_PROTOCOL, host_protocol);
+    if (host_protocol == "HTTP" || host_protocol == "HTTPS") {
+        http_ = true;
+    }
+
     ecf::environment::get("ECF_HOSTFILE", host_file_);
 
     ecf::environment::get(ecf::environment::ECF_RID, remote_id_);

libs/client/src/ecflow/client/ClientInvoker.cpp

@@ -84,6 +84,15 @@
 
 #include "ecflow/base/HttpClient.hpp"
 
+#if defined(ADD)
+// undefine to avoid conflict with /usr/include/arpa/nameser_compat.h #define ADD ns_uop_add
+    #undef ADD
+#endif
+#if defined(STATUS)
+// undefine to avoid conflict with /usr/include/arpa/nameser_compat.h #define STATUS ns_o_status
+    #undef STATUS
+#endif
+
 using namespace std;
 using namespace ecf;
 using namespace boost::posix_time;

libs/core/src/ecflow/core/Ecf.cpp

@@ -13,8 +13,8 @@
 bool Ecf::server_                            = false;
 bool Ecf::debug_equality_                    = false;
 unsigned int Ecf::debug_level_               = 0;
-thread_local Ecf::atomic_counter_t Ecf::state_change_no_  = 0;
-thread_local Ecf::atomic_counter_t Ecf::modify_change_no_ = 0;
+Ecf::atomic_counter_t Ecf::state_change_no_  = 0;
+Ecf::atomic_counter_t Ecf::modify_change_no_ = 0;
 bool DebugEquality::ignore_server_variables_ = false;
 
 const char* Ecf::SERVER_NAME() {
@@ -80,20 +80,6 @@ const std::string& Ecf::URL() {
     return URL;
 }
 
-Ecf::counter_t Ecf::incr_state_change_no() {
-    if (server_) {
-        return ++state_change_no_;
-    }
-    return state_change_no_;
-}
-
-Ecf::counter_t Ecf::incr_modify_change_no() {
-    if (server_) {
-        return ++modify_change_no_;
-    }
-    return modify_change_no_;
-}
-
 // =======================================================
 
 EcfPreserveChangeNo::EcfPreserveChangeNo()

libs/core/src/ecflow/core/Ecf.hpp

@@ -16,6 +16,7 @@
 ///
 
 #include <atomic>
+#include <iostream>
 #include <string>
 
 /**
@@ -47,14 +48,32 @@ class Ecf {
     const Ecf& operator=(const Ecf&) = delete;
 
     /// Increment and then return state change no
-    static counter_t incr_state_change_no();
-    static counter_t state_change_no() { return state_change_no_; }
-    static void set_state_change_no(counter_t x) { state_change_no_ = x; }
+    static counter_t incr_state_change_no() {
+        if (server_) {
+            ++state_change_no_;
+        }
+        return state_change_no_;
+    }
+    static counter_t state_change_no() {
+        return state_change_no_;
+    }
+    static void set_state_change_no(counter_t x) {
+        state_change_no_ = x;
+    }
 
     /// The modify_change_no_ is used for node addition and deletion and re-ordering
-    static counter_t incr_modify_change_no();
-    static counter_t modify_change_no() { return modify_change_no_; }
-    static void set_modify_change_no(counter_t x) { modify_change_no_ = x; }
+    static counter_t incr_modify_change_no() {
+        if (server_) {
+            ++modify_change_no_;
+        }
+        return modify_change_no_;
+    }
+    static counter_t modify_change_no() {
+        return modify_change_no_;
+    }
+    static void set_modify_change_no(counter_t x) {
+        modify_change_no_ = x;
+    }
 
     /// Returns true if we are on the server side.
     /// Only in server side do we increment state/modify numbers
@@ -91,8 +110,8 @@ class Ecf {
     static bool server_;
     static bool debug_equality_;
     static unsigned int debug_level_;
-    static thread_local atomic_counter_t state_change_no_;
-    static thread_local atomic_counter_t modify_change_no_;
+    static atomic_counter_t state_change_no_;
+    static atomic_counter_t modify_change_no_;
 };
 
 /// Make sure the Ecf number don't change

libs/rest/src/ecflow/http/HttpServer.cpp

@@ -84,7 +84,7 @@ void HttpServer::parse_args(int argc, char** argv) const {
         opts.no_ssl = true;
     }
     if (backend_http) {
-        opts.host_protocol = "http";
+        opts.host_protocol = "HTTPS";
     }
 
     setenv("ECF_HOST", opts.ecflow_host.c_str(), 1);

libs/rest/test/TestApiV1.cpp

@@ -9,9 +9,12 @@
  */
 
 #ifdef ECF_OPENSSL
-    #define CPPHTTPLIB_OPENSSL_SUPPORT
+    #define CPPHTTPLIB_OPENSSL_SUPPORT 1
 #endif
 
+#define CPPHTTPLIB_THREAD_POOL_COUNT 1
+#define CPPHTTPLIB_ZLIB_SUPPORT 1
+
 #include <httplib.h>
 
 #include <boost/test/unit_test.hpp>

libs/server/CMakeLists.txt

@@ -20,6 +20,9 @@ ecbuild_add_executable(
     ecflow_all
     $<$<BOOL:${OPENSSL_FOUND}>:OpenSSL::SSL>
     Threads::Threads
+    $<$<BOOL:${ZLIB_FOUND}>:ZLIB::ZLIB>
+  DEFINITIONS
+    $<$<BOOL:${ZLIB_FOUND}>:ECF_HTTP_COMPRESSION>
 )
 target_clangformat(ecflow_server)