From 7839841f052506e4ef362de63f50dfbe24f56f2a Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Wed, 21 Feb 2024 06:41:34 +0000 Subject: [PATCH] Enable HTTP connection between Client/Server Re ECFLOW-1957 --- libs/CMakeLists.txt | 2 + libs/base/src/ecflow/base/HttpClient.cpp | 49 ++++++++++++++ libs/base/src/ecflow/base/HttpClient.hpp | 49 ++++++++++++++ .../src/ecflow/client/ClientEnvironment.hpp | 6 +- .../src/ecflow/client/ClientInvoker.cpp | 22 +++++++ .../src/ecflow/client/ClientOptions.cpp | 9 +++ libs/rest/src/ecflow/http/HttpServer.cpp | 2 +- libs/server/src/ecflow/server/Server.hpp | 64 +++++++++++++++++++ .../src/ecflow/server/ServerEnvironment.cpp | 2 + .../src/ecflow/server/ServerEnvironment.hpp | 3 + libs/server/src/ecflow/server/ServerMain.cpp | 18 +++++- .../src/ecflow/server/ServerOptions.cpp | 33 ++++++---- 12 files changed, 242 insertions(+), 17 deletions(-) create mode 100644 libs/base/src/ecflow/base/HttpClient.cpp create mode 100644 libs/base/src/ecflow/base/HttpClient.hpp diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index bb9c9f91e..43b8b524f 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -62,6 +62,7 @@ set(srcs base/src/ecflow/base/Cmd.hpp base/src/ecflow/base/Connection.hpp base/src/ecflow/base/Gnuplot.hpp + base/src/ecflow/base/HttpClient.hpp base/src/ecflow/base/ServerReply.hpp base/src/ecflow/base/ServerToClientResponse.hpp base/src/ecflow/base/Stats.hpp @@ -137,6 +138,7 @@ set(srcs base/src/ecflow/base/ClientToServerRequest.cpp base/src/ecflow/base/Connection.cpp base/src/ecflow/base/Gnuplot.cpp + base/src/ecflow/base/HttpClient.cpp base/src/ecflow/base/ServerReply.cpp base/src/ecflow/base/ServerToClientResponse.cpp base/src/ecflow/base/Stats.cpp diff --git a/libs/base/src/ecflow/base/HttpClient.cpp b/libs/base/src/ecflow/base/HttpClient.cpp new file mode 100644 index 000000000..3b0184297 --- /dev/null +++ b/libs/base/src/ecflow/base/HttpClient.cpp @@ -0,0 +1,49 @@ +/* + * Copyright 2009- ECMWF. + * + * This software is licensed under the terms of the Apache Licence version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "ecflow/base/HttpClient.hpp" + +#include +#include +#include + +#include "ecflow/base/stc/StcCmd.hpp" +#include "ecflow/core/Converter.hpp" + +HttpClient::HttpClient(Cmd_ptr cmd_ptr, const std::string& host, const std::string& port, int timeout) + : stopped_(false), + host_(host), + port_(port), + client_(host, ecf::convert_to(port)) { + + if (!cmd_ptr.get()) { + throw std::runtime_error("Client::Client: No request specified !"); + } + + outbound_request_.set_cmd(cmd_ptr); +} + +void HttpClient::run() { + std::string outbound; + ecf::save_as_string(outbound, outbound_request_); + + auto result = client_.Post("/v1/ecflow", outbound, "application/json"); + auto response = result.value(); + + ecf::restore_from_string(response.body, inbound_response_); +}; + +bool HttpClient::handle_server_response(ServerReply& server_reply, bool debug) const { + if (debug) { + std::cout << " Client::handle_server_response" << std::endl; + } + server_reply.set_host_port(host_, port_); // client context, needed by some commands, ie. SServerLoadCmd + return inbound_response_.handle_server_response(server_reply, outbound_request_.get_cmd(), debug); +} diff --git a/libs/base/src/ecflow/base/HttpClient.hpp b/libs/base/src/ecflow/base/HttpClient.hpp new file mode 100644 index 000000000..500b2f204 --- /dev/null +++ b/libs/base/src/ecflow/base/HttpClient.hpp @@ -0,0 +1,49 @@ +/* + * Copyright 2009- ECMWF. + * + * This software is licensed under the terms of the Apache Licence version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#ifndef ecflow_base_HttpClient_HPP +#define ecflow_base_HttpClient_HPP + +#include + +#include "ecflow/base/ClientToServerRequest.hpp" +#include "ecflow/base/Connection.hpp" +#include "ecflow/base/ServerToClientResponse.hpp" + +/// +/// \brief This class acts as the client part. ( in client/server architecture) +/// +/// \note The plug command can move a node to another server hence the server +/// itself will NEED to ACT as a client. This is why client lives in Base and +/// not the Client project +/// + +class HttpClient { +public: + /// Constructor starts the asynchronous connect operation. + HttpClient(Cmd_ptr cmd_ptr, const std::string& host, const std::string& port, int timout = 0); + + void run(); + + /// Client side, get the server response, handles reply from server + /// Returns true if all is ok, else false if further client action is required + /// will throw std::runtime_error for errors + bool handle_server_response(ServerReply&, bool debug) const; + +private: + bool stopped_; + std::string host_; /// the servers name + std::string port_; /// the port on the server + httplib::Client client_; + ClientToServerRequest outbound_request_; /// The request we will send to the server + ServerToClientResponse inbound_response_; /// The response we get back from the server +}; + +#endif /* ecflow_base_HttpClient_HPP */ diff --git a/libs/client/src/ecflow/client/ClientEnvironment.hpp b/libs/client/src/ecflow/client/ClientEnvironment.hpp index ec939f83c..ba47d7760 100644 --- a/libs/client/src/ecflow/client/ClientEnvironment.hpp +++ b/libs/client/src/ecflow/client/ClientEnvironment.hpp @@ -90,6 +90,9 @@ class ClientEnvironment final : public AbstractClientEnv { // Used by python to enable debug of client api void set_debug(bool flag); + bool http() const { return http_; } + void enable_http() { http_ = true; } + #ifdef ECF_OPENSSL /// return true if this is a ssl enabled server ecf::Openssl& openssl() { return ssl_; } @@ -168,7 +171,8 @@ class ClientEnvironment final : public AbstractClientEnv { bool denied_{false}; // ECF_DENIED.If the server denies the communication, then the child command can be set to fail // immediately bool no_ecf_{false}; // NO_ECF. if defined then abort cmd immediately. useful when test jobs stand-alone - bool debug_{false}; // For live debug, enabled by env variable ECF_CLIENT_DEBUG or set by option -d|--debug + bool http_{false}; + bool debug_{false}; // For live debug, enabled by env variable ECF_CLIENT_DEBUG or set by option -d|--debug bool under_test_{false}; // Used in testing client interface bool host_file_read_{false}; // to ensure we read host file only once bool gui_{false}; diff --git a/libs/client/src/ecflow/client/ClientInvoker.cpp b/libs/client/src/ecflow/client/ClientInvoker.cpp index 40b7eef5f..3360b098a 100644 --- a/libs/client/src/ecflow/client/ClientInvoker.cpp +++ b/libs/client/src/ecflow/client/ClientInvoker.cpp @@ -82,6 +82,8 @@ #define NEXT_HOST_POLL_PERIOD 30 #endif +#include "ecflow/base/HttpClient.hpp" + using namespace std; using namespace ecf; using namespace boost::posix_time; @@ -426,6 +428,26 @@ int ClientInvoker::do_invoke_cmd(Cmd_ptr cts_cmd) const { return 1; } } + else if (clientEnv_.http()) { + HttpClient theClient(cts_cmd, clientEnv_.host(), clientEnv_.port()); + theClient.run(); + + if (clientEnv_.debug()) + cout << TimeStamp::now() << "ClientInvoker: >>> After: io_service.run() <<<" << endl; + + /// Let see how the server responded if at all. + try { + /// will return false if further action required + if (theClient.handle_server_response(server_reply_, clientEnv_.debug())) { + // The normal response. RoundTriprecorder will record in rtt_ + return 0; // the normal exit path + } + } + catch (std::exception& e) { + server_reply_.set_error_msg(e.what()); + return 1; + } + } else { #endif Client theClient( diff --git a/libs/client/src/ecflow/client/ClientOptions.cpp b/libs/client/src/ecflow/client/ClientOptions.cpp index 91b11ff22..2dedaf97b 100644 --- a/libs/client/src/ecflow/client/ClientOptions.cpp +++ b/libs/client/src/ecflow/client/ClientOptions.cpp @@ -78,6 +78,9 @@ ClientOptions::ClientOptions() { "Enables the use of SSL when contacting the server.\n" "When specified overrides the environment variable ECF_SSL."); #endif + desc_->add_options()( + "http", + "Enables communication over HTTP between client/server.\n"); // clang-format on } @@ -185,6 +188,12 @@ Cmd_ptr ClientOptions::parse(const CommandLine& cl, ClientEnvironment* env) cons } #endif + if (vm.count("http")) { + if (env->debug()) + std::cout << " http set via command line\n"; + env->enable_http(); + } + // Defer the parsing of the command , to the command. This allows // all cmd functionality to be centralised with the command // This can throw std::runtime_error if arg's don't parse diff --git a/libs/rest/src/ecflow/http/HttpServer.cpp b/libs/rest/src/ecflow/http/HttpServer.cpp index 2aaf7ed9f..0fe5307d1 100644 --- a/libs/rest/src/ecflow/http/HttpServer.cpp +++ b/libs/rest/src/ecflow/http/HttpServer.cpp @@ -235,7 +235,7 @@ void HttpServer::run() const { const std::string key = opts.cert_directory + "/server.key"; httplib::SSLServer http_server(cert.c_str(), key.c_str()); - apply_listeners(dynamic_cast(http_server)); + apply_listeners(http_server); start_server(http_server); } else diff --git a/libs/server/src/ecflow/server/Server.hpp b/libs/server/src/ecflow/server/Server.hpp index 88d067f4d..cf10de47b 100644 --- a/libs/server/src/ecflow/server/Server.hpp +++ b/libs/server/src/ecflow/server/Server.hpp @@ -11,6 +11,10 @@ #ifndef ecflow_server_Server_HPP #define ecflow_server_Server_HPP +#include + +#include "ecflow/base/stc/PreAllocatedReply.hpp" +#include "ecflow/core/Converter.hpp" #include "ecflow/server/BaseServer.hpp" #include "ecflow/server/ServerEnvironment.hpp" #include "ecflow/server/SslTcpServer.hpp" @@ -41,4 +45,64 @@ class DefaultServer : public BaseServer { using Server = DefaultServer; using SslServer = DefaultServer; +class HttpServer : public BaseServer { +public: + explicit HttpServer(boost::asio::io_service& service, ServerEnvironment& env) : BaseServer(service, env) {} + ~HttpServer() override = default; + + std::string ssl() const override { return ""; } + + void run() { + httplib::Server server; + + server.Post("/v1/ecflow", [this](const httplib::Request& request, httplib::Response& response) { + // Buffers to hold request/response + ClientToServerRequest inbound_request; + ServerToClientResponse outbound_response; + + // 1) unserialize request body into inbound_request + ecf::restore_from_string(request.body, inbound_request); + + // 2) handle request, as per TcpBaseServer::handle_request() + { + // See what kind of message we got from the client + if (serverEnv_.debug()) { + std::cout << " TcpBaseServer::handle_request : client request " << inbound_request << std::endl; + } + + try { + // Service the in bound request, handling the request will populate the outbound_response_ + // Note:: Handle request will first authenticate + outbound_response.set_cmd(inbound_request.handleRequest(this)); + } + catch (std::exception& e) { + outbound_response.set_cmd(PreAllocatedReply::error_cmd(e.what())); + } + + // Do any necessary clean up after inbound_request_ has run. i.e like re-claiming memory + inbound_request.cleanup(); + } + + // 3) serialize outbound_response into response body + std::string outbound; + ecf::save_as_string(outbound, outbound_response); + + // 4) ship response + response.set_content(outbound, "text/plain"); + }); + + try { + auto [host, port] = serverEnv_.hostPort(); + + bool ret = server.listen(host, ecf::convert_to(port)); + if (ret == false) { + throw std::runtime_error(std::string("Failed to start HTTP server")); + } + } + catch (const std::exception& e) { + std::cout << "[HttpServer] Failure running HTTP server:" << e.what() << std::endl; + } + } +}; + #endif /* ecflow_server_Server_HPP */ diff --git a/libs/server/src/ecflow/server/ServerEnvironment.cpp b/libs/server/src/ecflow/server/ServerEnvironment.cpp index 92920a6e4..1ccbdfef0 100644 --- a/libs/server/src/ecflow/server/ServerEnvironment.cpp +++ b/libs/server/src/ecflow/server/ServerEnvironment.cpp @@ -67,6 +67,7 @@ ServerEnvironment::ServerEnvironment(int argc, char* argv[]) submitJobsInterval_(defaultSubmitJobsInterval), ecf_prune_node_log_(0), jobGeneration_(true), + http_(false), debug_(false), help_option_(false), version_option_(false), @@ -88,6 +89,7 @@ ServerEnvironment::ServerEnvironment(int argc, char* argv[], const std::string& submitJobsInterval_(defaultSubmitJobsInterval), ecf_prune_node_log_(0), jobGeneration_(true), + http_(false), debug_(false), help_option_(false), version_option_(false), diff --git a/libs/server/src/ecflow/server/ServerEnvironment.hpp b/libs/server/src/ecflow/server/ServerEnvironment.hpp index 25592d686..767cfa620 100644 --- a/libs/server/src/ecflow/server/ServerEnvironment.hpp +++ b/libs/server/src/ecflow/server/ServerEnvironment.hpp @@ -69,6 +69,8 @@ class ServerEnvironment { void enable_ssl() { ssl_.enable(serverHost_, the_port()); } // search server.crt first, then ..crt #endif + bool http() const { return http_; } + /// returns the server port. This has a default value defined in server_environment.cfg /// but can be overridden by the environment variable ECF_PORT int port() const { return serverPort_; } @@ -208,6 +210,7 @@ class ServerEnvironment { int submitJobsInterval_; int ecf_prune_node_log_; bool jobGeneration_; // used in debug/test mode only + bool http_; bool debug_; bool help_option_; bool version_option_; diff --git a/libs/server/src/ecflow/server/ServerMain.cpp b/libs/server/src/ecflow/server/ServerMain.cpp index e81f01a58..bf687e80e 100644 --- a/libs/server/src/ecflow/server/ServerMain.cpp +++ b/libs/server/src/ecflow/server/ServerMain.cpp @@ -81,6 +81,12 @@ int run(BaseServer& server) { return run_boost_services(server.io_, server.serverEnv_); } +int run(HttpServer& server) { + server.run(); + return 0; +} + + int main(int argc, char* argv[]) { try { @@ -103,15 +109,21 @@ int main(int argc, char* argv[]) { boost::asio::io_context io; - // Launching SSL server + // Launching Http server + if (server_environment.http()) { + HttpServer theServer(io, server_environment); + return run(theServer); + } + + // Launching custom TCP/IP (SSL) server if constexpr (ECF_OPENSSL == 1) { if (server_environment.ssl()) { SslServer theServer(io, server_environment); // This throws exception, if bind address in use return run(theServer); } } - - // Launching non-SSL server + + // Launching custom TCP/IP (non-SSL) server Server theServer(io, server_environment); // This throws exception, if bind address in use return run(theServer); } diff --git a/libs/server/src/ecflow/server/ServerOptions.cpp b/libs/server/src/ecflow/server/ServerOptions.cpp index e8cfd9b05..6af35c83e 100644 --- a/libs/server/src/ecflow/server/ServerOptions.cpp +++ b/libs/server/src/ecflow/server/ServerOptions.cpp @@ -120,7 +120,8 @@ ServerOptions::ServerOptions(int argc, char* argv[], ServerEnvironment* env) { #ifdef ECF_OPENSSL ("ssl", ecf::Openssl::ssl_info()) #endif - ("dis_job_gen", "Disable job generation. For DEBUG/Test only.")("debug,d", "Enable debug output.")( + ("http", "Enable server/client HTTP communication")( + "dis_job_gen", "Disable job generation. For DEBUG/Test only.")("debug,d", "Enable debug output.")( "version,v", "Show ecflow version number,boost library version, compiler used and compilation date, then exit"); @@ -137,43 +138,51 @@ ServerOptions::ServerOptions(int argc, char* argv[], ServerEnvironment* env) { if (vm_.count("debug")) env->debug_ = true; + if (vm_.count("http")) { + if (env->debug_) { + cout << "ServerOptions:: Deploying HTTP Server\n"; + } + env->http_ = true; + } + if (vm_.count("port")) { - if (env->debug_) + if (env->debug_) { cout << "ServerOptions:: The port number set to '" << vm_["port"].as() << "'\n"; + } env->serverPort_ = vm_["port"].as(); } if (vm_.count("ecfinterval")) { - if (env->debug_) + if (env->debug_) { cout << "ServerOptions: The ecfinterval set to '" << vm_["ecfinterval"].as() << "'\n"; + } env->submitJobsInterval_ = vm_["ecfinterval"].as(); } if (vm_.count("v6")) { - if (env->debug_) + if (env->debug_) { cout << "ServerOptions: The tcp protocol set to v6\n"; + } env->tcp_protocol_ = boost::asio::ip::tcp::v6(); } if (vm_.count("dis_job_gen")) { - if (env->debug_) + if (env->debug_) { cout << "ServerOptions: The dis_job_gen is set\n"; + } env->jobGeneration_ = false; } #ifdef ECF_OPENSSL if (vm_.count("ssl")) { - if (env->debug_) + if (env->debug_) { cout << "ServerOptions: ssl server \n"; + } env->enable_ssl(); // search server.crt first, then ..crt } #endif } bool ServerOptions::help_option() const { - if (vm_.count("help")) - return true; - return false; + return vm_.count("help"); } bool ServerOptions::version_option() const { - if (vm_.count("version")) - return true; - return false; + return vm_.count("version"); }