feat: support follow_redirects, tls (#1896)

alibaba · Nov 26, 2024 · 3d216eb · 3d216eb
1 parent 5c63146
commit 3d216eb
Show file tree

Hide file tree

Showing 13 changed files with 261 additions and 39 deletions.
diff --git a/core/common/http/AsynCurlRunner.cpp b/core/common/http/AsynCurlRunner.cpp
@@ -88,7 +88,10 @@ bool AsynCurlRunner::AddRequestToClient(unique_ptr<AsynHttpRequest>&& request) {
                                    headers,
                                    request->mTimeout,
                                    AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(),
-                                   AppConfig::GetInstance()->GetBindInterface());
+                                   AppConfig::GetInstance()->GetBindInterface(),
+                                   request->mFollowRedirects,
+                                   request->mTls);
+
     if (curl == nullptr) {
         LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get()));
         request->OnSendDone(request->mResponse);
@@ -135,9 +138,7 @@ void AsynCurlRunner::DoRun() {
             }
         }
 
-        struct timeval timeout {
-            1, 0
-        };
+        struct timeval timeout{1, 0};
         long curlTimeout = -1;
         if ((mc = curl_multi_timeout(mClient, &curlTimeout)) != CURLM_OK) {
             LOG_WARNING(

diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp
@@ -69,7 +69,9 @@ CURL* CreateCurlHandler(const std::string& method,
                         curl_slist*& headers,
                         uint32_t timeout,
                         bool replaceHostWithIp,
-                        const std::string& intf) {
+                        const std::string& intf,
+                        bool followRedirects,
+                        std::optional<CurlTLS> tls) {
     static DnsCache* dnsCache = DnsCache::GetInstance();
 
     CURL* curl = curl_easy_init();
@@ -102,11 +104,28 @@ CURL* CreateCurlHandler(const std::string& method,
         curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size());
     }
 
+    if (followRedirects) {
+        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+    }
+
     if (httpsFlag) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
     }
 
+    if (tls.has_value()) {
+        if (!tls->mCaFile.empty()) {
+            curl_easy_setopt(curl, CURLOPT_CAINFO, tls->mCaFile.c_str());
+        }
+        if (!tls->mCertFile.empty()) {
+            curl_easy_setopt(curl, CURLOPT_SSLCERT, tls->mCertFile.c_str());
+        }
+        if (!tls->mKeyFile.empty()) {
+            curl_easy_setopt(curl, CURLOPT_SSLKEY, tls->mKeyFile.c_str());
+        }
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, tls->mInsecureSkipVerify ? 0 : 1);
+    }
+
     curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
     if (!intf.empty()) {
         curl_easy_setopt(curl, CURLOPT_INTERFACE, intf.c_str());
@@ -139,7 +158,9 @@ bool SendHttpRequest(std::unique_ptr<HttpRequest>&& request, HttpResponse& respo
                                    headers,
                                    request->mTimeout,
                                    AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(),
-                                   AppConfig::GetInstance()->GetBindInterface());
+                                   AppConfig::GetInstance()->GetBindInterface(),
+                                   request->mFollowRedirects,
+                                   request->mTls);
     if (curl == NULL) {
         LOG_ERROR(sLogger,
                   ("failed to init curl handler", "failed to init curl client")("request address", request.get()));

diff --git a/core/common/http/Curl.h b/core/common/http/Curl.h
@@ -20,8 +20,8 @@
 
 #include <cstdint>
 #include <map>
-#include <string>
 #include <memory>
+#include <string>
 
 #include "common/http/HttpRequest.h"
 #include "common/http/HttpResponse.h"
@@ -40,7 +40,9 @@ CURL* CreateCurlHandler(const std::string& method,
                         curl_slist*& headers,
                         uint32_t timeout,
                         bool replaceHostWithIp = true,
-                        const std::string& intf = "");
+                        const std::string& intf = "",
+                        bool followRedirects = false,
+                        std::optional<CurlTLS> tls = std::nullopt);
 
 bool SendHttpRequest(std::unique_ptr<HttpRequest>&& request, HttpResponse& response);
 

diff --git a/core/common/http/HttpRequest.h b/core/common/http/HttpRequest.h
@@ -20,6 +20,7 @@
 #include <cstdint>
 #include <map>
 #include <string>
+#include <utility>
 
 #include "common/Flags.h"
 #include "common/http/HttpResponse.h"
@@ -29,6 +30,13 @@ DECLARE_FLAG_INT32(default_http_request_max_try_cnt);
 
 namespace logtail {
 
+struct CurlTLS {
+    std::string mCaFile;
+    std::string mCertFile;
+    std::string mKeyFile;
+    bool mInsecureSkipVerify = true;
+};
+
 struct HttpRequest {
     std::string mMethod;
     // TODO: upgrade curl to 7.62, and replace the following 4 members
@@ -43,6 +51,8 @@ struct HttpRequest {
     int32_t mPort;
     uint32_t mTimeout = static_cast<uint32_t>(INT32_FLAG(default_http_request_timeout_secs));
     uint32_t mMaxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt));
+    bool mFollowRedirects = false;
+    std::optional<CurlTLS> mTls = std::nullopt;
 
     uint32_t mTryCnt = 1;
     std::chrono::system_clock::time_point mLastSendTime;
@@ -56,7 +66,9 @@ struct HttpRequest {
                 const std::map<std::string, std::string>& header,
                 const std::string& body,
                 uint32_t timeout = static_cast<uint32_t>(INT32_FLAG(default_http_request_timeout_secs)),
-                uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)))
+                uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)),
+                bool followRedirects = false,
+                std::optional<CurlTLS> tls = std::nullopt)
         : mMethod(method),
           mHTTPSFlag(httpsFlag),
           mUrl(url),
@@ -66,7 +78,9 @@ struct HttpRequest {
           mHost(host),
           mPort(port),
           mTimeout(timeout),
-          mMaxTryCnt(maxTryCnt) {}
+          mMaxTryCnt(maxTryCnt),
+          mFollowRedirects(followRedirects),
+          mTls(std::move(tls)) {}
     virtual ~HttpRequest() = default;
 };
 
@@ -85,8 +99,11 @@ struct AsynHttpRequest : public HttpRequest {
                     const std::string& body,
                     HttpResponse&& response = HttpResponse(),
                     uint32_t timeout = static_cast<uint32_t>(INT32_FLAG(default_http_request_timeout_secs)),
-                    uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)))
-        : HttpRequest(method, httpsFlag, host, port, url, query, header, body, timeout, maxTryCnt),
+                    uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)),
+                    bool followRedirects = false,
+                    std::optional<CurlTLS> tls = std::nullopt)
+        : HttpRequest(
+              method, httpsFlag, host, port, url, query, header, body, timeout, maxTryCnt, followRedirects, std::move(tls)),
           mResponse(std::move(response)) {}
 
     virtual bool IsContextValid() const = 0;

diff --git a/core/common/http/HttpResponse.h b/core/common/http/HttpResponse.h
@@ -26,6 +26,8 @@ class curl_slist;
 
 namespace logtail {
 
+struct CurlTLS;
+
 bool caseInsensitiveComp(const char lhs, const char rhs);
 
 bool compareHeader(const std::string& lhs, const std::string& rhs);
@@ -45,7 +47,9 @@ class HttpResponse {
                                    curl_slist*& headers,
                                    uint32_t timeout,
                                    bool replaceHostWithIp,
-                                   const std::string& intf);
+                                   const std::string& intf,
+                                   bool followRedirects,
+                                   std::optional<CurlTLS> tls);
 
 public:
     HttpResponse()

diff --git a/core/prometheus/Constants.h b/core/prometheus/Constants.h
@@ -69,6 +69,14 @@ const char* const PASSWORD_FILE = "password_file";
 const char* const BASIC_PREFIX = "Basic ";
 const char* const HONOR_LABELS = "honor_labels";
 const char* const HONOR_TIMESTAMPS = "honor_timestamps";
+const char* const FOLLOW_REDIRECTS = "follow_redirects";
+const char* const TLS_CONFIG = "tls_config";
+const char* const CA_FILE = "ca_file";
+const char* const CERT_FILE = "cert_file";
+const char* const KEY_FILE = "key_file";
+const char* const SERVER_NAME = "server_name";
+const char* const HOST = "Host";
+const char* const INSECURE_SKIP_VERIFY = "insecure_skip_verify";
 
 // scrape protocols, from https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
 // text/plain, application/openmetrics-text will be used

diff --git a/core/prometheus/async/PromHttpRequest.cpp b/core/prometheus/async/PromHttpRequest.cpp
@@ -3,6 +3,7 @@
 #include <chrono>
 #include <cstdint>
 #include <string>
+#include <utility>
 
 #include "common/http/HttpRequest.h"
 
@@ -20,7 +21,9 @@ PromHttpRequest::PromHttpRequest(const std::string& method,
                                  uint32_t timeout,
                                  uint32_t maxTryCnt,
                                  std::shared_ptr<PromFuture<HttpResponse&, uint64_t>> future,
-                                 std::shared_ptr<PromFuture<>> isContextValidFuture)
+                                 std::shared_ptr<PromFuture<>> isContextValidFuture,
+                                 bool followRedirects,
+                                 std::optional<CurlTLS> tls)
     : AsynHttpRequest(method,
                       httpsFlag,
                       host,
@@ -31,7 +34,9 @@ PromHttpRequest::PromHttpRequest(const std::string& method,
                       body,
                       std::move(response),
                       timeout,
-                      maxTryCnt),
+                      maxTryCnt,
+                      followRedirects,
+                      std::move(tls)),
       mFuture(std::move(future)),
       mIsContextValidFuture(std::move(isContextValidFuture)) {
 }

diff --git a/core/prometheus/async/PromHttpRequest.h b/core/prometheus/async/PromHttpRequest.h
@@ -23,7 +23,9 @@ class PromHttpRequest : public AsynHttpRequest {
                     uint32_t timeout,
                     uint32_t maxTryCnt,
                     std::shared_ptr<PromFuture<HttpResponse&, uint64_t>> future,
-                    std::shared_ptr<PromFuture<>> isContextValidFuture = nullptr);
+                    std::shared_ptr<PromFuture<>> isContextValidFuture = nullptr,
+                    bool followRedirects = false,
+                    std::optional<CurlTLS> tls = std::nullopt);
     PromHttpRequest(const PromHttpRequest&) = default;
     ~PromHttpRequest() override = default;
 

diff --git a/core/prometheus/schedulers/ScrapeConfig.cpp b/core/prometheus/schedulers/ScrapeConfig.cpp
@@ -22,10 +22,13 @@ ScrapeConfig::ScrapeConfig()
       mHonorLabels(false),
       mHonorTimestamps(true),
       mScheme("http"),
+      mFollowRedirects(true),
+      mEnableTLS(false),
       mMaxScrapeSizeBytes(0),
       mSampleLimit(0),
       mSeriesLimit(0) {
 }
+
 bool ScrapeConfig::Init(const Json::Value& scrapeConfig) {
     if (!InitStaticConfig(scrapeConfig)) {
         return false;
@@ -41,6 +44,17 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) {
         InitScrapeProtocols(nullJson);
     }
 
+    if (scrapeConfig.isMember(prometheus::FOLLOW_REDIRECTS) && scrapeConfig[prometheus::FOLLOW_REDIRECTS].isBool()) {
+        mFollowRedirects = scrapeConfig[prometheus::FOLLOW_REDIRECTS].asBool();
+    }
+
+    if (scrapeConfig.isMember(prometheus::TLS_CONFIG) && scrapeConfig[prometheus::TLS_CONFIG].isObject()) {
+        if (!InitTLSConfig(scrapeConfig[prometheus::TLS_CONFIG])) {
+            LOG_ERROR(sLogger, ("tls config error", ""));
+            return false;
+        }
+    }
+
     if (scrapeConfig.isMember(prometheus::ENABLE_COMPRESSION)
         && scrapeConfig[prometheus::ENABLE_COMPRESSION].isBool()) {
         // InitEnableCompression(scrapeConfig[prometheus::ENABLE_COMPRESSION].asBool());
@@ -338,4 +352,49 @@ void ScrapeConfig::InitEnableCompression(bool enableCompression) {
     }
 }
 
+bool ScrapeConfig::InitTLSConfig(const Json::Value& tlsConfig) {
+    if (tlsConfig.isMember(prometheus::CA_FILE)) {
+        if (tlsConfig[prometheus::CA_FILE].isString()) {
+            mTLS.mCaFile = tlsConfig[prometheus::CA_FILE].asString();
+        } else {
+            LOG_ERROR(sLogger, ("tls config error", ""));
+            return false;
+        }
+    }
+    if (tlsConfig.isMember(prometheus::CERT_FILE)) {
+        if (tlsConfig[prometheus::CERT_FILE].isString()) {
+            mTLS.mCertFile = tlsConfig[prometheus::CERT_FILE].asString();
+        } else {
+            LOG_ERROR(sLogger, ("tls config error", ""));
+            return false;
+        }
+    }
+    if (tlsConfig.isMember(prometheus::KEY_FILE)) {
+        if (tlsConfig[prometheus::KEY_FILE].isString()) {
+            mTLS.mKeyFile = tlsConfig[prometheus::KEY_FILE].asString();
+        } else {
+            LOG_ERROR(sLogger, ("tls config error", ""));
+            return false;
+        }
+    }
+    if (tlsConfig.isMember(prometheus::SERVER_NAME)) {
+        if (tlsConfig[prometheus::SERVER_NAME].isString()) {
+            mRequestHeaders[prometheus::HOST] = tlsConfig[prometheus::SERVER_NAME].asString();
+        } else {
+            LOG_ERROR(sLogger, ("tls config error", ""));
+            return false;
+        }
+    }
+    if (tlsConfig.isMember(prometheus::INSECURE_SKIP_VERIFY)) {
+        if (tlsConfig[prometheus::INSECURE_SKIP_VERIFY].isBool()) {
+            mTLS.mInsecureSkipVerify = tlsConfig[prometheus::INSECURE_SKIP_VERIFY].asBool();
+        } else {
+            LOG_ERROR(sLogger, ("tls config error", ""));
+            return false;
+        }
+    }
+    mEnableTLS = true;
+    return true;
+}
+
 } // namespace logtail
diff --git a/core/prometheus/schedulers/ScrapeConfig.h b/core/prometheus/schedulers/ScrapeConfig.h
@@ -7,6 +7,7 @@
 #include <string>
 #include <vector>
 
+#include "common/http/HttpRequest.h"
 #include "prometheus/labels/Relabel.h"
 
 
@@ -28,6 +29,10 @@ class ScrapeConfig {
     // enable_compression Accept-Encoding header: gzip, identity
     std::map<std::string, std::string> mRequestHeaders;
 
+    bool mFollowRedirects;
+    bool mEnableTLS;
+    CurlTLS mTLS;
+
     uint64_t mMaxScrapeSizeBytes;
     uint64_t mSampleLimit;
     uint64_t mSeriesLimit;
@@ -47,6 +52,7 @@ class ScrapeConfig {
     bool InitAuthorization(const Json::Value& authorization);
     bool InitScrapeProtocols(const Json::Value& scrapeProtocols);
     void InitEnableCompression(bool enableCompression);
+    bool InitTLSConfig(const Json::Value& tlsConfig);
 
 #ifdef APSARA_UNIT_TEST_MAIN
     friend class ScrapeConfigUnittest;