From c75af2550e57c9073c10664837b4d361e7a79daf Mon Sep 17 00:00:00 2001 From: "zhanghaoxiang.zhx" Date: Tue, 31 Dec 2024 16:12:35 +0800 Subject: [PATCH] refactor sls client manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 本次代码评审主要增加了对企业配置提供者中区域端点的支持，引入了新的类和方法来管理端点信息，并优化了一些现有功能以提高性能和可靠性。 Link: https://code.alibaba-inc.com/sls/ilogtail/codereview/19609308 --- core/common/EnterpriseEndpointUtil.cpp | 81 + core/common/EnterpriseEndpointUtil.h | 33 + .../provider/EnterpriseConfigProvider.cpp | 179 +-- .../provider/EnterpriseConfigProvider.h | 10 +- core/config_sdk/ConfigClient.cpp | 24 +- .../queue/SLSProfileSenderQueueItem.h | 26 +- .../sls/EnterpriseFlusherSLSMonitor.cpp | 136 +- .../flusher/sls/EnterpriseFlusherSLSMonitor.h | 14 +- .../sls/EnterpriseSLSClientManager.cpp | 1192 +++++++++++++- .../flusher/sls/EnterpriseSLSClientManager.h | 226 ++- core/shennong/MetricSender.cpp | 6 +- .../EnterpriseConfigProviderUnittest.cpp | 8 +- .../EnterpriseSLSClientManagerUnittest.cpp | 1378 ++++++++++++++++- .../flusher/SLSNetworkRequestMock.cpp | 160 ++ core/unittest/flusher/SLSNetworkRequestMock.h | 76 + .../test_cases/framework/backpressure.feature | 127 ++ 16 files changed, 3350 insertions(+), 326 deletions(-) create mode 100644 core/common/EnterpriseEndpointUtil.cpp create mode 100644 core/common/EnterpriseEndpointUtil.h create mode 100644 core/unittest/flusher/SLSNetworkRequestMock.cpp create mode 100644 core/unittest/flusher/SLSNetworkRequestMock.h create mode 100644 test/e2e_enterprise/test_cases/framework/backpressure.feature diff --git a/core/common/EnterpriseEndpointUtil.cpp b/core/common/EnterpriseEndpointUtil.cpp new file mode 100644 index 0000000000..aa9df1e565 --- /dev/null +++ b/core/common/EnterpriseEndpointUtil.cpp @@ -0,0 +1,81 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/EnterpriseEndpointUtil.h" + +#include "common/EndpointUtil.h" +#include "common/StringTools.h" +#include "logger/Logger.h" + +using namespace std; + +namespace logtail { + +const string kLogEndpointSuffix = ".log.aliyuncs.com"; +const string kAccelerationDataEndpoint = "log-global.aliyuncs.com"; + +bool IsCustomEndpoint(const string& endpoint) { + if (StartWith(endpoint, "log.") || StartWith(endpoint, "log-intranet.") || StartWith(endpoint, "log-internal.") + || EndWith(endpoint, ".aliyuncs.com") || EndWith(endpoint, ".aliyun-inc.com")) { + return false; + } + return true; +} + +EndpointAddressType GetEndpointAddressType(const string& address) { + // 一国一云 OXS区访问 & VPC访问 + if (StartWith(address, "log-intranet.") || StartWith(address, "log-internal.")) { + return EndpointAddressType::INTRANET; + } + // 一国一云 公网访问 + if (StartWith(address, "log.")) { + return EndpointAddressType::PUBLIC; + } + if (EndWith(address, "-intranet" + kLogEndpointSuffix) || EndWith(address, "-internal" + kLogEndpointSuffix)) { + return EndpointAddressType::INTRANET; + } + if (!EndWith(address, "-share" + kLogEndpointSuffix) && EndWith(address, kLogEndpointSuffix)) { + return EndpointAddressType::PUBLIC; + } + return EndpointAddressType::INNER; +} + +string CheckAddress(const string& definedAddress, const string& defaultAddress) { + // Domain name aliyun.com is going done, do not use it any more. + // Aone: https://aone.alibaba-inc.com/req/24401254. + if (definedAddress.find(".aliyun.com") != string::npos) { + const char* recommendedAddress = "http://cn-hangzhou-corp.sls.aliyuncs.com"; + LOG_INFO(sLogger, ("*.aliyun.com is deprecated", definedAddress)("new", recommendedAddress)); + return recommendedAddress; + } + + return StandardizeHost(definedAddress, defaultAddress); +} + +string CheckDataServerEndpoint(const string& endpoint) { + constexpr size_t logtailPrefixLen = sizeof("logtail.") - 1; + constexpr size_t configPrefixLen = sizeof("config.") - 1; + + size_t index = endpoint.find("logtail."); + if (index == 0) { + return endpoint.substr(logtailPrefixLen); + } + index = endpoint.find("config."); + if (index == 0) { + return endpoint.substr(configPrefixLen); + } + return endpoint; +} + +} // namespace logtail diff --git a/core/common/EnterpriseEndpointUtil.h b/core/common/EnterpriseEndpointUtil.h new file mode 100644 index 0000000000..0878540103 --- /dev/null +++ b/core/common/EnterpriseEndpointUtil.h @@ -0,0 +1,33 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace logtail { + +enum class EndpointAddressType { INNER, INTRANET, PUBLIC }; + +extern const std::string kLogEndpointSuffix; +extern const std::string kAccelerationDataEndpoint; + +bool IsCustomEndpoint(const std::string& endpoint); +EndpointAddressType GetEndpointAddressType(const std::string& address); +std::string CheckAddress(const std::string& definedAddress, const std::string& defaultAddress); +std::string CheckDataServerEndpoint(const std::string& definedAddress); + +} // namespace logtail diff --git a/core/config/provider/EnterpriseConfigProvider.cpp b/core/config/provider/EnterpriseConfigProvider.cpp index 61fc47ac79..b95f73e1f9 100644 --- a/core/config/provider/EnterpriseConfigProvider.cpp +++ b/core/config/provider/EnterpriseConfigProvider.cpp @@ -20,6 +20,7 @@ #include "app_config/AppConfig.h" #include "application/Application.h" #include "common/EndpointUtil.h" +#include "common/EnterpriseEndpointUtil.h" #include "common/FileSystemUtil.h" #include "common/Flags.h" #include "common/JsonUtil.h" @@ -36,7 +37,7 @@ #include "logger/Logger.h" #include "monitor/AlarmManager.h" #include "monitor/Monitor.h" -#include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #include "profile_sender/ProfileSender.h" DEFINE_FLAG_STRING(ilogtail_aliuid_env_name, "aliuid", "ALIYUN_LOGTAIL_USER_ID"); @@ -69,7 +70,6 @@ DEFINE_FLAG_STRING(user_log_config, "", "user_log_config.json"); DEFINE_FLAG_BOOL(enable_host_id, "", false); DECLARE_FLAG_STRING(profile_project_name); -DECLARE_FLAG_BOOL(global_network_success); using namespace std; @@ -92,8 +92,6 @@ const string CN_HANGZHOU_FINANCE = "cn-hangzhou-finance"; const string kConfigPrefix = "/etc/ilogtail/conf/"; const string kConfigSuffix = "/ilogtail_config.json"; -const string kLogEndpointSuffix = ".log.aliyuncs.com"; -const string kAccelerationDataEndpoint = "log-global.aliyuncs.com"; const string kConfigTemplate = R"({ "config_servers" : [ @@ -122,9 +120,6 @@ const string kConfigTemplate = R"({ "buffer_map_num" : 5 })"; -const string DEBUG_SERVER_FILE = "file"; -const string DEBUG_SERVER_NULL = "null"; - void AddHardCodeEndpointList(); bool ReadAliuidsFile(vector& aliuids); void GetRelatedRegionList(const string& region, vector& regionList); @@ -132,8 +127,6 @@ bool NeedCheckConfig(const unordered_map& regionConfigUpdateMap, c bool NeedCheckConfigSingle(const unordered_map& regionConfigUpdateMap, const string& region); bool IsValidAliuid(const string& aliuid); void InitHttpsAddr(const vector& srcConfigServerList, vector& desHttpsConfigServerList); -string CheckAddress(const string& definedAddress, const string& defaultAdderss); -string CheckDataServerAddress(const string& definedAddress, const string& defaultAdderss); void AddConfigInfoToRequest(const std::pair& configInfo, configserver::proto::v2::ConfigInfo* reqConfig); void EnterpriseConfigProvider::RegionConfigServerInfo::Reset(const string& region, @@ -343,12 +336,13 @@ void EnterpriseConfigProvider::LoadLegacyGlobalConfig(const Json::Value& confJso } } + unordered_map> regionEndpoints; mDataServer.clear(); bool findDataServerItem = false; if (confJson.isMember("data_server_address") && confJson["data_server_address"].isString()) { findDataServerItem = true; - mDataServer = CheckDataServerAddress(TrimString(confJson["data_server_address"].asString(), ' ', ' '), - STRING_FLAG(logtail_send_address)); + mDataServer = CheckAddress(TrimString(confJson["data_server_address"].asString(), ' ', ' '), + STRING_FLAG(logtail_send_address)); if (mIsPrivateCloud) { LOG_INFO(sLogger, ("data_server_mode", "local")("data_server_address", mDataServer)); FlusherSLS::SetDefaultRegion(STRING_FLAG(default_region_name)); @@ -357,9 +351,7 @@ void EnterpriseConfigProvider::LoadLegacyGlobalConfig(const Json::Value& confJso } LOG_INFO(sLogger, ("init default region from old config file, region", FlusherSLS::GetDefaultRegion())); - bool isProxy = mDataServer.find("sls-proxy.aliyun-inc.com") != string::npos; - SLSClientManager::GetInstance()->AddEndpointEntry( - FlusherSLS::GetDefaultRegion(), mDataServer, isProxy, SLSClientManager::EndpointSourceType::LOCAL); + regionEndpoints[FlusherSLS::GetDefaultRegion()].push_back(mDataServer); } string defaultRegion = FlusherSLS::GetDefaultRegion(); @@ -481,10 +473,7 @@ void EnterpriseConfigProvider::LoadLegacyGlobalConfig(const Json::Value& confJso string region = TrimString(dValue["cluster"].asString(), ' ', ' '); string endpoint = TrimString(dValue["endpoint"].asString(), ' ', ' '); if (region.size() > 0 && endpoint.size() > 0) { - endpoint = CheckDataServerAddress(endpoint, endpoint); - bool isProxy = endpoint.find("sls-proxy.aliyun-inc.com") != string::npos; - SLSClientManager::GetInstance()->AddEndpointEntry( - region, endpoint, isProxy, SLSClientManager::EndpointSourceType::LOCAL); + regionEndpoints[region].push_back(endpoint); // use first region as default region if (!findDataServerItem) { findDataServerItem = true; @@ -492,28 +481,26 @@ void EnterpriseConfigProvider::LoadLegacyGlobalConfig(const Json::Value& confJso LOG_INFO(sLogger, ("init default region from config file, region", FlusherSLS::GetDefaultRegion())); } - - // if (confJson["data_server_list"].size() == 1 && !findDataServerItem) - // mDefaultRegion = region; } } } } else if (!findDataServerItem) { mDataServer = STRING_FLAG(logtail_send_address); FlusherSLS::SetDefaultRegion(STRING_FLAG(default_region_name)); - SLSClientManager::GetInstance()->AddEndpointEntry(FlusherSLS::GetDefaultRegion(), - CheckDataServerAddress(mDataServer, mDataServer), - false, - SLSClientManager::EndpointSourceType::LOCAL); + regionEndpoints[FlusherSLS::GetDefaultRegion()].push_back(mDataServer); } } + for (const auto& item : regionEndpoints) { + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(item.first, item.second); + } + if (!mIsPrivateCloud) { AddHardCodeEndpointList(); } } bool EnterpriseConfigProvider::IsConfigServerInList(const string& configServer) { - return find(mConfigServerList.begin(), mConfigServerList.end(), configServer) != mConfigServerList.end(); + return std::find(mConfigServerList.begin(), mConfigServerList.end(), configServer) != mConfigServerList.end(); } void EnterpriseConfigProvider::AddConfigServerList(string configServer) { @@ -703,20 +690,14 @@ void EnterpriseConfigProvider::LoadGlobalConfig(const Json::Value& confJson) { continue; } + vector endpoints; for (const auto& endpointStr : dValue["endpoint_list"]) { if (!endpointStr.isString()) { continue; } - string endpoint = TrimString(endpointStr.asString(), ' ', ' '); - if (endpoint.empty()) { - continue; - } + endpoints.push_back(endpointStr.asString()); - endpoint = CheckDataServerAddress(endpoint, endpoint); - bool isProxy = endpoint.find("sls-proxy.aliyun-inc.com") != string::npos; - SLSClientManager::GetInstance()->AddEndpointEntry( - region, endpoint, isProxy, SLSClientManager::EndpointSourceType::LOCAL); // use first region as default region if (!findDataServerItem) { findDataServerItem = true; @@ -725,15 +706,14 @@ void EnterpriseConfigProvider::LoadGlobalConfig(const Json::Value& confJson) { ("init default region from config file, region", FlusherSLS::GetDefaultRegion())); } } + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region, endpoints); } } else if (!findDataServerItem) { mDataServer = STRING_FLAG(logtail_send_address); const string defaultRegion = STRING_FLAG(default_region_name); FlusherSLS::SetDefaultRegion(defaultRegion); - SLSClientManager::GetInstance()->AddEndpointEntry(FlusherSLS::GetDefaultRegion(), - CheckDataServerAddress(mDataServer, mDataServer), - false, - SLSClientManager::EndpointSourceType::LOCAL); + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo( + FlusherSLS::GetDefaultRegion(), {mDataServer}); } } if (!mIsPrivateCloud) { @@ -786,12 +766,7 @@ void EnterpriseConfigProvider::LoadRegionConfig() { ProfileSender::GetInstance()->SetProfileProjectName(profileRegion, profileProjectName); LOG_INFO(sLogger, ("set profiling info, region", profileRegion)("project", profileProjectName)); } - for (const auto& endPoint : endpointList) { - SLSClientManager::GetInstance()->AddEndpointEntry(profileRegion, - CheckDataServerAddress(endPoint, endPoint), - false, - SLSClientManager::EndpointSourceType::REMOTE); - } + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints(profileRegion, endpointList); if (!enterpriseResponseExtension.region_type().empty()) { string regionType = ToLowerCaseString(enterpriseResponseExtension.region_type()); if (regionType == "pub" && mRegionType != RegionType::REGION_PUB) { @@ -1024,8 +999,6 @@ void EnterpriseConfigProvider::GetConfigUpdate() { } isAllRegionsVisitedSuccessfully &= isVisitSuccessful; regionVisitStatusMap[FlusherSLS::GetDefaultRegion()] = isVisitSuccessful; - // the network condition of default region is always considered good - FlusherSLS::UpdateRegionStatus(FlusherSLS::GetDefaultRegion(), true); LOG_DEBUG(sLogger, ("begin to visit sub regions, size", mRegionConfigServerMap.size())); for (auto& iter : mRegionConfigServerMap) { @@ -1044,7 +1017,6 @@ void EnterpriseConfigProvider::GetConfigUpdate() { continue; } if (!NeedCheckConfig(regionVisitStatusMap, pInfo->mRegion)) { - FlusherSLS::UpdateRegionStatus(pInfo->mRegion, true); LOG_DEBUG(sLogger, ("skip sub region visit", "related regions are visited successfully, no need to visit")("region", pInfo->mRegion)); @@ -1078,7 +1050,6 @@ void EnterpriseConfigProvider::GetConfigUpdate() { modifiedInstanceConfigs, pInfo->mRegionConfigHash, pInfo->mSequenceNum++)) { - FlusherSLS::UpdateRegionStatus(pInfo->mRegion, true); pInfo->mNextGetConfigTime = 0; pInfo->mRetryInterval = 0; ++updatedRegionsCnt; @@ -1100,7 +1071,6 @@ void EnterpriseConfigProvider::GetConfigUpdate() { } // 疑惑：原来第一次出问题不会更新region status，是不是不太对？ if (!isVisitSuccessful) { - FlusherSLS::UpdateRegionStatus(pInfo->mRegion, false); if (pInfo->mRetryInterval > INT32_FLAG(global_pub_config_retry_interval_net_max)) { pInfo->mRetryInterval = INT32_FLAG(global_pub_config_retry_interval_net_max); } @@ -1108,7 +1078,6 @@ void EnterpriseConfigProvider::GetConfigUpdate() { ("config server visit is paused", "network connection failed")("region", pInfo->mRegion)( "retry interval secs", pInfo->mRetryInterval)); } else { - FlusherSLS::UpdateRegionStatus(pInfo->mRegion, true); if (pInfo->mRetryInterval > INT32_FLAG(global_pub_config_retry_interval_noconfig_max)) { pInfo->mRetryInterval = INT32_FLAG(global_pub_config_retry_interval_noconfig_max); } @@ -1118,7 +1087,6 @@ void EnterpriseConfigProvider::GetConfigUpdate() { } pInfo->mNextGetConfigTime = nowTime + pInfo->mRetryInterval; } else { - FlusherSLS::UpdateRegionStatus(pInfo->mRegion, true); pInfo->mRetryInterval = 0; pInfo->mNextGetConfigTime = 0; } @@ -1284,7 +1252,6 @@ bool EnterpriseConfigProvider::GetConfigUpdate(const string& region, return false; } - BOOL_FLAG(global_network_success) = true; string errorMsg; configserver::proto::enterprise::EnterpriseResponseExtension enterpriseResponseExtension; if (!enterpriseResponseExtension.ParseFromString(heartbeatResponse.opaque())) { @@ -1420,6 +1387,13 @@ void EnterpriseConfigProvider::MergeRegionList( bool isDefaultRegion, const int64_t& regionConfigHash) { // static const EndpointAddressType sConfigServerAddrNetType = mConfigServerAddressNetType; + string primaryRegion; + for (auto& item : regionList) { + if (item.primary()) { + primaryRegion = item.name(); + break; + } + } for (auto iter = regionList.begin(); iter != regionList.end(); ++iter) { const auto& regionInfo = iter; vector endPoints; @@ -1453,6 +1427,11 @@ void EnterpriseConfigProvider::MergeRegionList( string profileProject = regionInfo->profile_project(); if (CheckRegionInfoUpdate(region, endPoints, profileProject)) { UpdateRegionInfo(region, endPoints, profileProject, regionConfigHash); + // for corp scene, no need to copy + if (primaryRegion != CN_HANGZHOU_CORP && !regionInfo->primary()) { + EnterpriseSLSClientManager::GetInstance()->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted( + primaryRegion, region); + } } } } @@ -1692,10 +1671,7 @@ void EnterpriseConfigProvider::UpdateRegionInfo(const string& region, const string& profileProject, const int64_t& regionConfigHash) { ProfileSender::GetInstance()->SetProfileProjectName(region, profileProject); - for (const auto& endPoint : endPoints) { - SLSClientManager::GetInstance()->AddEndpointEntry( - region, CheckDataServerAddress(endPoint, endPoint), false, SLSClientManager::EndpointSourceType::REMOTE); - } + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints(region, endPoints); auto iter = mRegionConfigServerMap.find(region); RegionConfigServerInfo* pInfo = nullptr; if (iter == mRegionConfigServerMap.end()) { @@ -2162,25 +2138,6 @@ bool EnterpriseConfigProvider::GenerateAPPConfigByConfigPath(const string& fileP } } -EnterpriseConfigProvider::EndpointAddressType -EnterpriseConfigProvider::GetEndpointAddressType(const string& address) const { - // 一国一云 OXS区访问 & VPC访问 - if (StartWith(address, "log-intranet.") || StartWith(address, "log-internal.")) { - return EndpointAddressType::INTRANET; - } - // 一国一云 公网访问 - if (StartWith(address, "log.")) { - return EndpointAddressType::PUBLIC; - } - if (EndWith(address, "-intranet" + kLogEndpointSuffix) || EndWith(address, "-internal" + kLogEndpointSuffix)) { - return EndpointAddressType::INTRANET; - } - if (!EndWith(address, "-share" + kLogEndpointSuffix) && EndWith(address, kLogEndpointSuffix)) { - return EndpointAddressType::PUBLIC; - } - return EndpointAddressType::INNER; -} - string EnterpriseConfigProvider::GetInstanceId() { return Application::GetInstance()->GetInstanceId(); } @@ -2276,7 +2233,7 @@ EnterpriseConfigProvider::PrepareHeartbeat(const vector& regionList, if (!isDefaultRegion) { continue; } - } else if (find(regionList.begin(), regionList.end(), region) == regionList.end()) { + } else if (std::find(regionList.begin(), regionList.end(), region) == regionList.end()) { continue; } } @@ -2470,20 +2427,14 @@ void EnterpriseConfigProvider::FeedbackOnetimePipelineConfigStatus(const std::st void AddHardCodeEndpointList() { // hard code corp endpoint, // http://gitlab.alibaba-inc.com/sls/desin_doc/blob/master/Logtail/%E5%BC%B9%E5%86%85%E5%90%84%E7%A7%8D%E7%BD%91%E7%BB%9C%E7%8E%AF%E5%A2%83%E4%B8%8B%E8%87%AA%E5%8A%A8%E9%80%89%E6%8B%A9%E5%8F%AF%E7%94%A8%E8%AE%BF%E9%97%AE%E5%85%A5%E5%8F%A3.md - SLSClientManager::GetInstance()->AddEndpointEntry( - CN_HANGZHOU_CORP, "http://sls.aliyun-inc.com", false, SLSClientManager::EndpointSourceType::LOCAL); - SLSClientManager::GetInstance()->AddEndpointEntry(CN_HANGZHOU_CORP, - "http://cn-hangzhou-corp.sls.aliyuncs.com", - false, - SLSClientManager::EndpointSourceType::LOCAL); - SLSClientManager::GetInstance()->AddEndpointEntry(CN_SHANGHAI_CORP, - "http://cn-shanghai-intranet.log.aliyun-inc.com", - false, - SLSClientManager::EndpointSourceType::LOCAL); - SLSClientManager::GetInstance()->AddEndpointEntry(CN_SHANGHAI_CORP, - "http://cn-shanghai-corp.sls.aliyuncs.com", - false, - SLSClientManager::EndpointSourceType::LOCAL); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + CN_HANGZHOU_CORP, + {"sls.aliyun-inc.com", "cn-hangzhou-corp.sls.aliyuncs.com"}, + EnterpriseSLSClientManager::RemoteEndpointUpdateAction::APPEND); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + CN_SHANGHAI_CORP, + {"cn-shanghai-intranet.log.aliyun-inc.com", "cn-shanghai-corp.sls.aliyuncs.com"}, + EnterpriseSLSClientManager::RemoteEndpointUpdateAction::APPEND); } bool ReadAliuidsFile(vector& aliuids) { @@ -2632,54 +2583,6 @@ void InitHttpsAddr(const vector& srcConfigServerList, vector& de } } -/** - * @brief 检查并处理Address - * - * 该函数用于检查和处理给定的Address。它执行以下操作： - * 1. 检查是否为特殊调试Address（DEBUG_SERVER_FILE 或 DEBUG_SERVER_NULL）或空Address - * 2. 检查是否包含已弃用的 aliyun.com 域名 - * 3. 使用 StandardizeEndpoint 函数标准化Address - */ -string CheckAddress(const string& definedAddress, const string& defaultAdderss) { - if (definedAddress == DEBUG_SERVER_FILE || definedAddress == DEBUG_SERVER_NULL || definedAddress.empty()) - return definedAddress; - - // Domain name aliyun.com is going done, do not use it any more. - // Aone: https://aone.alibaba-inc.com/req/24401254. - if (definedAddress.find(".aliyun.com") != string::npos) { - const char* recommendedAddress = "http://cn-hangzhou-corp.sls.aliyuncs.com"; - LOG_INFO(sLogger, ("*.aliyun.com is deprecated", definedAddress)("new", recommendedAddress)); - return recommendedAddress; - } - - return StandardizeEndpoint(definedAddress, defaultAdderss); -} - -/** - * @brief 检查并处理DataServerAddress - * - * 该函数用于检查和处理给定的DataServerAddress。它执行以下操作： - * 1. 使用CheckAddress函数标准化输入的地址 - * 2. 移除地址中的"logtail."或"config."前缀（如果存在） - */ -string CheckDataServerAddress(const string& definedAddress, const string& defaultAdderss) { - constexpr size_t logtailPrefixLen = sizeof("logtail.") - 1; - constexpr size_t configPrefixLen = sizeof("config.") - 1; - - string endpoint = CheckAddress(definedAddress, defaultAdderss); - size_t index = endpoint.find("logtail."); - if (index != string::npos) { - endpoint = endpoint.substr(0, index) + endpoint.substr(index + logtailPrefixLen); - return endpoint; - } - index = endpoint.find("config."); - if (index != string::npos) { - endpoint = endpoint.substr(0, index) + endpoint.substr(index + configPrefixLen); - return endpoint; - } - return endpoint; -} - void AddConfigInfoToRequest(const std::pair& configInfo, configserver::proto::v2::ConfigInfo* reqConfig) { reqConfig->set_name(configInfo.second.name); diff --git a/core/config/provider/EnterpriseConfigProvider.h b/core/config/provider/EnterpriseConfigProvider.h index a00d1af545..312b945321 100644 --- a/core/config/provider/EnterpriseConfigProvider.h +++ b/core/config/provider/EnterpriseConfigProvider.h @@ -29,6 +29,7 @@ #include #include +#include "common/EnterpriseEndpointUtil.h" #include "common/Lock.h" #include "common/Thread.h" #include "common/WaitObject.h" @@ -80,8 +81,9 @@ class EnterpriseConfigProvider : public ConfigProvider, ConfigFeedbackable { void FeedbackContinuousPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status) override; void FeedbackInstanceConfigStatus(const std::string& name, ConfigFeedbackStatus status) override; - void - FeedbackOnetimePipelineConfigStatus(const std::string& type, const std::string& name, ConfigFeedbackStatus status) override; + void FeedbackOnetimePipelineConfigStatus(const std::string& type, + const std::string& name, + ConfigFeedbackStatus status) override; #ifdef APSARA_UNIT_TEST_MAIN void SetHttpsConfigServerList(const std::vector& configServerList) { @@ -99,7 +101,6 @@ class EnterpriseConfigProvider : public ConfigProvider, ConfigFeedbackable { private: bool isLegacyConfig(const Json::Value& confJson); - enum class EndpointAddressType { INNER, INTRANET, PUBLIC }; enum class RegionType { REGION_PUB, REGION_CORP }; enum class ConfirmSubConfigServerAddrsTaskStatus { OnUnitTest, @@ -164,7 +165,6 @@ class EnterpriseConfigProvider : public ConfigProvider, ConfigFeedbackable { void LoadLegacyGlobalConfig(const Json::Value& confJson); void LoadConfigFile(); bool UseHTTPSConfigServer() const { return mUseHTTPSConfigServer; } - EndpointAddressType GetEndpointAddressType(const std::string& address) const; void ReloadLogtailSysConf(); void CorrectionAliuidFile() const; @@ -330,11 +330,9 @@ class EnterpriseConfigProvider : public ConfigProvider, ConfigFeedbackable { std::map mBuiltInPipelineMap; -#ifdef __ENTERPRISE__ friend class FlusherSLSUnittest; friend class EnterpriseConfigProviderUnittest; friend class ProcessorTagNativeUnittest; -#endif }; } // namespace logtail diff --git a/core/config_sdk/ConfigClient.cpp b/core/config_sdk/ConfigClient.cpp index 1ab17eae8c..b41297ffdf 100644 --- a/core/config_sdk/ConfigClient.cpp +++ b/core/config_sdk/ConfigClient.cpp @@ -26,10 +26,12 @@ #include #include -#include "constants/Constants.h" #include "common/Flags.h" +#include "common/http/Constant.h" +#include "constants/Constants.h" #include "logger/Logger.h" -#include "sdk/Common.h" +#include "plugin/flusher/sls/SLSConstant.h" +#include "plugin/flusher/sls/SLSUtil.h" #include "shennong/sdk/sls_adapter.h" DEFINE_FLAG_BOOL(oas_requst_enable_trace, "", false); @@ -290,23 +292,23 @@ namespace config { struct curl_slist* headers = NULL; std::map headerMap; - headerMap[sdk::CONTENT_TYPE] = sdk::TYPE_LOG_PROTOBUF; - headerMap[sdk::USER_AGENT] = AGENT_NAME; - headerMap[sdk::X_LOG_APIVERSION] = sdk::LOG_API_VERSION; - headerMap[sdk::X_LOG_SIGNATUREMETHOD] = sdk::HMAC_SHA1; - headerMap[sdk::DATE] = sdk::GetDateString(); - headerMap[sdk::CONTENT_LENGTH] = std::to_string(config.size()); + headerMap[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; + headerMap[USER_AGENT] = AGENT_NAME; + headerMap[X_LOG_APIVERSION] = LOG_API_VERSION; + headerMap[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; + headerMap[DATE] = GetDateString(); + headerMap[CONTENT_LENGTH] = std::to_string(config.size()); for (const auto& pair : headerMap) { std::string header = pair.first + ":" + pair.second; headers = curl_slist_append(headers, header.c_str()); } - std::string signature = sdk::GetUrlSignature( - sdk::HTTP_POST, targetURL, headerMap, std::map{}, config, accessKey); + std::string signature = GetUrlSignature( + logtail::HTTP_POST, targetURL, headerMap, std::map{}, config, accessKey); string signature_header - = string(sdk::AUTHORIZATION) + ":" + string(sdk::LOG_HEADSIGNATURE_PREFIX) + accessKeyId + ":" + signature; + = string(AUTHORIZATION) + ":" + string(LOG_HEADSIGNATURE_PREFIX) + accessKeyId + ":" + signature; headers = curl_slist_append(headers, signature_header.c_str()); return SyncSendRequest(HTTP_POST, diff --git a/core/pipeline/queue/SLSProfileSenderQueueItem.h b/core/pipeline/queue/SLSProfileSenderQueueItem.h index e65c2ac24c..31ffa9323c 100644 --- a/core/pipeline/queue/SLSProfileSenderQueueItem.h +++ b/core/pipeline/queue/SLSProfileSenderQueueItem.h @@ -26,18 +26,26 @@ class Pipeline; struct SLSProfileSenderQueueItem : public SLSSenderQueueItem { // only profile data need project and region - std::string mProject; + std::string mProfileProject; std::string mRegion; SLSProfileSenderQueueItem(std::string&& data, - size_t rawSize, - Flusher* flusher, - QueueKey key, - const std::string& project, - const std::string& region, - const std::string& logstore) - : SLSSenderQueueItem(std::move(data), rawSize, flusher, key, logstore, RawDataType::EVENT_GROUP, "", RangeCheckpointPtr(), false), - mProject(project), + size_t rawSize, + Flusher* flusher, + QueueKey key, + const std::string& project, + const std::string& region, + const std::string& logstore) + : SLSSenderQueueItem(std::move(data), + rawSize, + flusher, + key, + logstore, + RawDataType::EVENT_GROUP, + "", + RangeCheckpointPtr(), + false), + mProfileProject(project), mRegion(region) {} SenderQueueItem* Clone() override { return new SLSProfileSenderQueueItem(*this); } diff --git a/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.cpp b/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.cpp index a5ea6bbedc..e854a4d42b 100644 --- a/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.cpp +++ b/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.cpp @@ -21,11 +21,15 @@ #include "app_config/AppConfig.h" #include "common/LogtailCommonFlags.h" #include "common/compression/CompressorFactory.h" +#include "common/http/Constant.h" #include "monitor/Monitor.h" #include "pipeline/queue/SLSProfileSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#include "pipeline/serializer/JsonSerializer.h" +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSConstant.h" #include "plugin/flusher/sls/SLSResponse.h" #include "provider/Provider.h" #include "runner/FlusherRunner.h" @@ -37,7 +41,6 @@ DECLARE_FLAG_INT32(merge_log_count_limit); DECLARE_FLAG_INT32(batch_send_metric_size); DECLARE_FLAG_INT32(max_send_log_group_size); DECLARE_FLAG_DOUBLE(sls_serialize_size_expansion_ratio); -DECLARE_FLAG_INT32(profile_data_send_retrytimes); namespace logtail { @@ -56,11 +59,11 @@ set GetRegionsByProjects(string projects) { return regions; } -std::string FormatDoubleCompact(double value) { - std::string str = ToString(value); +string FormatDoubleCompact(double value) { + string str = ToString(value); size_t decimalPos = str.find('.'); - if (decimalPos != std::string::npos) { - str.erase(str.find_last_not_of('0') + 1, std::string::npos); + if (decimalPos != string::npos) { + str.erase(str.find_last_not_of('0') + 1, string::npos); if (str.back() == '.') { str.pop_back(); } @@ -68,9 +71,9 @@ std::string FormatDoubleCompact(double value) { return str; } -void FillOutputCategory(const Json::Value& config, const string& key, std::set& outputCategorySet) { +void FillOutputCategory(const Json::Value& config, const string& key, set& outputCategorySet) { string errorMsg; - std::vector data; + vector data; if (GetOptionalListFilterParam(config, key, data, errorMsg)) { for (auto& category : data) { outputCategorySet.insert(category); @@ -80,14 +83,17 @@ void FillOutputCategory(const Json::Value& config, const string& key, std::set> FlusherSLSMonitor::sRegionCandidateHostsInfoMap; + // sls output -const std::string LOGSTORE_STATUS = "logtail_status_profile"; +const string LOGSTORE_STATUS = "logtail_status_profile"; const string METRIC_TOPIC_TYPE_STATUS = "loongcollector_status"; -const std::string LOGSTORE_SHENNONG = "shennong_log_profile"; +const string LOGSTORE_SHENNONG = "shennong_log_profile"; const string METRIC_TOPIC_TYPE_SHENNONG = "loongcollector_metric"; // file output -const std::string FileOutputPath = GetAgentLogDir() + "/self_metrics/self_metrics.log"; -const std::string FileOutputPattern = "[%Y-%m-%d %H:%M:%S.%f]\t%v"; +const string FileOutputPath = GetAgentLogDir() + "/self_metrics/self_metrics.log"; +const string FileOutputPattern = "[%Y-%m-%d %H:%M:%S.%f]\t%v"; const uint32_t FileOutputMaxFileSize = 1024 * 1024 * 10; const uint32_t FileOutputMaxFiles = 10; @@ -112,9 +118,9 @@ bool FlusherSLSMonitor::Init(const Json::Value& config, Json::Value& optionalGoP mSLSCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, mPluginID, CompressType::ZSTD); // init file output - auto file_sink = std::make_shared( + auto file_sink = make_shared( FileOutputPath, FileOutputMaxFileSize, FileOutputMaxFiles, true); - mFileWriter = std::make_shared( + mFileWriter = make_shared( sName, file_sink, spdlog::thread_pool(), spdlog::async_overflow_policy::block); mFileWriter->set_pattern(FileOutputPattern); mFileBatcher.Init(Json::Value(), this, DefaultFlushStrategyOptions{}); @@ -125,7 +131,7 @@ bool FlusherSLSMonitor::Init(const Json::Value& config, Json::Value& optionalGoP bool FlusherSLSMonitor::Send(PipelineEventGroup&& g) { unordered_map slsOutput; - PipelineEventGroup fileOutput(std::make_shared()); + PipelineEventGroup fileOutput(make_shared()); // 入：一个PipelineEventGroup（内部是MetricEvent），是完整的一分钟的指标，所以在这里可以对指标进行聚合、关联。 // 出： @@ -191,7 +197,7 @@ bool FlusherSLSMonitor::FilterPipelineEventGroup(PipelineEventGroup&& originGrou } // 写shennong if (mShennongOutputCategory.find(e.GetName().to_string()) != mShennongOutputCategory.end()) { - set regions = GetRegionsByProjects(e.GetTag("project").to_string()); + set regions = GetRegionsByProjects(e.GetTag("project").to_string()); for (auto& region : regions) { LogEvent* outEvent = CreateLogEvent(slsOutput, METRIC_TOPIC_TYPE_SHENNONG, region, LOGSTORE_SHENNONG); TransToLogEvent(e, outEvent); @@ -243,7 +249,7 @@ bool FlusherSLSMonitor::SerializeAndPushSLS(BatchedEventsList&& groupList) { return allSucceeded; } -bool FlusherSLSMonitor::SerializeAndPushSLS(std::vector&& groupLists) { +bool FlusherSLSMonitor::SerializeAndPushSLS(vector&& groupLists) { bool allSucceeded = true; for (auto& groupList : groupLists) { allSucceeded = SerializeAndPushSLS(std::move(groupList)) && allSucceeded; @@ -275,31 +281,61 @@ bool FlusherSLSMonitor::SerializeAndPushFile(vector&& groupLi return allSucceeded; } -bool FlusherSLSMonitor::BuildRequest(SenderQueueItem* item, - std::unique_ptr& req, - bool* keepItem) const { +bool FlusherSLSMonitor::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem, string* errMsg) { + SLSClientManager::AuthType type; + string accessKeyId, accessKeySecret; + if (!SLSClientManager::GetInstance()->GetAccessKey( + STRING_FLAG(logtail_profile_aliuid), type, accessKeyId, accessKeySecret)) { + *keepItem = false; + *errMsg = "failed to get access key"; + return false; + } + auto data = static_cast(item); - sdk::Client* sendClient - = SLSClientManager::GetInstance()->GetClient(data->mRegion, STRING_FLAG(logtail_profile_aliuid)); - - int32_t curTime = time(NULL); - static int32_t lastResetEndpointTime = 0; - data->mCurrentEndpoint = sendClient->GetRawSlsHost(); - if (data->mCurrentEndpoint.empty()) { - if (curTime - lastResetEndpointTime >= 30) { - SLSClientManager::GetInstance()->ResetClientEndpoint( - STRING_FLAG(logtail_profile_aliuid), data->mRegion, curTime); - data->mCurrentEndpoint = sendClient->GetRawSlsHost(); - lastResetEndpointTime = curTime; + auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo( + data->mRegion, data->mProfileProject, EndpointMode::DEFAULT); + { + lock_guard lock(sRegionCandidateHostsInfoMapMux); + auto it = sRegionCandidateHostsInfoMap.find(data->mRegion); + if (it == sRegionCandidateHostsInfoMap.end()) { + sRegionCandidateHostsInfoMap.try_emplace(data->mRegion, info); + } else if (info.get() != it->second.get()) { + it->second = info; } } - req = sendClient->CreatePostLogStoreLogsRequest( - data->mProject, data->mLogstore, sls_logs::SLS_CMP_ZSTD, data->mData, data->mRawSize, item); - if (!req) { - *keepItem = true; + data->mCurrentHost = info->GetCurrentHost(); + if (data->mCurrentHost.empty()) { + if (info->IsInitialized()) { + *keepItem = false; + } else { + *keepItem = true; + } + *errMsg = "failed to get available host"; return false; } + + string path, query; + map header; + PreparePostLogStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + data->mCurrentHost, + data->mRealIpFlag, + data->mProfileProject, + data->mLogstore, + CompressTypeToString(CompressType::ZSTD), + data->mType, + data->mData, + data->mRawSize, + "", + optional(), + path, + query, + header); + bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(data->mRegion); + req = make_unique( + HTTP_POST, httpsFlag, data->mCurrentHost, httpsFlag ? 443 : 80, path, query, header, data->mData, item); return true; } @@ -307,35 +343,25 @@ void FlusherSLSMonitor::OnSendDone(const HttpResponse& response, SenderQueueItem SLSResponse slsResponse; if (AppConfig::GetInstance()->IsResponseVerificationEnabled() && !IsSLSResponse(response)) { slsResponse.mStatusCode = 0; - slsResponse.mErrorCode = sdk::LOGE_REQUEST_ERROR; + slsResponse.mErrorCode = LOGE_REQUEST_ERROR; slsResponse.mErrorMsg = "invalid response body"; } else { slsResponse.Parse(response); } - auto data = static_cast(item); - string configName = HasContext() ? GetContext().GetConfigName() : ""; - if (slsResponse.mStatusCode == 200) { - SenderQueueManager::GetInstance()->DecreaseConcurrencyLimiterInSendingCnt(item->mQueueKey); - DealSenderQueueItemAfterSend(item, false); - } else { - bool shouldRetry = true; - if (data->mTryCnt >= static_cast(INT32_FLAG(profile_data_send_retrytimes))) { - shouldRetry = false; - } - SenderQueueManager::GetInstance()->DecreaseConcurrencyLimiterInSendingCnt(item->mQueueKey); - DealSenderQueueItemAfterSend(item, shouldRetry); + if (slsResponse.mStatusCode != 200) { + // TODO: print error log every xx minutes. } + DealSenderQueueItemAfterSend(item, false); } - -LogEvent* FlusherSLSMonitor::CreateLogEvent(std::unordered_map& slsOutput, - const std::string topic, - const std::string region, - const std::string logstore) { +LogEvent* FlusherSLSMonitor::CreateLogEvent(unordered_map& slsOutput, + const string topic, + const string region, + const string logstore) { string key = logstore + region; if (slsOutput.find(key) == slsOutput.end()) { - slsOutput.emplace(key, PipelineEventGroup(std::make_shared())); + slsOutput.emplace(key, PipelineEventGroup(make_shared())); slsOutput.at(key).SetTag(LOG_RESERVED_KEY_SOURCE, LoongCollectorMonitor::mIpAddr); slsOutput.at(key).SetTag(LOG_RESERVED_KEY_TOPIC, topic); slsOutput.at(key).SetTag(PROFILE_PROJECT, GetProfileSender()->GetProfileProjectName(region)); @@ -461,4 +487,4 @@ void FlusherSLSMonitor::StatusProfileAggregator::AddRunnerInfo(const MetricEvent } } -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.h b/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.h index c3169fefbf..f118fc5ee7 100644 --- a/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.h +++ b/core/plugin/flusher/sls/EnterpriseFlusherSLSMonitor.h @@ -18,15 +18,19 @@ #include +#include +#include +#include #include +#include #include #include #include "common/compression/Compressor.h" #include "pipeline/batch/Batcher.h" #include "pipeline/plugin/interface/HttpFlusher.h" -#include "pipeline/serializer/JsonSerializer.h" -#include "pipeline/serializer/SLSSerializer.h" +#include "pipeline/serializer/Serializer.h" +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" namespace logtail { @@ -40,7 +44,7 @@ class FlusherSLSMonitor : public HttpFlusher { bool Flush(size_t key) override; bool FlushAll() override; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const override; + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override; void OnSendDone(const HttpResponse& response, SenderQueueItem* item) override; CompressType GetCompressType() const { @@ -48,6 +52,9 @@ class FlusherSLSMonitor : public HttpFlusher { } private: + static std::mutex sRegionCandidateHostsInfoMapMux; + static std::unordered_map> sRegionCandidateHostsInfoMap; + bool FilterPipelineEventGroup(PipelineEventGroup&& originGroup, std::unordered_map& slsOutput, PipelineEventGroup& fileOutput); @@ -76,6 +83,7 @@ class FlusherSLSMonitor : public HttpFlusher { void AddAgentInfo(const MetricEvent& rawEvent); void AddRunnerInfo(const MetricEvent& rawEvent); }; + LogEvent* CreateLogEvent(std::unordered_map& slsOutput, const std::string topic, const std::string region, diff --git a/core/plugin/flusher/sls/EnterpriseSLSClientManager.cpp b/core/plugin/flusher/sls/EnterpriseSLSClientManager.cpp index 05c0b21c06..6ec69abdaf 100644 --- a/core/plugin/flusher/sls/EnterpriseSLSClientManager.cpp +++ b/core/plugin/flusher/sls/EnterpriseSLSClientManager.cpp @@ -1,4 +1,4 @@ -// Copyright 2023 iLogtail Authors +// Copyright 2024 iLogtail Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,25 +14,32 @@ #include "plugin/flusher/sls/EnterpriseSLSClientManager.h" -#include -#if defined(__linux__) +#ifdef __linux__ #include #endif #include "app_config/AppConfig.h" +#include "common/EndpointUtil.h" +#include "common/EnterpriseEndpointUtil.h" #include "common/FileSystemUtil.h" #include "common/Flags.h" +#include "common/HashUtil.h" #include "common/JsonUtil.h" +#include "common/LogtailCommonFlags.h" #include "common/RuntimeUtil.h" +#include "common/StringTools.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/version.h" #include "config/provider/EnterpriseConfigProvider.h" #include "config_sdk/ConfigClient.h" #include "logger/Logger.h" #include "monitor/Monitor.h" -// for windows compatability, to avoid conflict with the same function defined in windows.h -#ifdef SetPort -#undef SetPort -#endif +#include "pipeline/queue/SenderQueueItem.h" +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#include "plugin/flusher/sls/PackIdManager.h" +#include "plugin/flusher/sls/SLSConstant.h" +#include "plugin/flusher/sls/SLSUtil.h" DEFINE_FLAG_STRING(logtail_profile_access_key_id, "default user's accessKeyId", "94to3z418yupi6ikawqqd370"); DEFINE_FLAG_STRING(logtail_profile_access_key, "default user's LogtailAccessKey", "2D2CB141D9B49DBCCF47ED437E3AF287"); @@ -46,6 +53,14 @@ DEFINE_FLAG_STRING(https_ca_cert, "set CURLOPT_CAINFO for libcurl", "cacert.pem" DEFINE_FLAG_BOOL(https_verify_peer, "set CURLOPT_SSL_VERIFYPEER, CURLOPT_SSL_VERIFYHOST option for libcurl", true); DEFINE_FLAG_INT32(sls_client_max_ak_error_cnt, "", 3); DEFINE_FLAG_STRING(default_aliuid, "", ""); +DEFINE_FLAG_INT32(sls_hosts_probe_max_try_cnt, "", 3); +DEFINE_FLAG_INT32(sls_hosts_probe_timeout, "", 3); +DEFINE_FLAG_INT32(sls_uninit_hosts_probe_max_try_cnt, "", 1); +DEFINE_FLAG_INT32(sls_uninit_hosts_probe_timeout, "", 1); +DEFINE_FLAG_INT32(sls_all_hosts_probe_interval_sec, "seconds", 5 * 60); +DEFINE_FLAG_INT32(sls_hosts_probe_interval_sec, "", 60); +DEFINE_FLAG_INT32(send_switch_real_ip_interval, "seconds", 60); +DEFINE_FLAG_BOOL(send_prefer_real_ip, "use real ip to send data", false); DECLARE_FLAG_STRING(custom_user_agent); DECLARE_FLAG_STRING(default_access_key_id); @@ -58,8 +73,330 @@ namespace logtail { const string ACCESS_KEY_ID = "access_key_id"; const string ACCESS_KEY = "access_key"; +const string& EndpointModeToString(EndpointMode mode) { + switch (mode) { + case EndpointMode::CUSTOM: + static string customStr = "custom"; + return customStr; + case EndpointMode::ACCELERATE: + static string accelerateStr = "accelerate"; + return accelerateStr; + case EndpointMode::DEFAULT: + static string defaultStr = "default"; + return defaultStr; + default: + static string unknownStr = "unknown"; + return unknownStr; + } +} + +chrono::milliseconds HostInfo::GetLatency() const { + lock_guard lock(mLatencyMux); + return mLatency; +} + +void HostInfo::SetLatency(const chrono::milliseconds& latency) { + { + lock_guard lock(mLatencyMux); + mLatency = latency; + } + if (latency != chrono::milliseconds::max()) { + mContinuousErrorCnt = 0; + } +} + +bool HostInfo::UpdateStatus(bool isValid) { + if (isValid) { + mContinuousErrorCnt = 0; + return false; + } + ++mContinuousErrorCnt; + if (mContinuousErrorCnt == sMaxContinuousErrorCnt) { + lock_guard lock(mLatencyMux); + if (mLatency != chrono::milliseconds::max()) { + mLatency = chrono::milliseconds::max(); + return true; + } + } + return false; +} + +bool HostInfo::IsForbidden() const { + lock_guard lock(mLatencyMux); + return mLatency == chrono::milliseconds::max(); +} + +void CandidateHostsInfo::UpdateHosts(const CandidateEndpoints& regionEndpoints) { + lock_guard lock(mCandidateHostsMux); + switch (mMode) { + case EndpointMode::DEFAULT: { + vector endpoints(regionEndpoints.mLocalEndpoints); + for (const auto& endpoint : regionEndpoints.mRemoteEndpoints) { + bool found = false; + for (const auto& existedEndpoint : regionEndpoints.mLocalEndpoints) { + if (existedEndpoint == endpoint) { + found = true; + break; + } + } + if (!found) { + endpoints.emplace_back(endpoint); + } + } + vector> infos; + for (const auto& endpoint : endpoints) { + string host = mProject.empty() ? endpoint : mProject + "." + endpoint; + if (mCandidateHosts.empty()) { + infos.emplace_back().emplace_back(host); + } else { + bool found = false; + for (const auto& item : mCandidateHosts) { + if (!item.empty() && item[0].GetHostname() == host) { + found = true; + infos.emplace_back(item); + break; + } + } + if (!found) { + infos.emplace_back().emplace_back(host); + } + } + } + mCandidateHosts.swap(infos); + break; + } + case EndpointMode::ACCELERATE: { + vector endpoints{kAccelerationDataEndpoint}; + for (const auto& item : regionEndpoints.mRemoteEndpoints) { + if (GetEndpointAddressType(item) == EndpointAddressType::PUBLIC) { + endpoints.emplace_back(item); + } + } + vector infos; + for (const auto& endpoint : endpoints) { + string host = mProject.empty() ? endpoint : mProject + "." + endpoint; + if (mCandidateHosts.empty()) { + infos.emplace_back(host); + } else { + bool found = false; + for (const auto& item : mCandidateHosts[0]) { + if (item.GetHostname() == host) { + found = true; + infos.emplace_back(item); + break; + } + } + if (!found) { + infos.emplace_back(host); + } + } + } + if (mCandidateHosts.empty()) { + mCandidateHosts.emplace_back(infos); + } else { + mCandidateHosts[0].swap(infos); + } + break; + } + case EndpointMode::CUSTOM: { + vector infos; + for (const auto& endpoint : regionEndpoints.mLocalEndpoints) { + string host = mProject.empty() ? endpoint : mProject + "." + endpoint; + if (mCandidateHosts.empty()) { + infos.emplace_back(host); + } else { + bool found = false; + for (const auto& item : mCandidateHosts[0]) { + if (item.GetHostname() == host) { + found = true; + infos.emplace_back(item); + break; + } + } + if (!found) { + infos.emplace_back(host); + } + } + } + if (mCandidateHosts.empty()) { + mCandidateHosts.emplace_back(infos); + } else { + mCandidateHosts[0].swap(infos); + } + break; + } + default: + break; + } + + LOG_INFO(sLogger, + ("update candidate host info, region", + mRegion)("project", mProject)("mode", EndpointModeToString(mMode))("endpoints", GetAllHostsStr())); +} + +bool CandidateHostsInfo::UpdateHostLatency(const string& hostname, const chrono::milliseconds& latency) { + lock_guard lock(mCandidateHostsMux); + for (auto& item : mCandidateHosts) { + for (auto& entry : item) { + if (entry.GetHostname() == hostname) { + entry.SetLatency(latency); + if (latency != chrono::milliseconds::max()) { + LOG_DEBUG( + sLogger, + ("update host info, region", mRegion)("project", mProject)("mode", EndpointModeToString(mMode))( + "host", hostname)("latency", ToString(latency.count()) + "ms")); + } else { + LOG_DEBUG(sLogger, + ("host is invalid, region", + mRegion)("project", mProject)("mode", EndpointModeToString(mMode))("host", hostname)); + } + return true; + } + } + } + return false; +} + +bool CandidateHostsInfo::UpdateHostStatus(const std::string& hostname, bool isValid) { + lock_guard lock(mCandidateHostsMux); + for (auto& item : mCandidateHosts) { + for (auto& entry : item) { + if (entry.GetHostname() == hostname) { + if (entry.UpdateStatus(isValid)) { + LOG_INFO(sLogger, + ("host is invalid, region", + mRegion)("project", mProject)("mode", EndpointModeToString(mMode))("host", hostname)); + } + return true; + } + } + } + return false; +} + +void CandidateHostsInfo::GetProbeHosts(vector& hosts) const { + if (!HasValidHost()) { + return GetAllHosts(hosts); + } + { + lock_guard lock(mCandidateHostsMux); + for (const auto& item : mCandidateHosts) { + for (const auto& entry : item) { + if (!entry.IsForbidden()) { + hosts.emplace_back(entry.GetHostname()); + } + } + } + } +} + +void CandidateHostsInfo::GetAllHosts(vector& hosts) const { + lock_guard lock(mCandidateHostsMux); + for (const auto& item : mCandidateHosts) { + for (const auto& entry : item) { + hosts.emplace_back(entry.GetHostname()); + } + } +} + +std::string CandidateHostsInfo::GetAllHostsStr() const { + string res; + for (size_t i = 0; i < mCandidateHosts.size(); ++i) { + for (size_t j = 0; j < mCandidateHosts[i].size(); ++j) { + res += mCandidateHosts[i][j].GetHostname(); + if (j != mCandidateHosts[i].size() - 1) { + res += " = "; + } + } + if (i != mCandidateHosts.size() - 1) { + res += " > "; + } + } + return res; +} + +void CandidateHostsInfo::SelectBestHost() { + lock_guard lock(mCandidateHostsMux); + for (size_t i = 0; i < mCandidateHosts.size(); ++i) { + const auto& hosts = mCandidateHosts[i]; + chrono::milliseconds minLatency = chrono::milliseconds::max(); + size_t minIdx = numeric_limits::max(); + for (size_t j = 0; j < hosts.size(); ++j) { + if (!hosts[j].IsForbidden() && hosts[j].GetLatency() < minLatency) { + minLatency = hosts[j].GetLatency(); + minIdx = j; + } + } + if (minIdx != numeric_limits::max()) { + const auto& hostname = hosts[minIdx].GetHostname(); + if (GetCurrentHost() != hostname) { + SetCurrentHost(hostname); + LOG_INFO(sLogger, + ("switch to the best host", hostname)("latency", ToString(minLatency.count()) + "ms")( + "project", mProject)("region", mRegion)("endpoint mode", EndpointModeToString(mMode))); + } + return; + } + } + if (!GetCurrentHost().empty()) { + SetCurrentHost(""); + LOG_INFO(sLogger, + ("no valid host", "stop sending data and retry later")("project", mProject)("region", mRegion)( + "endpoint mode", EndpointModeToString(mMode))); + } +} + +string CandidateHostsInfo::GetCurrentHost() const { + lock_guard lock(mCurrentHostMux); + return mCurrentHost; +} + +string CandidateHostsInfo::GetFirstHost() const { + lock_guard lock(mCandidateHostsMux); + if (mCandidateHosts.empty() || mCandidateHosts[0].empty()) { + return ""; + } + return mCandidateHosts[0][0].GetHostname(); +} + +bool CandidateHostsInfo::HasValidHost() const { + lock_guard lock(mCurrentHostMux); + return !mCurrentHost.empty(); +} + +void CandidateHostsInfo::SetCurrentHost(const string& host) { + lock_guard lock(mCurrentHostMux); + mCurrentHost = host; +} + +EnterpriseSLSClientManager::ProbeNetworkHttpRequest::ProbeNetworkHttpRequest(const string& region, + const string& project, + EndpointMode mode, + const string& host, + bool httpsFlag, + uint32_t timeout, + uint32_t tryCnt) + : AsynHttpRequest( + HTTP_GET, httpsFlag, host, httpsFlag ? 443 : 80, HEALTH, "", {}, "", HttpResponse(), timeout, tryCnt), + mRegion(region), + mProject(project), + mMode(mode), + mHost(host) { +} + +void EnterpriseSLSClientManager::ProbeNetworkHttpRequest::OnSendDone(HttpResponse& response) { + if (mProject.empty()) { + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency(mRegion, mHost, response.GetResponseTime()); + } else { + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency( + mProject, mMode, mHost, response.GetResponseTime()); + } +} + EnterpriseSLSClientManager::EnterpriseSLSClientManager() - : SLSClientManager(), mGetAccessKeyFromSLS(config::ConfigClient::GetAccessKey) { + : SLSClientManager(), + mGetAccessKeyFromSLS(config::ConfigClient::GetAccessKey), + mGetEndpointRealIp(GetEndpointRealIp) { STRING_FLAG(default_access_key_id) = STRING_FLAG(logtail_profile_access_key_id); STRING_FLAG(default_access_key) = STRING_FLAG(logtail_profile_access_key); SetAccessKey(STRING_FLAG(logtail_profile_aliuid), @@ -72,6 +409,80 @@ EnterpriseSLSClientManager::EnterpriseSLSClientManager() STRING_FLAG(default_access_key)); } +EnterpriseSLSClientManager* EnterpriseSLSClientManager::GetInstance() { + static EnterpriseSLSClientManager instance; + return &instance; +} + +void EnterpriseSLSClientManager::Init() { + auto& config = AppConfig::GetInstance()->GetConfig(); + if (config.isMember("send_prefer_real_ip") && config["send_prefer_real_ip"].isBool()) { + BOOL_FLAG(send_prefer_real_ip) = config["send_prefer_real_ip"].asBool(); + } + + GenerateUserAgent(); + + mUnInitializedHostProbeClient = curl_multi_init(); + if (mUnInitializedHostProbeClient == nullptr) { + LOG_ERROR(sLogger, ("failed to init uninitialized host probe", "failed to init curl client")); + } else { + mUnInitializedHostProbeThreadRes + = async(launch::async, &EnterpriseSLSClientManager::UnInitializedHostProbeThread, this); + } + + mHostProbeClient = curl_multi_init(); + if (mHostProbeClient == nullptr) { + LOG_ERROR(sLogger, ("failed to init host probe", "failed to init curl client")); + } else { + mHostProbeThreadRes = async(launch::async, &EnterpriseSLSClientManager::HostProbeThread, this); + } + + if (BOOL_FLAG(send_prefer_real_ip)) { + mUpdateRealIpThreadRes = async(launch::async, &EnterpriseSLSClientManager::UpdateRealIpThread, this); + } +} + +void EnterpriseSLSClientManager::Stop() { + if (mUnInitializedHostProbeClient) { + lock_guard lock(mUnInitializedHostProbeThreadRunningMux); + mIsUnInitializedHostProbeThreadRunning = false; + } + if (mHostProbeClient) { + lock_guard lock(mHostProbeThreadRunningMux); + mIsHostProbeThreadRunning = false; + } + if (BOOL_FLAG(send_prefer_real_ip)) { + lock_guard lock(mUpdateRealIpThreadRunningMux); + mIsUpdateRealIpThreadRunning = false; + } + mStopCV.notify_all(); + + if (mUnInitializedHostProbeClient) { + future_status s = mUnInitializedHostProbeThreadRes.wait_for(chrono::seconds(3)); + if (s == future_status::ready) { + LOG_INFO(sLogger, ("sls uninitialized host probe", "stopped successfully")); + } else { + LOG_WARNING(sLogger, ("sls uninitialized host probe", "forced to stopped")); + } + } + if (mHostProbeClient) { + future_status s = mHostProbeThreadRes.wait_for(chrono::seconds(10)); + if (s == future_status::ready) { + LOG_INFO(sLogger, ("sls host probe", "stopped successfully")); + } else { + LOG_WARNING(sLogger, ("sls host probe", "forced to stopped")); + } + } + if (BOOL_FLAG(send_prefer_real_ip) && mUpdateRealIpThreadRes.valid()) { + future_status s = mUpdateRealIpThreadRes.wait_for(chrono::seconds(1)); + if (s == future_status::ready) { + LOG_INFO(sLogger, ("sls real ip update", "stopped successfully")); + } else { + LOG_WARNING(sLogger, ("sls real ip update", "forced to stopped")); + } + } +} + void EnterpriseSLSClientManager::UpdateAccessKeyStatus(const string& aliuid, bool success) { lock_guard lock(mAccessKeyInfoCacheMux); auto it = mAccessKeyInfoCache.find(aliuid); @@ -80,6 +491,8 @@ void EnterpriseSLSClientManager::UpdateAccessKeyStatus(const string& aliuid, boo it->second.mErrorCnt = 0; } else { if (++it->second.mErrorCnt >= static_cast(INT32_FLAG(sls_client_max_ak_error_cnt))) { + LOG_INFO(sLogger, + ("remove access key info from local cache", "error count exceeds limit")("aliuid", aliuid)); mAccessKeyInfoCache.erase(aliuid); } } @@ -285,6 +698,686 @@ void EnterpriseSLSClientManager::UpdateProjectAnonymousWriteStatus(const string& } } +void EnterpriseSLSClientManager::UpdateLocalRegionEndpointsAndHttpsInfo(const string& region, + const vector& rawEndpoints) { + vector endpoints; + for (const auto& item : rawEndpoints) { + auto tmp = CheckDataServerEndpoint(ExtractEndpoint(item)); + if (!tmp.empty()) { + endpoints.emplace_back(tmp); + } + } + + string endpointsStr; + EndpointMode mode; + { + lock_guard lock(mRegionCandidateEndpointsMapMux); + auto& candidate = mRegionCandidateEndpointsMap[region]; + candidate.mMode = EndpointMode::DEFAULT; + candidate.mLocalEndpoints.clear(); + for (const auto& item : endpoints) { + // if both acclerate and custom endpoints are given, we ignore custom endpoints + if (item == kAccelerationDataEndpoint) { + candidate.mMode = EndpointMode::ACCELERATE; + break; + } + if (IsCustomEndpoint(item)) { + candidate.mMode = EndpointMode::CUSTOM; + candidate.mLocalEndpoints.emplace_back(item); + } + } + if (candidate.mMode == EndpointMode::ACCELERATE) { + candidate.mLocalEndpoints.clear(); + } else if (candidate.mMode == EndpointMode::DEFAULT) { + candidate.mLocalEndpoints = endpoints; + } + endpointsStr = ToString(candidate.mLocalEndpoints); + mode = candidate.mMode; + } + + // as long as one endpoint is https, we treat the region as https + bool isHttps = false; + for (const auto& item : rawEndpoints) { + if (IsHttpsEndpoint(item)) { + isHttps = true; + break; + } + } + { + lock_guard lock(mHttpsRegionsMux); + if (isHttps) { + mHttpsRegions.insert(region); + } else { + mHttpsRegions.erase(region); + } + } + LOG_INFO(sLogger, + ("update local region endpoints info, region", + region)("mode", EndpointModeToString(mode))("endpoints", endpointsStr)("https", ToString(isHttps))); +} + +void EnterpriseSLSClientManager::CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted(const string& src, + const string& dst) { + string endpointsStr; + EndpointMode mode; + { + lock_guard lock(mRegionCandidateEndpointsMapMux); + auto srcIt = mRegionCandidateEndpointsMap.find(src); + if (srcIt == mRegionCandidateEndpointsMap.end()) { + // should not happen + return; + } + + auto dstIt = mRegionCandidateEndpointsMap.find(dst); + if (dstIt != mRegionCandidateEndpointsMap.end()) { + if (dstIt->second.mMode != EndpointMode::DEFAULT || !dstIt->second.mLocalEndpoints.empty()) { + return; + } + } else { + dstIt = mRegionCandidateEndpointsMap.try_emplace(dst, CandidateEndpoints()).first; + } + + dstIt->second.mMode = srcIt->second.mMode; + if (srcIt->second.mMode == EndpointMode::DEFAULT) { + for (const auto& item : srcIt->second.mLocalEndpoints) { + string newEndpoint = item; + auto it = newEndpoint.find(src); + if (it != string::npos) { + newEndpoint.replace(it, src.size(), dst); + } + dstIt->second.mLocalEndpoints.emplace_back(newEndpoint); + } + } else { + dstIt->second.mLocalEndpoints = srcIt->second.mLocalEndpoints; + } + mode = dstIt->second.mMode; + endpointsStr = ToString(dstIt->second.mLocalEndpoints); + } + bool isHttps = false; + { + lock_guard lock(mHttpsRegionsMux); + if (mHttpsRegions.find(src) != mHttpsRegions.end()) { + mHttpsRegions.insert(dst); + isHttps = true; + } + } + LOG_INFO(sLogger, + ("copy local region endpoints from", src)("into", dst)("mode", EndpointModeToString(mode))( + "endpoints", endpointsStr)("https", ToString(isHttps))); +} + +void EnterpriseSLSClientManager::UpdateRemoteRegionEndpoints(const string& region, + const vector& rawEndpoints, + RemoteEndpointUpdateAction action) { + vector endpoints; + for (const auto& item : rawEndpoints) { + auto tmp = CheckDataServerEndpoint(ExtractEndpoint(item)); + if (!tmp.empty()) { + endpoints.emplace_back(tmp); + } + } + + lock_guard lock(mRegionCandidateEndpointsMapMux); + if (action == RemoteEndpointUpdateAction::CREATE + && mRegionCandidateEndpointsMap.find(region) != mRegionCandidateEndpointsMap.end()) { + return; + } + auto& candidate = mRegionCandidateEndpointsMap[region]; + if (action == RemoteEndpointUpdateAction::APPEND) { + for (const auto& item : endpoints) { + bool found = false; + for (const auto& existed : candidate.mRemoteEndpoints) { + if (existed == item) { + found = true; + break; + } + } + if (!found) { + candidate.mRemoteEndpoints.emplace_back(item); + } + } + } else { + candidate.mRemoteEndpoints = endpoints; + } + LOG_INFO( + sLogger, + ("update remote region endpoints info, region", region)("endpoints", ToString(candidate.mRemoteEndpoints))); + + { + lock_guard lock(mCandidateHostsInfosMapMux); + for (auto& item : mRegionCandidateHostsInfosMap[region]) { + if (item.expired()) { + continue; + } + auto info = item.lock(); + info->UpdateHosts(candidate); + } + } + if (BOOL_FLAG(send_prefer_real_ip)) { + lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); + auto it = mRegionRealIpCandidateHostsInfosMap.find(region); + if (it == mRegionRealIpCandidateHostsInfosMap.end()) { + it = mRegionRealIpCandidateHostsInfosMap.try_emplace(region, "", region, EndpointMode::DEFAULT).first; + } + it->second.UpdateHosts(candidate); + } +} + +shared_ptr +EnterpriseSLSClientManager::GetCandidateHostsInfo(const string& region, const string& project, EndpointMode mode) { + if (mode == EndpointMode::DEFAULT) { + // when flusher does not specify the mode, we use the mode of the region, which is set before flusher init and + // will not change. + lock_guard lock(mRegionCandidateEndpointsMapMux); + auto it = mRegionCandidateEndpointsMap.find(region); + if (it != mRegionCandidateEndpointsMap.end()) { + mode = it->second.mMode; + } + } + { + lock_guard lock(mCandidateHostsInfosMapMux); + auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; + for (auto& item : hostsInfo) { + if (item.expired()) { + continue; + } + auto info = item.lock(); + if (info->GetMode() == mode) { + return info; + } + } + } + + auto info = make_shared(project, region, mode); + LOG_INFO(sLogger, + ("create candidate host info, region", region)("project", project)("mode", EndpointModeToString(mode))); + + CandidateEndpoints endpoints; + { + lock_guard lock(mRegionCandidateEndpointsMapMux); + auto it = mRegionCandidateEndpointsMap.find(region); + if (it != mRegionCandidateEndpointsMap.end()) { + endpoints = it->second; + } + } + info->UpdateHosts(endpoints); + { + lock_guard lock(mCandidateHostsInfosMapMux); + mProjectCandidateHostsInfosMap[project].emplace_back(info); + mRegionCandidateHostsInfosMap[region].emplace_back(info); + } + { + lock_guard lock(mUnInitializedCandidateHostsInfosMux); + mUnInitializedCandidateHostsInfos.emplace_back(info); + } + return info; +} + +bool EnterpriseSLSClientManager::UpdateHostLatency(const string& project, + EndpointMode mode, + const string& host, + const chrono::milliseconds& latency) { + lock_guard lock(mCandidateHostsInfosMapMux); + auto it = mProjectCandidateHostsInfosMap.find(project); + if (it == mProjectCandidateHostsInfosMap.end()) { + return false; + } + for (auto& entry : it->second) { + if (entry.expired()) { + continue; + } + auto info = entry.lock(); + if (info->GetMode() == mode) { + info->UpdateHostLatency(host, latency); + return true; + } + } + return false; +} + +bool EnterpriseSLSClientManager::UpdateHostStatus(const std::string& project, + EndpointMode mode, + const std::string& host, + bool isValid) { + lock_guard lock(mCandidateHostsInfosMapMux); + auto it = mProjectCandidateHostsInfosMap.find(project); + if (it == mProjectCandidateHostsInfosMap.end()) { + return false; + } + for (auto& entry : it->second) { + if (entry.expired()) { + continue; + } + auto info = entry.lock(); + if (info->GetMode() == mode) { + info->UpdateHostStatus(host, isValid); + return true; + } + } + return false; +} + +bool EnterpriseSLSClientManager::UpdateHostLatency(const string& region, + const string& host, + const chrono::milliseconds& latency) { + lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); + auto it = mRegionRealIpCandidateHostsInfosMap.find(region); + if (it == mRegionRealIpCandidateHostsInfosMap.end()) { + return false; + } + return it->second.UpdateHostLatency(host, latency); +} + +bool EnterpriseSLSClientManager::UsingHttps(const string& region) const { + if (AppConfig::GetInstance()->GetDataServerPort() == 443) { + return true; + } + lock_guard lock(mHttpsRegionsMux); + return mHttpsRegions.find(region) != mHttpsRegions.end(); +} + +void EnterpriseSLSClientManager::UnInitializedHostProbeThread() { + LOG_INFO(sLogger, ("sls uninitialized host probe", "started")); + unique_lock lock(mUnInitializedHostProbeThreadRunningMux); + while (mIsUnInitializedHostProbeThreadRunning) { + lock.unlock(); + DoProbeUnInitializedHost(); + lock.lock(); + if (mStopCV.wait_for( + lock, chrono::milliseconds(10), [this]() { return !mIsUnInitializedHostProbeThreadRunning; })) { + return; + } + } + + auto mc = curl_multi_cleanup(mUnInitializedHostProbeClient); + if (mc != CURLM_OK) { + LOG_ERROR(sLogger, ("failed to cleanup curl multi handle", "exit anyway")("errMsg", curl_multi_strerror(mc))); + } +} + +void EnterpriseSLSClientManager::DoProbeUnInitializedHost() { + vector> infos; + { + lock_guard lock(mUnInitializedCandidateHostsInfosMux); + infos.swap(mUnInitializedCandidateHostsInfos); + } + if (infos.empty()) { + return; + } +#ifndef APSARA_UNIT_TEST_MAIN + for (auto& item : infos) { + { + lock_guard lk(mUnInitializedHostProbeThreadRunningMux); + if (!mIsUnInitializedHostProbeThreadRunning) { + return; + } + } + if (item.expired()) { + continue; + } + auto info = item.lock(); + bool httpsFlag = UsingHttps(info->GetRegion()); + auto req = make_unique(info->GetRegion(), + info->GetProject(), + info->GetMode(), + info->GetFirstHost(), + httpsFlag, + INT32_FLAG(sls_uninit_hosts_probe_timeout), + INT32_FLAG(sls_uninit_hosts_probe_max_try_cnt)); + AddRequestToMultiCurlHandler(mUnInitializedHostProbeClient, std::move(req)); + } + SendAsynRequests(mUnInitializedHostProbeClient); +#else + for (auto& item : infos) { + if (item.expired()) { + continue; + } + auto info = item.lock(); + bool httpsFlag = UsingHttps(info->GetRegion()); + auto req = make_unique(info->GetRegion(), + info->GetProject(), + info->GetMode(), + info->GetFirstHost(), + httpsFlag, + INT32_FLAG(sls_uninit_hosts_probe_timeout), + INT32_FLAG(sls_uninit_hosts_probe_max_try_cnt)); + HttpResponse response = mDoProbeNetwork(req); + req->OnSendDone(response); + } +#endif + + for (auto& item : infos) { + if (item.expired()) { + continue; + } + auto info = item.lock(); + info->SelectBestHost(); + LOG_INFO( + sLogger, + ("candidate host info partially initialized, region", info->GetRegion())("project", info->GetProject())( + "mode", EndpointModeToString(info->GetMode()))("current host", info->GetCurrentHost())); + } + { + lock_guard lock(mPartiallyInitializedCandidateHostsInfosMux); + mPartiallyInitializedCandidateHostsInfos.insert( + mPartiallyInitializedCandidateHostsInfos.end(), infos.begin(), infos.end()); + } +} + +void EnterpriseSLSClientManager::HostProbeThread() { + LOG_INFO(sLogger, ("sls host probe", "started")); + unique_lock lock(mHostProbeThreadRunningMux); + while (mIsHostProbeThreadRunning) { + lock.unlock(); + DoProbeHost(); + lock.lock(); + if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsHostProbeThreadRunning; })) { + return; + } + } + + auto mc = curl_multi_cleanup(mHostProbeClient); + if (mc != CURLM_OK) { + LOG_ERROR(sLogger, ("failed to cleanup curl multi handle", "exit anyway")("errMsg", curl_multi_strerror(mc))); + } +} + +void EnterpriseSLSClientManager::DoProbeHost() { + static time_t lastProbeAllEndpointsTime = time(nullptr); + static time_t lastProbeEndpointsTime = time(nullptr); + bool shouldTestAllEndpoints = false; + bool shouldTestAvailableEndpoints = false; + time_t beginTime = time(nullptr); + if (HasPartiallyInitializedCandidateHostsInfos()) { + // do nothing + } else if (beginTime - lastProbeAllEndpointsTime >= INT32_FLAG(sls_all_hosts_probe_interval_sec)) { + shouldTestAllEndpoints = true; + lastProbeAllEndpointsTime = beginTime; + } else if (beginTime - lastProbeEndpointsTime >= INT32_FLAG(sls_hosts_probe_interval_sec)) { + shouldTestAvailableEndpoints = true; + lastProbeEndpointsTime = beginTime; + } else { + ClearExpiredCandidateHostsInfos(); + PackIdManager::GetInstance()->CleanTimeoutEntry(); + return; + } + + // collect hosts to be probed + map, pair>> projectHostMap; + map> regionHostsMap; // only for realip + if (shouldTestAllEndpoints || shouldTestAvailableEndpoints) { + { + lock_guard lk(mCandidateHostsInfosMapMux); + for (const auto& item : mProjectCandidateHostsInfosMap) { + for (const auto& entry : item.second) { + if (entry.expired()) { + continue; + } + auto info = entry.lock(); + auto& hosts = projectHostMap[make_pair(item.first, info->GetMode())]; + hosts.first = info->GetRegion(); + if (shouldTestAllEndpoints) { + info->GetAllHosts(hosts.second); + } else { + info->GetProbeHosts(hosts.second); + } + } + } + } + if (BOOL_FLAG(send_prefer_real_ip)) { + lock_guard lk(mRegionRealIpCandidateHostsInfosMapMux); + for (const auto& item : mRegionRealIpCandidateHostsInfosMap) { + auto& hosts = regionHostsMap[item.second.GetRegion()]; + if (shouldTestAllEndpoints) { + item.second.GetAllHosts(hosts); + } else { + item.second.GetProbeHosts(hosts); + } + } + } + } else { + lock_guard lk(mPartiallyInitializedCandidateHostsInfosMux); + for (const auto& item : mPartiallyInitializedCandidateHostsInfos) { + if (item.expired()) { + continue; + } + auto info = item.lock(); + auto& hosts = projectHostMap[make_pair(info->GetProject(), info->GetMode())]; + hosts.first = info->GetRegion(); + info->GetAllHosts(hosts.second); + } + mPartiallyInitializedCandidateHostsInfos.clear(); + } + + // do probe and update latency + for (const auto& item : projectHostMap) { + bool httpsFlag = UsingHttps(item.second.first); +#ifndef APSARA_UNIT_TEST_MAIN + for (const auto& host : item.second.second) { + { + lock_guard lk(mHostProbeThreadRunningMux); + if (!mIsHostProbeThreadRunning) { + return; + } + } + auto req = make_unique(item.second.first, + item.first.first, + item.first.second, + host, + httpsFlag, + INT32_FLAG(sls_hosts_probe_timeout), + INT32_FLAG(sls_hosts_probe_max_try_cnt)); + AddRequestToMultiCurlHandler(mHostProbeClient, std::move(req)); + } + SendAsynRequests(mHostProbeClient); +#else + for (const auto& host : item.second.second) { + auto req = make_unique(item.second.first, + item.first.first, + item.first.second, + host, + httpsFlag, + INT32_FLAG(sls_hosts_probe_timeout), + INT32_FLAG(sls_hosts_probe_max_try_cnt)); + HttpResponse response = mDoProbeNetwork(req); + req->OnSendDone(response); + } +#endif + } + if (BOOL_FLAG(send_prefer_real_ip)) { + for (const auto& item : regionHostsMap) { + bool httpsFlag = UsingHttps(item.first); +#ifndef APSARA_UNIT_TEST_MAIN + for (const auto& host : item.second) { + { + lock_guard lk(mHostProbeThreadRunningMux); + if (!mIsHostProbeThreadRunning) { + return; + } + } + auto req = make_unique(item.first, + "", + EndpointMode::DEFAULT, + host, + httpsFlag, + INT32_FLAG(sls_hosts_probe_timeout), + INT32_FLAG(sls_hosts_probe_max_try_cnt)); + AddRequestToMultiCurlHandler(mHostProbeClient, std::move(req)); + } + SendAsynRequests(mHostProbeClient); +#else + for (const auto& host : item.second) { + auto req = make_unique(item.first, + "", + EndpointMode::DEFAULT, + host, + httpsFlag, + INT32_FLAG(sls_hosts_probe_timeout), + INT32_FLAG(sls_hosts_probe_max_try_cnt)); + HttpResponse response = mDoProbeNetwork(req); + req->OnSendDone(response); + } +#endif + } + } + + // update selected host + { + lock_guard lk(mCandidateHostsInfosMapMux); + for (const auto& item : mProjectCandidateHostsInfosMap) { + for (auto& entry : item.second) { + if (auto p = entry.lock()) { + p->SelectBestHost(); + if (!p->IsInitialized()) { + p->SetInitialized(); + LOG_INFO(sLogger, + ("candidate host info completely initialized, region", + p->GetRegion())("project", p->GetProject())( + "mode", EndpointModeToString(p->GetMode()))("current host", p->GetCurrentHost())); + } + } + } + } + } + if (BOOL_FLAG(send_prefer_real_ip) && (shouldTestAllEndpoints || shouldTestAvailableEndpoints)) { + lock_guard lk(mRegionRealIpCandidateHostsInfosMapMux); + for (auto& item : mRegionRealIpCandidateHostsInfosMap) { + item.second.SelectBestHost(); + } + } +} + +bool EnterpriseSLSClientManager::HasPartiallyInitializedCandidateHostsInfos() const { + lock_guard lk(mPartiallyInitializedCandidateHostsInfosMux); + return !mPartiallyInitializedCandidateHostsInfos.empty(); +} + +void EnterpriseSLSClientManager::ClearExpiredCandidateHostsInfos() { + lock_guard lk(mCandidateHostsInfosMapMux); + for (auto iter = mProjectCandidateHostsInfosMap.begin(); iter != mProjectCandidateHostsInfosMap.end();) { + for (auto it = iter->second.begin(); it != iter->second.end();) { + if (it->expired()) { + it = iter->second.erase(it); + } else { + ++it; + } + } + if (iter->second.empty()) { + iter = mProjectCandidateHostsInfosMap.erase(iter); + } else { + ++iter; + } + } + for (auto iter = mRegionCandidateHostsInfosMap.begin(); iter != mRegionCandidateHostsInfosMap.end();) { + for (auto it = iter->second.begin(); it != iter->second.end();) { + if (it->expired()) { + it = iter->second.erase(it); + } else { + ++it; + } + } + if (iter->second.empty()) { + iter = mRegionCandidateHostsInfosMap.erase(iter); + } else { + ++iter; + } + } +} + +void EnterpriseSLSClientManager::UpdateRealIpThread() { + LOG_INFO(sLogger, ("sls real ip update", "started")); + unique_lock lock(mUpdateRealIpThreadRunningMux); + while (mIsUpdateRealIpThreadRunning) { + lock.unlock(); + DoUpdateRealIp(); + lock.lock(); + if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsUpdateRealIpThreadRunning; })) { + break; + } + } +} + +void EnterpriseSLSClientManager::DoUpdateRealIp() { + vector> hosts; + static time_t lastUpdateRealIpTime = 0; + if (time(nullptr) - lastUpdateRealIpTime >= INT32_FLAG(send_switch_real_ip_interval)) { + { + lock_guard lock(mOutdatedRealIpRegionsMux); + mOutdatedRealIpRegions.clear(); + } + { + lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); + for (const auto& item : mRegionRealIpCandidateHostsInfosMap) { + hosts.emplace_back(item.first, item.second.GetCurrentHost()); + } + } + lastUpdateRealIpTime = time(nullptr); + } else if (HasOutdatedRealIpRegions()) { + vector regions; + { + lock_guard lock(mOutdatedRealIpRegionsMux); + regions.swap(mOutdatedRealIpRegions); + } + { + lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); + for (const auto& item : regions) { + auto it = mRegionRealIpCandidateHostsInfosMap.find(item); + if (it != mRegionRealIpCandidateHostsInfosMap.end()) { + hosts.emplace_back(item, it->second.GetCurrentHost()); + } + } + } + } + for (const auto& item : hosts) { + { + lock_guard lk(mUpdateRealIpThreadRunningMux); + if (!mIsUpdateRealIpThreadRunning) { + return; + } + } + if (item.second.empty()) { + continue; + } + string ip; + if (!mGetEndpointRealIp(item.second, ip)) { + LOG_WARNING(sLogger, + ("failed to get real ip", "retry later")("region", item.first)("endpoint", item.second)); + continue; + } + SetRealIp(item.first, ip); + } +} + +bool EnterpriseSLSClientManager::HasOutdatedRealIpRegions() const { + lock_guard lock(mOutdatedRealIpRegionsMux); + return !mOutdatedRealIpRegions.empty(); +} + +void EnterpriseSLSClientManager::UpdateOutdatedRealIpRegions(const string& region) { + lock_guard lock(mOutdatedRealIpRegionsMux); + mOutdatedRealIpRegions.emplace_back(region); +} + +string EnterpriseSLSClientManager::GetRealIp(const string& region) const { + lock_guard lock(mRegionRealIpMux); + auto it = mRegionRealIpMap.find(region); + if (it == mRegionRealIpMap.end()) { + return ""; + } + return it->second; +} + +void EnterpriseSLSClientManager::SetRealIp(const string& region, const string& ip) { + lock_guard lock(mRegionRealIpMux); + auto it = mRegionRealIpMap.find(region); + if (it == mRegionRealIpMap.end()) { + mRegionRealIpMap.emplace(region, ip); + } else if (it->second != ip) { + LOG_INFO(sLogger, ("set real ip for region", region)("from", it->second)("to", ip)); + it->second = ip; + } +} + void EnterpriseSLSClientManager::GenerateUserAgent() { string os; #if defined(__linux__) @@ -310,7 +1403,7 @@ void EnterpriseSLSClientManager::GenerateUserAgent() { os = LoongCollectorMonitor::mOsDetail; #endif - mUserAgent = string(sdk::LOGTAIL_USER_AGENT) + "/" + ILOGTAIL_VERSION + " (" + os + ") ip/" + mUserAgent = string(LOGTAIL_USER_AGENT) + "/" + ILOGTAIL_VERSION + " (" + os + ") ip/" + LoongCollectorMonitor::mIpAddr + " env/" + GetRunningEnvironment(); if (!STRING_FLAG(custom_user_agent).empty()) { mUserAgent += " " + STRING_FLAG(custom_user_agent); @@ -329,7 +1422,7 @@ string EnterpriseSLSClientManager::GetRunningEnvironment() { // containers in K8S will possess the above env if (AppConfig::GetInstance()->IsPurageContainerMode()) { env = "K8S-Daemonset "; - } else if (TryCurlEndpoint("http://100.100.100.200/latest/meta-data")) { + } else if (PingEndpoint("100.100.100.200", "/latest/meta-data")) { // containers in ACK can be connected to the above address, see // https://help.aliyun.com/document_detail/108460.html#section-akf-lwh-1gb. // Note: we can not distinguish ACK from K8S built on ECS @@ -339,9 +1432,9 @@ string EnterpriseSLSClientManager::GetRunningEnvironment() { } } else if (AppConfig::GetInstance()->IsPurageContainerMode() || getenv("ALIYUN_LOGTAIL_CONFIG")) { env = "Docker"; - } else if (TryCurlEndpoint("http://100.100.100.200/latest/meta-data")) { + } else if (PingEndpoint("100.100.100.200", "/latest/meta-data")) { env = "ECS"; - } else if (TryCurlEndpoint("http://config.sls.aliyun-inc.com")) { + } else if (PingEndpoint("config.sls.aliyun-inc.com", "")) { env = "Corp"; } else { env = "Others"; @@ -349,4 +1442,79 @@ string EnterpriseSLSClientManager::GetRunningEnvironment() { return env; } +string EnterpriseSLSClientManager::GetRegionFromEndpoint(const string& endpoint) { + string standardEndpoint = CheckDataServerEndpoint(ExtractEndpoint(endpoint)); + lock_guard lock(mRegionCandidateEndpointsMapMux); + for (const auto& item : mRegionCandidateEndpointsMap) { + for (const auto& entry : item.second.mLocalEndpoints) { + if (entry == standardEndpoint) { + return item.first; + } + } + } + return STRING_FLAG(default_region_name); +} + +#ifdef APSARA_UNIT_TEST_MAIN +void EnterpriseSLSClientManager::Clear() { + mRegionCandidateEndpointsMap.clear(); + mHttpsRegions.clear(); + mRegionCandidateHostsInfosMap.clear(); + mProjectCandidateHostsInfosMap.clear(); + mUnInitializedCandidateHostsInfos.clear(); + mRegionRealIpMap.clear(); + mOutdatedRealIpRegions.clear(); + mRegionRealIpCandidateHostsInfosMap.clear(); +} +#endif + +bool GetEndpointRealIp(const string& endpoint, string& ip) { + static string body; + if (body.empty()) { + sls_logs::LogGroup logGroup; + logGroup.set_source(LoongCollectorMonitor::mIpAddr); + body = logGroup.SerializeAsString(); + } + + static string project = "logtail-real-ip-project"; + static string logstore = "logtail-real-ip-logstore"; + + EnterpriseSLSClientManager::AuthType type; + string accessKeyId, accessKeySecret; + EnterpriseSLSClientManager::GetInstance()->GetAccessKey("", type, accessKeyId, accessKeySecret); + + string path, query; + map header; + string host = project + "." + endpoint; + PreparePostLogStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + host, + false, + project, + logstore, + "", + RawDataType::EVENT_GROUP, + body, + body.size(), + "", + nullopt, + path, + query, + header); + HttpResponse response; + if (!SendHttpRequest(make_unique(HTTP_POST, false, host, 80, path, query, header, body), response)) { + return false; + } + if (response.GetStatusCode() != 200) { + auto it = response.GetHeader().find(X_LOG_HOSTIP); + if (it != response.GetHeader().end()) { + ip = it->second; + } + return true; + } + // should not happen + return true; +} + } // namespace logtail diff --git a/core/plugin/flusher/sls/EnterpriseSLSClientManager.h b/core/plugin/flusher/sls/EnterpriseSLSClientManager.h index 99aabbfdbe..8ba9545517 100644 --- a/core/plugin/flusher/sls/EnterpriseSLSClientManager.h +++ b/core/plugin/flusher/sls/EnterpriseSLSClientManager.h @@ -16,30 +16,169 @@ #pragma once +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include +#include +#include +#include "common/http/HttpRequest.h" +#include "common/http/HttpResponse.h" #include "plugin/flusher/sls/SLSClientManager.h" namespace logtail { +enum class EndpointMode { DEFAULT, ACCELERATE, CUSTOM }; +const std::string& EndpointModeToString(EndpointMode mode); + +struct CandidateEndpoints { + // currently only remote endpoints can be updated + EndpointMode mMode = EndpointMode::DEFAULT; + // when mMode = ACCELERATE, mLocalEndpoints should be empty + std::vector mLocalEndpoints; + std::vector mRemoteEndpoints; +}; + +class HostInfo { +public: + HostInfo(const std::string& hostname) : mHostname(hostname) {} + HostInfo(const HostInfo& rhs) : mHostname(rhs.mHostname) { mLatency = rhs.GetLatency(); } + + const std::string& GetHostname() const { return mHostname; } + std::chrono::milliseconds GetLatency() const; + void SetLatency(const std::chrono::milliseconds& latency); + // this is only called by FlusherSLS::OnSendDone, so it does not update latency + bool UpdateStatus(bool isValid); + bool IsForbidden() const; + +private: + static constexpr uint32_t sMaxContinuousErrorCnt = 3; + // normally in the form of ., except for the real ip scene, which equals to endpoint + const std::string mHostname; + + mutable std::mutex mLatencyMux; + std::chrono::milliseconds mLatency = std::chrono::milliseconds::max(); + + std::atomic_uint32_t mContinuousErrorCnt = 0; +}; + +class CandidateHostsInfo { +public: + CandidateHostsInfo(const std::string& project, const std::string& region, EndpointMode mode) + : mProject(project), mRegion(region), mMode(mode) {} + + void UpdateHosts(const CandidateEndpoints& regionEndpoints); + bool UpdateHostLatency(const std::string& hostname, const std::chrono::milliseconds& latency); + bool UpdateHostStatus(const std::string& hostname, bool isValid); + // return available hosts if at least one is available, otherwise, return all hosts + void GetProbeHosts(std::vector& hosts) const; + void GetAllHosts(std::vector& hosts) const; + std::string GetAllHostsStr() const; + void SelectBestHost(); + std::string GetCurrentHost() const; + std::string GetFirstHost() const; + const std::string& GetProject() const { return mProject; } + const std::string& GetRegion() const { return mRegion; } + EndpointMode GetMode() const { return mMode; } + bool IsInitialized() const { return mInitialized.load(); } + void SetInitialized() { mInitialized = true; } + +private: + bool HasValidHost() const; + void SetCurrentHost(const std::string& host); + + // for real ip scene, mProject is empty + const std::string mProject; + const std::string mRegion; + const EndpointMode mMode; + std::atomic_bool mInitialized = false; + + mutable std::mutex mCurrentHostMux; + std::string mCurrentHost; + + // the priority of host decreases from front to back, while hosts in the same element shares the same priority + // mMode = DEFAULT: each mCandidateHosts element has exactly one element + // mMode = ACCELERATE: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 2 + // mMode = CUSTOM: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 1 + mutable std::mutex mCandidateHostsMux; + std::vector> mCandidateHosts; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class CandidateHostsInfoUnittest; + friend class SLSClientManagerUnittest; + friend class EnterpriseSLSClientManagerUnittest; +#endif +}; + class EnterpriseSLSClientManager : public SLSClientManager { friend SLSClientManager* SLSClientManager::GetInstance(); public: + enum class RemoteEndpointUpdateAction { CREATE, OVERWRITE, APPEND }; + + static EnterpriseSLSClientManager* GetInstance(); + + void Init() override; + void Stop() override; + bool GetAccessKey(const std::string& aliuid, AuthType& type, std::string& accessKeyId, std::string& accessKeySecret) override; - void UpdateAccessKeyStatus(const std::string& aliuid, bool success) override; - + void UpdateAccessKeyStatus(const std::string& aliuid, bool success); bool GetAccessKeyIfProjectSupportsAnonymousWrite(const std::string& project, AuthType& type, std::string& accessKeyId, std::string& accessKeySecret); void UpdateProjectAnonymousWriteStatus(const std::string& project, bool status); + // does not support hot reloading + void UpdateLocalRegionEndpointsAndHttpsInfo(const std::string& region, const std::vector& endpoints); + // When region mode is changed due to sub region update, we should not update the related + // CandidateHostsInfo as we do in remote endpoint update. Instead, let the caller, i.e., FlusherSLS, calls + // GetCandidateHostsInfo to check if the cached CandidateHostsInfo is changed. This is mainly because the + // CandidateHostsInfo is shared, and the mode in CandidateHostsInfo is decided by both the mode in the pipeline + // config and the region mode. So flushers sharing the same CandidateHostsInfo before does not neccessarily + // share the same CandidateHostsInfo after the region mode is changed. + void CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted(const std::string& src, const std::string& dst); + void UpdateRemoteRegionEndpoints(const std::string& region, + const std::vector& endpoints, + RemoteEndpointUpdateAction action = RemoteEndpointUpdateAction::OVERWRITE); + + std::shared_ptr + GetCandidateHostsInfo(const std::string& region, const std::string& project, EndpointMode mode); + bool UpdateHostLatency(const std::string& project, + EndpointMode mode, + const std::string& host, + const std::chrono::milliseconds& latency); + bool UpdateHostStatus(const std::string& project, EndpointMode mode, const std::string& host, bool isValid); + // only for real ip + bool + UpdateHostLatency(const std::string& region, const std::string& host, const std::chrono::milliseconds& latency); + + bool UsingHttps(const std::string& region) const override; + + void UpdateOutdatedRealIpRegions(const std::string& region); + std::string GetRealIp(const std::string& region) const; + + std::string GetRegionFromEndpoint(const std::string& endpoint); // for backward compatibility + +#ifdef APSARA_UNIT_TEST_MAIN + void Clear(); +#endif + private: enum class ProjectAnonymousWriteStatus { PENDING, VALID, INVALID }; @@ -54,6 +193,24 @@ class EnterpriseSLSClientManager : public SLSClientManager { : mAuthType(type), mId(id), mSecret(secret), mExpireTime(expireTime) {} }; + struct ProbeNetworkHttpRequest : public AsynHttpRequest { + std::string mRegion; + std::string mProject; + EndpointMode mMode; + std::string mHost; + + ProbeNetworkHttpRequest(const std::string& region, + const std::string& project, + EndpointMode mode, + const std::string& host, + bool httpsFlag, + uint32_t timeout, + uint32_t tryCnt); + + bool IsContextValid() const override { return true; }; + void OnSendDone(HttpResponse& response) override; + }; + EnterpriseSLSClientManager(); void GenerateUserAgent() override; @@ -79,6 +236,18 @@ class EnterpriseSLSClientManager : public SLSClientManager { std::string& accessKeySecret, std::string& errorMsg) const; + void UnInitializedHostProbeThread(); + void DoProbeUnInitializedHost(); + void HostProbeThread(); + void DoProbeHost(); + bool HasPartiallyInitializedCandidateHostsInfos() const; + void ClearExpiredCandidateHostsInfos(); + + void UpdateRealIpThread(); + void DoUpdateRealIp(); + void SetRealIp(const std::string& region, const std::string& ip); + bool HasOutdatedRealIpRegions() const; + mutable std::mutex mAccessKeyInfoCacheMux; std::unordered_map mAccessKeyInfoCache; @@ -95,9 +264,62 @@ class EnterpriseSLSClientManager : public SLSClientManager { const std::string& caCert) = nullptr; + mutable std::mutex mRegionCandidateEndpointsMapMux; + std::map mRegionCandidateEndpointsMap; + + mutable std::mutex mHttpsRegionsMux; + std::unordered_set mHttpsRegions; + + mutable std::mutex mCandidateHostsInfosMapMux; + std::map>> mRegionCandidateHostsInfosMap; + // only one info for each mode is supported + std::map>> mProjectCandidateHostsInfosMap; + + CURLM* mUnInitializedHostProbeClient = nullptr; + mutable std::mutex mUnInitializedCandidateHostsInfosMux; + std::vector> mUnInitializedCandidateHostsInfos; + + std::future mUnInitializedHostProbeThreadRes; + mutable std::mutex mUnInitializedHostProbeThreadRunningMux; + bool mIsUnInitializedHostProbeThreadRunning = true; + + CURLM* mHostProbeClient = nullptr; + mutable std::mutex mPartiallyInitializedCandidateHostsInfosMux; + std::vector> mPartiallyInitializedCandidateHostsInfos; + + std::future mHostProbeThreadRes; + mutable std::mutex mHostProbeThreadRunningMux; + bool mIsHostProbeThreadRunning = true; +#ifdef APSARA_UNIT_TEST_MAIN + HttpResponse (*mDoProbeNetwork)(const std::unique_ptr& req) = nullptr; +#endif + + mutable std::mutex mRegionRealIpMux; + std::map mRegionRealIpMap; + mutable std::mutex mOutdatedRealIpRegionsMux; + std::vector mOutdatedRealIpRegions; + mutable std::mutex mRegionRealIpCandidateHostsInfosMapMux; + std::map mRegionRealIpCandidateHostsInfosMap; + bool (*mGetEndpointRealIp)(const std::string& endpoint, std::string& ip) = nullptr; + + std::future mUpdateRealIpThreadRes; + mutable std::mutex mUpdateRealIpThreadRunningMux; + bool mIsUpdateRealIpThreadRunning = true; + + mutable std::condition_variable mStopCV; + #ifdef APSARA_UNIT_TEST_MAIN friend class EnterpriseSLSClientManagerUnittest; + friend class ProbeNetworkMock; + friend class GetRealIpMock; + friend class FlusherSLSUnittest; #endif }; +bool GetEndpointRealIp(const std::string& endpoint, std::string& ip); + +#ifdef APSARA_UNIT_TEST_MAIN +extern HttpResponse (*DoGetRealIp)(const std::unique_ptr& req); +#endif + } // namespace logtail diff --git a/core/shennong/MetricSender.cpp b/core/shennong/MetricSender.cpp index 2c4d606a62..1691fa92d5 100644 --- a/core/shennong/MetricSender.cpp +++ b/core/shennong/MetricSender.cpp @@ -16,6 +16,7 @@ #include "Monitor.h" #include "common/Flags.h" +#include "common/LogtailCommonFlags.h" #include "logger/Logger.h" #include "protobuf/sls/sls_logs.pb.h" #include "sdk/RestfulApiCommon.h" @@ -26,8 +27,7 @@ using namespace std; using namespace sls_logs; -DECLARE_FLAG_INT32(sls_client_send_timeout); -DECLARE_FLAG_BOOL(sls_client_send_compress); +DECLARE_FLAG_INT32(default_http_request_timeout_sec); DEFINE_FLAG_INT32(metric_max_buffer_package, "", 1000); DECLARE_FLAG_INT32(send_retrytimes); DECLARE_FLAG_INT32(batch_send_interval); @@ -141,7 +141,7 @@ void MetricSender::SetMetricAddress(const string& addr) { sender->mSlsMetricClient = new shengnong_sdk::SLSClient(addr, "", "", - INT32_FLAG(sls_client_send_timeout), + INT32_FLAG(default_http_request_timeout_sec), LoongCollectorMonitor::mIpAddr, BOOL_FLAG(sls_client_send_compress)); sender->mSendToMetricServer = true; diff --git a/core/unittest/config/EnterpriseConfigProviderUnittest.cpp b/core/unittest/config/EnterpriseConfigProviderUnittest.cpp index a934f15984..0780fbbcf3 100644 --- a/core/unittest/config/EnterpriseConfigProviderUnittest.cpp +++ b/core/unittest/config/EnterpriseConfigProviderUnittest.cpp @@ -37,10 +37,9 @@ namespace logtail { class EnterpriseConfigProviderUnittest : public testing::Test { public: void TestReloadLogtailSysConf(); - void TestGetRegionAndProfileProjectFromJsonValue(); - void TestIsLegacyConfig(); + void TestRegionEndpoints(); protected: void SetUp() override { @@ -260,9 +259,14 @@ void EnterpriseConfigProviderUnittest::TestIsLegacyConfig() { } } +void EnterpriseConfigProviderUnittest::TestRegionEndpoints() { + +} + UNIT_TEST_CASE(EnterpriseConfigProviderUnittest, TestReloadLogtailSysConf) UNIT_TEST_CASE(EnterpriseConfigProviderUnittest, TestGetRegionAndProfileProjectFromJsonValue) UNIT_TEST_CASE(EnterpriseConfigProviderUnittest, TestIsLegacyConfig) +UNIT_TEST_CASE(EnterpriseConfigProviderUnittest, TestRegionEndpoints) } // namespace logtail diff --git a/core/unittest/flusher/EnterpriseSLSClientManagerUnittest.cpp b/core/unittest/flusher/EnterpriseSLSClientManagerUnittest.cpp index 7ec4e10751..3170419f6e 100644 --- a/core/unittest/flusher/EnterpriseSLSClientManagerUnittest.cpp +++ b/core/unittest/flusher/EnterpriseSLSClientManagerUnittest.cpp @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "app_config/AppConfig.h" +#include "common/EnterpriseEndpointUtil.h" #include "common/JsonUtil.h" #include "config/provider/EnterpriseConfigProvider.h" #include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #include "unittest/Unittest.h" +#include "unittest/flusher/SLSNetworkRequestMock.h" DECLARE_FLAG_STRING(default_access_key_id); DECLARE_FLAG_STRING(default_access_key); @@ -23,58 +26,454 @@ DECLARE_FLAG_STRING(logtail_profile_aliuid); DECLARE_FLAG_STRING(logtail_profile_access_key_id); DECLARE_FLAG_STRING(logtail_profile_access_key); DECLARE_FLAG_INT32(request_access_key_interval); +DECLARE_FLAG_INT32(sls_hosts_probe_interval_sec); +DECLARE_FLAG_INT32(sls_all_hosts_probe_interval_sec); +DECLARE_FLAG_BOOL(send_prefer_real_ip); +DECLARE_FLAG_INT32(send_switch_real_ip_interval); using namespace std; namespace logtail { -static uint32_t sRequestRecievedCnt = 0; +class HostInfoUnittest : public ::testing::Test { +public: + void TestHostname(); + void TestLatency(); + +private: + const std::string mHostname = "project.endpoint"; +}; + +void HostInfoUnittest::TestHostname() { + HostInfo hostInfo(mHostname); + APSARA_TEST_EQUAL(mHostname, hostInfo.GetHostname()); +} + +void HostInfoUnittest::TestLatency() { + HostInfo hostInfo(mHostname); + APSARA_TEST_TRUE(hostInfo.IsForbidden()); + + auto latency = chrono::milliseconds(100); + hostInfo.SetLatency(latency); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + APSARA_TEST_FALSE(hostInfo.IsForbidden()); + + // continuous send error will trigger the host to be forbidden + hostInfo.UpdateStatus(false); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + hostInfo.UpdateStatus(false); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + hostInfo.UpdateStatus(false); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), hostInfo.GetLatency()); + hostInfo.UpdateStatus(true); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), hostInfo.GetLatency()); + + hostInfo.SetLatency(latency); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + + // set latency will clear error cnt + hostInfo.UpdateStatus(false); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + hostInfo.SetLatency(latency); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + hostInfo.UpdateStatus(false); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + hostInfo.UpdateStatus(false); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + hostInfo.UpdateStatus(false); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), hostInfo.GetLatency()); +} + +UNIT_TEST_CASE(HostInfoUnittest, TestHostname) +UNIT_TEST_CASE(HostInfoUnittest, TestLatency) + +class CandidateHostsInfoUnittest : public ::testing::Test { +public: + void TestBasicInfo(); + void TestHostsInfo(); + void TestUpdateHosts(); + void TestFirstHost(); + +private: + const string mProject = "project"; + const string mRegion = "region"; + const EndpointMode mMode = EndpointMode::DEFAULT; +}; + +void CandidateHostsInfoUnittest::TestBasicInfo() { + CandidateHostsInfo info(mProject, mRegion, mMode); + APSARA_TEST_EQUAL(mProject, info.GetProject()); + APSARA_TEST_EQUAL(mRegion, info.GetRegion()); + APSARA_TEST_EQUAL(mMode, info.GetMode()); +} + +void CandidateHostsInfoUnittest::TestHostsInfo() { + const string host1 = mProject + ".endpoint_1"; + const string host21 = mProject + ".endpoint_2_1"; + const string host22 = mProject + ".endpoint_2_2"; + const string host3 = mProject + ".endpoint_3"; + + CandidateHostsInfo info(mProject, mRegion, mMode); + info.mCandidateHosts.push_back({host1}); + info.mCandidateHosts.push_back({host21, host22}); + info.mCandidateHosts.push_back({host3}); + + // initialized + APSARA_TEST_TRUE(info.GetCurrentHost().empty()); + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // no valid host + info.SelectBestHost(); + APSARA_TEST_TRUE(info.GetCurrentHost().empty()); + + // some hosts become valid + info.UpdateHostLatency(host21, chrono::milliseconds(100)); + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host21, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(1U, res.size()); + APSARA_TEST_EQUAL(host21, res[0]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // host with the same priority as the current one has lower latency + info.UpdateHostLatency(host22, chrono::milliseconds(50)); + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host22, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(2U, res.size()); + APSARA_TEST_EQUAL(host21, res[0]); + APSARA_TEST_EQUAL(host22, res[1]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // host with higher priority becomes valid + info.UpdateHostLatency(host1, chrono::milliseconds(200)); + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(3U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // no change + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(3U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // all hosts becomes invalid + info.UpdateHostLatency(host1, chrono::milliseconds::max()); + info.UpdateHostLatency(host21, chrono::milliseconds::max()); + info.UpdateHostLatency(host22, chrono::milliseconds::max()); + info.UpdateHostLatency(host3, chrono::milliseconds::max()); + info.SelectBestHost(); + + APSARA_TEST_TRUE(info.GetCurrentHost().empty()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } +} + +void CandidateHostsInfoUnittest::TestUpdateHosts() { + const string region = "region"; + const string publicEndpoint = region + ".log.aliyuncs.com"; + const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; + const string internalEndpoint = region + "-internal.log.aliyuncs.com"; + const string globalEndpoint = "log-global.aliyuncs.com"; + const string customEndpoint = "custom.endpoint"; + const string publicHost = mProject + "." + publicEndpoint; + const string privateHost = mProject + "." + privateEndpoint; + const string internalHost = mProject + "." + internalEndpoint; + const string globalHost = mProject + "." + globalEndpoint; + const string customHost = mProject + "." + customEndpoint; + { + // default mode + CandidateHostsInfo info("project", region, EndpointMode::DEFAULT); + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); -bool GetAccessKeyMock(const std::string& targetURL, - const std::string& intf, - bool httpsVerifyPeer, - const std::string& request, - std::string& response, - std::string& errorMessage, - const std::string& caCert) { - static unordered_map> sAccessKeyMap - = {{"1", {"id1", "secret1"}}, {"2", {"id2", "secret2"}}}; + info.UpdateHostLatency(publicHost, chrono::milliseconds(100)); - ++sRequestRecievedCnt; + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); - Json::Value req; - string errMsg; - ParseJsonTable(request, req, errMsg); - string aliuid = req["aliuid"].asString(); + info.UpdateHostLatency(publicHost, chrono::milliseconds(50)); + info.UpdateHostLatency(privateHost, chrono::milliseconds(150)); - auto it = sAccessKeyMap.find(aliuid); - if (it == sAccessKeyMap.end()) { - if (aliuid == "invalid_format") { - response = "{]"; - return true; - } else if (aliuid == "invalid_header") { - response = R"({"AccessKey": {"access_key_id": "id1", "access_key": "secret1"}})"; - return true; - } else if (aliuid == "invalid_id") { - response = R"({"GetAccessKey": {"access_key": "secret1"}})"; - return true; - } else if (aliuid == "invalid_secret") { - response = R"({"GetAccessKey": {"access_key_id": "id1"}})"; - return true; + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(3U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(internalHost, info.mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[2].size()); + APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[2][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[2][0].GetLatency()); + } + { + // accelerate mode + { + CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostLatency(globalHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + + info.UpdateHostLatency(globalHost, chrono::milliseconds(50)); + info.UpdateHostLatency(publicHost, chrono::milliseconds(150)); + + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); } - return false; + { + CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::ACCELERATE, {globalEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostLatency(globalHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + + info.UpdateHostLatency(globalHost, chrono::milliseconds(50)); + info.UpdateHostLatency(publicHost, chrono::milliseconds(150)); + + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); + } + { + CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostLatency(globalHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + + info.UpdateHostLatency(globalHost, chrono::milliseconds(50)); + info.UpdateHostLatency(publicHost, chrono::milliseconds(150)); + + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); + } + } + { + // custom mode + CandidateHostsInfo info("project", region, EndpointMode::CUSTOM); + + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostLatency(customHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); } +} + +void CandidateHostsInfoUnittest::TestFirstHost() { + const string host1 = mProject + ".endpoint_1"; + const string host2 = mProject + ".endpoint_2"; + + CandidateHostsInfo info(mProject, mRegion, mMode); + APSARA_TEST_EQUAL("", info.GetFirstHost()); - Json::Value res; - res["GetAccessKey"]["access_key_id"] = it->second.first; - res["GetAccessKey"]["access_key"] = it->second.second; - response = res.toStyledString(); - return true; + info.mCandidateHosts.push_back({host1, host2}); + APSARA_TEST_EQUAL(host1, info.GetFirstHost()); + + info.mCandidateHosts[0].clear(); + APSARA_TEST_EQUAL("", info.GetFirstHost()); } +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestBasicInfo) +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestHostsInfo) +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestUpdateHosts) +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestFirstHost) + class EnterpriseSLSClientManagerUnittest : public ::testing::Test { public: + void TestLocalRegionEndpoints(); + void TestCopyLocalRegionEndpoints(); + void TestRemoteRegionEndpoints(); + void TestGetCandidateHostsInfo(); + void TestUpdateHostInfo(); + void TestUsingHttps(); + void TestProbeNetwork(); + void TestRealIp(); void TestDefaultAccessKey(); void TestGetAccessKeyFromSLS(); void TestUpdateAccessKeyStatus(); @@ -84,26 +483,824 @@ class EnterpriseSLSClientManagerUnittest : public ::testing::Test { static void SetUpTestCase() { STRING_FLAG(logtail_profile_aliuid) = "1264425845278179"; EnterpriseConfigProvider::GetInstance()->SetHttpsConfigServerList({"https://example.com"}); + EnterpriseSLSClientManager::GetInstance()->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; + EnterpriseSLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIp; + EnterpriseSLSClientManager::GetInstance()->mGetAccessKeyFromSLS = GetAccessKeyMock::DoGetAccessKey; } - void SetUp() override { mManager.mGetAccessKeyFromSLS = GetAccessKeyMock; } - void TearDown() override { sRequestRecievedCnt = 0; } + void TearDown() override { + mManager->Clear(); + GetAccessKeyMock::sRequestRecievedCnt = 0; + } private: - EnterpriseSLSClientManager mManager; + EnterpriseSLSClientManager* mManager = EnterpriseSLSClientManager::GetInstance(); }; +void EnterpriseSLSClientManagerUnittest::TestLocalRegionEndpoints() { + const string region = "region"; + { + // default + const string endpoint = region + ".log.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, item.mMode); + APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); + APSARA_TEST_EQUAL(endpoint, item.mLocalEndpoints[0]); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // accelerate (+ default) + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = kAccelerationDataEndpoint; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); + APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // custom (+ default) + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = "custom.endpoint"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::CUSTOM, item.mMode); + APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); + APSARA_TEST_EQUAL(endpoint2, item.mLocalEndpoints[0]); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // accelerate + custom (+ default) + const string endpoint1 = "custom.endpoint"; + const string endpoint2 = region + "-intranet.log.aliyuncs.com"; + const string endpoint3 = kAccelerationDataEndpoint; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2, endpoint3}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); + APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // http -> https -> https + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = "https://" + kAccelerationDataEndpoint; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_TRUE(mManager->UsingHttps(region)); + + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_TRUE(mManager->UsingHttps(region)); + } + { + // https -> http -> http + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = kAccelerationDataEndpoint; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } +} + +void EnterpriseSLSClientManagerUnittest::TestCopyLocalRegionEndpoints() { + mManager->UpdateLocalRegionEndpointsAndHttpsInfo("cn-shenzhen", {"cn-shenzhen.log.aliyuncs.com"}); + mManager->UpdateLocalRegionEndpointsAndHttpsInfo("cn-shanghai", {kAccelerationDataEndpoint}); + mManager->UpdateLocalRegionEndpointsAndHttpsInfo("cn-hangzhou", {"custom.endpoint"}); + mManager->UpdateLocalRegionEndpointsAndHttpsInfo("cn-hangzhou-b", {"cn-hangzhou-b.log.aliyuncs.com"}); + mManager->UpdateLocalRegionEndpointsAndHttpsInfo("cn-hangzhou-c", {kAccelerationDataEndpoint}); + { + // src exists, dst not exist, and src mode is default + mManager->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("cn-shenzhen", "cn-shenzhen-b"); + auto& endpoints = mManager->mRegionCandidateEndpointsMap["cn-shenzhen-b"]; + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, endpoints.mMode); + APSARA_TEST_EQUAL(1U, endpoints.mLocalEndpoints.size()); + APSARA_TEST_EQUAL("cn-shenzhen-b.log.aliyuncs.com", endpoints.mLocalEndpoints[0]); + } + { + // src exists, dst not exist, and src mode is not default + mManager->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("cn-shanghai", "cn-shanghai-b"); + auto& endpoints = mManager->mRegionCandidateEndpointsMap["cn-shanghai-b"]; + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, endpoints.mMode); + APSARA_TEST_EQUAL(0U, endpoints.mLocalEndpoints.size()); + } + { + // src exists, dst exists, dst mode is default + mManager->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("cn-hangzhou", "cn-hangzhou-b"); + auto& endpoints = mManager->mRegionCandidateEndpointsMap["cn-hangzhou-b"]; + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, endpoints.mMode); + APSARA_TEST_EQUAL(1U, endpoints.mLocalEndpoints.size()); + APSARA_TEST_EQUAL("cn-hangzhou-b.log.aliyuncs.com", endpoints.mLocalEndpoints[0]); + } + { + // src exists, dst exists, dst mode is not default + mManager->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("cn-hangzhou", "cn-hangzhou-c"); + auto& endpoints = mManager->mRegionCandidateEndpointsMap["cn-hangzhou-c"]; + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, endpoints.mMode); + APSARA_TEST_EQUAL(0U, endpoints.mLocalEndpoints.size()); + } + { + // src not exist + mManager->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("cn-beijing", "cn-beijing-b"); + APSARA_TEST_EQUAL(mManager->mRegionCandidateEndpointsMap.end(), + mManager->mRegionCandidateEndpointsMap.find("cn-beijing")); + APSARA_TEST_EQUAL(mManager->mRegionCandidateEndpointsMap.end(), + mManager->mRegionCandidateEndpointsMap.find("cn-beijing-b")); + } +} + +void EnterpriseSLSClientManagerUnittest::TestRemoteRegionEndpoints() { + { + auto info1 = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); + auto info2 = mManager->GetCandidateHostsInfo("region_1", "project_2", EndpointMode::DEFAULT); + auto info3 = mManager->GetCandidateHostsInfo("region_2", "project_3", EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + + // create remote info + mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_1", "endpoint_2"}); + APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[1][0].GetHostname()); + + // update remote info with overwrite + mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_2", "endpoint_3"}); + APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + + // update remote info with create + mManager->UpdateRemoteRegionEndpoints( + "region_1", {"endpoint_4"}, EnterpriseSLSClientManager::RemoteEndpointUpdateAction::CREATE); + APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + + // update remote info with append + mManager->UpdateRemoteRegionEndpoints( + "region_1", {"endpoint_3", "endpoint_1"}, EnterpriseSLSClientManager::RemoteEndpointUpdateAction::APPEND); + APSARA_TEST_EQUAL(3U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(3U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[2].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[2].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[2][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[2][0].GetHostname()); + } + { + // get candidate host info after remote info is updated + auto info = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(3U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[2].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_1", info->mCandidateHosts[2][0].GetHostname()); + } +} + +void EnterpriseSLSClientManagerUnittest::TestGetCandidateHostsInfo() { + const string region = "region"; + const string project = "project"; + const string publicEndpoint = region + ".log.aliyuncs.com"; + const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; + const string internalEndpoint = region + "-internal.log.aliyuncs.com"; + const string globalEndpoint = kAccelerationDataEndpoint; + const string customEndpoint = "custom.endpoint"; + const string publicHost = project + "." + publicEndpoint; + const string privateHost = project + "." + privateEndpoint; + const string internalHost = project + "." + internalEndpoint; + const string globalHost = project + "." + globalEndpoint; + const string customHost = project + "." + customEndpoint; + { + // no candidate host info && no region endpoints + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); + APSARA_TEST_EQUAL(0U, info->mCandidateHosts.size()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // no candidate host info && with region endpoints && region endpoint mode is default + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(privateHost, info->mCandidateHosts[0][0].GetHostname()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // no candidate host info && with region endpoints && region endpoint mode is accelerate + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // no candidate host info && with region endpoints && region endpoint mode is custom + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(customHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto& weakInfo = *(++infos.begin()); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto& weakInfo = *(++infos.begin()); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // candidate host info exists && no region endpoints + auto info1 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + auto info2 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(info1.get(), info2.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info1.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + + mManager->Clear(); + } + { + // candidate host info exists && with region endpoints && region endpoint mode is default + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); + auto infoDefault = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(infoDefault.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(infoDefault.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + mManager->Clear(); + } + { + // candidate host info exists && with region endpoints && region endpoint mode is accelerate + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); + auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + mManager->Clear(); + } + { + // candidate host info exists && with region endpoints && region endpoint mode is custom + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); + auto infoCustom = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(infoCustom.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(infoCustom.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + mManager->Clear(); + } +} + +void EnterpriseSLSClientManagerUnittest::TestUpdateHostInfo() { + const string region = "region"; + const string project = "project"; + const EndpointMode mode = EndpointMode::DEFAULT; + const string endpoint1 = region + ".log.aliyuncs.com"; + const string endpoint2 = region + "-intranet.log.aliyuncs.com"; + const string host1 = project + "." + endpoint1; + const string host2 = project + "." + endpoint2; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + { + auto info = mManager->GetCandidateHostsInfo(region, project, mode); + + APSARA_TEST_TRUE(mManager->UpdateHostLatency(project, mode, host1, chrono::milliseconds(100))); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info->mCandidateHosts[0][0].GetLatency()); + } + + // expired info + APSARA_TEST_FALSE(mManager->UpdateHostLatency(project, mode, host1, chrono::milliseconds(50))); + + // unknown project + APSARA_TEST_FALSE(mManager->UpdateHostLatency("unknown_project", mode, host1, chrono::milliseconds(50))); +} + +void EnterpriseSLSClientManagerUnittest::TestUsingHttps() { + const string region = "region"; + const string host = region + ".log.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {host}); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + + AppConfig::GetInstance()->mSendDataPort = 443; + APSARA_TEST_TRUE(mManager->UsingHttps(region)); +} + +void EnterpriseSLSClientManagerUnittest::TestProbeNetwork() { + ProbeNetworkMock::Prepare(); + const string publicHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::publicEndpoint1; + const string privateHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::privateEndpoint1; + const string internalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::internalEndpoint1; + const string globalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::globalEndpoint; + const string publicHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::publicEndpoint2; + const string globalHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::globalEndpoint; + const string globalHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::globalEndpoint; + const string customHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::customEndpoint; + + // ! means not available + vector> infos; + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::DEFAULT)); // public !internal !private + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::ACCELERATE)); // !accelerate public + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region2, ProbeNetworkMock::project2, EndpointMode::DEFAULT)); // accelerate public + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::DEFAULT)); // custom + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::ACCELERATE)); // !accelerate + + // probe uninitialized host + mManager->DoProbeUnInitializedHost(); + APSARA_TEST_TRUE(mManager->mUnInitializedCandidateHostsInfos.empty()); + APSARA_TEST_EQUAL(infos.size(), mManager->mPartiallyInitializedCandidateHostsInfos.size()); + for (size_t i = 0; i < infos.size(); ++i) { + APSARA_TEST_EQUAL(infos[i].get(), mManager->mPartiallyInitializedCandidateHostsInfos[i].lock().get()); + } + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); + vector len = {1, 2, 1, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + + // probe partially initialized host + mManager->DoProbeHost(); + APSARA_TEST_TRUE(mManager->mPartiallyInitializedCandidateHostsInfos.empty()); + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); + len = {1, 1, 2, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + + vector> oldLatencies; + for (const auto& info : infos) { + auto& latency = oldLatencies.emplace_back(); + for (const auto& item : info->mCandidateHosts) { + for (const auto& entry : item) { + latency.push_back(entry.GetLatency()); + } + } + } + + // probe all avaialble hosts + INT32_FLAG(sls_hosts_probe_interval_sec) = 0; + mManager->DoProbeHost(); + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); + len = {1, 1, 2, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + + vector> newLatencies; + for (const auto& info : infos) { + auto& latency = newLatencies.emplace_back(); + for (const auto& item : info->mCandidateHosts) { + for (const auto& entry : item) { + latency.push_back(entry.GetLatency()); + } + } + } + + for (size_t i = 0; i < oldLatencies.size(); ++i) { + for (size_t j = 0; j < oldLatencies[i].size(); ++j) { + if (oldLatencies[i][j] != chrono::milliseconds::max()) { + APSARA_TEST_NOT_EQUAL(newLatencies[i][j], oldLatencies[i][j]); + } else { + APSARA_TEST_EQUAL(newLatencies[i][j], oldLatencies[i][j]); + } + } + } + INT32_FLAG(sls_hosts_probe_interval_sec) = 60; + + // probe all hosts + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; + ProbeNetworkMock::enableUnavailableHosts = true; + mManager->DoProbeHost(); + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost1, infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost3, infos[4]->GetCurrentHost()); + + len = {3, 2, 2, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + ProbeNetworkMock::enableUnavailableHosts = false; + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; + + // expired info + infos[2].reset(); + infos[4].reset(); + mManager->DoProbeHost(); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project1].size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project3].size()); + APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region1].size()); + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region3].size()); +} + +void EnterpriseSLSClientManagerUnittest::TestRealIp() { + BOOL_FLAG(send_prefer_real_ip) = true; + mManager->mDoProbeNetwork = GetRealIpMock::DoProbeNetwork; + + const string unknownRegion = "unknown_region"; + const string normalRegionEndpoint1 = GetRealIpMock::normalRegion + "-intranet.log.aliyuncs.com"; + const string normalRegionEndpoint2 = GetRealIpMock::normalRegion + ".log.aliyuncs.com"; + const string unavailableRegionEndpoint = GetRealIpMock::unavailableRegion + ".log.aliyuncs.com"; + const string unknownRegionEndpoint = unknownRegion + ".log.aliyuncs.com"; + mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::normalRegion, {normalRegionEndpoint1, normalRegionEndpoint2}); + mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::unavailableRegion, {unavailableRegionEndpoint}); + mManager->UpdateRemoteRegionEndpoints(unknownRegion, {unknownRegionEndpoint}); + APSARA_TEST_EQUAL(3U, mManager->mRegionRealIpCandidateHostsInfosMap.size()); + { + vector hosts; + mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::normalRegion).GetAllHosts(hosts); + APSARA_TEST_EQUAL(2U, hosts.size()); + APSARA_TEST_EQUAL(normalRegionEndpoint1, hosts[0]); + APSARA_TEST_EQUAL(normalRegionEndpoint2, hosts[1]); + } + { + vector hosts; + mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::unavailableRegion).GetAllHosts(hosts); + APSARA_TEST_EQUAL(1U, hosts.size()); + APSARA_TEST_EQUAL(unavailableRegionEndpoint, hosts[0]); + } + { + vector hosts; + mManager->mRegionRealIpCandidateHostsInfosMap.at(unknownRegion).GetAllHosts(hosts); + APSARA_TEST_EQUAL(1U, hosts.size()); + APSARA_TEST_EQUAL(unknownRegionEndpoint, hosts[0]); + } + + // no endpoint available + mManager->DoUpdateRealIp(); + APSARA_TEST_EQUAL(0U, mManager->mRegionRealIpMap.size()); + + // probe host + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; + mManager->DoProbeHost(); + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; + for (const auto& item : mManager->mRegionRealIpCandidateHostsInfosMap) { + APSARA_TEST_NOT_EQUAL("", item.second.GetCurrentHost()); + } + + // update all regions + INT32_FLAG(send_switch_real_ip_interval) = 0; + mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::unavailableRegion); + mManager->DoUpdateRealIp(); + APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); + APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); + APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[0], mManager->GetRealIp(GetRealIpMock::normalRegion)); + APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); + INT32_FLAG(send_switch_real_ip_interval) = 60; + + // update only outdated regions + mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::normalRegion); + mManager->DoUpdateRealIp(); + APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); + APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); + APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[1], mManager->GetRealIp(GetRealIpMock::normalRegion)); + APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); + + mManager->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; + BOOL_FLAG(send_prefer_real_ip) = false; +} + void EnterpriseSLSClientManagerUnittest::TestDefaultAccessKey() { - APSARA_TEST_EQUAL(2U, mManager.mAccessKeyInfoCache.size()); + APSARA_TEST_EQUAL(2U, mManager->mAccessKeyInfoCache.size()); { - auto& info = mManager.mAccessKeyInfoCache.at(""); + auto& info = mManager->mAccessKeyInfoCache.at(""); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, info.mAuthType); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key_id), info.mId); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), info.mSecret); APSARA_TEST_EQUAL(numeric_limits::max(), info.mExpireTime); } { - auto& info = mManager.mAccessKeyInfoCache.at(STRING_FLAG(logtail_profile_aliuid)); + auto& info = mManager->mAccessKeyInfoCache.at(STRING_FLAG(logtail_profile_aliuid)); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, info.mAuthType); APSARA_TEST_EQUAL(STRING_FLAG(logtail_profile_access_key_id), info.mId); APSARA_TEST_EQUAL(STRING_FLAG(logtail_profile_access_key), info.mSecret); @@ -112,18 +1309,18 @@ void EnterpriseSLSClientManagerUnittest::TestDefaultAccessKey() { } void EnterpriseSLSClientManagerUnittest::TestGetAccessKeyFromSLS() { - mManager.mAccessKeyInfoCache.clear(); + mManager->mAccessKeyInfoCache.clear(); SLSClientManager::AuthType type; string akId, akSecret; { // no cache, successful request - APSARA_TEST_TRUE(mManager.GetAccessKey("1", type, akId, akSecret)); + APSARA_TEST_TRUE(mManager->GetAccessKey("1", type, akId, akSecret)); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, type); APSARA_TEST_EQUAL("id1", akId); APSARA_TEST_EQUAL("secret1", akSecret); - APSARA_TEST_EQUAL(1U, sRequestRecievedCnt); - APSARA_TEST_EQUAL(1U, mManager.mAccessKeyInfoCache.size()); - auto& info = mManager.mAccessKeyInfoCache.at("1"); + APSARA_TEST_EQUAL(1U, GetAccessKeyMock::sRequestRecievedCnt); + APSARA_TEST_EQUAL(1U, mManager->mAccessKeyInfoCache.size()); + auto& info = mManager->mAccessKeyInfoCache.at("1"); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, info.mAuthType); APSARA_TEST_EQUAL("id1", info.mId); APSARA_TEST_EQUAL("secret1", info.mSecret); @@ -132,45 +1329,45 @@ void EnterpriseSLSClientManagerUnittest::TestGetAccessKeyFromSLS() { } { // with cache - APSARA_TEST_TRUE(mManager.GetAccessKey("1", type, akId, akSecret)); + APSARA_TEST_TRUE(mManager->GetAccessKey("1", type, akId, akSecret)); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, type); APSARA_TEST_EQUAL("id1", akId); APSARA_TEST_EQUAL("secret1", akSecret); - APSARA_TEST_EQUAL(1U, sRequestRecievedCnt); - APSARA_TEST_EQUAL(1U, mManager.mAccessKeyInfoCache.size()); + APSARA_TEST_EQUAL(1U, GetAccessKeyMock::sRequestRecievedCnt); + APSARA_TEST_EQUAL(1U, mManager->mAccessKeyInfoCache.size()); } { // invalid response - APSARA_TEST_FALSE(mManager.GetAccessKey("invalid_format", type, akId, akSecret)); - APSARA_TEST_EQUAL(2U, sRequestRecievedCnt); - APSARA_TEST_FALSE(mManager.GetAccessKey("invalid_header", type, akId, akSecret)); - APSARA_TEST_EQUAL(3U, sRequestRecievedCnt); - APSARA_TEST_FALSE(mManager.GetAccessKey("invalid_id", type, akId, akSecret)); - APSARA_TEST_EQUAL(4U, sRequestRecievedCnt); - APSARA_TEST_FALSE(mManager.GetAccessKey("invalid_secret", type, akId, akSecret)); - APSARA_TEST_EQUAL(5U, sRequestRecievedCnt); + APSARA_TEST_FALSE(mManager->GetAccessKey("invalid_format", type, akId, akSecret)); + APSARA_TEST_EQUAL(2U, GetAccessKeyMock::sRequestRecievedCnt); + APSARA_TEST_FALSE(mManager->GetAccessKey("invalid_header", type, akId, akSecret)); + APSARA_TEST_EQUAL(3U, GetAccessKeyMock::sRequestRecievedCnt); + APSARA_TEST_FALSE(mManager->GetAccessKey("invalid_id", type, akId, akSecret)); + APSARA_TEST_EQUAL(4U, GetAccessKeyMock::sRequestRecievedCnt); + APSARA_TEST_FALSE(mManager->GetAccessKey("invalid_secret", type, akId, akSecret)); + APSARA_TEST_EQUAL(5U, GetAccessKeyMock::sRequestRecievedCnt); } { // no cache, failed request - APSARA_TEST_FALSE(mManager.GetAccessKey("unknown", type, akId, akSecret)); - APSARA_TEST_EQUAL(6U, sRequestRecievedCnt); - APSARA_TEST_EQUAL(1U, mManager.mAccessKeyInfoCache.size()); + APSARA_TEST_FALSE(mManager->GetAccessKey("unknown", type, akId, akSecret)); + APSARA_TEST_EQUAL(6U, GetAccessKeyMock::sRequestRecievedCnt); + APSARA_TEST_EQUAL(1U, mManager->mAccessKeyInfoCache.size()); } { // no cache, request after network failure - APSARA_TEST_FALSE(mManager.GetAccessKey("2", type, akId, akSecret)); - APSARA_TEST_EQUAL(6U, sRequestRecievedCnt); + APSARA_TEST_FALSE(mManager->GetAccessKey("2", type, akId, akSecret)); + APSARA_TEST_EQUAL(6U, GetAccessKeyMock::sRequestRecievedCnt); } { // no cache, request after timeout INT32_FLAG(request_access_key_interval) = 0; - APSARA_TEST_TRUE(mManager.GetAccessKey("2", type, akId, akSecret)); + APSARA_TEST_TRUE(mManager->GetAccessKey("2", type, akId, akSecret)); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, type); APSARA_TEST_EQUAL("id2", akId); APSARA_TEST_EQUAL("secret2", akSecret); - APSARA_TEST_EQUAL(7U, sRequestRecievedCnt); - APSARA_TEST_EQUAL(2U, mManager.mAccessKeyInfoCache.size()); - auto& info = mManager.mAccessKeyInfoCache.at("2"); + APSARA_TEST_EQUAL(7U, GetAccessKeyMock::sRequestRecievedCnt); + APSARA_TEST_EQUAL(2U, mManager->mAccessKeyInfoCache.size()); + auto& info = mManager->mAccessKeyInfoCache.at("2"); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, info.mAuthType); APSARA_TEST_EQUAL("id2", info.mId); APSARA_TEST_EQUAL("secret2", info.mSecret); @@ -181,21 +1378,21 @@ void EnterpriseSLSClientManagerUnittest::TestGetAccessKeyFromSLS() { } void EnterpriseSLSClientManagerUnittest::TestUpdateAccessKeyStatus() { - mManager.mAccessKeyInfoCache.clear(); + mManager->mAccessKeyInfoCache.clear(); SLSClientManager::AuthType type; string akId, akSecret; - mManager.GetAccessKey("1", type, akId, akSecret); + mManager->GetAccessKey("1", type, akId, akSecret); - mManager.UpdateAccessKeyStatus("1", false); - APSARA_TEST_EQUAL(1U, mManager.mAccessKeyInfoCache.at("1").mErrorCnt); + mManager->UpdateAccessKeyStatus("1", false); + APSARA_TEST_EQUAL(1U, mManager->mAccessKeyInfoCache.at("1").mErrorCnt); - mManager.UpdateAccessKeyStatus("1", true); - APSARA_TEST_EQUAL(0U, mManager.mAccessKeyInfoCache.at("1").mErrorCnt); + mManager->UpdateAccessKeyStatus("1", true); + APSARA_TEST_EQUAL(0U, mManager->mAccessKeyInfoCache.at("1").mErrorCnt); - mManager.UpdateAccessKeyStatus("1", false); - mManager.UpdateAccessKeyStatus("1", false); - mManager.UpdateAccessKeyStatus("1", false); - APSARA_TEST_EQUAL(0U, mManager.mAccessKeyInfoCache.size()); + mManager->UpdateAccessKeyStatus("1", false); + mManager->UpdateAccessKeyStatus("1", false); + mManager->UpdateAccessKeyStatus("1", false); + APSARA_TEST_EQUAL(0U, mManager->mAccessKeyInfoCache.size()); } void EnterpriseSLSClientManagerUnittest::TestProjectAnonymousWrite() { @@ -204,35 +1401,46 @@ void EnterpriseSLSClientManagerUnittest::TestProjectAnonymousWrite() { SLSClientManager::AuthType type; // meet new project, give a try - APSARA_TEST_TRUE(mManager.GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); + APSARA_TEST_TRUE( + mManager->GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, type); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key_id), accessKeyId); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); // first try has not come back, forbidden for now APSARA_TEST_FALSE( - mManager.GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); + mManager->GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); // first try comes back - mManager.UpdateProjectAnonymousWriteStatus(project, true); - APSARA_TEST_TRUE(mManager.GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); + mManager->UpdateProjectAnonymousWriteStatus(project, true); + APSARA_TEST_TRUE( + mManager->GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, type); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key_id), accessKeyId); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); // more update does not work on project that has been confirmed - mManager.UpdateProjectAnonymousWriteStatus(project, false); - APSARA_TEST_TRUE(mManager.GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); + mManager->UpdateProjectAnonymousWriteStatus(project, false); + APSARA_TEST_TRUE( + mManager->GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)); APSARA_TEST_EQUAL(SLSClientManager::AuthType::ANONYMOUS, type); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key_id), accessKeyId); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); // update project that has not been tried does nothing - mManager.UpdateProjectAnonymousWriteStatus("unknown", true); - APSARA_TEST_EQUAL(mManager.mProjectAnonymousWriteStatusMap.end(), - mManager.mProjectAnonymousWriteStatusMap.find("unknown")); + mManager->UpdateProjectAnonymousWriteStatus("unknown", true); + APSARA_TEST_EQUAL(mManager->mProjectAnonymousWriteStatusMap.end(), + mManager->mProjectAnonymousWriteStatusMap.find("unknown")); } +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestLocalRegionEndpoints) +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestCopyLocalRegionEndpoints) +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestRemoteRegionEndpoints) +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestGetCandidateHostsInfo) +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestUpdateHostInfo) +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestUsingHttps) +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestProbeNetwork) +UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestRealIp) UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestDefaultAccessKey) UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestGetAccessKeyFromSLS) // UNIT_TEST_CASE(EnterpriseSLSClientManagerUnittest, TestUpdateAccessKeyStatus) diff --git a/core/unittest/flusher/SLSNetworkRequestMock.cpp b/core/unittest/flusher/SLSNetworkRequestMock.cpp new file mode 100644 index 0000000000..a5fcc9eafd --- /dev/null +++ b/core/unittest/flusher/SLSNetworkRequestMock.cpp @@ -0,0 +1,160 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "unittest/flusher/SLSNetworkRequestMock.h" + +#include "common/JsonUtil.h" + +using namespace std; + +namespace logtail { + +bool GetAccessKeyMock::DoGetAccessKey(const string& targetURL, + const string& intf, + bool httpsVerifyPeer, + const string& request, + string& response, + string& errorMessage, + const string& caCert) { + static unordered_map> sAccessKeyMap + = {{"1", {"id1", "secret1"}}, {"2", {"id2", "secret2"}}}; + + ++sRequestRecievedCnt; + + Json::Value req; + string errMsg; + ParseJsonTable(request, req, errMsg); + string aliuid = req["aliuid"].asString(); + + auto it = sAccessKeyMap.find(aliuid); + if (it == sAccessKeyMap.end()) { + if (aliuid == "invalid_format") { + response = "{]"; + return true; + } else if (aliuid == "invalid_header") { + response = R"({"AccessKey": {"access_key_id": "id1", "access_key": "secret1"}})"; + return true; + } else if (aliuid == "invalid_id") { + response = R"({"GetAccessKey": {"access_key": "secret1"}})"; + return true; + } else if (aliuid == "invalid_secret") { + response = R"({"GetAccessKey": {"access_key_id": "id1"}})"; + return true; + } + return false; + } + + Json::Value res; + res["GetAccessKey"]["access_key_id"] = it->second.first; + res["GetAccessKey"]["access_key"] = it->second.second; + response = res.toStyledString(); + return true; +} + +uint32_t GetAccessKeyMock::sRequestRecievedCnt = 0; + +HttpResponse +ProbeNetworkMock::DoProbeNetwork(const unique_ptr& req) { + HttpResponse response; + chrono::milliseconds latency = chrono::milliseconds::max(); + if (!enableUnavailableHosts) { + if (req->mProject == project1 && req->mHost == project1 + "." + publicEndpoint1) { + static uint32_t sec = 1; + latency = chrono::milliseconds(sec++); + } else if (req->mProject == project2) { + if (req->mHost == project2 + "." + publicEndpoint2) { + static uint32_t sec = 10; + latency = chrono::milliseconds(sec++); + } else if (req->mHost == project2 + "." + globalEndpoint) { + static uint32_t sec = 5; + latency = chrono::milliseconds(sec++); + } + } else if (req->mProject == project3 && req->mHost == project3 + "." + customEndpoint) { + static uint32_t sec = 20; + latency = chrono::milliseconds(sec++); + } + } else { + static uint32_t sec = 200; + latency = chrono::milliseconds(sec++); + } + + response.SetResponseTime(latency); + return response; +} + +void ProbeNetworkMock::Prepare() { + publicEndpoint1 = region1 + ".log.aliyuncs.com"; + privateEndpoint1 = region1 + "-intranet.log.aliyuncs.com"; + internalEndpoint1 = region1 + "-internal.log.aliyuncs.com"; + publicEndpoint2 = region2 + ".log.aliyuncs.com"; + privateEndpoint2 = region2 + "-intranet.log.aliyuncs.com"; + internalEndpoint2 = region2 + "-internal.log.aliyuncs.com"; + + // region_1: only public endpoint is available + // region_2: both accelerate and public endpoints are available + // region_3: only custom endpoint is available + // project_x belongs to region_x + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region1, {publicEndpoint1}); + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region2, {globalEndpoint}); + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region3, {customEndpoint}); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + region1, {internalEndpoint1, privateEndpoint1, publicEndpoint1}); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + region2, {internalEndpoint2, privateEndpoint2, publicEndpoint2}); +} + +bool ProbeNetworkMock::enableUnavailableHosts = false; +string ProbeNetworkMock::region1 = "region_1"; +string ProbeNetworkMock::region2 = "region_2"; +string ProbeNetworkMock::region3 = "region_3"; +string ProbeNetworkMock::project1 = "project_1"; +string ProbeNetworkMock::project2 = "project_2"; +string ProbeNetworkMock::project3 = "project_3"; +string ProbeNetworkMock::publicEndpoint1; +string ProbeNetworkMock::privateEndpoint1; +string ProbeNetworkMock::internalEndpoint1; +string ProbeNetworkMock::publicEndpoint2; +string ProbeNetworkMock::privateEndpoint2; +string ProbeNetworkMock::internalEndpoint2; +string ProbeNetworkMock::globalEndpoint = "log-global.aliyuncs.com"; +string ProbeNetworkMock::customEndpoint = "custom.endpoint"; + +bool GetRealIpMock::GetEndpointRealIp(const string& endpoint, string& ip) { + if (endpoint.find(normalRegion) != string::npos) { + static size_t i = 0; + ip = normalRegionIps[i]; + i = (i + 1) % normalRegionIps.size(); + return true; + } else if (endpoint.find(unavailableRegion) != string::npos) { + static size_t i = 0; + ip = unavailableRegionIps[i]; + i = (i + 1) % unavailableRegionIps.size(); + return true; + } else { + return false; + } +} + +HttpResponse GetRealIpMock::DoProbeNetwork(const unique_ptr& req) { + HttpResponse response; + response.SetResponseTime(chrono::milliseconds(100)); + return response; +} + +string GetRealIpMock::normalRegion = "region_1"; +string GetRealIpMock::unavailableRegion = "region_2"; +vector GetRealIpMock::normalRegionIps = {"192.168.0.1", "192.168.0.2"}; +vector GetRealIpMock::unavailableRegionIps = {"10.0.0.1", "10.0.0.2"}; + +} // namespace logtail diff --git a/core/unittest/flusher/SLSNetworkRequestMock.h b/core/unittest/flusher/SLSNetworkRequestMock.h new file mode 100644 index 0000000000..2bd4ff4533 --- /dev/null +++ b/core/unittest/flusher/SLSNetworkRequestMock.h @@ -0,0 +1,76 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" + +namespace logtail { + +class GetAccessKeyMock { +public: + static bool DoGetAccessKey(const std::string& targetURL, + const std::string& intf, + bool httpsVerifyPeer, + const std::string& request, + std::string& response, + std::string& errorMessage, + const std::string& caCert); + + static uint32_t sRequestRecievedCnt; +}; + +class ProbeNetworkMock { +public: + static HttpResponse DoProbeNetwork(const std::unique_ptr& req); + + static void Prepare(); + + static bool enableUnavailableHosts; + + static std::string project1; + static std::string project2; + static std::string project3; + static std::string region1; + static std::string region2; + static std::string region3; + static std::string publicEndpoint1; + static std::string privateEndpoint1; + static std::string internalEndpoint1; + static std::string publicEndpoint2; + static std::string privateEndpoint2; + static std::string internalEndpoint2; + static std::string globalEndpoint; + static std::string customEndpoint; +}; + +class GetRealIpMock { +public: + static bool GetEndpointRealIp(const std::string& endpoint, std::string& ip); + + static HttpResponse DoProbeNetwork(const std::unique_ptr& req); + + static std::string normalRegion; + static std::string unavailableRegion; + static std::vector normalRegionIps; + static std::vector unavailableRegionIps; +}; + +} // namespace logtail diff --git a/test/e2e_enterprise/test_cases/framework/backpressure.feature b/test/e2e_enterprise/test_cases/framework/backpressure.feature new file mode 100644 index 0000000000..fcb612081f --- /dev/null +++ b/test/e2e_enterprise/test_cases/framework/backpressure.feature @@ -0,0 +1,127 @@ +@input +Feature: pipeline + Test queue back pressure + + @e2e @host + Scenario: TestPureNativeBackPressure + Given {host} environment + Given subcribe data from {sls} with config + """ + """ + Given {back_pressure} local config as below + """ + enable: true + inputs: + - Type: input_file + FilePaths: + - /tmp/loongcollector/test.log + """ + When begin trigger + When generate {1000} regex logs to file {/tmp/loongcollector/test.log}, with interval {10}ms + Then wait {3} seconds + Given network lost package {100}% for ip {47.90.119.19} + Then wait {20} seconds + Given clean all chaos + + When query through {* | select count(1) as cnt} + Then wait {30} seconds + Then there is {1} logs + Then the log fields match kv + """ + cnt: "1000" + """ + + @e2e @host + Scenario: TestHybridBackPressure + Given {host} environment + Given subcribe data from {sls} with config + """ + """ + Given {back_pressure} local config as below + """ + enable: true + inputs: + - Type: input_file + FilePaths: + - /tmp/loongcollector/test.log + processors: + - Type: processor_drop + DropKeys: ["unknown"] + """ + When begin trigger + When generate {1000} regex logs to file {/tmp/loongcollector/test.log}, with interval {10}ms + Then wait {3} seconds + Given network lost package {100}% for ip {47.90.119.19} + Then wait {20} seconds + Given clean all chaos + + When query through {* | select count(1) as cnt} + Then wait {30} seconds + Then there is {1} logs + Then the log fields match kv + """ + cnt: "1000" + """ + + @e2e @k8s + Scenario: TestPureNativeBackPressure + Given {daemonset} environment + Given subcribe data from {sls} with config + """ + """ + Given {back_pressure} local config as below + """ + enable: true + inputs: + - Type: input_file + FilePaths: + - /tmp/loongcollector/test.log + EnableContainerDiscovery: true + """ + When begin trigger + When generate {1000} regex logs to file {/tmp/loongcollector/test.log}, with interval {10}ms + Then wait {3} seconds + Given network lost package {90}% for ip {47.90.119.19} + Then wait {20} seconds + Given clean all chaos + + When query through {* | select count(1) as cnt} + Then wait {30} seconds + Then there is {1} logs + Then the log fields match kv + """ + cnt: "1000" + """ + + @e2e @k8s + Scenario: TestHybridBackPressure + Given {daemonset} environment + Given subcribe data from {sls} with config + """ + """ + Given {back_pressure} local config as below + """ + enable: true + inputs: + - Type: input_file + FilePaths: + - /tmp/loongcollector/test.log + EnableContainerDiscovery: true + processors: + - Type: processor_drop + DropKeys: ["unknown"] + """ + When begin trigger + When generate {1000} regex logs to file {/tmp/loongcollector/test.log}, with interval {10}ms + Then wait {3} seconds + Given network lost package {90}% for ip {47.90.119.19} + Then wait {20} seconds + Given clean all chaos + + When query through {* | select count(1) as cnt} + Then wait {30} seconds + Then there is {1} logs + Then the log fields match kv + """ + cnt: "1000" + """