diff --git a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp index a8ac2f20cb..efe61f4ec1 100644 --- a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp +++ b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp @@ -173,18 +173,22 @@ void ProcessorPromRelabelMetricNative::AddAutoMetrics(PipelineEventGroup& eGroup LOG_ERROR(sLogger, ("scrape_timestamp_milliseconds is not set", "")); return; } + if (!eGroup.HasMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_ID)) { + LOG_ERROR(sLogger, ("prometheus stream id", "")); + return; + } + targetTags[prometheus::LC_TARGET_HASH] = eGroup.GetMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_ID); StringView scrapeTimestampMilliSecStr = eGroup.GetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_TIMESTAMP_MILLISEC); auto timestampMilliSec = StringTo(scrapeTimestampMilliSecStr.to_string()); auto timestamp = timestampMilliSec / 1000; auto nanoSec = timestampMilliSec % 1000 * 1000000; - AddMetric( eGroup, prometheus::SCRAPE_DURATION_SECONDS, autoMetric.mScrapeDurationSeconds, timestamp, nanoSec, targetTags); AddMetric(eGroup, - prometheus::SCRAPE_RESPONSE_SIZE_BYTES, + prometheus::SCRAPE_BODY_SIZE_BYTES, autoMetric.mScrapeResponseSizeBytes, timestamp, nanoSec, diff --git a/core/prometheus/Constants.h b/core/prometheus/Constants.h index e663fdf207..f155ebcdf5 100644 --- a/core/prometheus/Constants.h +++ b/core/prometheus/Constants.h @@ -13,6 +13,7 @@ const uint64_t RefeshIntervalSeconds = 5; const char* const META = "__meta_"; const char* const UNDEFINED = "undefined"; const std::string PROMETHEUS = "prometheus"; +const char* const LC_TARGET_HASH = "lc_target_hash"; // relabel config const char* const SOURCE_LABELS = "source_labels"; @@ -100,11 +101,13 @@ const char* const SCHEME_LABEL_NAME = "__scheme__"; const char* const METRICS_PATH_LABEL_NAME = "__metrics_path__"; const char* const PARAM_LABEL_NAME = "__param_"; const char* const LABELS = "labels"; +const char* const TARGET_HASH = "hash"; +const char* const REBALANCE_MS = "rebalance_ms"; // auto metrics const char* const SCRAPE_STATE = "scrape_state"; const char* const SCRAPE_DURATION_SECONDS = "scrape_duration_seconds"; -const char* const SCRAPE_RESPONSE_SIZE_BYTES = "scrape_response_size_bytes"; +const char* const SCRAPE_BODY_SIZE_BYTES = "scrape_body_size_bytes"; const char* const SCRAPE_SAMPLES_LIMIT = "scrape_samples_limit"; const char* const SCRAPE_SAMPLES_POST_METRIC_RELABELING = "scrape_samples_post_metric_relabeling"; const char* const SCRAPE_SAMPLES_SCRAPED = "scrape_samples_scraped"; @@ -119,4 +122,16 @@ const char* const ACCEPT_ENCODING = "Accept-Encoding"; const char* const GZIP = "gzip"; const char* const IDENTITY = "identity"; +const char* const AGENT_INFO = "AgentInfo"; +const char* const TARGETS_INFO = "TargetsInfo"; +const char* const CPU_LIMIT = "CpuLimit"; +const char* const CPU_USAGE = "CpuUsage"; +const char* const MEM_LIMIT = "MemLimit"; +const char* const MEM_USAGE = "MemUsage"; +const char* const HTTP_SINK_IN_ITEMS_TOTAL = "HttpSinkInItemsTotal"; +const char* const HTTP_SINK_OUT_FAILED = "HttpSinkOutFailed"; +const char* const HASH = "Hash"; +const char* const SIZE = "Size"; +const char* const SCRAPE_DELAY_SECONDS = "ScrapeDelaySeconds"; + } // namespace logtail::prometheus diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index b90fcbe248..ff130b7229 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -25,14 +25,12 @@ #include "common/Flags.h" #include "common/JsonUtil.h" #include "common/StringTools.h" -#include "common/TimeUtil.h" #include "common/http/AsynCurlRunner.h" #include "common/http/Constant.h" #include "common/http/Curl.h" #include "common/timer/Timer.h" #include "logger/Logger.h" #include "monitor/metric_constants/MetricConstants.h" -#include "plugin/flusher/sls/FlusherSLS.h" #include "prometheus/Constants.h" #include "prometheus/Utils.h" @@ -173,6 +171,8 @@ void PrometheusInputRunner::Init() { mUnRegisterMs = 0; } else { mUnRegisterMs.store(StringTo(tmpStr)); + // adjust unRegisterMs to scrape targets for zero-cost + mUnRegisterMs -= 1000; LOG_INFO(sLogger, ("unRegisterMs", ToString(mUnRegisterMs))); } } diff --git a/core/prometheus/component/StreamScraper.cpp b/core/prometheus/component/StreamScraper.cpp index 724c79eb5b..5ee62e9055 100644 --- a/core/prometheus/component/StreamScraper.cpp +++ b/core/prometheus/component/StreamScraper.cpp @@ -107,7 +107,7 @@ void StreamScraper::PushEventGroup(PipelineEventGroup&& eGroup) const { void StreamScraper::SendMetrics() { mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_TIMESTAMP_MILLISEC, ToString(mScrapeTimestampMilliSec)); - mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_ID, GetId() + ToString(mScrapeTimestampMilliSec)); + mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_ID, GetId()); SetTargetLabels(mEventGroup); PushEventGroup(std::move(mEventGroup)); @@ -132,7 +132,7 @@ void StreamScraper::SetAutoMetricMeta(double scrapeDurationSeconds, bool upState mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_DURATION, ToString(scrapeDurationSeconds)); mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_RESPONSE_SIZE, ToString(mRawSize)); mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_UP_STATE, ToString(upState)); - mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_ID, GetId() + ToString(mScrapeTimestampMilliSec)); + mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_ID, GetId()); mEventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_TOTAL, ToString(mStreamIndex)); } std::string StreamScraper::GetId() { diff --git a/core/prometheus/labels/Labels.cpp b/core/prometheus/labels/Labels.cpp index f701cabd82..60eb388ff7 100644 --- a/core/prometheus/labels/Labels.cpp +++ b/core/prometheus/labels/Labels.cpp @@ -111,10 +111,26 @@ void Labels::Range(const std::function& uint64_t Labels::Hash() { string hash; uint64_t sum = prometheus::OFFSET64; - Range([&hash](const string& k, const string& v) { hash += k + "\xff" + v + "\xff"; }); - for (auto i : hash) { - sum ^= (uint64_t)i; - sum *= prometheus::PRIME64; + vector names; + Range([&names](const string& k, const string&) { names.push_back(k); }); + sort(names.begin(), names.end()); + auto calc = [](uint64_t h, uint64_t c) { + h ^= (uint64_t)c; + h *= prometheus::PRIME64; + return h; + }; + auto calcString = [](uint64_t h, const string& s) { + for (auto c : s) { + h ^= (uint64_t)c; + h *= prometheus::PRIME64; + } + return h; + }; + for (const auto& name : names) { + sum = calcString(sum, name); + sum = calc(sum, 255); + sum = calcString(sum, Get(name)); + sum = calc(sum, 255); } return sum; } diff --git a/core/prometheus/schedulers/BaseScheduler.cpp b/core/prometheus/schedulers/BaseScheduler.cpp index 600b601d8d..7c3d2e9508 100644 --- a/core/prometheus/schedulers/BaseScheduler.cpp +++ b/core/prometheus/schedulers/BaseScheduler.cpp @@ -8,6 +8,10 @@ using namespace std; namespace logtail { void BaseScheduler::ExecDone() { mExecCount++; + while (mLatestExecTime > mFirstExecTime + chrono::seconds(mExecCount * mInterval)) { + mExecCount++; + } + mLatestExecTime = mFirstExecTime + chrono::seconds(mExecCount * mInterval); mLatestScrapeTime = mFirstScrapeTime + chrono::seconds(mExecCount * mInterval); } diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index 57856a19e2..d501b08536 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -40,27 +40,25 @@ using namespace std; namespace logtail { ScrapeScheduler::ScrapeScheduler(std::shared_ptr scrapeConfigPtr, - std::string host, + string host, int32_t port, std::string scheme, std::string metricsPath, uint64_t scrapeIntervalSeconds, uint64_t scrapeTimeoutSeconds, - Labels labels, QueueKey queueKey, - size_t inputIndex) + size_t inputIndex, + const PromTargetInfo& targetInfo) : mScrapeConfigPtr(std::move(scrapeConfigPtr)), mHost(std::move(host)), mPort(port), + mTargetInfo(targetInfo), mMetricsPath(std::move(metricsPath)), mScheme(std::move(scheme)), mScrapeTimeoutSeconds(scrapeTimeoutSeconds), mQueueKey(queueKey), mInputIndex(inputIndex), - mTargetLabels(labels) { - string tmpTargetURL = mScheme + "://" + mHost + ":" + ToString(mPort) + mMetricsPath; - mHash = mScrapeConfigPtr->mJobName + tmpTargetURL + ToString(labels.Hash()); - mInstance = mHost + ":" + ToString(mPort); + mScrapeResponseSizeBytes(-1) { mInterval = scrapeIntervalSeconds; } @@ -78,7 +76,9 @@ void ScrapeScheduler::OnMetricResult(HttpResponse& response, uint64_t) { const auto& networkStatus = response.GetNetworkStatus(); string scrapeState; - mUpState = false; + auto scrapeDurationSeconds = scrapeDurationMilliSeconds * sRate; + auto upState = false; + if (networkStatus.mCode != NetworkCode::Ok) { // not 0 means curl error scrapeState = prom::NetworkCodeToState(networkStatus.mCode); @@ -87,24 +87,23 @@ void ScrapeScheduler::OnMetricResult(HttpResponse& response, uint64_t) { } else { // 0 means success scrapeState = prom::NetworkCodeToState(NetworkCode::Ok); - mUpState = true; + upState = true; } - mScrapeDurationSeconds = scrapeDurationMilliSeconds * sRate; if (response.GetStatusCode() != 200) { LOG_WARNING(sLogger, - ("scrape failed, status code", - response.GetStatusCode())("target", mHash)("curl msg", response.GetNetworkStatus().mMessage)); + ("scrape failed, status code", response.GetStatusCode())("target", mTargetInfo.mHash)( + "curl msg", response.GetNetworkStatus().mMessage)); } - auto mScrapeDurationSeconds = scrapeDurationMilliSeconds * sRate; - streamScraper->mStreamIndex++; - if (mUpState) { + streamScraper->mStreamIndex++; + if (upState) { streamScraper->FlushCache(); } - streamScraper->SetAutoMetricMeta(mScrapeDurationSeconds, mUpState, scrapeState); + streamScraper->SetAutoMetricMeta(scrapeDurationSeconds, upState, scrapeState); streamScraper->SendMetrics(); + mScrapeResponseSizeBytes = streamScraper->mRawSize; streamScraper->Reset(); ADD_COUNTER(mPluginTotalDelayMs, scrapeDurationMilliSeconds); @@ -112,7 +111,7 @@ void ScrapeScheduler::OnMetricResult(HttpResponse& response, uint64_t) { string ScrapeScheduler::GetId() const { - return mHash; + return mTargetInfo.mHash; } uint64_t ScrapeScheduler::GetScrapeIntervalSeconds() const { @@ -137,6 +136,7 @@ void ScrapeScheduler::ScheduleNext() { return true; } this->DelayExecTime(1); + this->mExecDelayCount++; ADD_COUNTER(this->mPromDelayTotal, 1); this->ScheduleNext(); return false; @@ -185,7 +185,8 @@ std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono:: mScrapeConfigPtr->mRequestHeaders, "", HttpResponse( - new prom::StreamScraper(mTargetLabels, mQueueKey, mInputIndex, mHash, mEventPool, mLatestScrapeTime), + new prom::StreamScraper( + mTargetInfo.mLabels, mQueueKey, mInputIndex, mTargetInfo.mHash, mEventPool, mLatestScrapeTime), [](void* p) { delete static_cast(p); }, prom::StreamScraper::MetricWriteCallback), mScrapeTimeoutSeconds, @@ -215,7 +216,7 @@ void ScrapeScheduler::Cancel() { void ScrapeScheduler::InitSelfMonitor(const MetricLabels& defaultLabels) { mSelfMonitor = std::make_shared(); MetricLabels labels = defaultLabels; - labels.emplace_back(METRIC_LABEL_KEY_INSTANCE, mInstance); + labels.emplace_back(METRIC_LABEL_KEY_INSTANCE, mTargetInfo.mInstance); static const std::unordered_map sScrapeMetricKeys = {{METRIC_PLUGIN_OUT_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, diff --git a/core/prometheus/schedulers/ScrapeScheduler.h b/core/prometheus/schedulers/ScrapeScheduler.h index c73bc087a8..8a2816e0aa 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.h +++ b/core/prometheus/schedulers/ScrapeScheduler.h @@ -24,7 +24,6 @@ #include "common/http/HttpResponse.h" #include "monitor/metric_models/MetricTypes.h" #include "prometheus/PromSelfMonitor.h" -#include "prometheus/component/StreamScraper.h" #include "prometheus/schedulers/ScrapeConfig.h" #ifdef APSARA_UNIT_TEST_MAIN @@ -33,7 +32,16 @@ namespace logtail { +struct PromTargetInfo { + Labels mLabels; + std::string mInstance; + std::string mHash; + uint64_t mRebalanceMs = 0; +}; + class ScrapeScheduler : public BaseScheduler { + friend class TargetSubscriberScheduler; + public: ScrapeScheduler(std::shared_ptr scrapeConfigPtr, std::string host, @@ -42,9 +50,9 @@ class ScrapeScheduler : public BaseScheduler { std::string metricsPath, uint64_t scrapeIntervalSeconds, uint64_t scrapeTimeoutSeconds, - Labels labels, QueueKey queueKey, - size_t inputIndex); + size_t inputIndex, + const PromTargetInfo& targetInfo); ScrapeScheduler(const ScrapeScheduler&) = delete; ~ScrapeScheduler() override = default; @@ -54,7 +62,9 @@ class ScrapeScheduler : public BaseScheduler { uint64_t GetScrapeIntervalSeconds() const; void SetComponent(EventPool* eventPool); + int64_t GetLastScrapeSize() const { return mScrapeResponseSizeBytes; } + uint64_t GetReBalanceMs() const { return mTargetInfo.mRebalanceMs; } void ScheduleNext() override; void ScrapeOnce(std::chrono::steady_clock::time_point execTime); void Cancel() override; @@ -64,10 +74,10 @@ class ScrapeScheduler : public BaseScheduler { std::unique_ptr BuildScrapeTimerEvent(std::chrono::steady_clock::time_point execTime); std::shared_ptr mScrapeConfigPtr; - std::string mHash; + std::atomic_int mExecDelayCount = 0; std::string mHost; int32_t mPort; - std::string mInstance; + PromTargetInfo mTargetInfo; std::string mMetricsPath; std::string mScheme; uint64_t mScrapeTimeoutSeconds; @@ -76,13 +86,8 @@ class ScrapeScheduler : public BaseScheduler { QueueKey mQueueKey; size_t mInputIndex; - Labels mTargetLabels; - // auto metrics - uint64_t mScrapeTimestampMilliSec = 0; - double mScrapeDurationSeconds = 0; - uint64_t mScrapeResponseSizeBytes = 0; - bool mUpState = true; + std::atomic_int mScrapeResponseSizeBytes; // self monitor std::shared_ptr mSelfMonitor; diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index f9c046fa3e..500880f48b 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -16,18 +16,21 @@ #include "prometheus/schedulers/TargetSubscriberScheduler.h" +#include #include #include #include +#include "AppConfig.h" +#include "SelfMonitorMetricEvent.h" #include "common/JsonUtil.h" #include "common/StringTools.h" #include "common/TimeUtil.h" #include "common/http/Constant.h" #include "common/timer/HttpRequestTimerEvent.h" -#include "common/timer/Timer.h" #include "logger/Logger.h" +#include "monitor/Monitor.h" #include "monitor/metric_constants/MetricConstants.h" #include "prometheus/Constants.h" #include "prometheus/Utils.h" @@ -39,6 +42,8 @@ using namespace std; namespace logtail { +std::chrono::steady_clock::time_point TargetSubscriberScheduler::mLastUpdateTime = std::chrono::steady_clock::now(); +uint64_t TargetSubscriberScheduler::sDelaySeconds = 0; TargetSubscriberScheduler::TargetSubscriberScheduler() : mQueueKey(0), mInputIndex(0), mServicePort(0), mUnRegisterMs(0) { } @@ -74,7 +79,7 @@ void TargetSubscriberScheduler::OnSubscription(HttpResponse& response, uint64_t mETag = response.GetHeader().at(prometheus::ETAG); } const string& content = *response.GetBody(); - vector targetGroup; + vector targetGroup; if (!ParseScrapeSchedulerGroup(content, targetGroup)) { return; } @@ -104,19 +109,27 @@ void TargetSubscriberScheduler::UpdateScrapeScheduler( } // save new scrape work + auto added = 0; + auto total = 0; for (const auto& [k, v] : newScrapeSchedulerMap) { if (mScrapeSchedulerMap.find(k) == mScrapeSchedulerMap.end()) { + added++; mScrapeSchedulerMap[k] = v; auto tmpCurrentMilliSeconds = GetCurrentTimeInMilliSeconds(); auto tmpRandSleepMilliSec = GetRandSleepMilliSec(v->GetId(), v->GetScrapeIntervalSeconds(), tmpCurrentMilliSeconds); // zero-cost upgrade - if (mUnRegisterMs > 0 - && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - v->GetScrapeIntervalSeconds() * 1000 - > mUnRegisterMs) - && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - v->GetScrapeIntervalSeconds() * 1000 * 2 - < mUnRegisterMs)) { + if ((mUnRegisterMs > 0 + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - v->GetScrapeIntervalSeconds() * 1000 + > mUnRegisterMs) + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - v->GetScrapeIntervalSeconds() * 1000 * 2 + < mUnRegisterMs)) + || (v->GetReBalanceMs() > 0 + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - v->GetScrapeIntervalSeconds() * 1000 + > v->GetReBalanceMs()) + && (tmpCurrentMilliSeconds + tmpRandSleepMilliSec - v->GetScrapeIntervalSeconds() * 1000 * 2 + < v->GetReBalanceMs()))) { // scrape once just now LOG_INFO(sLogger, ("scrape zero cost", ToString(tmpCurrentMilliSeconds))); v->SetScrapeOnceTime(chrono::steady_clock::now(), chrono::system_clock::now()); @@ -124,11 +137,13 @@ void TargetSubscriberScheduler::UpdateScrapeScheduler( v->ScheduleNext(); } } + total = mScrapeSchedulerMap.size(); + LOG_INFO(sLogger, ("prom job", mJobName)("targets removed", toRemove.size())("added", added)("total", total)); } } bool TargetSubscriberScheduler::ParseScrapeSchedulerGroup(const std::string& content, - std::vector& scrapeSchedulerGroup) { + std::vector& scrapeSchedulerGroup) { string errs; Json::Value root; if (!ParseJsonTable(content, root, errs) || !root.isArray()) { @@ -161,8 +176,20 @@ bool TargetSubscriberScheduler::ParseScrapeSchedulerGroup(const std::string& con if (targets.empty()) { continue; } + PromTargetInfo targetInfo; // Parse labels https://www.robustperception.io/life-of-a-label/ Labels labels; + if (element.isMember(prometheus::LABELS) && element[prometheus::LABELS].isObject()) { + for (const string& labelKey : element[prometheus::LABELS].getMemberNames()) { + labels.Set(labelKey, element[prometheus::LABELS][labelKey].asString()); + } + } + std::ostringstream rawHashStream; + rawHashStream << std::setw(16) << std::setfill('0') << std::hex << labels.Hash(); + string rawAddress = labels.Get(prometheus::ADDRESS_LABEL_NAME); + targetInfo.mHash = mScrapeConfigPtr->mJobName + rawAddress + rawHashStream.str(); + targetInfo.mInstance = targets[0]; + for (const auto& pair : mScrapeConfigPtr->mParams) { if (!pair.second.empty()) { labels.Set(prometheus::PARAM_LABEL_NAME + pair.first, pair.second[0]); @@ -186,17 +213,19 @@ bool TargetSubscriberScheduler::ParseScrapeSchedulerGroup(const std::string& con if (labels.Get(prometheus::ADDRESS_LABEL_NAME).empty()) { continue; } - scrapeSchedulerGroup.push_back(labels); + + targetInfo.mLabels = labels; + scrapeSchedulerGroup.push_back(targetInfo); } return true; } std::unordered_map> -TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGroups) { +TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGroups) { std::unordered_map> scrapeSchedulerMap; - for (const auto& labels : targetGroups) { + for (auto& targetInfo : targetGroups) { // Relabel Config - Labels resultLabel = labels; + auto& resultLabel = targetInfo.mLabels; if (!mScrapeConfigPtr->mRelabelConfigs.Process(resultLabel)) { continue; } @@ -291,9 +320,9 @@ TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGr metricsPath, scrapeIntervalSeconds, scrapeTimeoutSeconds, - resultLabel, mQueueKey, - mInputIndex); + mInputIndex, + targetInfo); scrapeScheduler->SetComponent(mEventPool); @@ -365,6 +394,7 @@ TargetSubscriberScheduler::BuildSubscriberTimerEvent(std::chrono::steady_clock:: if (!mETag.empty()) { httpHeader[prometheus::IF_NONE_MATCH] = mETag; } + auto body = TargetsInfoToString(); auto request = std::make_unique(HTTP_GET, false, mServiceHost, @@ -372,7 +402,7 @@ TargetSubscriberScheduler::BuildSubscriberTimerEvent(std::chrono::steady_clock:: "/jobs/" + URLEncode(GetId()) + "/targets", "collector_id=" + mPodName, httpHeader, - "", + body, HttpResponse(), prometheus::RefeshIntervalSeconds, 1, @@ -382,6 +412,43 @@ TargetSubscriberScheduler::BuildSubscriberTimerEvent(std::chrono::steady_clock:: return timerEvent; } +string TargetSubscriberScheduler::TargetsInfoToString() const { + Json::Value root; + + SelfMonitorMetricEvent wantAgentEvent; + LoongCollectorMonitor::GetInstance()->GetAgentMetric(wantAgentEvent); + SelfMonitorMetricEvent wantRunnerEvent; + LoongCollectorMonitor::GetInstance()->GetRunnerMetric(METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK, wantRunnerEvent); + + root[prometheus::AGENT_INFO][prometheus::CPU_USAGE] = wantAgentEvent.GetGauge(METRIC_AGENT_CPU); // double + root[prometheus::AGENT_INFO][prometheus::CPU_LIMIT] = AppConfig::GetInstance()->GetCpuUsageUpLimit(); // float + root[prometheus::AGENT_INFO][prometheus::MEM_USAGE] = wantAgentEvent.GetGauge(METRIC_AGENT_MEMORY); // double + root[prometheus::AGENT_INFO][prometheus::MEM_LIMIT] = AppConfig::GetInstance()->GetMemUsageUpLimit(); // int64_t + root[prometheus::AGENT_INFO][prometheus::HTTP_SINK_IN_ITEMS_TOTAL] + = wantRunnerEvent.GetCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); // uint64_t + root[prometheus::AGENT_INFO][prometheus::HTTP_SINK_OUT_FAILED] + = wantRunnerEvent.GetCounter(METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL); // uint64_t + { + ReadLock lock(mRWLock); + for (const auto& [k, v] : mScrapeSchedulerMap) { + Json::Value targetInfo; + targetInfo[prometheus::HASH] = v->GetId(); + targetInfo[prometheus::SIZE] = v->GetLastScrapeSize(); + sDelaySeconds += v->mExecDelayCount; + v->mExecDelayCount = 0; + root[prometheus::TARGETS_INFO].append(targetInfo); + } + } + auto curTime = std::chrono::steady_clock::now(); + auto needToClear = curTime - mLastUpdateTime >= std::chrono::seconds(prometheus::RefeshIntervalSeconds); + root[prometheus::AGENT_INFO][prometheus::SCRAPE_DELAY_SECONDS] = sDelaySeconds; + if (needToClear) { + sDelaySeconds = 0; + mLastUpdateTime = curTime; + } + return root.toStyledString(); +} + void TargetSubscriberScheduler::CancelAllScrapeScheduler() { ReadLock lock(mRWLock); for (const auto& [k, v] : mScrapeSchedulerMap) { diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.h b/core/prometheus/schedulers/TargetSubscriberScheduler.h index 3b7777d5d5..3b31c930aa 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.h +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.h @@ -17,12 +17,11 @@ #pragma once #include +#include #include #include -#include "json/json.h" - #include "collection_pipeline/queue/QueueKey.h" #include "common/http/HttpResponse.h" #include "common/timer/Timer.h" @@ -61,21 +60,24 @@ class TargetSubscriberScheduler : public BaseScheduler { // zero cost upgrade uint64_t mUnRegisterMs; + static std::chrono::steady_clock::time_point mLastUpdateTime; + static uint64_t sDelaySeconds; private: - bool ParseScrapeSchedulerGroup(const std::string& content, std::vector& scrapeSchedulerGroup); + bool ParseScrapeSchedulerGroup(const std::string& content, std::vector& scrapeSchedulerGroup); std::unordered_map> - BuildScrapeSchedulerSet(std::vector& scrapeSchedulerGroup); + BuildScrapeSchedulerSet(std::vector& scrapeSchedulerGroup); std::unique_ptr BuildSubscriberTimerEvent(std::chrono::steady_clock::time_point execTime); + std::string TargetsInfoToString() const; void UpdateScrapeScheduler(std::unordered_map>&); void CancelAllScrapeScheduler(); std::shared_ptr mScrapeConfigPtr; - ReadWriteLock mRWLock; + mutable ReadWriteLock mRWLock; std::unordered_map> mScrapeSchedulerMap; std::string mJobName; diff --git a/core/unittest/processor/ProcessorPromRelabelMetricNativeUnittest.cpp b/core/unittest/processor/ProcessorPromRelabelMetricNativeUnittest.cpp index aa55e26726..4ffd816a5a 100644 --- a/core/unittest/processor/ProcessorPromRelabelMetricNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorPromRelabelMetricNativeUnittest.cpp @@ -215,6 +215,7 @@ test_metric8{k1="v1", k3="v2", } 9.9410452992e+10 1715829785083 eventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_RESPONSE_SIZE, ToString(2325)); eventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_UP_STATE, ToString(true)); eventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_STATE, string("OK")); + eventGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_STREAM_ID, string("123")); eventGroup.SetTag(string("instance"), "localhost:8080"); eventGroup.SetTag(string("job"), "test_job"); processor.UpdateAutoMetrics(eventGroup, autoMetric); @@ -235,6 +236,7 @@ test_metric8{k1="v1", k3="v2", } 9.9410452992e+10 1715829785083 APSARA_TEST_EQUAL(1, eventGroup.GetEvents().at(14).Cast().GetValue()->mValue); APSARA_TEST_EQUAL("localhost:8080", eventGroup.GetEvents().at(14).Cast().GetTag("instance")); APSARA_TEST_EQUAL("test_job", eventGroup.GetEvents().at(14).Cast().GetTag("job")); + APSARA_TEST_EQUAL("123", eventGroup.GetEvents().at(14).Cast().GetTag("lc_target_hash")); } void ProcessorPromRelabelMetricNativeUnittest::TestHonorLabels() { diff --git a/core/unittest/prometheus/LabelsUnittest.cpp b/core/unittest/prometheus/LabelsUnittest.cpp index cccc0dc109..0f84ca1698 100644 --- a/core/unittest/prometheus/LabelsUnittest.cpp +++ b/core/unittest/prometheus/LabelsUnittest.cpp @@ -58,26 +58,6 @@ void LabelsUnittest::TestRemoveMetaLabels() { APSARA_TEST_EQUAL("", labels.Get("__meta_port")); } -void LabelsUnittest::TestHash() { - Labels labels; - - labels.Set("host", "172.17.0.3:9100"); - labels.Set("ip", "172.17.0.3"); - labels.Set("port", "9100"); - uint64_t hash = labels.Hash(); - - uint64_t expect = prometheus::OFFSET64; - string raw; - raw = raw + "host" + "\xff" + "172.17.0.3:9100" + "\xff" + "ip" + "\xff" + "172.17.0.3" + "\xff" + "port" + "\xff" - + "9100" + "\xff"; - for (auto i : raw) { - expect ^= (uint64_t)i; - expect *= prometheus::PRIME64; - } - - APSARA_TEST_EQUAL(expect, hash); -} - void LabelsUnittest::TestGet() { Labels labels; labels.Set("host", "172.17.0.3:9100"); @@ -117,7 +97,6 @@ void LabelsUnittest::TestRange() { UNIT_TEST_CASE(LabelsUnittest, TestGet) UNIT_TEST_CASE(LabelsUnittest, TestSet) UNIT_TEST_CASE(LabelsUnittest, TestRange) -UNIT_TEST_CASE(LabelsUnittest, TestHash) UNIT_TEST_CASE(LabelsUnittest, TestRemoveMetaLabels) diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index 804706cab3..157c97973b 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -24,6 +24,7 @@ #include "models/RawEvent.h" #include "prometheus/Constants.h" #include "prometheus/async/PromFuture.h" +#include "prometheus/component/StreamScraper.h" #include "prometheus/labels/Labels.h" #include "prometheus/schedulers/ScrapeConfig.h" #include "prometheus/schedulers/ScrapeScheduler.h" @@ -63,8 +64,14 @@ class ScrapeSchedulerUnittest : public testing::Test { void ScrapeSchedulerUnittest::TestInitscrapeScheduler() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); - ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, labels, 0, 0); - APSARA_TEST_EQUAL(event.GetId(), "test_jobhttp://localhost:8080/metrics" + ToString(labels.Hash())); + labels.Set("testb", "valueb"); + labels.Set("testa", "localhost:8080"); + PromTargetInfo targetInfo; + targetInfo.mLabels = labels; + targetInfo.mHash = "test_joblocalhost:8080887d0db7cce49fc7"; + ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, 0, 0, targetInfo); + + APSARA_TEST_EQUAL(event.GetId(), "test_joblocalhost:8080887d0db7cce49fc7"); } void ScrapeSchedulerUnittest::TestProcess() { @@ -73,12 +80,15 @@ void ScrapeSchedulerUnittest::TestProcess() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); - ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, labels, 0, 0); + PromTargetInfo targetInfo; + targetInfo.mLabels = labels; + targetInfo.mHash = "test_hash"; + ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, 0, 0, targetInfo); auto streamScraper = prom::StreamScraper(labels, 0, 0, event.GetId(), nullptr, std::chrono::system_clock::now()); HttpResponse httpResponse = HttpResponse(&streamScraper, [](void*) {}, prom::StreamScraper::MetricWriteCallback); auto defaultLabels = MetricLabels(); event.InitSelfMonitor(defaultLabels); - APSARA_TEST_EQUAL(event.GetId(), "test_jobhttp://localhost:8080/metrics" + ToString(labels.Hash())); + // APSARA_TEST_EQUAL(event.GetId(), "test_jobhttp://localhost:8080/metrics" + ToString(labels.Hash())); // if status code is not 200, no data will be processed // but will continue running, sending self-monitoring metrics httpResponse.SetStatusCode(503); @@ -132,10 +142,13 @@ void ScrapeSchedulerUnittest::TestStreamMetricWriteCallback() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); - ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, labels, 0, 0); + PromTargetInfo targetInfo; + targetInfo.mLabels = labels; + targetInfo.mHash = "test_hash"; + ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, 0, 0, targetInfo); auto streamScraper = prom::StreamScraper(labels, 0, 0, event.GetId(), nullptr, std::chrono::system_clock::now()); HttpResponse httpResponse = HttpResponse(&streamScraper, [](void*) {}, prom::StreamScraper::MetricWriteCallback); - APSARA_TEST_EQUAL(event.GetId(), "test_jobhttp://localhost:8080/metrics" + ToString(labels.Hash())); + // APSARA_TEST_EQUAL(event.GetId(), "test_jobhttp://localhost:8080/metrics" + ToString(labels.Hash())); string body1 = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" "# TYPE go_gc_duration_seconds summary\n" @@ -192,8 +205,11 @@ void ScrapeSchedulerUnittest::TestReceiveMessage() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); + PromTargetInfo targetInfo; + targetInfo.mLabels = labels; + targetInfo.mHash = "test_hash"; auto event - = make_shared(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, labels, 0, 0); + = make_shared(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, 0, 0, targetInfo); // before @@ -207,7 +223,10 @@ void ScrapeSchedulerUnittest::TestReceiveMessage() { void ScrapeSchedulerUnittest::TestScheduler() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); - ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, labels, 0, 0); + PromTargetInfo targetInfo; + targetInfo.mLabels = labels; + targetInfo.mHash = "test_hash"; + ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, 0, 0, targetInfo); auto timer = make_shared(); EventPool eventPool{true}; event.SetComponent(&eventPool); @@ -224,7 +243,10 @@ void ScrapeSchedulerUnittest::TestScheduler() { void ScrapeSchedulerUnittest::TestQueueIsFull() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); - ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, labels, 0, 0); + PromTargetInfo targetInfo; + targetInfo.mLabels = labels; + targetInfo.mHash = "test_hash"; + ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 15, 15, 0, 0, targetInfo); auto defaultLabels = MetricLabels(); event.InitSelfMonitor(defaultLabels); EventPool eventPool{true}; @@ -249,7 +271,10 @@ void ScrapeSchedulerUnittest::TestQueueIsFull() { void ScrapeSchedulerUnittest::TestExactlyScrape() { Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); - ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 10, 10, labels, 0, 0); + PromTargetInfo targetInfo; + targetInfo.mLabels = labels; + targetInfo.mHash = "test_hash"; + ScrapeScheduler event(mScrapeConfig, "localhost", 8080, "http", "/metrics", 10, 10, 0, 0, targetInfo); auto defaultLabels = MetricLabels(); event.InitSelfMonitor(defaultLabels); EventPool eventPool{true}; diff --git a/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp b/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp index e2e47df884..3be26fda97 100644 --- a/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp +++ b/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp @@ -14,13 +14,15 @@ * limitations under the License. */ +#include + #include #include #include -#include "json/json.h" - +#include "ScrapeScheduler.h" #include "common/JsonUtil.h" +#include "prometheus/Constants.h" #include "prometheus/labels/Labels.h" #include "prometheus/schedulers/TargetSubscriberScheduler.h" #include "unittest/Unittest.h" @@ -36,6 +38,7 @@ class TargetSubscriberSchedulerUnittest : public ::testing::Test { void TestParseTargetGroups(); void TestBuildScrapeSchedulerSet(); void TestTargetLabels(); + void TestTargetsInfoToString(); protected: void SetUp() override { @@ -47,7 +50,7 @@ class TargetSubscriberSchedulerUnittest : public ::testing::Test { "enable_http2": true, "follow_redirects": true, "honor_timestamps": false, - "job_name": "_kube-state-metrics", + "job_name": "loong-collector/demo-podmonitor-500/0", "kubernetes_sd_configs": [ { "enable_http2": true, @@ -80,34 +83,141 @@ class TargetSubscriberSchedulerUnittest : public ::testing::Test { *mHttpResponse.GetBody() = R"JSON([ { "targets": [ - "192.168.22.7:8080" + "10.0.2.81:8080" ], "labels": { - "__meta_kubernetes_pod_controller_kind": "ReplicaSet", - "__meta_kubernetes_pod_container_image": "registry-vpc.cn-hangzhou.aliyuncs.com/acs/kube-state-metrics:v2.3.0-a71f78c-aliyun", - "__meta_kubernetes_namespace": "arms-prom", - "__meta_kubernetes_pod_labelpresent_pod_template_hash": "true", - "__meta_kubernetes_pod_uid": "00d1897f-d442-47c4-8423-e9bf32dea173", + "__meta_kubernetes_pod_labelpresent_label_key_47": "true", + "__meta_kubernetes_pod_labelpresent_label_key_07": "true", + "__meta_kubernetes_pod_labelpresent_label_key_18": "true", + "__meta_kubernetes_pod_annotationpresent_prometheus_io_port": "true", + "__meta_kubernetes_pod_labelpresent_label_key_08": "true", + "__meta_kubernetes_pod_labelpresent_label_key_15": "true", + "__meta_kubernetes_pod_labelpresent_label_key_32": "true", + "__meta_kubernetes_pod_label_label_key_07": "label_value_07", + "__meta_kubernetes_pod_labelpresent_label_key_33": "true", + "__meta_kubernetes_pod_labelpresent_label_key_26": "true", + "__meta_kubernetes_pod_label_label_key_17": "label_value_17", + "__meta_kubernetes_pod_label_label_key_38": "label_value_38", + "__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape": "true", "__meta_kubernetes_pod_container_init": "false", - "__meta_kubernetes_pod_container_port_protocol": "TCP", - "__meta_kubernetes_pod_host_ip": "192.168.21.234", - "__meta_kubernetes_pod_controller_name": "kube-state-metrics-64cf88c8f4", - "__meta_kubernetes_pod_annotation_k8s_aliyun_com_pod_ips": "192.168.22.7", + "__meta_kubernetes_pod_labelpresent_label_key_35": "true", + "__meta_kubernetes_pod_label_label_key_13": "label_value_13", + "__meta_kubernetes_pod_name": "demo-app-500-5c97455f77-brddj", + "__meta_kubernetes_pod_label_label_key_32": "label_value_32", + "__meta_kubernetes_pod_labelpresent_label_key_01": "true", + "__meta_kubernetes_pod_label_label_key_12": "label_value_12", + "__meta_kubernetes_pod_label_label_key_11": "label_value_11", + "__meta_kubernetes_pod_label_label_key_04": "label_value_04", + "__meta_kubernetes_pod_labelpresent_label_key_42": "true", + "__meta_kubernetes_pod_uid": "c640e01c-0c1e-487e-9d1b-a743b88bb01a", + "__meta_kubernetes_pod_labelpresent_label_key_10": "true", + "__meta_kubernetes_pod_label_label_key_36": "label_value_36", + "__meta_kubernetes_pod_label_label_key_43": "label_value_43", + "__meta_kubernetes_pod_labelpresent_label_key_24": "true", + "__meta_kubernetes_pod_labelpresent_label_key_04": "true", + "__meta_kubernetes_pod_label_label_key_01": "label_value_01", + "__meta_kubernetes_pod_label_label_key_09": "label_value_09", + "__meta_kubernetes_pod_label_label_key_00": "label_value_00", + "__meta_kubernetes_pod_labelpresent_label_key_34": "true", + "__meta_kubernetes_pod_labelpresent_pod_template_hash": "true", + "__meta_kubernetes_pod_labelpresent_label_key_41": "true", + "__meta_kubernetes_pod_label_label_key_19": "label_value_19", + "__meta_kubernetes_pod_label_label_key_10": "label_value_10", + "__meta_kubernetes_pod_label_label_key_35": "label_value_35", + "__meta_kubernetes_pod_labelpresent_app": "true", + "__meta_kubernetes_pod_controller_kind": "ReplicaSet", + "__meta_kubernetes_pod_label_label_key_49": "label_value_49", + "__meta_kubernetes_pod_labelpresent_label_key_05": "true", + "__meta_kubernetes_pod_label_label_key_25": "label_value_25", + "__meta_kubernetes_pod_labelpresent_label_key_49": "true", + "__meta_kubernetes_pod_labelpresent_label_key_06": "true", + "__meta_kubernetes_pod_label_label_key_23": "label_value_23", + "__meta_kubernetes_pod_labelpresent_label_key_27": "true", + "__meta_kubernetes_pod_labelpresent_label_key_25": "true", + "__meta_kubernetes_pod_labelpresent_label_key_38": "true", + "__meta_kubernetes_pod_label_label_key_02": "label_value_02", + "__meta_kubernetes_pod_label_label_key_22": "label_value_22", + "__meta_kubernetes_pod_annotation_prometheus_io_port": "8080", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_label_label_key_27": "label_value_27", + "__meta_kubernetes_pod_labelpresent_label_key_28": "true", + "__meta_kubernetes_pod_label_label_key_41": "label_value_41", + "__meta_kubernetes_pod_label_label_key_24": "label_value_24", + "__meta_kubernetes_pod_label_label_key_20": "label_value_20", + "__meta_kubernetes_pod_label_pod_template_hash": "5c97455f77", + "__meta_kubernetes_pod_label_label_key_33": "label_value_33", + "__meta_kubernetes_pod_labelpresent_label_key_40": "true", + "__meta_kubernetes_pod_labelpresent_label_key_03": "true", + "__meta_kubernetes_pod_label_label_key_08": "label_value_08", + "__meta_kubernetes_pod_container_port_number": "8080", + "__meta_kubernetes_pod_labelpresent_label_key_31": "true", + "__meta_kubernetes_pod_label_label_key_48": "label_value_48", + "__meta_kubernetes_pod_label_label_key_40": "label_value_40", + "__meta_kubernetes_pod_label_label_key_28": "label_value_28", + "__meta_kubernetes_pod_labelpresent_label_key_23": "true", + "__meta_kubernetes_pod_labelpresent_label_key_48": "true", + "__meta_kubernetes_pod_labelpresent_label_key_43": "true", + "__meta_kubernetes_pod_labelpresent_label_key_46": "true", + "__meta_kubernetes_pod_label_label_key_46": "label_value_46", + "__meta_kubernetes_pod_controller_name": "demo-app-500-5c97455f77", + "__meta_kubernetes_pod_host_ip": "10.0.4.54", + "__meta_kubernetes_pod_labelpresent_label_key_44": "true", + "__meta_kubernetes_pod_label_label_key_44": "label_value_44", + "__meta_kubernetes_pod_node_name": "cn-heyuan.10.0.4.54", + "__address__": "10.0.2.81:8080", + "__meta_kubernetes_pod_container_name": "demo-app-500", + "__meta_kubernetes_pod_labelpresent_label_key_30": "true", + "__meta_kubernetes_pod_labelpresent_label_key_17": "true", + "__meta_kubernetes_pod_label_label_key_29": "label_value_29", + "__meta_kubernetes_pod_labelpresent_label_key_02": "true", + "__meta_kubernetes_pod_label_label_key_39": "label_value_39", + "__meta_kubernetes_pod_label_label_key_21": "label_value_21", + "__meta_kubernetes_pod_label_label_key_37": "label_value_37", + "__meta_kubernetes_pod_labelpresent_label_key_37": "true", + "__meta_kubernetes_pod_labelpresent_label_key_45": "true", + "__meta_kubernetes_pod_label_label_key_45": "label_value_45", + "__meta_kubernetes_pod_label_label_key_26": "label_value_26", + "__meta_kubernetes_pod_labelpresent_label_key_12": "true", + "__meta_kubernetes_pod_labelpresent_label_key_11": "true", + "__meta_kubernetes_pod_label_app": "demo-app-500", + "__meta_kubernetes_pod_labelpresent_label_key_21": "true", + "__meta_kubernetes_pod_labelpresent_label_key_13": "true", + "__meta_kubernetes_pod_ip": "10.0.2.81", + "__meta_kubernetes_pod_label_label_key_42": "label_value_42", + "__meta_kubernetes_pod_annotation_k8s_aliyun_com_pod_ips": "10.0.2.81", + "__meta_kubernetes_pod_label_label_key_34": "label_value_34", + "__meta_kubernetes_pod_label_label_key_06": "label_value_06", + "__meta_kubernetes_pod_label_label_key_31": "label_value_31", + "__meta_kubernetes_pod_annotation_prometheus_io_path": "/metrics", + "__meta_kubernetes_pod_container_id": "containerd://788da13840e1e8711f71b42015cdaabf590b10e8658524a9e3de910a4f373532", "__meta_kubernetes_pod_ready": "true", - "__meta_kubernetes_pod_node_name": "cn-hangzhou.192.168.21.234", + "__meta_kubernetes_pod_labelpresent_label_key_00": "true", + "__meta_kubernetes_pod_label_label_key_03": "label_value_03", + "__meta_kubernetes_pod_labelpresent_label_key_16": "true", + "__meta_kubernetes_pod_container_port_name": "", + "__meta_kubernetes_pod_label_label_key_16": "label_value_16", + "__meta_kubernetes_pod_label_label_key_15": "label_value_15", + "__meta_kubernetes_pod_container_image": "arms-deploy-registry.cn-hangzhou.cr.aliyuncs.com/arms-deploy-repo/prometheus-sample-app:latest", + "__meta_kubernetes_pod_annotation_prometheus_io_scrape": "true", + "__meta_kubernetes_pod_labelpresent_label_key_20": "true", + "__meta_kubernetes_pod_labelpresent_label_key_19": "true", + "__meta_kubernetes_pod_label_label_key_18": "label_value_18", + "__meta_kubernetes_pod_annotationpresent_prometheus_io_path": "true", "__meta_kubernetes_pod_annotationpresent_k8s_aliyun_com_pod_ips": "true", - "__address__": "192.168.22.7:8080", - "__meta_kubernetes_pod_labelpresent_k8s_app": "true", - "__meta_kubernetes_pod_label_k8s_app": "kube-state-metrics", - "__meta_kubernetes_pod_container_id": "containerd://57c4dfd8d9ea021defb248dfbc5cc3bd3758072c4529be351b8cc6838bdff02f", - "__meta_kubernetes_pod_container_port_number": "8080", - "__meta_kubernetes_pod_ip": "192.168.22.7", - "__meta_kubernetes_pod_phase": "Running", - "__meta_kubernetes_pod_container_name": "kube-state-metrics", - "__meta_kubernetes_pod_container_port_name": "http-metrics", - "__meta_kubernetes_pod_label_pod_template_hash": "64cf88c8f4", - "__meta_kubernetes_pod_name": "kube-state-metrics-64cf88c8f4-jtn6v" - } + "__meta_kubernetes_pod_labelpresent_label_key_39": "true", + "__meta_kubernetes_pod_label_label_key_30": "label_value_30", + "__meta_kubernetes_pod_labelpresent_label_key_14": "true", + "__meta_kubernetes_pod_label_label_key_05": "label_value_05", + "__meta_kubernetes_pod_labelpresent_label_key_22": "true", + "__meta_kubernetes_pod_container_port_protocol": "TCP", + "__meta_kubernetes_pod_labelpresent_label_key_09": "true", + "__meta_kubernetes_pod_labelpresent_label_key_36": "true", + "__meta_kubernetes_pod_label_label_key_47": "label_value_47", + "__meta_kubernetes_pod_label_label_key_14": "label_value_14", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_labelpresent_label_key_29": "true" + }, + "Load": 425 }, { "targets": [ @@ -164,7 +274,7 @@ void TargetSubscriberSchedulerUnittest::OnInitScrapeJobEvent() { APSARA_TEST_TRUE(targetSubscriber->Init(mConfig["ScrapeConfig"])); APSARA_TEST_NOT_EQUAL(targetSubscriber->mScrapeConfigPtr.get(), nullptr); - APSARA_TEST_EQUAL(targetSubscriber->mJobName, "_kube-state-metrics"); + APSARA_TEST_EQUAL(targetSubscriber->mJobName, "loong-collector/demo-podmonitor-500/0"); } void TargetSubscriberSchedulerUnittest::TestProcess() { @@ -188,7 +298,7 @@ void TargetSubscriberSchedulerUnittest::TestParseTargetGroups() { std::shared_ptr targetSubscriber = std::make_shared(); APSARA_TEST_TRUE(targetSubscriber->Init(mConfig["ScrapeConfig"])); - std::vector newScrapeSchedulerSet; + std::vector newScrapeSchedulerSet; APSARA_TEST_TRUE( targetSubscriber->ParseScrapeSchedulerGroup(*mHttpResponse.GetBody(), newScrapeSchedulerSet)); APSARA_TEST_EQUAL(3UL, newScrapeSchedulerSet.size()); @@ -198,7 +308,7 @@ void TargetSubscriberSchedulerUnittest::TestBuildScrapeSchedulerSet() { // prepare data std::shared_ptr targetSubscriber = std::make_shared(); APSARA_TEST_TRUE(targetSubscriber->Init(mConfig["ScrapeConfig"])); - std::vector newScrapeSchedulerSet; + std::vector newScrapeSchedulerSet; APSARA_TEST_TRUE( targetSubscriber->ParseScrapeSchedulerGroup(*mHttpResponse.GetBody(), newScrapeSchedulerSet)); APSARA_TEST_EQUAL(3UL, newScrapeSchedulerSet.size()); @@ -214,6 +324,8 @@ void TargetSubscriberSchedulerUnittest::TestBuildScrapeSchedulerSet() { APSARA_TEST_NOT_EQUAL(startTimeList[0].second, startTimeList[1].second); APSARA_TEST_NOT_EQUAL(startTimeList[1].second, startTimeList[2].second); APSARA_TEST_NOT_EQUAL(startTimeList[0].second, startTimeList[2].second); + + APSARA_TEST_EQUAL(1UL, result.count("loong-collector/demo-podmonitor-500/010.0.2.81:808093796c8e4493906d")); } void TargetSubscriberSchedulerUnittest::TestTargetLabels() { @@ -228,7 +340,7 @@ void TargetSubscriberSchedulerUnittest::TestTargetLabels() { int32_t port) { std::shared_ptr targetSubscriber = std::make_shared(); APSARA_TEST_TRUE(targetSubscriber->Init(scrapeConfig)); - std::vector newScrapeSchedulerSet; + std::vector newScrapeSchedulerSet; APSARA_TEST_TRUE(targetSubscriber->ParseScrapeSchedulerGroup(targetResponse, newScrapeSchedulerSet)); APSARA_TEST_EQUAL(1UL, newScrapeSchedulerSet.size()); @@ -467,11 +579,29 @@ void TargetSubscriberSchedulerUnittest::TestTargetLabels() { 80); } +void TargetSubscriberSchedulerUnittest::TestTargetsInfoToString() { + std::shared_ptr targetSubscriber = std::make_shared(); + auto metricLabels = MetricLabels(); + APSARA_TEST_TRUE(targetSubscriber->Init(mConfig["ScrapeConfig"])); + targetSubscriber->InitSelfMonitor(metricLabels); + // if status code is 200 + mHttpResponse.SetStatusCode(200); + targetSubscriber->OnSubscription(mHttpResponse, 0); + APSARA_TEST_EQUAL(3UL, targetSubscriber->mScrapeSchedulerMap.size()); + auto res = targetSubscriber->TargetsInfoToString(); + string errorMsg; + Json::Value data; + ParseJsonTable(res, data, errorMsg); + APSARA_TEST_EQUAL(2.0, data[prometheus::AGENT_INFO][prometheus::CPU_LIMIT].asFloat()); + APSARA_TEST_EQUAL((uint64_t)3, data[prometheus::TARGETS_INFO].size()); +} + UNIT_TEST_CASE(TargetSubscriberSchedulerUnittest, OnInitScrapeJobEvent) UNIT_TEST_CASE(TargetSubscriberSchedulerUnittest, TestProcess) UNIT_TEST_CASE(TargetSubscriberSchedulerUnittest, TestParseTargetGroups) UNIT_TEST_CASE(TargetSubscriberSchedulerUnittest, TestBuildScrapeSchedulerSet) UNIT_TEST_CASE(TargetSubscriberSchedulerUnittest, TestTargetLabels) +UNIT_TEST_CASE(TargetSubscriberSchedulerUnittest, TestTargetsInfoToString) } // namespace logtail