From 2b9398410e523462e0cbd58c954c1551bef41144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E9=A3=8F?= Date: Wed, 6 Nov 2024 11:47:58 +0000 Subject: [PATCH 1/6] change metric struct --- core/common/compression/Compressor.cpp | 2 +- core/common/compression/CompressorFactory.cpp | 3 +- .../ContainerDiscoveryOptions.cpp | 2 +- .../ContainerDiscoveryOptions.h | 2 +- core/ebpf/SelfMonitor.cpp | 2 +- core/ebpf/SelfMonitor.h | 2 +- core/ebpf/eBPFServer.cpp | 10 +- core/ebpf/eBPFServer.h | 4 +- core/file_server/FileServer.cpp | 5 +- core/monitor/LogtailMetric.cpp | 142 +++++++++++------- core/monitor/LogtailMetric.h | 40 ++++- core/monitor/MetricExportor.cpp | 48 ++++-- core/monitor/Monitor.cpp | 3 +- core/monitor/PluginMetricManager.cpp | 4 +- core/monitor/PluginMetricManager.h | 14 +- .../monitor/metric_constants/AgentMetrics.cpp | 18 +-- .../metric_constants/ComponentMetrics.cpp | 57 ++++--- .../metric_constants/MetricConstants.h | 10 -- .../metric_constants/PipelineMetrics.cpp | 21 ++- .../metric_constants/PluginMetrics.cpp | 103 ++++++------- .../metric_constants/RunnerMetrics.cpp | 55 ++++--- core/pipeline/Pipeline.cpp | 6 +- core/pipeline/batch/Batcher.h | 4 +- core/pipeline/plugin/interface/Plugin.h | 5 +- core/pipeline/queue/QueueInterface.h | 5 +- core/pipeline/route/Router.cpp | 5 +- core/pipeline/serializer/Serializer.h | 5 +- core/plugin/input/InputContainerStdio.cpp | 7 +- core/plugin/input/InputFile.cpp | 4 +- core/plugin/input/InputFileSecurity.cpp | 2 +- core/plugin/input/InputFileSecurity.h | 2 +- core/plugin/input/InputNetworkObserver.cpp | 2 +- core/plugin/input/InputNetworkObserver.h | 2 +- core/plugin/input/InputNetworkSecurity.cpp | 2 +- core/plugin/input/InputNetworkSecurity.h | 2 +- core/plugin/input/InputProcessSecurity.cpp | 2 +- core/plugin/input/InputProcessSecurity.h | 2 +- core/prometheus/PromSelfMonitor.cpp | 2 +- core/prometheus/PrometheusInputRunner.cpp | 3 +- core/runner/FlusherRunner.cpp | 5 +- core/runner/ProcessorRunner.cpp | 6 +- core/runner/sink/http/HttpSink.cpp | 19 +-- .../monitor/PluginMetricManagerUnittest.cpp | 5 +- 43 files changed, 363 insertions(+), 281 deletions(-) diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp index 050d62123b..b445ac687a 100644 --- a/core/common/compression/Compressor.cpp +++ b/core/common/compression/Compressor.cpp @@ -24,7 +24,7 @@ namespace logtail { void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), MetricCategory::METRIC_CATEGORY_COMPONENT); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); diff --git a/core/common/compression/CompressorFactory.cpp b/core/common/compression/CompressorFactory.cpp index ed3fe2703f..c1d4ff0e78 100644 --- a/core/common/compression/CompressorFactory.cpp +++ b/core/common/compression/CompressorFactory.cpp @@ -15,9 +15,9 @@ #include "common/compression/CompressorFactory.h" #include "common/ParamExtractor.h" -#include "monitor/metric_constants/MetricConstants.h" #include "common/compression/LZ4Compressor.h" #include "common/compression/ZstdCompressor.h" +#include "monitor/metric_constants/MetricConstants.h" using namespace std; @@ -64,7 +64,6 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, compressor->SetMetricRecordRef({{METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, flusherId}}); return compressor; } diff --git a/core/container_manager/ContainerDiscoveryOptions.cpp b/core/container_manager/ContainerDiscoveryOptions.cpp index f9cdb6111a..ce59ef41f5 100644 --- a/core/container_manager/ContainerDiscoveryOptions.cpp +++ b/core/container_manager/ContainerDiscoveryOptions.cpp @@ -200,7 +200,7 @@ bool ContainerDiscoveryOptions::Init(const Json::Value& config, const PipelineCo void ContainerDiscoveryOptions::GenerateContainerMetaFetchingGoPipeline(Json::Value& res, const FileDiscoveryOptions* fileDiscovery, - const PluginInstance::PluginMeta pluginMeta) const { + const PluginInstance::PluginMeta& pluginMeta) const { Json::Value plugin(Json::objectValue); Json::Value detail(Json::objectValue); Json::Value object(Json::objectValue); diff --git a/core/container_manager/ContainerDiscoveryOptions.h b/core/container_manager/ContainerDiscoveryOptions.h index 1b1b06ccaf..2a106c819f 100644 --- a/core/container_manager/ContainerDiscoveryOptions.h +++ b/core/container_manager/ContainerDiscoveryOptions.h @@ -52,7 +52,7 @@ struct ContainerDiscoveryOptions { bool Init(const Json::Value& config, const PipelineContext& ctx, const std::string& pluginType); void GenerateContainerMetaFetchingGoPipeline(Json::Value& res, const FileDiscoveryOptions* fileDiscovery = nullptr, - const PluginInstance::PluginMeta pluginMeta = {"0"}) const; + const PluginInstance::PluginMeta& pluginMeta = {"0"}) const; }; using ContainerDiscoveryConfig = std::pair; diff --git a/core/ebpf/SelfMonitor.cpp b/core/ebpf/SelfMonitor.cpp index ad49fce33c..d352cfbaeb 100644 --- a/core/ebpf/SelfMonitor.cpp +++ b/core/ebpf/SelfMonitor.cpp @@ -179,7 +179,7 @@ void NetworkObserverSelfMonitor::HandleStatistic(nami::eBPFStatistics& stats) { eBPFSelfMonitorMgr::eBPFSelfMonitorMgr() : mSelfMonitors({}), mInited({}) {} -void eBPFSelfMonitorMgr::Init(const nami::PluginType type, std::shared_ptr mgr, const std::string& name, const std::string& logstore) { +void eBPFSelfMonitorMgr::Init(const nami::PluginType type, PluginMetricManagerPtr mgr, const std::string& name, const std::string& logstore) { if (mInited[int(type)]) return; WriteLock lk(mLock); diff --git a/core/ebpf/SelfMonitor.h b/core/ebpf/SelfMonitor.h index 4d59934266..4b8551f1ef 100644 --- a/core/ebpf/SelfMonitor.h +++ b/core/ebpf/SelfMonitor.h @@ -122,7 +122,7 @@ class FileSecuritySelfMonitor : public BaseBPFMonitor { class eBPFSelfMonitorMgr { public: eBPFSelfMonitorMgr(); - void Init(const nami::PluginType type, std::shared_ptr mgr, const std::string& name, const std::string& project); + void Init(const nami::PluginType type, PluginMetricManagerPtr mgr, const std::string& name, const std::string& project); void Release(const nami::PluginType type); void Suspend(const nami::PluginType type); void HandleStatistic(std::vector&& stats); diff --git a/core/ebpf/eBPFServer.cpp b/core/ebpf/eBPFServer.cpp index 874a29ec5a..ff5842503e 100644 --- a/core/ebpf/eBPFServer.cpp +++ b/core/ebpf/eBPFServer.cpp @@ -152,9 +152,9 @@ void eBPFServer::Init() { DynamicMetricLabels dynamicLabels; dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); }); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}, - std::move(dynamicLabels)); + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER}}, + std::move(dynamicLabels), + MetricCategory::METRIC_CATEGORY_RUNNER); mStartPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL); mStopPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL); @@ -202,7 +202,7 @@ void eBPFServer::Stop() { bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr) { + const std::variant options, PluginMetricManagerPtr mgr) { std::string prev_pipeline_name = CheckLoadedPipelineName(type); if (prev_pipeline_name.size() && prev_pipeline_name != pipeline_name) { @@ -314,7 +314,7 @@ bool eBPFServer::HasRegisteredPlugins() const { bool eBPFServer::EnablePlugin(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr) { + const std::variant options, PluginMetricManagerPtr mgr) { if (!IsSupportedEnv(type)) { return false; } diff --git a/core/ebpf/eBPFServer.h b/core/ebpf/eBPFServer.h index 4887080a79..9141a9ec9a 100644 --- a/core/ebpf/eBPFServer.h +++ b/core/ebpf/eBPFServer.h @@ -72,7 +72,7 @@ class eBPFServer : public InputRunner { bool EnablePlugin(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr); + const std::variant options, PluginMetricManagerPtr mgr); bool DisablePlugin(const std::string& pipeline_name, nami::PluginType type); @@ -88,7 +88,7 @@ class eBPFServer : public InputRunner { bool StartPluginInternal(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr); + const std::variant options, PluginMetricManagerPtr mgr); eBPFServer() = default; ~eBPFServer() = default; diff --git a/core/file_server/FileServer.cpp b/core/file_server/FileServer.cpp index 5a072c6c0f..0dd65a5780 100644 --- a/core/file_server/FileServer.cpp +++ b/core/file_server/FileServer.cpp @@ -34,8 +34,9 @@ namespace logtail { FileServer::FileServer() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}); + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}}, + {}, + MetricCategory::METRIC_CATEGORY_RUNNER); } // 启动文件服务，包括加载配置、处理检查点、注册事件等 diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index e10c44fbb1..7463b215a8 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -25,11 +25,19 @@ using namespace sls_logs; namespace logtail { -const std::string LABEL_PREFIX = "label."; -const std::string VALUE_PREFIX = "value."; +const std::string METRIC_KEY_LABEL = "label"; +const std::string METRIC_KEY_VALUE = "value"; +const std::string METRIC_KEY_CATEGORY = "category"; +const std::string MetricCategory::METRIC_CATEGORY_UNKNOWN = "unknown"; +const std::string MetricCategory::METRIC_CATEGORY_AGENT = "agent"; +const std::string MetricCategory::METRIC_CATEGORY_RUNNER = "runner"; +const std::string MetricCategory::METRIC_CATEGORY_PIPELINE = "pipeline"; +const std::string MetricCategory::METRIC_CATEGORY_COMPONENT = "component"; +const std::string MetricCategory::METRIC_CATEGORY_PLUGIN = "plugin"; +const std::string MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE = "plugin_source"; -MetricsRecord::MetricsRecord(MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels) - : mLabels(labels), mDynamicLabels(dynamicLabels), mDeleted(false) { +MetricsRecord::MetricsRecord(MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels, std::string category) + : mLabels(labels), mDynamicLabels(dynamicLabels), mCategory(category), mDeleted(false) { } CounterPtr MetricsRecord::CreateCounter(const std::string& name) { @@ -64,6 +72,10 @@ bool MetricsRecord::IsDeleted() const { return mDeleted; } +const std::string MetricsRecord::GetCategory() const { + return mCategory; +} + const MetricLabelsPtr& MetricsRecord::GetLabels() const { return mLabels; } @@ -89,7 +101,7 @@ const std::vector& MetricsRecord::GetDoubleGauges() const { } MetricsRecord* MetricsRecord::Collect() { - MetricsRecord* metrics = new MetricsRecord(mLabels, mDynamicLabels); + MetricsRecord* metrics = new MetricsRecord(mLabels, mDynamicLabels, mCategory); for (auto& item : mCounters) { CounterPtr newPtr(item->Collect()); metrics->mCounters.emplace_back(newPtr); @@ -127,6 +139,10 @@ void MetricsRecordRef::SetMetricsRecord(MetricsRecord* metricRecord) { mMetrics = metricRecord; } +const std::string MetricsRecordRef::GetCategory() const { + return mMetrics->GetCategory(); +} + const MetricLabelsPtr& MetricsRecordRef::GetLabels() const { return mMetrics->GetLabels(); } @@ -171,8 +187,12 @@ bool MetricsRecordRef::HasLabel(const std::string& key, const std::string& value #endif // ReentrantMetricsRecord相关操作可以无锁，因为mCounters、mGauges只在初始化时会添加内容，后续只允许Get操作 -void ReentrantMetricsRecord::Init(MetricLabels& labels, std::unordered_map& metricKeys) { - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); +void ReentrantMetricsRecord::Init(std::string category, + MetricLabels& labels, + DynamicMetricLabels& dynamicLabels, + std::unordered_map& metricKeys) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), category); for (auto metric : metricKeys) { switch (metric.second) { case MetricType::METRIC_TYPE_COUNTER: @@ -238,16 +258,18 @@ WriteMetrics::~WriteMetrics() { void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, - DynamicMetricLabels&& dynamicLabels) { - CreateMetricsRecordRef(ref, std::move(labels), std::move(dynamicLabels)); + DynamicMetricLabels&& dynamicLabels, + std::string category) { + CreateMetricsRecordRef(ref, std::move(labels), std::move(dynamicLabels), std::move(category)); CommitMetricsRecordRef(ref); } void WriteMetrics::CreateMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, - DynamicMetricLabels&& dynamicLabels) { - MetricsRecord* cur = new MetricsRecord(std::make_shared(labels), - std::make_shared(dynamicLabels)); + DynamicMetricLabels&& dynamicLabels, + std::string category) { + MetricsRecord* cur = new MetricsRecord( + std::make_shared(labels), std::make_shared(dynamicLabels), category); ref.SetMetricsRecord(cur); } @@ -395,42 +417,53 @@ void ReadMetrics::ReadAsLogGroup(const std::string& regionFieldName, auto now = GetCurrentLogtailTime(); SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec); - for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { - std::pair pair = *item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(LABEL_PREFIX + pair.first); - contentPtr->set_value(pair.second); - } - for (auto item = tmp->GetDynamicLabels()->begin(); item != tmp->GetDynamicLabels()->end(); ++item) { - std::pair> pair = *item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(LABEL_PREFIX + pair.first); - contentPtr->set_value(pair.second()); - } - - for (auto& item : tmp->GetCounters()) { - CounterPtr counter = item; + { // category Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + counter->GetName()); - contentPtr->set_value(ToString(counter->GetValue())); + contentPtr->set_key(METRIC_KEY_CATEGORY); + contentPtr->set_value(tmp->GetCategory()); } - for (auto& item : tmp->GetTimeCounters()) { - TimeCounterPtr counter = item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + counter->GetName()); - contentPtr->set_value(ToString(counter->GetValue())); - } - for (auto& item : tmp->GetIntGauges()) { - IntGaugePtr gauge = item; + { // label + Json::Value metricsRecordLabel; + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + metricsRecordLabel[pair.first] = pair.second; + } + for (auto item = tmp->GetDynamicLabels()->begin(); item != tmp->GetDynamicLabels()->end(); ++item) { + std::pair> pair = *item; + metricsRecordLabel[pair.first] = pair.second(); + } + Json::StreamWriterBuilder writer; + writer["indentation"] = ""; + std::string jsonString = Json::writeString(writer, metricsRecordLabel); Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + gauge->GetName()); - contentPtr->set_value(ToString(gauge->GetValue())); + contentPtr->set_key(METRIC_KEY_LABEL); + contentPtr->set_value(jsonString); } - for (auto& item : tmp->GetDoubleGauges()) { - DoubleGaugePtr gauge = item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + gauge->GetName()); - contentPtr->set_value(ToString(gauge->GetValue())); + { // value + for (auto& item : tmp->GetCounters()) { + CounterPtr counter = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(counter->GetName()); + contentPtr->set_value(ToString(counter->GetValue())); + } + for (auto& item : tmp->GetTimeCounters()) { + TimeCounterPtr counter = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(counter->GetName()); + contentPtr->set_value(ToString(counter->GetValue())); + } + for (auto& item : tmp->GetIntGauges()) { + IntGaugePtr gauge = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(gauge->GetName()); + contentPtr->set_value(ToString(gauge->GetValue())); + } + for (auto& item : tmp->GetDoubleGauges()) { + DoubleGaugePtr gauge = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(gauge->GetName()); + contentPtr->set_value(ToString(gauge->GetValue())); + } } tmp = tmp->GetNext(); } @@ -443,40 +476,43 @@ void ReadMetrics::ReadAsFileBuffer(std::string& metricsContent) const { MetricsRecord* tmp = mHead; while (tmp) { - Json::Value metricsRecordValue; + Json::Value metricsRecordJson, metricsRecordLabel; auto now = GetCurrentLogtailTime(); - metricsRecordValue["time"] + metricsRecordJson["time"] = AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec; + metricsRecordJson[METRIC_KEY_CATEGORY] = tmp->GetCategory(); + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { std::pair pair = *item; - metricsRecordValue[LABEL_PREFIX + pair.first] = pair.second; + metricsRecordLabel[pair.first] = pair.second; } for (auto item = tmp->GetDynamicLabels()->begin(); item != tmp->GetDynamicLabels()->end(); ++item) { std::pair> pair = *item; - metricsRecordValue[LABEL_PREFIX + pair.first] = pair.second(); + metricsRecordLabel[pair.first] = pair.second(); } + metricsRecordJson[METRIC_KEY_LABEL] = metricsRecordLabel; for (auto& item : tmp->GetCounters()) { CounterPtr counter = item; - metricsRecordValue[VALUE_PREFIX + counter->GetName()] = ToString(counter->GetValue()); + metricsRecordJson[counter->GetName()] = ToString(counter->GetValue()); } for (auto& item : tmp->GetTimeCounters()) { TimeCounterPtr counter = item; - metricsRecordValue[VALUE_PREFIX + counter->GetName()] = ToString(counter->GetValue()); + metricsRecordJson[counter->GetName()] = ToString(counter->GetValue()); } for (auto& item : tmp->GetIntGauges()) { IntGaugePtr gauge = item; - metricsRecordValue[VALUE_PREFIX + gauge->GetName()] = ToString(gauge->GetValue()); + metricsRecordJson[gauge->GetName()] = ToString(gauge->GetValue()); } for (auto& item : tmp->GetDoubleGauges()) { DoubleGaugePtr gauge = item; - metricsRecordValue[VALUE_PREFIX + gauge->GetName()] = ToString(gauge->GetValue()); + metricsRecordJson[gauge->GetName()] = ToString(gauge->GetValue()); } Json::StreamWriterBuilder writer; writer["indentation"] = ""; - std::string jsonString = Json::writeString(writer, metricsRecordValue); + std::string jsonString = Json::writeString(writer, metricsRecordJson); oss << jsonString << '\n'; tmp = tmp->GetNext(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 1c02fa4814..79783af4a5 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -28,22 +28,41 @@ namespace logtail { +extern const std::string METRIC_KEY_LABEL; +extern const std::string METRIC_KEY_VALUE; +extern const std::string METRIC_KEY_CATEGORY; +class MetricCategory { +public: + static const std::string METRIC_CATEGORY_UNKNOWN; + static const std::string METRIC_CATEGORY_AGENT; + static const std::string METRIC_CATEGORY_RUNNER; + static const std::string METRIC_CATEGORY_PIPELINE; + static const std::string METRIC_CATEGORY_COMPONENT; + static const std::string METRIC_CATEGORY_PLUGIN; + static const std::string METRIC_CATEGORY_PLUGIN_SOURCE; +}; + class MetricsRecord { private: MetricLabelsPtr mLabels; DynamicMetricLabelsPtr mDynamicLabels; - std::atomic_bool mDeleted; + std::string mCategory; std::vector mCounters; std::vector mTimeCounters; std::vector mIntGauges; std::vector mDoubleGauges; + + std::atomic_bool mDeleted; MetricsRecord* mNext = nullptr; public: - MetricsRecord(MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels = nullptr); + MetricsRecord(MetricLabelsPtr labels, + DynamicMetricLabelsPtr dynamicLabels = nullptr, + std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN); MetricsRecord() = default; void MarkDeleted(); bool IsDeleted() const; + const std::string GetCategory() const; const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; const std::vector& GetCounters() const; @@ -75,6 +94,7 @@ class MetricsRecordRef { MetricsRecordRef(MetricsRecordRef&&) = delete; MetricsRecordRef& operator=(MetricsRecordRef&&) = delete; void SetMetricsRecord(MetricsRecord* metricRecord); + const std::string GetCategory() const; const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; CounterPtr CreateCounter(const std::string& name); @@ -114,7 +134,10 @@ class ReentrantMetricsRecord { std::unordered_map mDoubleGauges; public: - void Init(MetricLabels& labels, std::unordered_map& metricKeys); + void Init(std::string category, + MetricLabels& labels, + DynamicMetricLabels& dynamicLabels, + std::unordered_map& metricKeys); const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; CounterPtr GetCounter(const std::string& name); @@ -140,9 +163,14 @@ class WriteMetrics { return ptr; } - void - PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); - void CreateMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + void PrepareMetricsRecordRef(MetricsRecordRef& ref, + MetricLabels&& labels, + DynamicMetricLabels&& dynamicLabels = {}, + std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN); + void CreateMetricsRecordRef(MetricsRecordRef& ref, + MetricLabels&& labels, + DynamicMetricLabels&& dynamicLabels = {}, + std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN); void CommitMetricsRecordRef(MetricsRecordRef& ref); MetricsRecord* DoSnapshot(); diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index 5c4356389a..98087b400a 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -16,7 +16,6 @@ #include -#include "app_config/AppConfig.h" #include "LogFileProfiler.h" #include "LogtailMetric.h" #include "MetricConstants.h" @@ -35,7 +34,6 @@ DECLARE_FLAG_STRING(metrics_report_method); namespace logtail { -const string METRIC_REGION_FIELD_NAME = "region"; const string METRIC_REGION_DEFAULT = "default"; const string METRIC_SLS_LOGSTORE_NAME = "shennong_log_profile"; const string METRIC_TOPIC_TYPE = "loong_collector_metric"; @@ -66,7 +64,7 @@ void MetricExportor::PushCppMetrics() { if ("sls" == STRING_FLAG(metrics_report_method)) { std::map logGroupMap; - ReadMetrics::GetInstance()->ReadAsLogGroup(METRIC_REGION_FIELD_NAME, METRIC_REGION_DEFAULT, logGroupMap); + ReadMetrics::GetInstance()->ReadAsLogGroup(METRIC_LABEL_KEY_REGION, METRIC_REGION_DEFAULT, logGroupMap); SendToSLS(logGroupMap); } else if ("file" == STRING_FLAG(metrics_report_method)) { std::string metricsContent; @@ -193,15 +191,15 @@ void MetricExportor::SerializeGoDirectMetricsListToLogGroupMap( std::string configName = ""; std::string region = METRIC_REGION_DEFAULT; { - // get the config_name label + // get the pipeline_name label for (const auto& metric : metrics) { - if (metric.first == "label.config_name") { + if (metric.first == METRIC_KEY_LABEL + "." + METRIC_LABEL_KEY_PIPELINE_NAME) { configName = metric.second; break; } } if (!configName.empty()) { - // get region info by config_name + // get region info by pipeline_name shared_ptr p = PipelineManager::GetInstance()->FindConfigByName(configName); if (p) { FlusherSLS* pConfig = NULL; @@ -225,11 +223,32 @@ void MetricExportor::SerializeGoDirectMetricsListToLogGroupMap( auto now = GetCurrentLogtailTime(); SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec); + + Json::Value metricsRecordLabel; for (const auto& metric : metrics) { + // category + if (metric.first.compare("label.metric_category") == 0) { + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(METRIC_KEY_CATEGORY); + contentPtr->set_value(metric.second); + continue; + } + // label + if (metric.first.compare(0, METRIC_KEY_LABEL.length(), METRIC_KEY_LABEL)) { + metricsRecordLabel[metric.first.substr(METRIC_KEY_LABEL.length() + 1)] = metric.second; + continue; + } + // value Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(metric.first); contentPtr->set_value(metric.second); } + Json::StreamWriterBuilder writer; + writer["indentation"] = ""; + std::string jsonString = Json::writeString(writer, metricsRecordLabel); + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(METRIC_KEY_LABEL); + contentPtr->set_value(jsonString); } } @@ -238,16 +257,25 @@ void MetricExportor::SerializeGoDirectMetricsListToString(std::vectorEnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec; for (const auto& metric : metrics) { - metricsRecordValue[metric.first] = metric.second; + if (metric.first.compare("label.metric_category") == 0) { + metricsRecordJson[METRIC_KEY_CATEGORY] = metric.second; + continue; + } + if (metric.first.compare(0, METRIC_KEY_LABEL.length(), METRIC_KEY_LABEL) == 0) { + metricsRecordLabel[metric.first.substr(METRIC_KEY_LABEL.length() + 1)] = metric.second; + continue; + } + metricsRecordJson[metric.first.substr(METRIC_KEY_VALUE.length() + 1)] = metric.second; } + metricsRecordJson[METRIC_KEY_LABEL] = metricsRecordLabel; Json::StreamWriterBuilder writer; writer["indentation"] = ""; - std::string jsonString = Json::writeString(writer, metricsRecordValue); + std::string jsonString = Json::writeString(writer, metricsRecordJson); oss << jsonString << '\n'; } metricsContent = oss.str(); diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 7eca928e46..9accba15e3 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -693,7 +693,6 @@ LoongCollectorMonitor* LoongCollectorMonitor::GetInstance() { void LoongCollectorMonitor::Init() { // create metric record MetricLabels labels; - labels.emplace_back(METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_AGENT); labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId()); labels.emplace_back(METRIC_LABEL_KEY_START_TIME, LogFileProfiler::mStartTime); labels.emplace_back(METRIC_LABEL_KEY_OS, OS_NAME); @@ -710,7 +709,7 @@ void LoongCollectorMonitor::Init() { }); #endif WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), MetricCategory::METRIC_CATEGORY_AGENT); // init value mAgentCpu = mMetricsRecordRef.CreateDoubleGauge(METRIC_AGENT_CPU); mAgentMemory = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_MEMORY); diff --git a/core/monitor/PluginMetricManager.cpp b/core/monitor/PluginMetricManager.cpp index b47b5ddcad..f39506971e 100644 --- a/core/monitor/PluginMetricManager.cpp +++ b/core/monitor/PluginMetricManager.cpp @@ -17,7 +17,7 @@ namespace logtail { -ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecordRef(MetricLabels labels) { +ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecordRef(MetricLabels labels, DynamicMetricLabels dynamicLabels) { std::lock_guard lock(mutex); std::string key = GenerateKey(labels); @@ -31,7 +31,7 @@ ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecord newLabels.insert(newLabels.end(), labels.begin(), labels.end()); ReentrantMetricsRecordRef ptr = std::make_shared(); - ptr->Init(newLabels, mMetricKeys); + ptr->Init(mDefaultCategory, newLabels, dynamicLabels, mMetricKeys); mReentrantMetricsRecordRefsMap.emplace(key, ptr); if (mSizeGauge != nullptr) { diff --git a/core/monitor/PluginMetricManager.h b/core/monitor/PluginMetricManager.h index db043fceda..dfa0a23ecb 100644 --- a/core/monitor/PluginMetricManager.h +++ b/core/monitor/PluginMetricManager.h @@ -23,10 +23,15 @@ namespace logtail { class PluginMetricManager { public: - PluginMetricManager(const MetricLabelsPtr defaultLabels, std::unordered_map metricKeys) - : mDefaultLabels(defaultLabels->begin(), defaultLabels->end()), mMetricKeys(metricKeys) {} - - ReentrantMetricsRecordRef GetOrCreateReentrantMetricsRecordRef(MetricLabels labels); + PluginMetricManager(const MetricLabelsPtr defaultLabels, + std::unordered_map metricKeys, + std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN) + : mDefaultLabels(defaultLabels->begin(), defaultLabels->end()), + mMetricKeys(metricKeys), + mDefaultCategory(category) {} + + ReentrantMetricsRecordRef GetOrCreateReentrantMetricsRecordRef(MetricLabels labels, + DynamicMetricLabels dynamicLabels = {}); void ReleaseReentrantMetricsRecordRef(MetricLabels labels); void RegisterSizeGauge(IntGaugePtr ptr) { mSizeGauge = ptr; } @@ -36,6 +41,7 @@ class PluginMetricManager { MetricLabels mDefaultLabels; std::unordered_map mMetricKeys; + std::string mDefaultCategory; std::unordered_map mReentrantMetricsRecordRefsMap; mutable std::mutex mutex; diff --git a/core/monitor/metric_constants/AgentMetrics.cpp b/core/monitor/metric_constants/AgentMetrics.cpp index f2e49630ac..6f81fcc437 100644 --- a/core/monitor/metric_constants/AgentMetrics.cpp +++ b/core/monitor/metric_constants/AgentMetrics.cpp @@ -19,7 +19,6 @@ using namespace std; namespace logtail { // label keys -const string METRIC_LABEL_KEY_METRIC_CATEGORY = "metric_category"; const string METRIC_LABEL_KEY_ALIUIDS = "aliuids"; const string METRIC_LABEL_KEY_INSTANCE_ID = "instance_id"; const string METRIC_LABEL_KEY_START_TIME = "start_time"; @@ -30,16 +29,13 @@ const string METRIC_LABEL_KEY_USER_DEFINED_ID = "user_defined_id"; const string METRIC_LABEL_KEY_UUID = "uuid"; const string METRIC_LABEL_KEY_VERSION = "version"; -// label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_AGENT = "agent"; - // metric keys -const string METRIC_AGENT_CPU = "agent_cpu_percent"; -const string METRIC_AGENT_GO_ROUTINES_TOTAL = "agent_go_routines_total"; -const string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "agent_instance_config_total"; // Not Implemented -const string METRIC_AGENT_MEMORY = "agent_memory_used_mb"; -const string METRIC_AGENT_MEMORY_GO = "agent_go_memory_used_mb"; -const string METRIC_AGENT_OPEN_FD_TOTAL = "agent_open_fd_total"; -const string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "agent_pipeline_config_total"; +const string METRIC_AGENT_CPU = "cpu_percent"; +const string METRIC_AGENT_GO_ROUTINES_TOTAL = "go_routines_total"; +const string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "instance_config_total"; // Not Implemented +const string METRIC_AGENT_MEMORY = "memory_used_mb"; +const string METRIC_AGENT_MEMORY_GO = "go_memory_used_mb"; +const string METRIC_AGENT_OPEN_FD_TOTAL = "open_fd_total"; +const string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "pipeline_config_total"; } // namespace logtail diff --git a/core/monitor/metric_constants/ComponentMetrics.cpp b/core/monitor/metric_constants/ComponentMetrics.cpp index 006602e751..a11ff6cb33 100644 --- a/core/monitor/metric_constants/ComponentMetrics.cpp +++ b/core/monitor/metric_constants/ComponentMetrics.cpp @@ -34,7 +34,6 @@ const string METRIC_LABEL_KEY_EXACTLY_ONCE_ENABLED = "exactly_once_enabled"; const string METRIC_LABEL_KEY_GROUP_BATCH_ENABLED = "group_batch_enabled"; // label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT = "component"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER = "batcher"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR = "compressor"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE = "process_queue"; @@ -43,46 +42,46 @@ const string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE = "sender_queue"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER = "serializer"; // metric keys -const string METRIC_COMPONENT_IN_EVENTS_TOTAL = "component_in_events_total"; -const string METRIC_COMPONENT_IN_SIZE_BYTES = "component_in_size_bytes"; -const string METRIC_COMPONENT_IN_ITEMS_TOTAL = "component_in_items_total"; -const string METRIC_COMPONENT_OUT_EVENTS_TOTAL = "component_out_events_total"; -const string METRIC_COMPONENT_OUT_ITEMS_TOTAL = "component_out_items_total"; -const string METRIC_COMPONENT_OUT_SIZE_BYTES = "component_out_size_bytes"; -const string METRIC_COMPONENT_TOTAL_DELAY_MS = "component_total_delay_ms"; -const string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = "component_total_process_time_ms"; -const string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = "component_discarded_items_total"; -const string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "component_discarded_item_size_bytes"; +const string METRIC_COMPONENT_IN_EVENTS_TOTAL = "in_events_total"; +const string METRIC_COMPONENT_IN_SIZE_BYTES = "in_size_bytes"; +const string METRIC_COMPONENT_IN_ITEMS_TOTAL = "in_items_total"; +const string METRIC_COMPONENT_OUT_EVENTS_TOTAL = "out_events_total"; +const string METRIC_COMPONENT_OUT_ITEMS_TOTAL = "out_items_total"; +const string METRIC_COMPONENT_OUT_SIZE_BYTES = "out_size_bytes"; +const string METRIC_COMPONENT_TOTAL_DELAY_MS = "total_delay_ms"; +const string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; +const string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = "discarded_items_total"; +const string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "discarded_item_size_bytes"; /********************************************************** * batcher **********************************************************/ -const string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL = "component_event_batches_total"; -const string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_TOTAL = "component_buffered_groups_total"; -const string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_TOTAL = "component_buffered_events_total"; -const string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "component_buffered_size_bytes"; -const string METRIC_COMPONENT_BATCHER_TOTAL_ADD_TIME_MS = "component_total_add_time_ms"; +const string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL = "event_batches_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_TOTAL = "buffered_groups_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_TOTAL = "buffered_events_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "buffered_size_bytes"; +const string METRIC_COMPONENT_BATCHER_TOTAL_ADD_TIME_MS = "total_add_time_ms"; /********************************************************** * queue **********************************************************/ -const string METRIC_COMPONENT_QUEUE_SIZE = "component_queue_size"; -const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "component_queue_size_bytes"; -const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "component_valid_to_push_status"; -const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "component_extra_buffer_size"; -const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "component_extra_buffer_size_bytes"; -const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = "component_discarded_events_total"; +const string METRIC_COMPONENT_QUEUE_SIZE = "queue_size"; +const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "queue_size_bytes"; +const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "valid_to_push_status"; +const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "extra_buffer_size"; +const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "extra_buffer_size_bytes"; +const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = "discarded_events_total"; -const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "component_fetched_items_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_ATTEMPTS_TOTAL = "component_fetch_attempts_total"; -const string METRIC_COMPONENT_QUEUE_SUCCESSFUL_FETCH_TIMES_TOTAL = "component_successful_fetch_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "fetched_items_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_ATTEMPTS_TOTAL = "fetch_attempts_total"; +const string METRIC_COMPONENT_QUEUE_SUCCESSFUL_FETCH_TIMES_TOTAL = "successful_fetch_times_total"; const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_REGION_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_region_limiter_times_total"; + = "fetch_rejected_by_region_limiter_times_total"; const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_PROJECT_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_project_limiter_times_total"; + = "fetch_rejected_by_project_limiter_times_total"; const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_LOGSTORE_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_logstore_limiter_times_total"; + = "fetch_rejected_by_logstore_limiter_times_total"; const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_rate_limiter_times_total"; + = "fetch_rejected_by_rate_limiter_times_total"; } // namespace logtail diff --git a/core/monitor/metric_constants/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h index 988c8b4afa..ffd74721a8 100644 --- a/core/monitor/metric_constants/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -19,16 +19,6 @@ namespace logtail { -// label keys -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY; - -// label values -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_AGENT; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_PIPELINE; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_PLUGIN; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER; - ////////////////////////////////////////////////////////////////////////// // agent ////////////////////////////////////////////////////////////////////////// diff --git a/core/monitor/metric_constants/PipelineMetrics.cpp b/core/monitor/metric_constants/PipelineMetrics.cpp index 85fd7f3e8f..83f76c7690 100644 --- a/core/monitor/metric_constants/PipelineMetrics.cpp +++ b/core/monitor/metric_constants/PipelineMetrics.cpp @@ -23,18 +23,15 @@ const string METRIC_LABEL_KEY_LOGSTORE = "logstore"; const string METRIC_LABEL_KEY_PIPELINE_NAME = "pipeline_name"; const string METRIC_LABEL_KEY_REGION = "region"; -// label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_PIPELINE = "pipeline"; - // metric keys -const string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "pipeline_processors_in_events_total"; -const string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "pipeline_processors_in_event_groups_total"; -const string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES = "pipeline_processors_in_size_bytes"; -const string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS = "pipeline_processors_total_process_time_ms"; -const string METRIC_PIPELINE_FLUSHERS_IN_EVENTS_TOTAL = "pipeline_flushers_in_events_total"; -const string METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL = "pipeline_flushers_in_event_groups_total"; -const string METRIC_PIPELINE_FLUSHERS_IN_SIZE_BYTES = "pipeline_flushers_in_size_bytes"; -const string METRIC_PIPELINE_FLUSHERS_TOTAL_PACKAGE_TIME_MS = "pipeline_flushers_total_package_time_ms"; -const string METRIC_PIPELINE_START_TIME = "pipeline_start_time"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "processors_in_events_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "processors_in_event_groups_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES = "processors_in_size_bytes"; +const string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS = "processors_total_process_time_ms"; +const string METRIC_PIPELINE_FLUSHERS_IN_EVENTS_TOTAL = "flushers_in_events_total"; +const string METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL = "flushers_in_event_groups_total"; +const string METRIC_PIPELINE_FLUSHERS_IN_SIZE_BYTES = "flushers_in_size_bytes"; +const string METRIC_PIPELINE_FLUSHERS_TOTAL_PACKAGE_TIME_MS = "flushers_total_package_time_ms"; +const string METRIC_PIPELINE_START_TIME = "start_time"; } // namespace logtail diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index 448a551cae..eebe3892d9 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -22,18 +22,15 @@ namespace logtail { const string METRIC_LABEL_KEY_PLUGIN_ID = "plugin_id"; const string METRIC_LABEL_KEY_PLUGIN_TYPE = "plugin_type"; -// label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_PLUGIN = "plugin"; - // metric keys -const string METRIC_PLUGIN_IN_EVENTS_TOTAL = "plugin_in_events_total"; -const string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = "plugin_in_event_groups_total"; -const string METRIC_PLUGIN_IN_SIZE_BYTES = "plugin_in_size_bytes"; -const string METRIC_PLUGIN_OUT_EVENTS_TOTAL = "plugin_out_events_total"; -const string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "plugin_out_event_groups_total"; -const string METRIC_PLUGIN_OUT_SIZE_BYTES = "plugin_out_size_bytes"; -const string METRIC_PLUGIN_TOTAL_DELAY_MS = "plugin_total_delay_ms"; -const string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = "plugin_total_process_time_ms"; +const string METRIC_PLUGIN_IN_EVENTS_TOTAL = "in_events_total"; +const string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; +const string METRIC_PLUGIN_IN_SIZE_BYTES = "in_size_bytes"; +const string METRIC_PLUGIN_OUT_EVENTS_TOTAL = "out_events_total"; +const string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "out_event_groups_total"; +const string METRIC_PLUGIN_OUT_SIZE_BYTES = "out_size_bytes"; +const string METRIC_PLUGIN_TOTAL_DELAY_MS = "total_delay_ms"; +const string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; /********************************************************** * input_file @@ -43,9 +40,9 @@ const string METRIC_LABEL_KEY_FILE_DEV = "file_dev"; const string METRIC_LABEL_KEY_FILE_INODE = "file_inode"; const string METRIC_LABEL_KEY_FILE_NAME = "file_name"; -const string METRIC_PLUGIN_MONITOR_FILE_TOTAL = "plugin_monitor_file_total"; -const string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "plugin_source_read_offset_bytes"; -const string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "plugin_source_size_bytes"; +const string METRIC_PLUGIN_MONITOR_FILE_TOTAL = "monitor_file_total"; +const string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "source_read_offset_bytes"; +const string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "source_size_bytes"; /********************************************************** * input_prometheus @@ -57,11 +54,11 @@ const std::string METRIC_LABEL_KEY_SERVICE_PORT = "service_port"; const std::string METRIC_LABEL_KEY_STATUS = "status"; const std::string METRIC_LABEL_KEY_INSTANCE = "instance"; -const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS = "plugin_prom_subscribe_targets"; -const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL = "plugin_prom_subscribe_total"; -const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TIME_MS = "plugin_prom_subscribe_time_ms"; -const std::string METRIC_PLUGIN_PROM_SCRAPE_TIME_MS = "plugin_prom_scrape_time_ms"; -const std::string METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL = "plugin_prom_scrape_delay_total"; +const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS = "prom_subscribe_targets"; +const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL = "prom_subscribe_total"; +const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TIME_MS = "prom_subscribe_time_ms"; +const std::string METRIC_PLUGIN_PROM_SCRAPE_TIME_MS = "prom_scrape_time_ms"; +const std::string METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL = "prom_scrape_delay_total"; /********************************************************** * input_ebpf @@ -92,69 +89,69 @@ const string METRIC_LABEL_VALUE_PLUGIN_TYPE_FILE_SECURITY = "file_security"; const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_OBSERVER = "process_observer"; const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_SECURITY = "process_security"; -const string METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL = "plugin_ebpf_loss_kernel_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM = "plugin_network_observer_conntracker_num"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL = "plugin_network_observer_worker_handle_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL = "plugin_network_observer_parse_records_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL = "plugin_network_observer_aggregate_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM = "plugin_network_observer_aggregate_key_num"; -const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM = "plugin_process_cache_entries_num"; -const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "plugin_process_cache_miss_total"; +const string METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL = "ebpf_loss_kernel_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM = "network_observer_conntracker_num"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL + = "network_observer_worker_handle_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL = "network_observer_parse_records_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL = "network_observer_aggregate_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM = "network_observer_aggregate_key_num"; +const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM = "process_cache_entries_num"; +const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "process_cache_miss_total"; /********************************************************** * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ -const string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = "plugin_discarded_events_total"; -const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "plugin_out_failed_events_total"; -const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "plugin_out_key_not_found_events_total"; -const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "plugin_out_successful_events_total"; +const string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = "discarded_events_total"; +const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "out_failed_events_total"; +const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "out_key_not_found_events_total"; +const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "out_successful_events_total"; /********************************************************** * processor_parse_apsara_native * processor_parse_timestamp_native **********************************************************/ -const string METRIC_PLUGIN_HISTORY_FAILURE_TOTAL = "plugin_history_failure_total"; +const string METRIC_PLUGIN_HISTORY_FAILURE_TOTAL = "history_failure_total"; /********************************************************** * processor_split_multiline_log_string_native **********************************************************/ -const string METRIC_PLUGIN_MATCHED_EVENTS_TOTAL = "plugin_matched_events_total"; -const string METRIC_PLUGIN_MATCHED_LINES_TOTAL = "plugin_matched_lines_total"; -const string METRIC_PLUGIN_UNMATCHED_LINES_TOTAL = "plugin_unmatched_lines_total"; +const string METRIC_PLUGIN_MATCHED_EVENTS_TOTAL = "matched_events_total"; +const string METRIC_PLUGIN_MATCHED_LINES_TOTAL = "matched_lines_total"; +const string METRIC_PLUGIN_UNMATCHED_LINES_TOTAL = "unmatched_lines_total"; /********************************************************** * processor_merge_multiline_log_native **********************************************************/ -const string METRIC_PLUGIN_MERGED_EVENTS_TOTAL = "plugin_merged_events_total"; -const string METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL = "plugin_unmatched_events_total"; +const string METRIC_PLUGIN_MERGED_EVENTS_TOTAL = "merged_events_total"; +const string METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL = "unmatched_events_total"; /********************************************************** * processor_parse_container_log_native **********************************************************/ -const string METRIC_PLUGIN_PARSE_STDERR_TOTAL = "plugin_parse_stderr_total"; -const string METRIC_PLUGIN_PARSE_STDOUT_TOTAL = "plugin_parse_stdout_total"; +const string METRIC_PLUGIN_PARSE_STDERR_TOTAL = "parse_stderr_total"; +const string METRIC_PLUGIN_PARSE_STDOUT_TOTAL = "parse_stdout_total"; /********************************************************** * all flusher （所有发送插件通用指标） **********************************************************/ -const string METRIC_PLUGIN_FLUSHER_TOTAL_PACKAGE_TIME_MS = "plugin_flusher_total_package_time_ms"; -const string METRIC_PLUGIN_FLUSHER_OUT_EVENT_GROUPS_TOTAL = "plugin_flusher_send_total"; -const string METRIC_PLUGIN_FLUSHER_SEND_DONE_TOTAL = "plugin_flusher_send_done_total"; -const string METRIC_PLUGIN_FLUSHER_SUCCESS_TOTAL = "plugin_flusher_success_total"; -const string METRIC_PLUGIN_FLUSHER_NETWORK_ERROR_TOTAL = "plugin_flusher_network_error_total"; -const string METRIC_PLUGIN_FLUSHER_SERVER_ERROR_TOTAL = "plugin_flusher_server_error_total"; -const string METRIC_PLUGIN_FLUSHER_UNAUTH_ERROR_TOTAL = "plugin_flusher_unauth_error_total"; -const string METRIC_PLUGIN_FLUSHER_PARAMS_ERROR_TOTAL = "plugin_flusher_params_error_total"; -const string METRIC_PLUGIN_FLUSHER_OTHER_ERROR_TOTAL = "plugin_flusher_other_error_total"; +const string METRIC_PLUGIN_FLUSHER_TOTAL_PACKAGE_TIME_MS = "total_package_time_ms"; +const string METRIC_PLUGIN_FLUSHER_OUT_EVENT_GROUPS_TOTAL = "send_total"; +const string METRIC_PLUGIN_FLUSHER_SEND_DONE_TOTAL = "send_done_total"; +const string METRIC_PLUGIN_FLUSHER_SUCCESS_TOTAL = "success_total"; +const string METRIC_PLUGIN_FLUSHER_NETWORK_ERROR_TOTAL = "network_error_total"; +const string METRIC_PLUGIN_FLUSHER_SERVER_ERROR_TOTAL = "server_error_total"; +const string METRIC_PLUGIN_FLUSHER_UNAUTH_ERROR_TOTAL = "unauth_error_total"; +const string METRIC_PLUGIN_FLUSHER_PARAMS_ERROR_TOTAL = "params_error_total"; +const string METRIC_PLUGIN_FLUSHER_OTHER_ERROR_TOTAL = "other_error_total"; /********************************************************** * flusher_sls **********************************************************/ -const string METRIC_PLUGIN_FLUSHER_SLS_SHARD_WRITE_QUOTA_ERROR_TOTAL - = "plugin_flusher_sls_shard_write_quota_error_total"; -const string METRIC_PLUGIN_FLUSHER_SLS_PROJECT_QUOTA_ERROR_TOTAL = "plugin_flusher_sls_project_quota_error_total"; -const string METRIC_PLUGIN_FLUSHER_SLS_SEQUENCE_ID_ERROR_TOTAL = "plugin_flusher_sls_sequence_id_error_total"; -const string METRIC_PLUGIN_FLUSHER_SLS_REQUEST_EXPRIRED_ERROR_TOTAL = "plugin_flusher_sls_request_exprired_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_SHARD_WRITE_QUOTA_ERROR_TOTAL = "shard_write_quota_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_PROJECT_QUOTA_ERROR_TOTAL = "project_quota_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_SEQUENCE_ID_ERROR_TOTAL = "sequence_id_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_REQUEST_EXPRIRED_ERROR_TOTAL = "request_exprired_error_total"; } // namespace logtail \ No newline at end of file diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index 8fb8519167..e6b8aa7505 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -22,7 +22,6 @@ namespace logtail { const string METRIC_LABEL_KEY_RUNNER_NAME = "runner_name"; // label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER = "runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER = "file_server"; const string METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER = "flusher_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK = "http_sink"; @@ -31,49 +30,49 @@ const string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS = "prometheus_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER = "ebpf_server"; // metric keys -const string METRIC_RUNNER_IN_EVENTS_TOTAL = "runner_in_events_total"; -const string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = "runner_in_event_groups_total"; -const string METRIC_RUNNER_IN_SIZE_BYTES = "runner_in_size_bytes"; -const string METRIC_RUNNER_IN_ITEMS_TOTAL = "runner_in_items_total"; -const string METRIC_RUNNER_LAST_RUN_TIME = "runner_last_run_time"; -const string METRIC_RUNNER_OUT_ITEMS_TOTAL = "runner_out_items_total"; -const string METRIC_RUNNER_TOTAL_DELAY_MS = "runner_total_delay_ms"; -const string METRIC_RUNNER_CLIENT_REGISTER_STATE = "runner_client_register_state"; -const string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL = "runner_client_register_retry_total"; -const string METRIC_RUNNER_JOBS_TOTAL = "runner_jobs_total"; +const string METRIC_RUNNER_IN_EVENTS_TOTAL = "in_events_total"; +const string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; +const string METRIC_RUNNER_IN_SIZE_BYTES = "in_size_bytes"; +const string METRIC_RUNNER_IN_ITEMS_TOTAL = "in_items_total"; +const string METRIC_RUNNER_LAST_RUN_TIME = "last_run_time"; +const string METRIC_RUNNER_OUT_ITEMS_TOTAL = "out_items_total"; +const string METRIC_RUNNER_TOTAL_DELAY_MS = "total_delay_ms"; +const string METRIC_RUNNER_CLIENT_REGISTER_STATE = "client_register_state"; +const string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL = "client_register_retry_total"; +const string METRIC_RUNNER_JOBS_TOTAL = "jobs_total"; /********************************************************** * all sinks **********************************************************/ -const string METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL = "runner_out_successful_items_total"; -const string METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL = "runner_out_failed_items_total"; -const string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS = "runner_successful_item_total_response_time_ms"; -const string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS = "runner_failed_item_total_response_time_ms"; -const string METRIC_RUNNER_SINK_SENDING_ITEMS_TOTAL = "runner_sending_items_total"; -const string METRIC_RUNNER_SINK_SEND_CONCURRENCY = "runner_send_concurrency"; +const string METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL = "out_successful_items_total"; +const string METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL = "out_failed_items_total"; +const string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS = "successful_item_total_response_time_ms"; +const string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS = "failed_item_total_response_time_ms"; +const string METRIC_RUNNER_SINK_SENDING_ITEMS_TOTAL = "sending_items_total"; +const string METRIC_RUNNER_SINK_SEND_CONCURRENCY = "send_concurrency"; /********************************************************** * flusher runner **********************************************************/ -const string METRIC_RUNNER_FLUSHER_IN_RAW_SIZE_BYTES = "runner_in_raw_size_bytes"; -const string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL = "runner_waiting_items_total"; +const string METRIC_RUNNER_FLUSHER_IN_RAW_SIZE_BYTES = "in_raw_size_bytes"; +const string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL = "waiting_items_total"; /********************************************************** * file server **********************************************************/ -const string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL = "runner_watched_dirs_total"; -const string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL = "runner_active_readers_total"; +const string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL = "watched_dirs_total"; +const string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL = "active_readers_total"; const string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG - = "runner_enable_file_included_by_multi_configs"; -const string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "runner_polling_modify_cache_size"; -const string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "runner_polling_dir_cache_size"; -const string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "runner_polling_file_cache_size"; + = "enable_file_included_by_multi_configs"; +const string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "polling_modify_cache_size"; +const string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "polling_dir_cache_size"; +const string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "polling_file_cache_size"; /********************************************************** * ebpf server **********************************************************/ -const string METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL = "runner_start_plugin_total"; -const string METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL = "runner_stop_plugin_total"; -const string METRIC_RUNNER_EBPF_SUSPEND_PLUGIN_TOTAL = "runner_suspend_plugin_total"; +const string METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL = "start_plugin_total"; +const string METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL = "stop_plugin_total"; +const string METRIC_RUNNER_EBPF_SUSPEND_PLUGIN_TOTAL = "suspend_plugin_total"; } // namespace logtail \ No newline at end of file diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 40dec54bd6..8a13886888 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -318,9 +318,9 @@ bool Pipeline::Init(PipelineConfig&& config) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, - {METRIC_LABEL_KEY_PIPELINE_NAME, mName}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_PIPELINE}}); + {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mName}}, + {}, + MetricCategory::METRIC_CATEGORY_PIPELINE); mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); mProcessorsInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL); mProcessorsInGroupsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL); diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index 523e390d1f..3a9ad01442 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -103,14 +103,14 @@ class Batcher { {METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, flusher->GetPluginID()}}; if (enableGroupBatch) { labels.emplace_back(METRIC_LABEL_KEY_GROUP_BATCH_ENABLED, "true"); } else { labels.emplace_back(METRIC_LABEL_KEY_GROUP_BATCH_ENABLED, "false"); } - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, std::move(labels), {}, MetricCategory::METRIC_CATEGORY_COMPONENT); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_EVENTS_TOTAL); diff --git a/core/pipeline/plugin/interface/Plugin.h b/core/pipeline/plugin/interface/Plugin.h index 087a44cd41..a9b7012c0a 100644 --- a/core/pipeline/plugin/interface/Plugin.h +++ b/core/pipeline/plugin/interface/Plugin.h @@ -41,9 +41,10 @@ class Plugin { mMetricsRecordRef, {{METRIC_LABEL_KEY_PROJECT, mContext->GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mContext->GetConfigName()}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_PLUGIN}, {METRIC_LABEL_KEY_PLUGIN_TYPE, name}, - {METRIC_LABEL_KEY_PLUGIN_ID, id}}); + {METRIC_LABEL_KEY_PLUGIN_ID, id}}, + {}, + MetricCategory::METRIC_CATEGORY_PLUGIN); } protected: diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index e7cf3a37a1..a4cf487524 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -31,8 +31,9 @@ class QueueInterface { { {METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, - }); + }, + {}, + MetricCategory::METRIC_CATEGORY_COMPONENT); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index 08abd08cca..6fe40a0865 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -39,8 +39,9 @@ bool Router::Init(std::vector> configs, const P mMetricsRecordRef, {{METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, - {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}}); + {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER}}, + {}, + MetricCategory::METRIC_CATEGORY_COMPONENT); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); return true; diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index a9a99c737e..8b40607398 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -52,8 +52,9 @@ class Serializer { {{METRIC_LABEL_KEY_PROJECT, f->GetContext().GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, f->GetContext().GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, - {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, f->GetPluginID()}}); + {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, f->GetPluginID()}}, + {}, + MetricCategory::METRIC_CATEGORY_COMPONENT); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); diff --git a/core/plugin/input/InputContainerStdio.cpp b/core/plugin/input/InputContainerStdio.cpp index 46a018e067..6c3f5d49a4 100644 --- a/core/plugin/input/InputContainerStdio.cpp +++ b/core/plugin/input/InputContainerStdio.cpp @@ -69,7 +69,8 @@ bool InputContainerStdio::Init(const Json::Value& config, Json::Value& optionalG if (!mContainerDiscovery.Init(config, *mContext, sName)) { return false; } - mContainerDiscovery.GenerateContainerMetaFetchingGoPipeline(optionalGoPipeline, nullptr, mContext->GetPipeline().GenNextPluginMeta(false)); + mContainerDiscovery.GenerateContainerMetaFetchingGoPipeline( + optionalGoPipeline, nullptr, mContext->GetPipeline().GenNextPluginMeta(false)); if (!mFileReader.Init(config, *mContext, sName)) { return false; @@ -167,8 +168,8 @@ bool InputContainerStdio::Init(const Json::Value& config, Json::Value& optionalG {METRIC_PLUGIN_SOURCE_SIZE_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, {METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, }; - mPluginMetricManager - = std::make_shared(GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys); + mPluginMetricManager = std::make_shared( + GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); // Register a Gauge metric to record PluginMetricManager‘s map size mMonitorFileTotal = GetMetricsRecordRef().CreateIntGauge(METRIC_PLUGIN_MONITOR_FILE_TOTAL); mPluginMetricManager->RegisterSizeGauge(mMonitorFileTotal); diff --git a/core/plugin/input/InputFile.cpp b/core/plugin/input/InputFile.cpp index c5b0039a19..b8186797b8 100644 --- a/core/plugin/input/InputFile.cpp +++ b/core/plugin/input/InputFile.cpp @@ -165,8 +165,8 @@ bool InputFile::Init(const Json::Value& config, Json::Value& optionalGoPipeline) {METRIC_PLUGIN_SOURCE_SIZE_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, {METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, }; - mPluginMetricManager - = std::make_shared(GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys); + mPluginMetricManager = std::make_shared( + GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); mPluginMetricManager->RegisterSizeGauge(mMonitorFileTotal); return CreateInnerProcessors(); diff --git a/core/plugin/input/InputFileSecurity.cpp b/core/plugin/input/InputFileSecurity.cpp index 3510833830..5c15536721 100644 --- a/core/plugin/input/InputFileSecurity.cpp +++ b/core/plugin/input/InputFileSecurity.cpp @@ -43,7 +43,7 @@ bool InputFileSecurity::Init(const Json::Value& config, Json::Value& optionalGoP {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::FILE, config, mContext, sName); } diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index e096e585f9..fea0b459fc 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -36,7 +36,7 @@ class InputFileSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputNetworkObserver.cpp b/core/plugin/input/InputNetworkObserver.cpp index c2b997d54f..ce9c4218ff 100644 --- a/core/plugin/input/InputNetworkObserver.cpp +++ b/core/plugin/input/InputNetworkObserver.cpp @@ -49,7 +49,7 @@ bool InputNetworkObserver::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return ebpf::InitObserverNetworkOption(config, mNetworkOption, mContext, sName); } diff --git a/core/plugin/input/InputNetworkObserver.h b/core/plugin/input/InputNetworkObserver.h index 7cc3d98ca6..7f204a2c90 100644 --- a/core/plugin/input/InputNetworkObserver.h +++ b/core/plugin/input/InputNetworkObserver.h @@ -36,7 +36,7 @@ class InputNetworkObserver : public Input { bool SupportAck() const override { return false; } nami::ObserverNetworkOption mNetworkOption; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputNetworkSecurity.cpp b/core/plugin/input/InputNetworkSecurity.cpp index 409d40f77c..ccadcc26ca 100644 --- a/core/plugin/input/InputNetworkSecurity.cpp +++ b/core/plugin/input/InputNetworkSecurity.cpp @@ -45,7 +45,7 @@ bool InputNetworkSecurity::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::NETWORK, config, mContext, sName); } diff --git a/core/plugin/input/InputNetworkSecurity.h b/core/plugin/input/InputNetworkSecurity.h index 20e3025246..cda3a7c170 100644 --- a/core/plugin/input/InputNetworkSecurity.h +++ b/core/plugin/input/InputNetworkSecurity.h @@ -35,7 +35,7 @@ class InputNetworkSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputProcessSecurity.cpp b/core/plugin/input/InputProcessSecurity.cpp index cc11afa854..3baa6cbed8 100644 --- a/core/plugin/input/InputProcessSecurity.cpp +++ b/core/plugin/input/InputProcessSecurity.cpp @@ -42,7 +42,7 @@ bool InputProcessSecurity::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::PROCESS, config, mContext, sName); } diff --git a/core/plugin/input/InputProcessSecurity.h b/core/plugin/input/InputProcessSecurity.h index b45fec9b5c..d26d7a95e3 100644 --- a/core/plugin/input/InputProcessSecurity.h +++ b/core/plugin/input/InputProcessSecurity.h @@ -35,7 +35,7 @@ class InputProcessSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/prometheus/PromSelfMonitor.cpp b/core/prometheus/PromSelfMonitor.cpp index 39c1473014..5852fc45ae 100644 --- a/core/prometheus/PromSelfMonitor.cpp +++ b/core/prometheus/PromSelfMonitor.cpp @@ -13,7 +13,7 @@ namespace logtail { void PromSelfMonitorUnsafe::InitMetricManager(const std::unordered_map& metricKeys, const MetricLabels& labels) { auto metricLabels = std::make_shared(labels); - mPluginMetricManagerPtr = std::make_shared(metricLabels, metricKeys); + mPluginMetricManagerPtr = std::make_shared(metricLabels, metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); } void PromSelfMonitorUnsafe::AddCounter(const std::string& metricName, uint64_t statusCode, uint64_t val) { diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index c033b5c2d1..fee0c58a33 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -53,7 +53,6 @@ PrometheusInputRunner::PrometheusInputRunner() // self monitor MetricLabels labels; - labels.emplace_back(METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER); labels.emplace_back(METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS); labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId()); labels.emplace_back(METRIC_LABEL_KEY_POD_NAME, mPodName); @@ -64,7 +63,7 @@ PrometheusInputRunner::PrometheusInputRunner() dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); }); WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), MetricCategory::METRIC_CATEGORY_RUNNER); mPromRegisterState = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_CLIENT_REGISTER_STATE); mPromJobNum = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_JOBS_TOTAL); diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index ae1d55dc1a..6c4f6e3842 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -46,8 +46,9 @@ bool FlusherRunner::Init() { srand(time(nullptr)); WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}); + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER}}, + {}, + MetricCategory::METRIC_CATEGORY_RUNNER); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_OUT_ITEMS_TOTAL); diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index 7b4177ade9..1aad1dfbd2 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -89,9 +89,9 @@ void ProcessorRunner::Run(uint32_t threadNo) { // thread local metrics should be initialized in each thread WriteMetrics::GetInstance()->PrepareMetricsRecordRef( sMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}, - {"thread_no", ToString(threadNo)}}); + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, {"thread_no", ToString(threadNo)}}, + {}, + MetricCategory::METRIC_CATEGORY_RUNNER); sInGroupsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL); sInEventsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENTS_TOTAL); sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index d1670ffeb9..3581a8765e 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -40,8 +40,9 @@ bool HttpSink::Init() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}); + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK}}, + {}, + MetricCategory::METRIC_CATEGORY_RUNNER); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); mOutSuccessfulItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL); @@ -272,13 +273,13 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { static_cast(request->mItem->mFlusher) ->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); - LOG_DEBUG( - sLogger, - ("failed to send http request", "abort")("item address", request->mItem)( - "config-flusher-dst", - QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( - "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))( - "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); + LOG_DEBUG(sLogger, + ("failed to send http request", "abort")("item address", request->mItem)( + "config-flusher-dst", + QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( + "response time", ToString(responseTimeMs) + "ms")("try cnt", + ToString(request->mTryCnt))( + "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); } mOutFailedItemsTotal->Add(1); mFailedItemTotalResponseTimeMs->Add(responseTime); diff --git a/core/unittest/monitor/PluginMetricManagerUnittest.cpp b/core/unittest/monitor/PluginMetricManagerUnittest.cpp index 2e0db37022..9f2dd38df8 100644 --- a/core/unittest/monitor/PluginMetricManagerUnittest.cpp +++ b/core/unittest/monitor/PluginMetricManagerUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/metric_constants/MetricConstants.h" #include "monitor/PluginMetricManager.h" +#include "monitor/metric_constants/MetricConstants.h" #include "unittest/Unittest.h" namespace logtail { @@ -32,7 +32,8 @@ class PluginMetricManagerUnittest : public ::testing::Test { std::unordered_map metricKeys; metricKeys.emplace("default_counter", MetricType::METRIC_TYPE_COUNTER); metricKeys.emplace("default_gauge", MetricType::METRIC_TYPE_INT_GAUGE); - pluginMetricManager = std::make_shared(mMetricsRecordRef->GetLabels(), metricKeys); + pluginMetricManager = std::make_shared( + mMetricsRecordRef->GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); } void TearDown() {} From 51215578b55d1f208acdfc4bdf0965ec964022af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E9=A3=8F?= Date: Thu, 7 Nov 2024 09:50:47 +0800 Subject: [PATCH 2/6] fix ut --- core/unittest/batch/BatcherUnittest.cpp | 2 +- core/unittest/compression/CompressorFactoryUnittest.cpp | 2 +- core/unittest/pipeline/PipelineUnittest.cpp | 2 +- core/unittest/queue/BoundedProcessQueueUnittest.cpp | 2 +- core/unittest/queue/CircularProcessQueueUnittest.cpp | 2 +- core/unittest/queue/SenderQueueUnittest.cpp | 2 +- core/unittest/route/RouterUnittest.cpp | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/unittest/batch/BatcherUnittest.cpp b/core/unittest/batch/BatcherUnittest.cpp index d75b2ed9ec..26333aabee 100644 --- a/core/unittest/batch/BatcherUnittest.cpp +++ b/core/unittest/batch/BatcherUnittest.cpp @@ -576,7 +576,7 @@ void BatcherUnittest::TestMetric() { vector res; batch.Add(std::move(g), res); - APSARA_TEST_EQUAL(6U, batch.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(5U, batch.mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER)); diff --git a/core/unittest/compression/CompressorFactoryUnittest.cpp b/core/unittest/compression/CompressorFactoryUnittest.cpp index 3a9aefb8eb..e9244c3904 100644 --- a/core/unittest/compression/CompressorFactoryUnittest.cpp +++ b/core/unittest/compression/CompressorFactoryUnittest.cpp @@ -95,7 +95,7 @@ void CompressorFactoryUnittest::TestCompressTypeToString() { void CompressorFactoryUnittest::TestMetric() { auto compressor = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); - APSARA_TEST_EQUAL(5U, compressor->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(4U, compressor->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR)); diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 1cf92420d5..7c980dc386 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -124,7 +124,7 @@ void PipelineUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL(QueueKeyManager::GetInstance()->GetKey("test_config-flusher_sls-test_project#test_logstore"), pipeline->GetContext().GetLogstoreKey()); APSARA_TEST_EQUAL(0, pipeline->mInProcessCnt.load()); - APSARA_TEST_EQUAL(3U, pipeline->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(2U, pipeline->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, configName)); APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "test_project")); diff --git a/core/unittest/queue/BoundedProcessQueueUnittest.cpp b/core/unittest/queue/BoundedProcessQueueUnittest.cpp index cb14522036..08a178b207 100644 --- a/core/unittest/queue/BoundedProcessQueueUnittest.cpp +++ b/core/unittest/queue/BoundedProcessQueueUnittest.cpp @@ -118,7 +118,7 @@ void BoundedProcessQueueUnittest::TestPop() { } void BoundedProcessQueueUnittest::TestMetric() { - APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index 3708fdc309..cd80c823ad 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -151,7 +151,7 @@ void CircularProcessQueueUnittest::TestReset() { } void CircularProcessQueueUnittest::TestMetric() { - APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); diff --git a/core/unittest/queue/SenderQueueUnittest.cpp b/core/unittest/queue/SenderQueueUnittest.cpp index b31c68edda..78cfa287d6 100644 --- a/core/unittest/queue/SenderQueueUnittest.cpp +++ b/core/unittest/queue/SenderQueueUnittest.cpp @@ -174,7 +174,7 @@ void SenderQueueUnittest::TestGetAvailableItems() { } void SenderQueueUnittest::TestMetric() { - APSARA_TEST_EQUAL(6U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE)); diff --git a/core/unittest/route/RouterUnittest.cpp b/core/unittest/route/RouterUnittest.cpp index 4d7ae94531..edc78c7a0b 100644 --- a/core/unittest/route/RouterUnittest.cpp +++ b/core/unittest/route/RouterUnittest.cpp @@ -169,7 +169,7 @@ void RouterUnittest::TestMetric() { Router router; router.Init(configs, ctx); - APSARA_TEST_EQUAL(4U, router.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(3U, router.mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE( From 2fc6cfefbe80320d4039de683cc905f884d9bbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E9=A3=8F?= Date: Thu, 7 Nov 2024 09:31:22 +0000 Subject: [PATCH 3/6] polish --- core/common/compression/Compressor.cpp | 2 +- core/ebpf/eBPFServer.cpp | 4 +- core/file_server/FileServer.cpp | 5 +- core/monitor/LogtailMetric.cpp | 12 ++-- core/monitor/LogtailMetric.h | 8 +-- core/monitor/MetricExportor.cpp | 4 +- core/monitor/Monitor.cpp | 2 +- .../monitor/metric_constants/AgentMetrics.cpp | 2 +- .../metric_constants/ComponentMetrics.cpp | 12 ++-- .../metric_constants/PipelineMetrics.cpp | 16 ++--- .../metric_constants/PluginMetrics.cpp | 11 ++-- .../metric_constants/RunnerMetrics.cpp | 4 +- core/pipeline/Pipeline.cpp | 5 +- core/pipeline/batch/Batcher.h | 2 +- core/pipeline/plugin/interface/Plugin.h | 5 +- core/pipeline/queue/QueueInterface.h | 5 +- core/pipeline/route/Router.cpp | 5 +- core/pipeline/serializer/Serializer.h | 5 +- core/prometheus/PrometheusInputRunner.cpp | 2 +- .../prometheus/schedulers/ScrapeScheduler.cpp | 2 +- .../schedulers/TargetSubscriberScheduler.cpp | 2 +- core/runner/FlusherRunner.cpp | 5 +- core/runner/ProcessorRunner.cpp | 5 +- core/runner/sink/http/HttpSink.cpp | 5 +- .../monitor/LogtailMetricUnittest.cpp | 14 ++--- .../monitor/PluginMetricManagerUnittest.cpp | 2 +- core/unittest/pipeline/PipelineUnittest.cpp | 6 +- .../prometheus/PromSelfMonitorUnittest.cpp | 4 +- pkg/helper/self_metrics_agent_constants.go | 4 +- pkg/helper/self_metrics_plugin_constants.go | 58 +++++++++---------- pkg/helper/self_metrics_runner_constants.go | 16 ++--- 31 files changed, 110 insertions(+), 124 deletions(-) diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp index b445ac687a..fd3527e1e6 100644 --- a/core/common/compression/Compressor.cpp +++ b/core/common/compression/Compressor.cpp @@ -24,7 +24,7 @@ namespace logtail { void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), MetricCategory::METRIC_CATEGORY_COMPONENT); + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_COMPONENT, std::move(labels), std::move(dynamicLabels)); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); diff --git a/core/ebpf/eBPFServer.cpp b/core/ebpf/eBPFServer.cpp index ff5842503e..501f833806 100644 --- a/core/ebpf/eBPFServer.cpp +++ b/core/ebpf/eBPFServer.cpp @@ -152,9 +152,9 @@ void eBPFServer::Init() { DynamicMetricLabels dynamicLabels; dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); }); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mRef, + MetricCategory::METRIC_CATEGORY_RUNNER, {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER}}, - std::move(dynamicLabels), - MetricCategory::METRIC_CATEGORY_RUNNER); + std::move(dynamicLabels)); mStartPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL); mStopPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL); diff --git a/core/file_server/FileServer.cpp b/core/file_server/FileServer.cpp index 0dd65a5780..0741efefca 100644 --- a/core/file_server/FileServer.cpp +++ b/core/file_server/FileServer.cpp @@ -34,9 +34,8 @@ namespace logtail { FileServer::FileServer() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}}, - {}, - MetricCategory::METRIC_CATEGORY_RUNNER); + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}}); } // 启动文件服务，包括加载配置、处理检查点、注册事件等 diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index 7463b215a8..c2152e4c1f 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -192,7 +192,7 @@ void ReentrantMetricsRecord::Init(std::string category, DynamicMetricLabels& dynamicLabels, std::unordered_map& metricKeys) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), category); + mMetricsRecordRef, category, std::move(labels), std::move(dynamicLabels)); for (auto metric : metricKeys) { switch (metric.second) { case MetricType::METRIC_TYPE_COUNTER: @@ -257,17 +257,17 @@ WriteMetrics::~WriteMetrics() { } void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, + std::string category, MetricLabels&& labels, - DynamicMetricLabels&& dynamicLabels, - std::string category) { - CreateMetricsRecordRef(ref, std::move(labels), std::move(dynamicLabels), std::move(category)); + DynamicMetricLabels&& dynamicLabels) { + CreateMetricsRecordRef(ref, std::move(category), std::move(labels), std::move(dynamicLabels)); CommitMetricsRecordRef(ref); } void WriteMetrics::CreateMetricsRecordRef(MetricsRecordRef& ref, + std::string category, MetricLabels&& labels, - DynamicMetricLabels&& dynamicLabels, - std::string category) { + DynamicMetricLabels&& dynamicLabels) { MetricsRecord* cur = new MetricsRecord( std::make_shared(labels), std::make_shared(dynamicLabels), category); ref.SetMetricsRecord(cur); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 79783af4a5..8b7d3962cc 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -164,13 +164,13 @@ class WriteMetrics { } void PrepareMetricsRecordRef(MetricsRecordRef& ref, + std::string category, MetricLabels&& labels, - DynamicMetricLabels&& dynamicLabels = {}, - std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN); + DynamicMetricLabels&& dynamicLabels = {}); void CreateMetricsRecordRef(MetricsRecordRef& ref, + std::string category, MetricLabels&& labels, - DynamicMetricLabels&& dynamicLabels = {}, - std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN); + DynamicMetricLabels&& dynamicLabels = {}); void CommitMetricsRecordRef(MetricsRecordRef& ref); MetricsRecord* DoSnapshot(); diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index 98087b400a..99c829d8d1 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -173,10 +173,10 @@ void MetricExportor::PushGoCppProvidedMetrics(std::vectorSetAgentGoMemory(std::stoi(metric.second)); } - if (metric.first == METRIC_AGENT_GO_ROUTINES_TOTAL) { + if (metric.first == METRIC_KEY_VALUE + "." + METRIC_AGENT_GO_ROUTINES_TOTAL) { LoongCollectorMonitor::GetInstance()->SetAgentGoRoutinesTotal(std::stoi(metric.second)); } LogtailMonitor::GetInstance()->UpdateMetric(metric.first, metric.second); diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 9accba15e3..3bb00943f1 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -709,7 +709,7 @@ void LoongCollectorMonitor::Init() { }); #endif WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), MetricCategory::METRIC_CATEGORY_AGENT); + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_AGENT, std::move(labels), std::move(dynamicLabels)); // init value mAgentCpu = mMetricsRecordRef.CreateDoubleGauge(METRIC_AGENT_CPU); mAgentMemory = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_MEMORY); diff --git a/core/monitor/metric_constants/AgentMetrics.cpp b/core/monitor/metric_constants/AgentMetrics.cpp index 6f81fcc437..ad17130e6c 100644 --- a/core/monitor/metric_constants/AgentMetrics.cpp +++ b/core/monitor/metric_constants/AgentMetrics.cpp @@ -30,7 +30,7 @@ const string METRIC_LABEL_KEY_UUID = "uuid"; const string METRIC_LABEL_KEY_VERSION = "version"; // metric keys -const string METRIC_AGENT_CPU = "cpu_percent"; +const string METRIC_AGENT_CPU = "cpu"; const string METRIC_AGENT_GO_ROUTINES_TOTAL = "go_routines_total"; const string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "instance_config_total"; // Not Implemented const string METRIC_AGENT_MEMORY = "memory_used_mb"; diff --git a/core/monitor/metric_constants/ComponentMetrics.cpp b/core/monitor/metric_constants/ComponentMetrics.cpp index 2ae4825644..2d4d382164 100644 --- a/core/monitor/metric_constants/ComponentMetrics.cpp +++ b/core/monitor/metric_constants/ComponentMetrics.cpp @@ -75,13 +75,9 @@ const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = "discarded_events_t const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "fetched_items_total"; const string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL = "fetch_times_total"; const string METRIC_COMPONENT_QUEUE_VALID_FETCH_TIMES_TOTAL = "valid_fetch_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_REGION_LIMITER_TIMES_TOTAL - = "fetch_rejected_by_region_limiter_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_PROJECT_LIMITER_TIMES_TOTAL - = "fetch_rejected_by_project_limiter_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_LOGSTORE_LIMITER_TIMES_TOTAL - = "fetch_rejected_by_logstore_limiter_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL - = "fetch_rejected_by_rate_limiter_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_REGION_LIMITER_TIMES_TOTAL = "region_rejects_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_PROJECT_LIMITER_TIMES_TOTAL = "project_rejects_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_LOGSTORE_LIMITER_TIMES_TOTAL = "logstore_rejects_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL = "rate_rejects_times_total"; } // namespace logtail diff --git a/core/monitor/metric_constants/PipelineMetrics.cpp b/core/monitor/metric_constants/PipelineMetrics.cpp index 83f76c7690..b82904101c 100644 --- a/core/monitor/metric_constants/PipelineMetrics.cpp +++ b/core/monitor/metric_constants/PipelineMetrics.cpp @@ -24,14 +24,14 @@ const string METRIC_LABEL_KEY_PIPELINE_NAME = "pipeline_name"; const string METRIC_LABEL_KEY_REGION = "region"; // metric keys -const string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "processors_in_events_total"; -const string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "processors_in_event_groups_total"; -const string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES = "processors_in_size_bytes"; -const string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS = "processors_total_process_time_ms"; -const string METRIC_PIPELINE_FLUSHERS_IN_EVENTS_TOTAL = "flushers_in_events_total"; -const string METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL = "flushers_in_event_groups_total"; -const string METRIC_PIPELINE_FLUSHERS_IN_SIZE_BYTES = "flushers_in_size_bytes"; -const string METRIC_PIPELINE_FLUSHERS_TOTAL_PACKAGE_TIME_MS = "flushers_total_package_time_ms"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "processor_in_events_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "processor_in_event_groups_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES = "processor_in_size_bytes"; +const string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS = "processor_total_process_time_ms"; +const string METRIC_PIPELINE_FLUSHERS_IN_EVENTS_TOTAL = "flusher_in_events_total"; +const string METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL = "flusher_in_event_groups_total"; +const string METRIC_PIPELINE_FLUSHERS_IN_SIZE_BYTES = "flusher_in_size_bytes"; +const string METRIC_PIPELINE_FLUSHERS_TOTAL_PACKAGE_TIME_MS = "flusher_total_package_time_ms"; const string METRIC_PIPELINE_START_TIME = "start_time"; } // namespace logtail diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index eebe3892d9..c01b6ff7a4 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -90,12 +90,11 @@ const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_OBSERVER = "process_observer const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_SECURITY = "process_security"; const string METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL = "ebpf_loss_kernel_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM = "network_observer_conntracker_num"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL - = "network_observer_worker_handle_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL = "network_observer_parse_records_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL = "network_observer_aggregate_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM = "network_observer_aggregate_key_num"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM = "conntracker_num"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL = "handle_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL = "parse_records_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL = "aggregate_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM = "aggregate_key_num"; const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM = "process_cache_entries_num"; const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "process_cache_miss_total"; diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index e6b8aa7505..94ab9c8302 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -46,8 +46,8 @@ const string METRIC_RUNNER_JOBS_TOTAL = "jobs_total"; **********************************************************/ const string METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL = "out_successful_items_total"; const string METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL = "out_failed_items_total"; -const string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS = "successful_item_total_response_time_ms"; -const string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS = "failed_item_total_response_time_ms"; +const string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS = "successful_response_time_ms"; +const string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS = "failed_response_time_ms"; const string METRIC_RUNNER_SINK_SENDING_ITEMS_TOTAL = "sending_items_total"; const string METRIC_RUNNER_SINK_SEND_CONCURRENCY = "send_concurrency"; diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 8a13886888..d80ebc5064 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -318,9 +318,8 @@ bool Pipeline::Init(PipelineConfig&& config) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mName}}, - {}, - MetricCategory::METRIC_CATEGORY_PIPELINE); + MetricCategory::METRIC_CATEGORY_PIPELINE, + {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mName}}); mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); mProcessorsInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL); mProcessorsInGroupsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL); diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index 0e4cf304ac..2b1898fd55 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -111,7 +111,7 @@ class Batcher { labels.emplace_back(METRIC_LABEL_KEY_GROUP_BATCH_ENABLED, "false"); } WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), {}, MetricCategory::METRIC_CATEGORY_COMPONENT); + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_COMPONENT, std::move(labels)); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_EVENTS_TOTAL); diff --git a/core/pipeline/plugin/interface/Plugin.h b/core/pipeline/plugin/interface/Plugin.h index a9b7012c0a..1dbd872756 100644 --- a/core/pipeline/plugin/interface/Plugin.h +++ b/core/pipeline/plugin/interface/Plugin.h @@ -39,12 +39,11 @@ class Plugin { void SetMetricsRecordRef(const std::string& name, const std::string& id) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_PLUGIN, {{METRIC_LABEL_KEY_PROJECT, mContext->GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mContext->GetConfigName()}, {METRIC_LABEL_KEY_PLUGIN_TYPE, name}, - {METRIC_LABEL_KEY_PLUGIN_ID, id}}, - {}, - MetricCategory::METRIC_CATEGORY_PLUGIN); + {METRIC_LABEL_KEY_PLUGIN_ID, id}}); } protected: diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index a4cf487524..b0ea6d0e47 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -28,12 +28,11 @@ class QueueInterface { public: QueueInterface(QueueKey key, size_t cap, const PipelineContext& ctx) : mKey(key), mCapacity(cap) { WriteMetrics::GetInstance()->CreateMetricsRecordRef(mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_COMPONENT, { {METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, - }, - {}, - MetricCategory::METRIC_CATEGORY_COMPONENT); + }); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index 6fe40a0865..521f4b0404 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -37,11 +37,10 @@ bool Router::Init(std::vector> configs, const P WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_COMPONENT, {{METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, - {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER}}, - {}, - MetricCategory::METRIC_CATEGORY_COMPONENT); + {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER}}); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); return true; diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index 8b40607398..ba03eb65f8 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -49,12 +49,11 @@ class Serializer { Serializer(Flusher* f) : mFlusher(f) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_COMPONENT, {{METRIC_LABEL_KEY_PROJECT, f->GetContext().GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, f->GetContext().GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER}, - {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, f->GetPluginID()}}, - {}, - MetricCategory::METRIC_CATEGORY_COMPONENT); + {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, f->GetPluginID()}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index fee0c58a33..59cc58d4d5 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -63,7 +63,7 @@ PrometheusInputRunner::PrometheusInputRunner() dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); }); WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels), MetricCategory::METRIC_CATEGORY_RUNNER); + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_RUNNER, std::move(labels), std::move(dynamicLabels)); mPromRegisterState = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_CLIENT_REGISTER_STATE); mPromJobNum = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_JOBS_TOTAL); diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index 33ef0886ad..0908cb6ea7 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -221,7 +221,7 @@ void ScrapeScheduler::InitSelfMonitor(const MetricLabels& defaultLabels) { mSelfMonitor->InitMetricManager(sScrapeMetricKeys, labels); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE, std::move(labels)); mPromDelayTotal = mMetricsRecordRef.CreateCounter(METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL); mPluginTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PLUGIN_TOTAL_DELAY_MS); } diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index 96de4f8ee6..d44948e469 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -334,7 +334,7 @@ void TargetSubscriberScheduler::InitSelfMonitor(const MetricLabels& defaultLabel mSelfMonitor = std::make_shared(); mSelfMonitor->InitMetricManager(sSubscriberMetricKeys, mDefaultLabels); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(mDefaultLabels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE, std::move(mDefaultLabels)); mPromSubscriberTargets = mMetricsRecordRef.CreateIntGauge(METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS); mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PLUGIN_TOTAL_DELAY_MS); } diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 99a66ab120..c98acd7bf2 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -45,9 +45,8 @@ bool FlusherRunner::Init() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER}}, - {}, - MetricCategory::METRIC_CATEGORY_RUNNER); + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_OUT_ITEMS_TOTAL); diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index 31d328e419..9f5580d4b1 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -89,9 +89,8 @@ void ProcessorRunner::Run(uint32_t threadNo) { // thread local metrics should be initialized in each thread WriteMetrics::GetInstance()->PrepareMetricsRecordRef( sMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, {"thread_no", ToString(threadNo)}}, - {}, - MetricCategory::METRIC_CATEGORY_RUNNER); + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, {"thread_no", ToString(threadNo)}}); sInGroupsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL); sInEventsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENTS_TOTAL); sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index e64d7e6566..4429ce40ec 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -40,9 +40,8 @@ bool HttpSink::Init() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK}}, - {}, - MetricCategory::METRIC_CATEGORY_RUNNER); + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); mOutSuccessfulItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL); diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 9a1c680f51..30d5822e47 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -54,7 +54,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); APSARA_TEST_EQUAL(fileMetric->GetLabels()->size(), 3); @@ -92,7 +92,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric2; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter2 = fileMetric2.CreateCounter("filed2"); fileCounter2->Add(222UL); } @@ -103,7 +103,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("logstore", "logstore1")); labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric3; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter3 = fileMetric3.CreateCounter("filed3"); fileCounter3->Add(333UL); } @@ -144,7 +144,7 @@ void createMetrics(int count) { labels.emplace_back(std::make_pair("count", std::to_string(count))); labels.emplace_back(std::make_pair("region", "cn-beijing")); MetricsRecordRef fileMetric; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric.CreateCounter("filed1"); fileCounter->Add(111UL); } @@ -212,7 +212,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test1")); labels.emplace_back(std::make_pair("logstore", "test1")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric1->CreateCounter("filed1"); fileCounter->Add(111UL); @@ -221,7 +221,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test2")); labels.emplace_back(std::make_pair("logstore", "test2")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric2->CreateCounter("filed1"); fileCounter->Add(111UL); } @@ -231,7 +231,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test3")); labels.emplace_back(std::make_pair("logstore", "test3")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric3->CreateCounter("filed1"); fileCounter->Add(111UL); } diff --git a/core/unittest/monitor/PluginMetricManagerUnittest.cpp b/core/unittest/monitor/PluginMetricManagerUnittest.cpp index 9f2dd38df8..41ef503c75 100644 --- a/core/unittest/monitor/PluginMetricManagerUnittest.cpp +++ b/core/unittest/monitor/PluginMetricManagerUnittest.cpp @@ -28,7 +28,7 @@ class PluginMetricManagerUnittest : public ::testing::Test { defaultLabels->emplace_back(METRIC_LABEL_KEY_PIPELINE_NAME, "default_config"); defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_TYPE, "default_plugin"); defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_ID, "default_id"); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(*defaultLabels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(*defaultLabels)); std::unordered_map metricKeys; metricKeys.emplace("default_counter", MetricType::METRIC_TYPE_COUNTER); metricKeys.emplace("default_gauge", MetricType::METRIC_TYPE_INT_GAUGE); diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 7c980dc386..3adda9f2a3 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2702,7 +2702,7 @@ void PipelineUnittest::TestProcess() const { processor->Init(Json::Value(), ctx); pipeline.mProcessorLine.emplace_back(std::move(processor)); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, {}); pipeline.mProcessorsInEventsTotal = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL); pipeline.mProcessorsInGroupsTotal @@ -2750,7 +2750,7 @@ void PipelineUnittest::TestSend() const { configs.emplace_back(1, nullptr); pipeline.mRouter.Init(configs, ctx); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, {}); pipeline.mFlushersInGroupsTotal = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL); pipeline.mFlushersInEventsTotal @@ -2816,7 +2816,7 @@ void PipelineUnittest::TestSend() const { configs.emplace_back(configJson.size(), nullptr); pipeline.mRouter.Init(configs, ctx); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, {}); pipeline.mFlushersInGroupsTotal = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL); pipeline.mFlushersInEventsTotal diff --git a/core/unittest/prometheus/PromSelfMonitorUnittest.cpp b/core/unittest/prometheus/PromSelfMonitorUnittest.cpp index 45501ca356..b262b7d8ec 100644 --- a/core/unittest/prometheus/PromSelfMonitorUnittest.cpp +++ b/core/unittest/prometheus/PromSelfMonitorUnittest.cpp @@ -23,7 +23,7 @@ void PromSelfMonitorUnittest::TestCounterAdd() { // check result auto metric = selfMonitor->mPromStatusMap["2XX"]->GetCounter(METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL); - APSARA_TEST_EQUAL("plugin_prom_subscribe_total", metric->GetName()); + APSARA_TEST_EQUAL("prom_subscribe_total", metric->GetName()); APSARA_TEST_EQUAL(999ULL, metric->GetValue()); selfMonitor->AddCounter(METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL, 200); APSARA_TEST_EQUAL(1000ULL, metric->GetValue()); @@ -41,7 +41,7 @@ void PromSelfMonitorUnittest::TestIntGaugeSet() { // check result auto metric = selfMonitor->mPromStatusMap["2XX"]->GetIntGauge(METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS); - APSARA_TEST_EQUAL("plugin_prom_subscribe_targets", metric->GetName()); + APSARA_TEST_EQUAL("prom_subscribe_targets", metric->GetName()); APSARA_TEST_EQUAL(999ULL, metric->GetValue()); selfMonitor->SetIntGauge(METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS, 200, 0); APSARA_TEST_EQUAL(0ULL, metric->GetValue()); diff --git a/pkg/helper/self_metrics_agent_constants.go b/pkg/helper/self_metrics_agent_constants.go index 52ed601c2e..e236e748da 100644 --- a/pkg/helper/self_metrics_agent_constants.go +++ b/pkg/helper/self_metrics_agent_constants.go @@ -20,6 +20,6 @@ package helper // metric keys const ( - MetricAgentMemoryGo = "agent_go_memory_used_mb" - MetricAgentGoRoutinesTotal = "agent_go_routines_total" + MetricAgentMemoryGo = "go_memory_used_mb" + MetricAgentGoRoutinesTotal = "go_routines_total" ) diff --git a/pkg/helper/self_metrics_plugin_constants.go b/pkg/helper/self_metrics_plugin_constants.go index e7638b215f..8b258bddec 100644 --- a/pkg/helper/self_metrics_plugin_constants.go +++ b/pkg/helper/self_metrics_plugin_constants.go @@ -37,28 +37,28 @@ const ( // metric keys const ( - MetricPluginInEventsTotal = "plugin_in_events_total" - MetricPluginInEventGroupsTotal = "plugin_in_event_groups_total" - MetricPluginInSizeBytes = "plugin_in_size_bytes" - MetricPluginOutEventsTotal = "plugin_out_events_total" - MetricPluginOutEventGroupsTotal = "plugin_out_event_groups_total" - MetricPluginOutSizeBytes = "plugin_out_size_bytes" - MetricPluginTotalDelayMs = "plugin_total_delay_ms" - MetricPluginTotalProcessTimeMs = "plugin_total_process_time_ms" + MetricPluginInEventsTotal = "in_events_total" + MetricPluginInEventGroupsTotal = "in_event_groups_total" + MetricPluginInSizeBytes = "in_size_bytes" + MetricPluginOutEventsTotal = "out_events_total" + MetricPluginOutEventGroupsTotal = "out_event_groups_total" + MetricPluginOutSizeBytes = "out_size_bytes" + MetricPluginTotalDelayMs = "total_delay_ms" + MetricPluginTotalProcessTimeMs = "total_process_time_ms" ) /********************************************************** * input_canal **********************************************************/ const ( - MetricPluginBinlogRotate = "plugin_binlog_rotate" - MetricPluginBinlogSync = "plugin_binlog_sync" - MetricPluginBinlogDdl = "plugin_binlog_ddl" - MetricPluginBinlogRow = "plugin_binlog_row" - MetricPluginBinlogXgid = "plugin_binlog_xgid" - MetricPluginBinlogCheckpoint = "plugin_binlog_checkpoint" - MetricPluginBinlogFilename = "plugin_binlog_filename" - MetricPluginBinlogGtid = "plugin_binlog_gtid" + MetricPluginBinlogRotate = "binlog_rotate" + MetricPluginBinlogSync = "binlog_sync" + MetricPluginBinlogDdl = "binlog_ddl" + MetricPluginBinlogRow = "binlog_row" + MetricPluginBinlogXgid = "binlog_xgid" + MetricPluginBinlogCheckpoint = "binlog_checkpoint" + MetricPluginBinlogFilename = "binlog_filename" + MetricPluginBinlogGtid = "binlog_gtid" ) /********************************************************** @@ -66,10 +66,10 @@ const ( * service_docker_stdout_v2 **********************************************************/ const ( - MetricPluginContainerTotal = "plugin_container_total" - MetricPluginAddContainerTotal = "plugin_add_container_total" - MetricPluginRemoveContainerTotal = "plugin_remove_container_total" - MetricPluginUpdateContainerTotal = "plugin_update_container_total" + MetricPluginContainerTotal = "container_total" + MetricPluginAddContainerTotal = "add_container_total" + MetricPluginRemoveContainerTotal = "remove_container_total" + MetricPluginUpdateContainerTotal = "update_container_total" ) /********************************************************** @@ -77,26 +77,26 @@ const ( * service_rdb **********************************************************/ const ( - MetricPluginCollectAvgCostTimeMs = "plugin_collect_avg_cost_time_ms" - MetricPluginCollectTotal = "plugin_collect_total" + MetricPluginCollectAvgCostTimeMs = "collect_avg_cost_time_ms" + MetricPluginCollectTotal = "collect_total" ) /********************************************************** * service_k8s_meta **********************************************************/ const ( - MetricCollectEntityTotal = "plugin_collect_entity_total" - MetricCollectLinkTotal = "plugin_collect_link_total" + MetricCollectEntityTotal = "collect_entity_total" + MetricCollectLinkTotal = "collect_link_total" ) /********************************************************** * all processor（所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ const ( - MetricPluginDiscardedEventsTotal = "plugin_discarded_events_total" - MetricPluginOutFailedEventsTotal = "plugin_out_failed_events_total" - MetricPluginOutKeyNotFoundEventsTotal = "plugin_out_key_not_found_events_total" - MetricPluginOutSuccessfulEventsTotal = "plugin_out_successful_events_total" + MetricPluginDiscardedEventsTotal = "discarded_events_total" + MetricPluginOutFailedEventsTotal = "out_failed_events_total" + MetricPluginOutKeyNotFoundEventsTotal = "out_key_not_found_events_total" + MetricPluginOutSuccessfulEventsTotal = "out_successful_events_total" ) /********************************************************** @@ -105,7 +105,7 @@ const ( * processor_string_replace **********************************************************/ const ( - PluginPairsPerLogTotal = "plugin_pairs_per_log_total" + PluginPairsPerLogTotal = "pairs_per_log_total" ) func GetPluginCommonLabels(context pipeline.Context, pluginMeta *pipeline.PluginMeta) []pipeline.LabelPair { diff --git a/pkg/helper/self_metrics_runner_constants.go b/pkg/helper/self_metrics_runner_constants.go index 317ac7f23d..29ce3d81bb 100644 --- a/pkg/helper/self_metrics_runner_constants.go +++ b/pkg/helper/self_metrics_runner_constants.go @@ -27,12 +27,12 @@ const ( * k8s meta **********************************************************/ const ( - MetricRunnerK8sMetaAddEventTotal = "runner_k8s_meta_add_event_total" - MetricRunnerK8sMetaUpdateEventTotal = "runner_k8s_meta_update_event_total" - MetricRunnerK8sMetaDeleteEventTotal = "runner_k8s_meta_delete_event_total" - MetricRunnerK8sMetaCacheSize = "runner_k8s_meta_cache_size" - MetricRunnerK8sMetaQueueSize = "runner_k8s_meta_queue_size" - MetricRunnerK8sMetaHTTPRequestTotal = "runner_k8s_meta_http_request_total" - MetricRunnerK8sMetaHTTPAvgDelayMs = "runner_k8s_meta_avg_delay_ms" - MetricRunnerK8sMetaHTTPMaxDelayMs = "runner_k8s_meta_max_delay_ms" + MetricRunnerK8sMetaAddEventTotal = "add_event_total" + MetricRunnerK8sMetaUpdateEventTotal = "update_event_total" + MetricRunnerK8sMetaDeleteEventTotal = "delete_event_total" + MetricRunnerK8sMetaCacheSize = "cache_size" + MetricRunnerK8sMetaQueueSize = "queue_size" + MetricRunnerK8sMetaHTTPRequestTotal = "http_request_total" + MetricRunnerK8sMetaHTTPAvgDelayMs = "avg_delay_ms" + MetricRunnerK8sMetaHTTPMaxDelayMs = "max_delay_ms" ) From b94c4df0a401dd73cdbf608434270a6c1a6eb733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E9=A3=8F?= Date: Mon, 11 Nov 2024 18:10:17 +0800 Subject: [PATCH 4/6] polish --- core/common/compression/Compressor.cpp | 2 +- .../metric_constants/ComponentMetrics.cpp | 22 ++++++------ .../MetricCommonConstants.cpp | 35 ++++++++++++++++++ .../metric_constants/MetricCommonConstants.h | 36 +++++++++++++++++++ .../metric_constants/MetricConstants.h | 3 +- .../metric_constants/PluginMetrics.cpp | 22 ++++++------ .../metric_constants/RunnerMetrics.cpp | 12 +++---- core/pipeline/serializer/Serializer.h | 2 +- pkg/helper/k8smeta/k8s_meta_manager.go | 10 +++--- pkg/helper/self_metrics_plugin_constants.go | 4 +-- pkg/helper/self_metrics_runner_constants.go | 19 +++++++++- 11 files changed, 128 insertions(+), 39 deletions(-) create mode 100644 core/monitor/metric_constants/MetricCommonConstants.cpp create mode 100644 core/monitor/metric_constants/MetricCommonConstants.h diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp index fd3527e1e6..134e6e4737 100644 --- a/core/common/compression/Compressor.cpp +++ b/core/common/compression/Compressor.cpp @@ -31,7 +31,7 @@ void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_SIZE_BYTES); mTotalProcessMs = mMetricsRecordRef.CreateTimeCounter(METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS); mDiscardedItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL); - mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_SIZE_BYTES); } bool Compressor::DoCompress(const string& input, string& output, string& errorMsg) { diff --git a/core/monitor/metric_constants/ComponentMetrics.cpp b/core/monitor/metric_constants/ComponentMetrics.cpp index 2d4d382164..39002ffea7 100644 --- a/core/monitor/metric_constants/ComponentMetrics.cpp +++ b/core/monitor/metric_constants/ComponentMetrics.cpp @@ -42,16 +42,16 @@ const string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE = "sender_queue"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER = "serializer"; // metric keys -const string METRIC_COMPONENT_IN_EVENTS_TOTAL = "in_events_total"; -const string METRIC_COMPONENT_IN_SIZE_BYTES = "in_size_bytes"; -const string METRIC_COMPONENT_IN_ITEMS_TOTAL = "in_items_total"; -const string METRIC_COMPONENT_OUT_EVENTS_TOTAL = "out_events_total"; -const string METRIC_COMPONENT_OUT_ITEMS_TOTAL = "out_items_total"; -const string METRIC_COMPONENT_OUT_SIZE_BYTES = "out_size_bytes"; -const string METRIC_COMPONENT_TOTAL_DELAY_MS = "total_delay_ms"; -const string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; -const string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = "discarded_items_total"; -const string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "discarded_item_size_bytes"; +const string METRIC_COMPONENT_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; +const string METRIC_COMPONENT_IN_SIZE_BYTES = IN_SIZE_BYTES; +const string METRIC_COMPONENT_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; +const string METRIC_COMPONENT_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; +const string METRIC_COMPONENT_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; +const string METRIC_COMPONENT_OUT_SIZE_BYTES = OUT_SIZE_BYTES; +const string METRIC_COMPONENT_TOTAL_DELAY_MS = TOTAL_DELAY_MS; +const string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; +const string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = DISCARDED_ITEMS_TOTAL; +const string METRIC_COMPONENT_DISCARDED_SIZE_BYTES = DISCARDED_SIZE_BYTES; /********************************************************** * batcher @@ -70,7 +70,7 @@ const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "queue_size_bytes"; const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "valid_to_push_status"; const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "extra_buffer_size"; const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "extra_buffer_size_bytes"; -const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = "discarded_events_total"; +const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "fetched_items_total"; const string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL = "fetch_times_total"; diff --git a/core/monitor/metric_constants/MetricCommonConstants.cpp b/core/monitor/metric_constants/MetricCommonConstants.cpp new file mode 100644 index 0000000000..a0d3180c65 --- /dev/null +++ b/core/monitor/metric_constants/MetricCommonConstants.cpp @@ -0,0 +1,35 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "MetricCommonConstants.h" + +using namespace std; + +namespace logtail { + +const string DISCARDED_EVENTS_TOTAL = "discarded_events_total"; +const string DISCARDED_ITEMS_TOTAL = "discarded_items_total"; +const string DISCARDED_SIZE_BYTES = "discarded_size_bytes"; +const string IN_EVENTS_TOTAL = "in_events_total"; +const string IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; +const string IN_ITEMS_TOTAL = "in_items_total"; +const string IN_SIZE_BYTES = "in_size_bytes"; +const string OUT_EVENTS_TOTAL = "out_events_total"; +const string OUT_EVENT_GROUPS_TOTAL = "out_event_groups_total"; +const string OUT_ITEMS_TOTAL = "out_items_total"; +const string OUT_SIZE_BYTES = "out_size_bytes"; +const string TOTAL_DELAY_MS = "total_delay_ms"; +const string TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; + +} \ No newline at end of file diff --git a/core/monitor/metric_constants/MetricCommonConstants.h b/core/monitor/metric_constants/MetricCommonConstants.h new file mode 100644 index 0000000000..5de8543efe --- /dev/null +++ b/core/monitor/metric_constants/MetricCommonConstants.h @@ -0,0 +1,36 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace logtail { + +extern const std::string DISCARDED_EVENTS_TOTAL; +extern const std::string DISCARDED_ITEMS_TOTAL; +extern const std::string DISCARDED_SIZE_BYTES; +extern const std::string IN_EVENTS_TOTAL; +extern const std::string IN_EVENT_GROUPS_TOTAL; +extern const std::string IN_ITEMS_TOTAL; +extern const std::string IN_SIZE_BYTES; +extern const std::string OUT_EVENTS_TOTAL; +extern const std::string OUT_EVENT_GROUPS_TOTAL; +extern const std::string OUT_ITEMS_TOTAL; +extern const std::string OUT_SIZE_BYTES; +extern const std::string TOTAL_DELAY_MS; +extern const std::string TOTAL_PROCESS_TIME_MS; + +} \ No newline at end of file diff --git a/core/monitor/metric_constants/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h index 7199843ce0..453f0bc6da 100644 --- a/core/monitor/metric_constants/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -16,6 +16,7 @@ #pragma once #include +#include "MetricCommonConstants.h" namespace logtail { @@ -229,7 +230,7 @@ extern const std::string METRIC_COMPONENT_OUT_SIZE_BYTES; extern const std::string METRIC_COMPONENT_TOTAL_DELAY_MS; extern const std::string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS; extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_DISCARDED_SIZE_BYTES; /********************************************************** * batcher diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index c01b6ff7a4..e89a25ffbd 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -23,14 +23,14 @@ const string METRIC_LABEL_KEY_PLUGIN_ID = "plugin_id"; const string METRIC_LABEL_KEY_PLUGIN_TYPE = "plugin_type"; // metric keys -const string METRIC_PLUGIN_IN_EVENTS_TOTAL = "in_events_total"; -const string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; -const string METRIC_PLUGIN_IN_SIZE_BYTES = "in_size_bytes"; -const string METRIC_PLUGIN_OUT_EVENTS_TOTAL = "out_events_total"; -const string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "out_event_groups_total"; -const string METRIC_PLUGIN_OUT_SIZE_BYTES = "out_size_bytes"; -const string METRIC_PLUGIN_TOTAL_DELAY_MS = "total_delay_ms"; -const string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; +const string METRIC_PLUGIN_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; +const string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; +const string METRIC_PLUGIN_IN_SIZE_BYTES = IN_SIZE_BYTES; +const string METRIC_PLUGIN_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; +const string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = OUT_EVENT_GROUPS_TOTAL; +const string METRIC_PLUGIN_OUT_SIZE_BYTES = OUT_SIZE_BYTES; +const string METRIC_PLUGIN_TOTAL_DELAY_MS = TOTAL_DELAY_MS; +const string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; /********************************************************** * input_file @@ -41,8 +41,8 @@ const string METRIC_LABEL_KEY_FILE_INODE = "file_inode"; const string METRIC_LABEL_KEY_FILE_NAME = "file_name"; const string METRIC_PLUGIN_MONITOR_FILE_TOTAL = "monitor_file_total"; -const string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "source_read_offset_bytes"; -const string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "source_size_bytes"; +const string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "read_offset_bytes"; +const string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "size_bytes"; /********************************************************** * input_prometheus @@ -101,7 +101,7 @@ const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "process_cache_miss_t /********************************************************** * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ -const string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = "discarded_events_total"; +const string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "out_failed_events_total"; const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "out_key_not_found_events_total"; const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "out_successful_events_total"; diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index 94ab9c8302..d37ab38322 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -30,13 +30,13 @@ const string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS = "prometheus_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER = "ebpf_server"; // metric keys -const string METRIC_RUNNER_IN_EVENTS_TOTAL = "in_events_total"; -const string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; -const string METRIC_RUNNER_IN_SIZE_BYTES = "in_size_bytes"; -const string METRIC_RUNNER_IN_ITEMS_TOTAL = "in_items_total"; +const string METRIC_RUNNER_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; +const string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; +const string METRIC_RUNNER_IN_SIZE_BYTES = IN_SIZE_BYTES; +const string METRIC_RUNNER_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; const string METRIC_RUNNER_LAST_RUN_TIME = "last_run_time"; -const string METRIC_RUNNER_OUT_ITEMS_TOTAL = "out_items_total"; -const string METRIC_RUNNER_TOTAL_DELAY_MS = "total_delay_ms"; +const string METRIC_RUNNER_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; +const string METRIC_RUNNER_TOTAL_DELAY_MS = TOTAL_DELAY_MS; const string METRIC_RUNNER_CLIENT_REGISTER_STATE = "client_register_state"; const string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL = "client_register_retry_total"; const string METRIC_RUNNER_JOBS_TOTAL = "jobs_total"; diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index ba03eb65f8..27cc2e0847 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -60,7 +60,7 @@ class Serializer { mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_SIZE_BYTES); mTotalProcessMs = mMetricsRecordRef.CreateTimeCounter(METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS); mDiscardedItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL); - mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_SIZE_BYTES); } virtual ~Serializer() = default; diff --git a/pkg/helper/k8smeta/k8s_meta_manager.go b/pkg/helper/k8smeta/k8s_meta_manager.go index 017e57782e..1432b1561e 100644 --- a/pkg/helper/k8smeta/k8s_meta_manager.go +++ b/pkg/helper/k8smeta/k8s_meta_manager.go @@ -226,18 +226,18 @@ func GetMetaManagerMetrics() []map[string]string { manager.metricRecord.Labels = []pipeline.Label{ { Key: helper.MetricLabelKeyMetricCategory, - Value: helper.MetricLabelKeyMetricCategoryRunner, + Value: helper.MetricLabelValueMetricCategoryRunner, }, { - Key: "cluster_id", + Key: helper.MetricLabelKeyClusterID, Value: *flags.ClusterID, }, { - Key: "runner_name", - Value: "k8s_meta_manager", + Key: helper.MetricLabelKeyRunnerName, + Value: helper.MetricLabelValueRunnerNameK8sMeta, }, { - Key: "project", + Key: helper.MetricLabelKeyProject, Value: strings.Join(projectName, " "), }, } diff --git a/pkg/helper/self_metrics_plugin_constants.go b/pkg/helper/self_metrics_plugin_constants.go index 8b258bddec..4233904edf 100644 --- a/pkg/helper/self_metrics_plugin_constants.go +++ b/pkg/helper/self_metrics_plugin_constants.go @@ -32,7 +32,7 @@ const ( // label values const ( - MetricLabelKeyMetricCategoryPlugin = "plugin" + MetricLabelValueMetricCategoryPlugin = "plugin" ) // metric keys @@ -110,7 +110,7 @@ const ( func GetPluginCommonLabels(context pipeline.Context, pluginMeta *pipeline.PluginMeta) []pipeline.LabelPair { labels := make([]pipeline.LabelPair, 0) - labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyMetricCategory, Value: MetricLabelKeyMetricCategoryPlugin}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyMetricCategory, Value: MetricLabelValueMetricCategoryPlugin}) labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyProject, Value: context.GetProject()}) labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyLogstore, Value: context.GetLogstore()}) labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyPipelineName, Value: context.GetConfigName()}) diff --git a/pkg/helper/self_metrics_runner_constants.go b/pkg/helper/self_metrics_runner_constants.go index 29ce3d81bb..fedcb15899 100644 --- a/pkg/helper/self_metrics_runner_constants.go +++ b/pkg/helper/self_metrics_runner_constants.go @@ -18,14 +18,31 @@ package helper // runner ////////////////////////////////////////////////////////////////////////// +// lebel keys +const ( + MetricLabelKeyRunnerName = "runner_name" +) + // label values const ( - MetricLabelKeyMetricCategoryRunner = "runner" + MetricLabelValueMetricCategoryRunner = "runner" ) /********************************************************** * k8s meta **********************************************************/ + +// label keys +const ( + MetricLabelKeyClusterID = "cluster_id" +) + +// label values +const ( + MetricLabelValueRunnerNameK8sMeta = "k8s_meta" +) + +// metric keys const ( MetricRunnerK8sMetaAddEventTotal = "add_event_total" MetricRunnerK8sMetaUpdateEventTotal = "update_event_total" From 9d2344b954c7bfd61fafff33ed3fd0a1b62b534a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E9=A3=8F?= Date: Tue, 12 Nov 2024 10:55:13 +0800 Subject: [PATCH 5/6] polish --- core/monitor/LogtailMetric.cpp | 20 +++--- core/monitor/LogtailMetric.h | 16 +++-- .../metric_constants/ComponentMetrics.cpp | 23 +++---- .../metric_constants/MetricConstants.h | 63 ++++++++++--------- .../metric_constants/PluginMetrics.cpp | 19 +++--- .../metric_constants/RunnerMetrics.cpp | 16 ++--- 6 files changed, 80 insertions(+), 77 deletions(-) diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index c2152e4c1f..239f6f8c9c 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -36,8 +36,8 @@ const std::string MetricCategory::METRIC_CATEGORY_COMPONENT = "component"; const std::string MetricCategory::METRIC_CATEGORY_PLUGIN = "plugin"; const std::string MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE = "plugin_source"; -MetricsRecord::MetricsRecord(MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels, std::string category) - : mLabels(labels), mDynamicLabels(dynamicLabels), mCategory(category), mDeleted(false) { +MetricsRecord::MetricsRecord(const std::string& category, MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels) + : mCategory(category), mLabels(labels), mDynamicLabels(dynamicLabels), mDeleted(false) { } CounterPtr MetricsRecord::CreateCounter(const std::string& name) { @@ -72,7 +72,7 @@ bool MetricsRecord::IsDeleted() const { return mDeleted; } -const std::string MetricsRecord::GetCategory() const { +const std::string& MetricsRecord::GetCategory() const { return mCategory; } @@ -101,7 +101,7 @@ const std::vector& MetricsRecord::GetDoubleGauges() const { } MetricsRecord* MetricsRecord::Collect() { - MetricsRecord* metrics = new MetricsRecord(mLabels, mDynamicLabels, mCategory); + MetricsRecord* metrics = new MetricsRecord(mCategory, mLabels, mDynamicLabels); for (auto& item : mCounters) { CounterPtr newPtr(item->Collect()); metrics->mCounters.emplace_back(newPtr); @@ -139,7 +139,7 @@ void MetricsRecordRef::SetMetricsRecord(MetricsRecord* metricRecord) { mMetrics = metricRecord; } -const std::string MetricsRecordRef::GetCategory() const { +const std::string& MetricsRecordRef::GetCategory() const { return mMetrics->GetCategory(); } @@ -187,7 +187,7 @@ bool MetricsRecordRef::HasLabel(const std::string& key, const std::string& value #endif // ReentrantMetricsRecord相关操作可以无锁，因为mCounters、mGauges只在初始化时会添加内容，后续只允许Get操作 -void ReentrantMetricsRecord::Init(std::string category, +void ReentrantMetricsRecord::Init(const std::string& category, MetricLabels& labels, DynamicMetricLabels& dynamicLabels, std::unordered_map& metricKeys) { @@ -257,19 +257,19 @@ WriteMetrics::~WriteMetrics() { } void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, - std::string category, + const std::string& category, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { - CreateMetricsRecordRef(ref, std::move(category), std::move(labels), std::move(dynamicLabels)); + CreateMetricsRecordRef(ref, category, std::move(labels), std::move(dynamicLabels)); CommitMetricsRecordRef(ref); } void WriteMetrics::CreateMetricsRecordRef(MetricsRecordRef& ref, - std::string category, + const std::string& category, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { MetricsRecord* cur = new MetricsRecord( - std::make_shared(labels), std::make_shared(dynamicLabels), category); + category, std::make_shared(labels), std::make_shared(dynamicLabels)); ref.SetMetricsRecord(cur); } diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 8b7d3962cc..2065f2184f 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -44,9 +44,9 @@ class MetricCategory { class MetricsRecord { private: + std::string mCategory; MetricLabelsPtr mLabels; DynamicMetricLabelsPtr mDynamicLabels; - std::string mCategory; std::vector mCounters; std::vector mTimeCounters; std::vector mIntGauges; @@ -56,13 +56,11 @@ class MetricsRecord { MetricsRecord* mNext = nullptr; public: - MetricsRecord(MetricLabelsPtr labels, - DynamicMetricLabelsPtr dynamicLabels = nullptr, - std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN); + MetricsRecord(const std::string& category, MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels = nullptr); MetricsRecord() = default; void MarkDeleted(); bool IsDeleted() const; - const std::string GetCategory() const; + const std::string& GetCategory() const; const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; const std::vector& GetCounters() const; @@ -94,7 +92,7 @@ class MetricsRecordRef { MetricsRecordRef(MetricsRecordRef&&) = delete; MetricsRecordRef& operator=(MetricsRecordRef&&) = delete; void SetMetricsRecord(MetricsRecord* metricRecord); - const std::string GetCategory() const; + const std::string& GetCategory() const; const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; CounterPtr CreateCounter(const std::string& name); @@ -134,7 +132,7 @@ class ReentrantMetricsRecord { std::unordered_map mDoubleGauges; public: - void Init(std::string category, + void Init(const std::string& category, MetricLabels& labels, DynamicMetricLabels& dynamicLabels, std::unordered_map& metricKeys); @@ -164,11 +162,11 @@ class WriteMetrics { } void PrepareMetricsRecordRef(MetricsRecordRef& ref, - std::string category, + const std::string& category, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); void CreateMetricsRecordRef(MetricsRecordRef& ref, - std::string category, + const std::string& category, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); void CommitMetricsRecordRef(MetricsRecordRef& ref); diff --git a/core/monitor/metric_constants/ComponentMetrics.cpp b/core/monitor/metric_constants/ComponentMetrics.cpp index 39002ffea7..9a8e164096 100644 --- a/core/monitor/metric_constants/ComponentMetrics.cpp +++ b/core/monitor/metric_constants/ComponentMetrics.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "MetricCommonConstants.h" #include "MetricConstants.h" using namespace std; @@ -42,16 +43,16 @@ const string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE = "sender_queue"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER = "serializer"; // metric keys -const string METRIC_COMPONENT_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; -const string METRIC_COMPONENT_IN_SIZE_BYTES = IN_SIZE_BYTES; -const string METRIC_COMPONENT_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; -const string METRIC_COMPONENT_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; -const string METRIC_COMPONENT_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; -const string METRIC_COMPONENT_OUT_SIZE_BYTES = OUT_SIZE_BYTES; -const string METRIC_COMPONENT_TOTAL_DELAY_MS = TOTAL_DELAY_MS; -const string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; -const string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = DISCARDED_ITEMS_TOTAL; -const string METRIC_COMPONENT_DISCARDED_SIZE_BYTES = DISCARDED_SIZE_BYTES; +const string& METRIC_COMPONENT_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; +const string& METRIC_COMPONENT_IN_SIZE_BYTES = IN_SIZE_BYTES; +const string& METRIC_COMPONENT_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; +const string& METRIC_COMPONENT_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; +const string& METRIC_COMPONENT_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; +const string& METRIC_COMPONENT_OUT_SIZE_BYTES = OUT_SIZE_BYTES; +const string& METRIC_COMPONENT_TOTAL_DELAY_MS = TOTAL_DELAY_MS; +const string& METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; +const string& METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = DISCARDED_ITEMS_TOTAL; +const string& METRIC_COMPONENT_DISCARDED_SIZE_BYTES = DISCARDED_SIZE_BYTES; /********************************************************** * batcher @@ -70,7 +71,7 @@ const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "queue_size_bytes"; const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "valid_to_push_status"; const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "extra_buffer_size"; const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "extra_buffer_size_bytes"; -const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; +const string& METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "fetched_items_total"; const string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL = "fetch_times_total"; diff --git a/core/monitor/metric_constants/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h index 453f0bc6da..e6627f8714 100644 --- a/core/monitor/metric_constants/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -16,7 +16,6 @@ #pragma once #include -#include "MetricCommonConstants.h" namespace logtail { @@ -72,14 +71,14 @@ extern const std::string METRIC_LABEL_KEY_PLUGIN_ID; extern const std::string METRIC_LABEL_KEY_PLUGIN_TYPE; // metric keys -extern const std::string METRIC_PLUGIN_IN_EVENTS_TOTAL; -extern const std::string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_PLUGIN_IN_SIZE_BYTES; -extern const std::string METRIC_PLUGIN_OUT_EVENTS_TOTAL; -extern const std::string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_PLUGIN_OUT_SIZE_BYTES; -extern const std::string METRIC_PLUGIN_TOTAL_DELAY_MS; -extern const std::string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS; +extern const std::string& METRIC_PLUGIN_IN_EVENTS_TOTAL; +extern const std::string& METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL; +extern const std::string& METRIC_PLUGIN_IN_SIZE_BYTES; +extern const std::string& METRIC_PLUGIN_OUT_EVENTS_TOTAL; +extern const std::string& METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL; +extern const std::string& METRIC_PLUGIN_OUT_SIZE_BYTES; +extern const std::string& METRIC_PLUGIN_TOTAL_DELAY_MS; +extern const std::string& METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS; /********************************************************** * input_file @@ -150,7 +149,7 @@ extern const std::string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL; /********************************************************** * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ -extern const std::string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL; +extern const std::string& METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL; @@ -221,16 +220,16 @@ extern const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE; extern const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER; // metric keys -extern const std::string METRIC_COMPONENT_IN_EVENTS_TOTAL; -extern const std::string METRIC_COMPONENT_IN_SIZE_BYTES; -extern const std::string METRIC_COMPONENT_IN_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_OUT_EVENTS_TOTAL; -extern const std::string METRIC_COMPONENT_OUT_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_OUT_SIZE_BYTES; -extern const std::string METRIC_COMPONENT_TOTAL_DELAY_MS; -extern const std::string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS; -extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_DISCARDED_SIZE_BYTES; +extern const std::string& METRIC_COMPONENT_IN_EVENTS_TOTAL; +extern const std::string& METRIC_COMPONENT_IN_SIZE_BYTES; +extern const std::string& METRIC_COMPONENT_IN_ITEMS_TOTAL; +extern const std::string& METRIC_COMPONENT_OUT_EVENTS_TOTAL; +extern const std::string& METRIC_COMPONENT_OUT_ITEMS_TOTAL; +extern const std::string& METRIC_COMPONENT_OUT_SIZE_BYTES; +extern const std::string& METRIC_COMPONENT_TOTAL_DELAY_MS; +extern const std::string& METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS; +extern const std::string& METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL; +extern const std::string& METRIC_COMPONENT_DISCARDED_SIZE_BYTES; /********************************************************** * batcher @@ -249,7 +248,7 @@ extern const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES; extern const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG; extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE; extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES; -extern const std::string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL; +extern const std::string& METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL; extern const std::string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL; extern const std::string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL; @@ -275,22 +274,26 @@ extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS; extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER; // metric keys -extern const std::string METRIC_RUNNER_IN_EVENTS_TOTAL; -extern const std::string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_RUNNER_IN_SIZE_BYTES; -extern const std::string METRIC_RUNNER_IN_ITEMS_TOTAL; +extern const std::string& METRIC_RUNNER_IN_EVENTS_TOTAL; +extern const std::string& METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL; +extern const std::string& METRIC_RUNNER_IN_SIZE_BYTES; +extern const std::string& METRIC_RUNNER_IN_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_LAST_RUN_TIME; -extern const std::string METRIC_RUNNER_OUT_ITEMS_TOTAL; -extern const std::string METRIC_RUNNER_TOTAL_DELAY_MS; +extern const std::string& METRIC_RUNNER_OUT_ITEMS_TOTAL; +extern const std::string& METRIC_RUNNER_TOTAL_DELAY_MS; +extern const std::string METRIC_RUNNER_CLIENT_REGISTER_STATE; +extern const std::string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL; +extern const std::string METRIC_RUNNER_JOBS_TOTAL; + +/********************************************************** + * all sinks + **********************************************************/ extern const std::string METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS; extern const std::string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS; extern const std::string METRIC_RUNNER_SINK_SENDING_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_SINK_SEND_CONCURRENCY; -extern const std::string METRIC_RUNNER_CLIENT_REGISTER_STATE; -extern const std::string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL; -extern const std::string METRIC_RUNNER_JOBS_TOTAL; /********************************************************** * flusher runner diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index e89a25ffbd..808e59ee68 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "MetricCommonConstants.h" #include "MetricConstants.h" using namespace std; @@ -23,14 +24,14 @@ const string METRIC_LABEL_KEY_PLUGIN_ID = "plugin_id"; const string METRIC_LABEL_KEY_PLUGIN_TYPE = "plugin_type"; // metric keys -const string METRIC_PLUGIN_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; -const string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; -const string METRIC_PLUGIN_IN_SIZE_BYTES = IN_SIZE_BYTES; -const string METRIC_PLUGIN_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; -const string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = OUT_EVENT_GROUPS_TOTAL; -const string METRIC_PLUGIN_OUT_SIZE_BYTES = OUT_SIZE_BYTES; -const string METRIC_PLUGIN_TOTAL_DELAY_MS = TOTAL_DELAY_MS; -const string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; +const string& METRIC_PLUGIN_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; +const string& METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; +const string& METRIC_PLUGIN_IN_SIZE_BYTES = IN_SIZE_BYTES; +const string& METRIC_PLUGIN_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; +const string& METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = OUT_EVENT_GROUPS_TOTAL; +const string& METRIC_PLUGIN_OUT_SIZE_BYTES = OUT_SIZE_BYTES; +const string& METRIC_PLUGIN_TOTAL_DELAY_MS = TOTAL_DELAY_MS; +const string& METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; /********************************************************** * input_file @@ -101,7 +102,7 @@ const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "process_cache_miss_t /********************************************************** * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ -const string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; +const string& METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "out_failed_events_total"; const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "out_key_not_found_events_total"; const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "out_successful_events_total"; diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index d37ab38322..577f489198 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "MetricCommonConstants.h" #include "MetricConstants.h" using namespace std; @@ -30,13 +31,13 @@ const string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS = "prometheus_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER = "ebpf_server"; // metric keys -const string METRIC_RUNNER_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; -const string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; -const string METRIC_RUNNER_IN_SIZE_BYTES = IN_SIZE_BYTES; -const string METRIC_RUNNER_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; +const string& METRIC_RUNNER_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; +const string& METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; +const string& METRIC_RUNNER_IN_SIZE_BYTES = IN_SIZE_BYTES; +const string& METRIC_RUNNER_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; const string METRIC_RUNNER_LAST_RUN_TIME = "last_run_time"; -const string METRIC_RUNNER_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; -const string METRIC_RUNNER_TOTAL_DELAY_MS = TOTAL_DELAY_MS; +const string& METRIC_RUNNER_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; +const string& METRIC_RUNNER_TOTAL_DELAY_MS = TOTAL_DELAY_MS; const string METRIC_RUNNER_CLIENT_REGISTER_STATE = "client_register_state"; const string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL = "client_register_retry_total"; const string METRIC_RUNNER_JOBS_TOTAL = "jobs_total"; @@ -62,8 +63,7 @@ const string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL = "waiting_items_total"; **********************************************************/ const string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL = "watched_dirs_total"; const string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL = "active_readers_total"; -const string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG - = "enable_file_included_by_multi_configs"; +const string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG = "enable_multi_configs"; const string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "polling_modify_cache_size"; const string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "polling_dir_cache_size"; const string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "polling_file_cache_size"; From b83df3f2102ba23aeb4467a5d8225e7cdacc6a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E9=A3=8F?= Date: Wed, 13 Nov 2024 02:21:33 +0000 Subject: [PATCH 6/6] polish --- .../metric_constants/ComponentMetrics.cpp | 30 +++++++++---------- .../MetricCommonConstants.cpp | 26 ++++++++-------- .../metric_constants/MetricCommonConstants.h | 26 ++++++++-------- .../metric_constants/MetricConstants.h | 1 + .../metric_constants/PluginMetrics.cpp | 18 +++++------ .../metric_constants/RunnerMetrics.cpp | 13 ++++---- core/runner/ProcessorRunner.cpp | 5 ++-- 7 files changed, 61 insertions(+), 58 deletions(-) diff --git a/core/monitor/metric_constants/ComponentMetrics.cpp b/core/monitor/metric_constants/ComponentMetrics.cpp index 9a8e164096..358a7a6a36 100644 --- a/core/monitor/metric_constants/ComponentMetrics.cpp +++ b/core/monitor/metric_constants/ComponentMetrics.cpp @@ -43,16 +43,16 @@ const string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE = "sender_queue"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER = "serializer"; // metric keys -const string& METRIC_COMPONENT_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; -const string& METRIC_COMPONENT_IN_SIZE_BYTES = IN_SIZE_BYTES; -const string& METRIC_COMPONENT_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; -const string& METRIC_COMPONENT_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; -const string& METRIC_COMPONENT_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; -const string& METRIC_COMPONENT_OUT_SIZE_BYTES = OUT_SIZE_BYTES; -const string& METRIC_COMPONENT_TOTAL_DELAY_MS = TOTAL_DELAY_MS; -const string& METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; -const string& METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = DISCARDED_ITEMS_TOTAL; -const string& METRIC_COMPONENT_DISCARDED_SIZE_BYTES = DISCARDED_SIZE_BYTES; +const string& METRIC_COMPONENT_IN_EVENTS_TOTAL = METRIC_IN_EVENTS_TOTAL; +const string& METRIC_COMPONENT_IN_SIZE_BYTES = METRIC_IN_SIZE_BYTES; +const string& METRIC_COMPONENT_IN_ITEMS_TOTAL = METRIC_IN_ITEMS_TOTAL; +const string& METRIC_COMPONENT_OUT_EVENTS_TOTAL = METRIC_OUT_EVENTS_TOTAL; +const string& METRIC_COMPONENT_OUT_ITEMS_TOTAL = METRIC_OUT_ITEMS_TOTAL; +const string& METRIC_COMPONENT_OUT_SIZE_BYTES = METRIC_OUT_SIZE_BYTES; +const string& METRIC_COMPONENT_TOTAL_DELAY_MS = METRIC_TOTAL_DELAY_MS; +const string& METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = METRIC_TOTAL_PROCESS_TIME_MS; +const string& METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = METRIC_DISCARDED_ITEMS_TOTAL; +const string& METRIC_COMPONENT_DISCARDED_SIZE_BYTES = METRIC_DISCARDED_SIZE_BYTES; /********************************************************** * batcher @@ -71,14 +71,14 @@ const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "queue_size_bytes"; const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "valid_to_push_status"; const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "extra_buffer_size"; const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "extra_buffer_size_bytes"; -const string& METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; +const string& METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = METRIC_DISCARDED_EVENTS_TOTAL; const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "fetched_items_total"; const string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL = "fetch_times_total"; const string METRIC_COMPONENT_QUEUE_VALID_FETCH_TIMES_TOTAL = "valid_fetch_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_REGION_LIMITER_TIMES_TOTAL = "region_rejects_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_PROJECT_LIMITER_TIMES_TOTAL = "project_rejects_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_LOGSTORE_LIMITER_TIMES_TOTAL = "logstore_rejects_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL = "rate_rejects_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_REGION_LIMITER_TIMES_TOTAL = "region_reject_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_PROJECT_LIMITER_TIMES_TOTAL = "project_reject_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_LOGSTORE_LIMITER_TIMES_TOTAL = "logstore_reject_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL = "rate_reject_times_total"; } // namespace logtail diff --git a/core/monitor/metric_constants/MetricCommonConstants.cpp b/core/monitor/metric_constants/MetricCommonConstants.cpp index a0d3180c65..a2a1e3542d 100644 --- a/core/monitor/metric_constants/MetricCommonConstants.cpp +++ b/core/monitor/metric_constants/MetricCommonConstants.cpp @@ -18,18 +18,18 @@ using namespace std; namespace logtail { -const string DISCARDED_EVENTS_TOTAL = "discarded_events_total"; -const string DISCARDED_ITEMS_TOTAL = "discarded_items_total"; -const string DISCARDED_SIZE_BYTES = "discarded_size_bytes"; -const string IN_EVENTS_TOTAL = "in_events_total"; -const string IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; -const string IN_ITEMS_TOTAL = "in_items_total"; -const string IN_SIZE_BYTES = "in_size_bytes"; -const string OUT_EVENTS_TOTAL = "out_events_total"; -const string OUT_EVENT_GROUPS_TOTAL = "out_event_groups_total"; -const string OUT_ITEMS_TOTAL = "out_items_total"; -const string OUT_SIZE_BYTES = "out_size_bytes"; -const string TOTAL_DELAY_MS = "total_delay_ms"; -const string TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; +const string METRIC_DISCARDED_EVENTS_TOTAL = "discarded_events_total"; +const string METRIC_DISCARDED_ITEMS_TOTAL = "discarded_items_total"; +const string METRIC_DISCARDED_SIZE_BYTES = "discarded_size_bytes"; +const string METRIC_IN_EVENTS_TOTAL = "in_events_total"; +const string METRIC_IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; +const string METRIC_IN_ITEMS_TOTAL = "in_items_total"; +const string METRIC_IN_SIZE_BYTES = "in_size_bytes"; +const string METRIC_OUT_EVENTS_TOTAL = "out_events_total"; +const string METRIC_OUT_EVENT_GROUPS_TOTAL = "out_event_groups_total"; +const string METRIC_OUT_ITEMS_TOTAL = "out_items_total"; +const string METRIC_OUT_SIZE_BYTES = "out_size_bytes"; +const string METRIC_TOTAL_DELAY_MS = "total_delay_ms"; +const string METRIC_TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; } \ No newline at end of file diff --git a/core/monitor/metric_constants/MetricCommonConstants.h b/core/monitor/metric_constants/MetricCommonConstants.h index 5de8543efe..4a78b4e9d1 100644 --- a/core/monitor/metric_constants/MetricCommonConstants.h +++ b/core/monitor/metric_constants/MetricCommonConstants.h @@ -19,18 +19,18 @@ namespace logtail { -extern const std::string DISCARDED_EVENTS_TOTAL; -extern const std::string DISCARDED_ITEMS_TOTAL; -extern const std::string DISCARDED_SIZE_BYTES; -extern const std::string IN_EVENTS_TOTAL; -extern const std::string IN_EVENT_GROUPS_TOTAL; -extern const std::string IN_ITEMS_TOTAL; -extern const std::string IN_SIZE_BYTES; -extern const std::string OUT_EVENTS_TOTAL; -extern const std::string OUT_EVENT_GROUPS_TOTAL; -extern const std::string OUT_ITEMS_TOTAL; -extern const std::string OUT_SIZE_BYTES; -extern const std::string TOTAL_DELAY_MS; -extern const std::string TOTAL_PROCESS_TIME_MS; +extern const std::string METRIC_DISCARDED_EVENTS_TOTAL; +extern const std::string METRIC_DISCARDED_ITEMS_TOTAL; +extern const std::string METRIC_DISCARDED_SIZE_BYTES; +extern const std::string METRIC_IN_EVENTS_TOTAL; +extern const std::string METRIC_IN_EVENT_GROUPS_TOTAL; +extern const std::string METRIC_IN_ITEMS_TOTAL; +extern const std::string METRIC_IN_SIZE_BYTES; +extern const std::string METRIC_OUT_EVENTS_TOTAL; +extern const std::string METRIC_OUT_EVENT_GROUPS_TOTAL; +extern const std::string METRIC_OUT_ITEMS_TOTAL; +extern const std::string METRIC_OUT_SIZE_BYTES; +extern const std::string METRIC_TOTAL_DELAY_MS; +extern const std::string METRIC_TOTAL_PROCESS_TIME_MS; } \ No newline at end of file diff --git a/core/monitor/metric_constants/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h index e6627f8714..73a62d30bf 100644 --- a/core/monitor/metric_constants/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -264,6 +264,7 @@ extern const std::string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_T // label keys extern const std::string METRIC_LABEL_KEY_RUNNER_NAME; +extern const std::string METRIC_LABEL_KEY_THREAD_NO; // label values extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER; diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index 808e59ee68..329e487539 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -24,14 +24,14 @@ const string METRIC_LABEL_KEY_PLUGIN_ID = "plugin_id"; const string METRIC_LABEL_KEY_PLUGIN_TYPE = "plugin_type"; // metric keys -const string& METRIC_PLUGIN_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; -const string& METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; -const string& METRIC_PLUGIN_IN_SIZE_BYTES = IN_SIZE_BYTES; -const string& METRIC_PLUGIN_OUT_EVENTS_TOTAL = OUT_EVENTS_TOTAL; -const string& METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = OUT_EVENT_GROUPS_TOTAL; -const string& METRIC_PLUGIN_OUT_SIZE_BYTES = OUT_SIZE_BYTES; -const string& METRIC_PLUGIN_TOTAL_DELAY_MS = TOTAL_DELAY_MS; -const string& METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = TOTAL_PROCESS_TIME_MS; +const string& METRIC_PLUGIN_IN_EVENTS_TOTAL = METRIC_IN_EVENTS_TOTAL; +const string& METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = METRIC_IN_EVENT_GROUPS_TOTAL; +const string& METRIC_PLUGIN_IN_SIZE_BYTES = METRIC_IN_SIZE_BYTES; +const string& METRIC_PLUGIN_OUT_EVENTS_TOTAL = METRIC_OUT_EVENTS_TOTAL; +const string& METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = METRIC_OUT_EVENT_GROUPS_TOTAL; +const string& METRIC_PLUGIN_OUT_SIZE_BYTES = METRIC_OUT_SIZE_BYTES; +const string& METRIC_PLUGIN_TOTAL_DELAY_MS = METRIC_TOTAL_DELAY_MS; +const string& METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = METRIC_TOTAL_PROCESS_TIME_MS; /********************************************************** * input_file @@ -102,7 +102,7 @@ const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "process_cache_miss_t /********************************************************** * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ -const string& METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = DISCARDED_EVENTS_TOTAL; +const string& METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = METRIC_DISCARDED_EVENTS_TOTAL; const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "out_failed_events_total"; const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "out_key_not_found_events_total"; const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "out_successful_events_total"; diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index 577f489198..c1aead8643 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -21,6 +21,7 @@ namespace logtail { // label keys const string METRIC_LABEL_KEY_RUNNER_NAME = "runner_name"; +const string METRIC_LABEL_KEY_THREAD_NO = "thread_no"; // label values const string METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER = "file_server"; @@ -31,13 +32,13 @@ const string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS = "prometheus_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER = "ebpf_server"; // metric keys -const string& METRIC_RUNNER_IN_EVENTS_TOTAL = IN_EVENTS_TOTAL; -const string& METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = IN_EVENT_GROUPS_TOTAL; -const string& METRIC_RUNNER_IN_SIZE_BYTES = IN_SIZE_BYTES; -const string& METRIC_RUNNER_IN_ITEMS_TOTAL = IN_ITEMS_TOTAL; +const string& METRIC_RUNNER_IN_EVENTS_TOTAL = METRIC_IN_EVENTS_TOTAL; +const string& METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = METRIC_IN_EVENT_GROUPS_TOTAL; +const string& METRIC_RUNNER_IN_SIZE_BYTES = METRIC_IN_SIZE_BYTES; +const string& METRIC_RUNNER_IN_ITEMS_TOTAL = METRIC_IN_ITEMS_TOTAL; const string METRIC_RUNNER_LAST_RUN_TIME = "last_run_time"; -const string& METRIC_RUNNER_OUT_ITEMS_TOTAL = OUT_ITEMS_TOTAL; -const string& METRIC_RUNNER_TOTAL_DELAY_MS = TOTAL_DELAY_MS; +const string& METRIC_RUNNER_OUT_ITEMS_TOTAL = METRIC_OUT_ITEMS_TOTAL; +const string& METRIC_RUNNER_TOTAL_DELAY_MS = METRIC_TOTAL_DELAY_MS; const string METRIC_RUNNER_CLIENT_REGISTER_STATE = "client_register_state"; const string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL = "client_register_retry_total"; const string METRIC_RUNNER_JOBS_TOTAL = "jobs_total"; diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index 9f5580d4b1..7a0963d542 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -90,7 +90,8 @@ void ProcessorRunner::Run(uint32_t threadNo) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( sMetricsRecordRef, MetricCategory::METRIC_CATEGORY_RUNNER, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, {"thread_no", ToString(threadNo)}}); + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, + {METRIC_LABEL_KEY_THREAD_NO, ToString(threadNo)}}); sInGroupsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL); sInEventsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENTS_TOTAL); sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); @@ -137,7 +138,7 @@ void ProcessorRunner::Run(uint32_t threadNo) { pipeline->Process(eventGroupList, item->mInputIndex); if (pipeline->IsFlushingThroughGoPipeline()) { - // TODO: + // TODO: // 1. allow all event types to be sent to Go pipelines // 2. use event group protobuf instead if (isLog) {