From eaf69cbd16c5a035fe2ba1372592e8e37759fd6d Mon Sep 17 00:00:00 2001 From: Takuka0311 <1914426213@qq.com> Date: Wed, 13 Nov 2024 10:22:17 +0800 Subject: [PATCH] change metric struct (#1862) --- core/common/compression/Compressor.cpp | 4 +- core/common/compression/CompressorFactory.cpp | 3 +- .../ContainerDiscoveryOptions.cpp | 2 +- .../ContainerDiscoveryOptions.h | 2 +- core/ebpf/SelfMonitor.cpp | 2 +- core/ebpf/SelfMonitor.h | 2 +- core/ebpf/eBPFServer.cpp | 8 +- core/ebpf/eBPFServer.h | 4 +- core/file_server/FileServer.cpp | 4 +- core/monitor/LogtailMetric.cpp | 138 +++++++++++------- core/monitor/LogtailMetric.h | 38 ++++- core/monitor/MetricExportor.cpp | 52 +++++-- core/monitor/Monitor.cpp | 3 +- core/monitor/PluginMetricManager.cpp | 4 +- core/monitor/PluginMetricManager.h | 14 +- .../monitor/metric_constants/AgentMetrics.cpp | 18 +-- .../metric_constants/ComponentMetrics.cpp | 62 ++++---- .../MetricCommonConstants.cpp | 35 +++++ .../metric_constants/MetricCommonConstants.h | 36 +++++ .../metric_constants/MetricConstants.h | 73 +++++---- .../metric_constants/PipelineMetrics.cpp | 21 ++- .../metric_constants/PluginMetrics.cpp | 103 +++++++------ .../metric_constants/RunnerMetrics.cpp | 58 ++++---- core/pipeline/Pipeline.cpp | 5 +- core/pipeline/batch/Batcher.h | 4 +- core/pipeline/plugin/interface/Plugin.h | 2 +- core/pipeline/queue/QueueInterface.h | 2 +- core/pipeline/route/Router.cpp | 4 +- core/pipeline/serializer/Serializer.h | 4 +- core/plugin/input/InputContainerStdio.cpp | 7 +- core/plugin/input/InputFile.cpp | 4 +- core/plugin/input/InputFileSecurity.cpp | 2 +- core/plugin/input/InputFileSecurity.h | 2 +- core/plugin/input/InputNetworkObserver.cpp | 2 +- core/plugin/input/InputNetworkObserver.h | 2 +- core/plugin/input/InputNetworkSecurity.cpp | 2 +- core/plugin/input/InputNetworkSecurity.h | 2 +- core/plugin/input/InputProcessSecurity.cpp | 2 +- core/plugin/input/InputProcessSecurity.h | 2 +- core/prometheus/PromSelfMonitor.cpp | 2 +- core/prometheus/PrometheusInputRunner.cpp | 3 +- .../prometheus/schedulers/ScrapeScheduler.cpp | 2 +- .../schedulers/TargetSubscriberScheduler.cpp | 2 +- core/runner/FlusherRunner.cpp | 4 +- core/runner/ProcessorRunner.cpp | 6 +- core/runner/sink/http/HttpSink.cpp | 4 +- core/unittest/batch/BatcherUnittest.cpp | 2 +- .../compression/CompressorFactoryUnittest.cpp | 2 +- .../monitor/LogtailMetricUnittest.cpp | 14 +- .../monitor/PluginMetricManagerUnittest.cpp | 7 +- core/unittest/pipeline/PipelineUnittest.cpp | 8 +- .../prometheus/PromSelfMonitorUnittest.cpp | 4 +- .../queue/BoundedProcessQueueUnittest.cpp | 2 +- .../queue/CircularProcessQueueUnittest.cpp | 2 +- core/unittest/queue/SenderQueueUnittest.cpp | 2 +- core/unittest/route/RouterUnittest.cpp | 2 +- pkg/helper/k8smeta/k8s_meta_manager.go | 10 +- pkg/helper/self_metrics_agent_constants.go | 4 +- pkg/helper/self_metrics_plugin_constants.go | 62 ++++---- pkg/helper/self_metrics_runner_constants.go | 35 +++-- 60 files changed, 538 insertions(+), 375 deletions(-) create mode 100644 core/monitor/metric_constants/MetricCommonConstants.cpp create mode 100644 core/monitor/metric_constants/MetricCommonConstants.h diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp index 050d62123b..134e6e4737 100644 --- a/core/common/compression/Compressor.cpp +++ b/core/common/compression/Compressor.cpp @@ -24,14 +24,14 @@ namespace logtail { void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_COMPONENT, std::move(labels), std::move(dynamicLabels)); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_SIZE_BYTES); mTotalProcessMs = mMetricsRecordRef.CreateTimeCounter(METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS); mDiscardedItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL); - mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_SIZE_BYTES); } bool Compressor::DoCompress(const string& input, string& output, string& errorMsg) { diff --git a/core/common/compression/CompressorFactory.cpp b/core/common/compression/CompressorFactory.cpp index ed3fe2703f..c1d4ff0e78 100644 --- a/core/common/compression/CompressorFactory.cpp +++ b/core/common/compression/CompressorFactory.cpp @@ -15,9 +15,9 @@ #include "common/compression/CompressorFactory.h" #include "common/ParamExtractor.h" -#include "monitor/metric_constants/MetricConstants.h" #include "common/compression/LZ4Compressor.h" #include "common/compression/ZstdCompressor.h" +#include "monitor/metric_constants/MetricConstants.h" using namespace std; @@ -64,7 +64,6 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, compressor->SetMetricRecordRef({{METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, flusherId}}); return compressor; } diff --git a/core/container_manager/ContainerDiscoveryOptions.cpp b/core/container_manager/ContainerDiscoveryOptions.cpp index f9cdb6111a..ce59ef41f5 100644 --- a/core/container_manager/ContainerDiscoveryOptions.cpp +++ b/core/container_manager/ContainerDiscoveryOptions.cpp @@ -200,7 +200,7 @@ bool ContainerDiscoveryOptions::Init(const Json::Value& config, const PipelineCo void ContainerDiscoveryOptions::GenerateContainerMetaFetchingGoPipeline(Json::Value& res, const FileDiscoveryOptions* fileDiscovery, - const PluginInstance::PluginMeta pluginMeta) const { + const PluginInstance::PluginMeta& pluginMeta) const { Json::Value plugin(Json::objectValue); Json::Value detail(Json::objectValue); Json::Value object(Json::objectValue); diff --git a/core/container_manager/ContainerDiscoveryOptions.h b/core/container_manager/ContainerDiscoveryOptions.h index 1b1b06ccaf..2a106c819f 100644 --- a/core/container_manager/ContainerDiscoveryOptions.h +++ b/core/container_manager/ContainerDiscoveryOptions.h @@ -52,7 +52,7 @@ struct ContainerDiscoveryOptions { bool Init(const Json::Value& config, const PipelineContext& ctx, const std::string& pluginType); void GenerateContainerMetaFetchingGoPipeline(Json::Value& res, const FileDiscoveryOptions* fileDiscovery = nullptr, - const PluginInstance::PluginMeta pluginMeta = {"0"}) const; + const PluginInstance::PluginMeta& pluginMeta = {"0"}) const; }; using ContainerDiscoveryConfig = std::pair; diff --git a/core/ebpf/SelfMonitor.cpp b/core/ebpf/SelfMonitor.cpp index ad49fce33c..d352cfbaeb 100644 --- a/core/ebpf/SelfMonitor.cpp +++ b/core/ebpf/SelfMonitor.cpp @@ -179,7 +179,7 @@ void NetworkObserverSelfMonitor::HandleStatistic(nami::eBPFStatistics& stats) { eBPFSelfMonitorMgr::eBPFSelfMonitorMgr() : mSelfMonitors({}), mInited({}) {} -void eBPFSelfMonitorMgr::Init(const nami::PluginType type, std::shared_ptr mgr, const std::string& name, const std::string& logstore) { +void eBPFSelfMonitorMgr::Init(const nami::PluginType type, PluginMetricManagerPtr mgr, const std::string& name, const std::string& logstore) { if (mInited[int(type)]) return; WriteLock lk(mLock); diff --git a/core/ebpf/SelfMonitor.h b/core/ebpf/SelfMonitor.h index 4d59934266..4b8551f1ef 100644 --- a/core/ebpf/SelfMonitor.h +++ b/core/ebpf/SelfMonitor.h @@ -122,7 +122,7 @@ class FileSecuritySelfMonitor : public BaseBPFMonitor { class eBPFSelfMonitorMgr { public: eBPFSelfMonitorMgr(); - void Init(const nami::PluginType type, std::shared_ptr mgr, const std::string& name, const std::string& project); + void Init(const nami::PluginType type, PluginMetricManagerPtr mgr, const std::string& name, const std::string& project); void Release(const nami::PluginType type); void Suspend(const nami::PluginType type); void HandleStatistic(std::vector&& stats); diff --git a/core/ebpf/eBPFServer.cpp b/core/ebpf/eBPFServer.cpp index 874a29ec5a..501f833806 100644 --- a/core/ebpf/eBPFServer.cpp +++ b/core/ebpf/eBPFServer.cpp @@ -152,8 +152,8 @@ void eBPFServer::Init() { DynamicMetricLabels dynamicLabels; dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); }); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}, + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER}}, std::move(dynamicLabels)); mStartPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL); @@ -202,7 +202,7 @@ void eBPFServer::Stop() { bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr) { + const std::variant options, PluginMetricManagerPtr mgr) { std::string prev_pipeline_name = CheckLoadedPipelineName(type); if (prev_pipeline_name.size() && prev_pipeline_name != pipeline_name) { @@ -314,7 +314,7 @@ bool eBPFServer::HasRegisteredPlugins() const { bool eBPFServer::EnablePlugin(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr) { + const std::variant options, PluginMetricManagerPtr mgr) { if (!IsSupportedEnv(type)) { return false; } diff --git a/core/ebpf/eBPFServer.h b/core/ebpf/eBPFServer.h index 4887080a79..9141a9ec9a 100644 --- a/core/ebpf/eBPFServer.h +++ b/core/ebpf/eBPFServer.h @@ -72,7 +72,7 @@ class eBPFServer : public InputRunner { bool EnablePlugin(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr); + const std::variant options, PluginMetricManagerPtr mgr); bool DisablePlugin(const std::string& pipeline_name, nami::PluginType type); @@ -88,7 +88,7 @@ class eBPFServer : public InputRunner { bool StartPluginInternal(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options, std::shared_ptr mgr); + const std::variant options, PluginMetricManagerPtr mgr); eBPFServer() = default; ~eBPFServer() = default; diff --git a/core/file_server/FileServer.cpp b/core/file_server/FileServer.cpp index 5a072c6c0f..0741efefca 100644 --- a/core/file_server/FileServer.cpp +++ b/core/file_server/FileServer.cpp @@ -34,8 +34,8 @@ namespace logtail { FileServer::FileServer() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}); + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}}); } // 启动文件服务，包括加载配置、处理检查点、注册事件等 diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index e10c44fbb1..239f6f8c9c 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -25,11 +25,19 @@ using namespace sls_logs; namespace logtail { -const std::string LABEL_PREFIX = "label."; -const std::string VALUE_PREFIX = "value."; +const std::string METRIC_KEY_LABEL = "label"; +const std::string METRIC_KEY_VALUE = "value"; +const std::string METRIC_KEY_CATEGORY = "category"; +const std::string MetricCategory::METRIC_CATEGORY_UNKNOWN = "unknown"; +const std::string MetricCategory::METRIC_CATEGORY_AGENT = "agent"; +const std::string MetricCategory::METRIC_CATEGORY_RUNNER = "runner"; +const std::string MetricCategory::METRIC_CATEGORY_PIPELINE = "pipeline"; +const std::string MetricCategory::METRIC_CATEGORY_COMPONENT = "component"; +const std::string MetricCategory::METRIC_CATEGORY_PLUGIN = "plugin"; +const std::string MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE = "plugin_source"; -MetricsRecord::MetricsRecord(MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels) - : mLabels(labels), mDynamicLabels(dynamicLabels), mDeleted(false) { +MetricsRecord::MetricsRecord(const std::string& category, MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels) + : mCategory(category), mLabels(labels), mDynamicLabels(dynamicLabels), mDeleted(false) { } CounterPtr MetricsRecord::CreateCounter(const std::string& name) { @@ -64,6 +72,10 @@ bool MetricsRecord::IsDeleted() const { return mDeleted; } +const std::string& MetricsRecord::GetCategory() const { + return mCategory; +} + const MetricLabelsPtr& MetricsRecord::GetLabels() const { return mLabels; } @@ -89,7 +101,7 @@ const std::vector& MetricsRecord::GetDoubleGauges() const { } MetricsRecord* MetricsRecord::Collect() { - MetricsRecord* metrics = new MetricsRecord(mLabels, mDynamicLabels); + MetricsRecord* metrics = new MetricsRecord(mCategory, mLabels, mDynamicLabels); for (auto& item : mCounters) { CounterPtr newPtr(item->Collect()); metrics->mCounters.emplace_back(newPtr); @@ -127,6 +139,10 @@ void MetricsRecordRef::SetMetricsRecord(MetricsRecord* metricRecord) { mMetrics = metricRecord; } +const std::string& MetricsRecordRef::GetCategory() const { + return mMetrics->GetCategory(); +} + const MetricLabelsPtr& MetricsRecordRef::GetLabels() const { return mMetrics->GetLabels(); } @@ -171,8 +187,12 @@ bool MetricsRecordRef::HasLabel(const std::string& key, const std::string& value #endif // ReentrantMetricsRecord相关操作可以无锁，因为mCounters、mGauges只在初始化时会添加内容，后续只允许Get操作 -void ReentrantMetricsRecord::Init(MetricLabels& labels, std::unordered_map& metricKeys) { - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); +void ReentrantMetricsRecord::Init(const std::string& category, + MetricLabels& labels, + DynamicMetricLabels& dynamicLabels, + std::unordered_map& metricKeys) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, category, std::move(labels), std::move(dynamicLabels)); for (auto metric : metricKeys) { switch (metric.second) { case MetricType::METRIC_TYPE_COUNTER: @@ -237,17 +257,19 @@ WriteMetrics::~WriteMetrics() { } void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, + const std::string& category, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { - CreateMetricsRecordRef(ref, std::move(labels), std::move(dynamicLabels)); + CreateMetricsRecordRef(ref, category, std::move(labels), std::move(dynamicLabels)); CommitMetricsRecordRef(ref); } void WriteMetrics::CreateMetricsRecordRef(MetricsRecordRef& ref, + const std::string& category, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { - MetricsRecord* cur = new MetricsRecord(std::make_shared(labels), - std::make_shared(dynamicLabels)); + MetricsRecord* cur = new MetricsRecord( + category, std::make_shared(labels), std::make_shared(dynamicLabels)); ref.SetMetricsRecord(cur); } @@ -395,42 +417,53 @@ void ReadMetrics::ReadAsLogGroup(const std::string& regionFieldName, auto now = GetCurrentLogtailTime(); SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec); - for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { - std::pair pair = *item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(LABEL_PREFIX + pair.first); - contentPtr->set_value(pair.second); - } - for (auto item = tmp->GetDynamicLabels()->begin(); item != tmp->GetDynamicLabels()->end(); ++item) { - std::pair> pair = *item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(LABEL_PREFIX + pair.first); - contentPtr->set_value(pair.second()); - } - - for (auto& item : tmp->GetCounters()) { - CounterPtr counter = item; + { // category Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + counter->GetName()); - contentPtr->set_value(ToString(counter->GetValue())); + contentPtr->set_key(METRIC_KEY_CATEGORY); + contentPtr->set_value(tmp->GetCategory()); } - for (auto& item : tmp->GetTimeCounters()) { - TimeCounterPtr counter = item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + counter->GetName()); - contentPtr->set_value(ToString(counter->GetValue())); - } - for (auto& item : tmp->GetIntGauges()) { - IntGaugePtr gauge = item; + { // label + Json::Value metricsRecordLabel; + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + metricsRecordLabel[pair.first] = pair.second; + } + for (auto item = tmp->GetDynamicLabels()->begin(); item != tmp->GetDynamicLabels()->end(); ++item) { + std::pair> pair = *item; + metricsRecordLabel[pair.first] = pair.second(); + } + Json::StreamWriterBuilder writer; + writer["indentation"] = ""; + std::string jsonString = Json::writeString(writer, metricsRecordLabel); Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + gauge->GetName()); - contentPtr->set_value(ToString(gauge->GetValue())); + contentPtr->set_key(METRIC_KEY_LABEL); + contentPtr->set_value(jsonString); } - for (auto& item : tmp->GetDoubleGauges()) { - DoubleGaugePtr gauge = item; - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key(VALUE_PREFIX + gauge->GetName()); - contentPtr->set_value(ToString(gauge->GetValue())); + { // value + for (auto& item : tmp->GetCounters()) { + CounterPtr counter = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(counter->GetName()); + contentPtr->set_value(ToString(counter->GetValue())); + } + for (auto& item : tmp->GetTimeCounters()) { + TimeCounterPtr counter = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(counter->GetName()); + contentPtr->set_value(ToString(counter->GetValue())); + } + for (auto& item : tmp->GetIntGauges()) { + IntGaugePtr gauge = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(gauge->GetName()); + contentPtr->set_value(ToString(gauge->GetValue())); + } + for (auto& item : tmp->GetDoubleGauges()) { + DoubleGaugePtr gauge = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(gauge->GetName()); + contentPtr->set_value(ToString(gauge->GetValue())); + } } tmp = tmp->GetNext(); } @@ -443,40 +476,43 @@ void ReadMetrics::ReadAsFileBuffer(std::string& metricsContent) const { MetricsRecord* tmp = mHead; while (tmp) { - Json::Value metricsRecordValue; + Json::Value metricsRecordJson, metricsRecordLabel; auto now = GetCurrentLogtailTime(); - metricsRecordValue["time"] + metricsRecordJson["time"] = AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec; + metricsRecordJson[METRIC_KEY_CATEGORY] = tmp->GetCategory(); + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { std::pair pair = *item; - metricsRecordValue[LABEL_PREFIX + pair.first] = pair.second; + metricsRecordLabel[pair.first] = pair.second; } for (auto item = tmp->GetDynamicLabels()->begin(); item != tmp->GetDynamicLabels()->end(); ++item) { std::pair> pair = *item; - metricsRecordValue[LABEL_PREFIX + pair.first] = pair.second(); + metricsRecordLabel[pair.first] = pair.second(); } + metricsRecordJson[METRIC_KEY_LABEL] = metricsRecordLabel; for (auto& item : tmp->GetCounters()) { CounterPtr counter = item; - metricsRecordValue[VALUE_PREFIX + counter->GetName()] = ToString(counter->GetValue()); + metricsRecordJson[counter->GetName()] = ToString(counter->GetValue()); } for (auto& item : tmp->GetTimeCounters()) { TimeCounterPtr counter = item; - metricsRecordValue[VALUE_PREFIX + counter->GetName()] = ToString(counter->GetValue()); + metricsRecordJson[counter->GetName()] = ToString(counter->GetValue()); } for (auto& item : tmp->GetIntGauges()) { IntGaugePtr gauge = item; - metricsRecordValue[VALUE_PREFIX + gauge->GetName()] = ToString(gauge->GetValue()); + metricsRecordJson[gauge->GetName()] = ToString(gauge->GetValue()); } for (auto& item : tmp->GetDoubleGauges()) { DoubleGaugePtr gauge = item; - metricsRecordValue[VALUE_PREFIX + gauge->GetName()] = ToString(gauge->GetValue()); + metricsRecordJson[gauge->GetName()] = ToString(gauge->GetValue()); } Json::StreamWriterBuilder writer; writer["indentation"] = ""; - std::string jsonString = Json::writeString(writer, metricsRecordValue); + std::string jsonString = Json::writeString(writer, metricsRecordJson); oss << jsonString << '\n'; tmp = tmp->GetNext(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 1c02fa4814..2065f2184f 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -28,22 +28,39 @@ namespace logtail { +extern const std::string METRIC_KEY_LABEL; +extern const std::string METRIC_KEY_VALUE; +extern const std::string METRIC_KEY_CATEGORY; +class MetricCategory { +public: + static const std::string METRIC_CATEGORY_UNKNOWN; + static const std::string METRIC_CATEGORY_AGENT; + static const std::string METRIC_CATEGORY_RUNNER; + static const std::string METRIC_CATEGORY_PIPELINE; + static const std::string METRIC_CATEGORY_COMPONENT; + static const std::string METRIC_CATEGORY_PLUGIN; + static const std::string METRIC_CATEGORY_PLUGIN_SOURCE; +}; + class MetricsRecord { private: + std::string mCategory; MetricLabelsPtr mLabels; DynamicMetricLabelsPtr mDynamicLabels; - std::atomic_bool mDeleted; std::vector mCounters; std::vector mTimeCounters; std::vector mIntGauges; std::vector mDoubleGauges; + + std::atomic_bool mDeleted; MetricsRecord* mNext = nullptr; public: - MetricsRecord(MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels = nullptr); + MetricsRecord(const std::string& category, MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels = nullptr); MetricsRecord() = default; void MarkDeleted(); bool IsDeleted() const; + const std::string& GetCategory() const; const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; const std::vector& GetCounters() const; @@ -75,6 +92,7 @@ class MetricsRecordRef { MetricsRecordRef(MetricsRecordRef&&) = delete; MetricsRecordRef& operator=(MetricsRecordRef&&) = delete; void SetMetricsRecord(MetricsRecord* metricRecord); + const std::string& GetCategory() const; const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; CounterPtr CreateCounter(const std::string& name); @@ -114,7 +132,10 @@ class ReentrantMetricsRecord { std::unordered_map mDoubleGauges; public: - void Init(MetricLabels& labels, std::unordered_map& metricKeys); + void Init(const std::string& category, + MetricLabels& labels, + DynamicMetricLabels& dynamicLabels, + std::unordered_map& metricKeys); const MetricLabelsPtr& GetLabels() const; const DynamicMetricLabelsPtr& GetDynamicLabels() const; CounterPtr GetCounter(const std::string& name); @@ -140,9 +161,14 @@ class WriteMetrics { return ptr; } - void - PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); - void CreateMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + void PrepareMetricsRecordRef(MetricsRecordRef& ref, + const std::string& category, + MetricLabels&& labels, + DynamicMetricLabels&& dynamicLabels = {}); + void CreateMetricsRecordRef(MetricsRecordRef& ref, + const std::string& category, + MetricLabels&& labels, + DynamicMetricLabels&& dynamicLabels = {}); void CommitMetricsRecordRef(MetricsRecordRef& ref); MetricsRecord* DoSnapshot(); diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index 5c4356389a..99c829d8d1 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -16,7 +16,6 @@ #include -#include "app_config/AppConfig.h" #include "LogFileProfiler.h" #include "LogtailMetric.h" #include "MetricConstants.h" @@ -35,7 +34,6 @@ DECLARE_FLAG_STRING(metrics_report_method); namespace logtail { -const string METRIC_REGION_FIELD_NAME = "region"; const string METRIC_REGION_DEFAULT = "default"; const string METRIC_SLS_LOGSTORE_NAME = "shennong_log_profile"; const string METRIC_TOPIC_TYPE = "loong_collector_metric"; @@ -66,7 +64,7 @@ void MetricExportor::PushCppMetrics() { if ("sls" == STRING_FLAG(metrics_report_method)) { std::map logGroupMap; - ReadMetrics::GetInstance()->ReadAsLogGroup(METRIC_REGION_FIELD_NAME, METRIC_REGION_DEFAULT, logGroupMap); + ReadMetrics::GetInstance()->ReadAsLogGroup(METRIC_LABEL_KEY_REGION, METRIC_REGION_DEFAULT, logGroupMap); SendToSLS(logGroupMap); } else if ("file" == STRING_FLAG(metrics_report_method)) { std::string metricsContent; @@ -175,10 +173,10 @@ void MetricExportor::PushGoCppProvidedMetrics(std::vectorSetAgentGoMemory(std::stoi(metric.second)); } - if (metric.first == METRIC_AGENT_GO_ROUTINES_TOTAL) { + if (metric.first == METRIC_KEY_VALUE + "." + METRIC_AGENT_GO_ROUTINES_TOTAL) { LoongCollectorMonitor::GetInstance()->SetAgentGoRoutinesTotal(std::stoi(metric.second)); } LogtailMonitor::GetInstance()->UpdateMetric(metric.first, metric.second); @@ -193,15 +191,15 @@ void MetricExportor::SerializeGoDirectMetricsListToLogGroupMap( std::string configName = ""; std::string region = METRIC_REGION_DEFAULT; { - // get the config_name label + // get the pipeline_name label for (const auto& metric : metrics) { - if (metric.first == "label.config_name") { + if (metric.first == METRIC_KEY_LABEL + "." + METRIC_LABEL_KEY_PIPELINE_NAME) { configName = metric.second; break; } } if (!configName.empty()) { - // get region info by config_name + // get region info by pipeline_name shared_ptr p = PipelineManager::GetInstance()->FindConfigByName(configName); if (p) { FlusherSLS* pConfig = NULL; @@ -225,11 +223,32 @@ void MetricExportor::SerializeGoDirectMetricsListToLogGroupMap( auto now = GetCurrentLogtailTime(); SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec); + + Json::Value metricsRecordLabel; for (const auto& metric : metrics) { + // category + if (metric.first.compare("label.metric_category") == 0) { + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(METRIC_KEY_CATEGORY); + contentPtr->set_value(metric.second); + continue; + } + // label + if (metric.first.compare(0, METRIC_KEY_LABEL.length(), METRIC_KEY_LABEL)) { + metricsRecordLabel[metric.first.substr(METRIC_KEY_LABEL.length() + 1)] = metric.second; + continue; + } + // value Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(metric.first); contentPtr->set_value(metric.second); } + Json::StreamWriterBuilder writer; + writer["indentation"] = ""; + std::string jsonString = Json::writeString(writer, metricsRecordLabel); + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(METRIC_KEY_LABEL); + contentPtr->set_value(jsonString); } } @@ -238,16 +257,25 @@ void MetricExportor::SerializeGoDirectMetricsListToString(std::vectorEnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec; for (const auto& metric : metrics) { - metricsRecordValue[metric.first] = metric.second; + if (metric.first.compare("label.metric_category") == 0) { + metricsRecordJson[METRIC_KEY_CATEGORY] = metric.second; + continue; + } + if (metric.first.compare(0, METRIC_KEY_LABEL.length(), METRIC_KEY_LABEL) == 0) { + metricsRecordLabel[metric.first.substr(METRIC_KEY_LABEL.length() + 1)] = metric.second; + continue; + } + metricsRecordJson[metric.first.substr(METRIC_KEY_VALUE.length() + 1)] = metric.second; } + metricsRecordJson[METRIC_KEY_LABEL] = metricsRecordLabel; Json::StreamWriterBuilder writer; writer["indentation"] = ""; - std::string jsonString = Json::writeString(writer, metricsRecordValue); + std::string jsonString = Json::writeString(writer, metricsRecordJson); oss << jsonString << '\n'; } metricsContent = oss.str(); diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 7eca928e46..3bb00943f1 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -693,7 +693,6 @@ LoongCollectorMonitor* LoongCollectorMonitor::GetInstance() { void LoongCollectorMonitor::Init() { // create metric record MetricLabels labels; - labels.emplace_back(METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_AGENT); labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId()); labels.emplace_back(METRIC_LABEL_KEY_START_TIME, LogFileProfiler::mStartTime); labels.emplace_back(METRIC_LABEL_KEY_OS, OS_NAME); @@ -710,7 +709,7 @@ void LoongCollectorMonitor::Init() { }); #endif WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_AGENT, std::move(labels), std::move(dynamicLabels)); // init value mAgentCpu = mMetricsRecordRef.CreateDoubleGauge(METRIC_AGENT_CPU); mAgentMemory = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_MEMORY); diff --git a/core/monitor/PluginMetricManager.cpp b/core/monitor/PluginMetricManager.cpp index b47b5ddcad..f39506971e 100644 --- a/core/monitor/PluginMetricManager.cpp +++ b/core/monitor/PluginMetricManager.cpp @@ -17,7 +17,7 @@ namespace logtail { -ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecordRef(MetricLabels labels) { +ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecordRef(MetricLabels labels, DynamicMetricLabels dynamicLabels) { std::lock_guard lock(mutex); std::string key = GenerateKey(labels); @@ -31,7 +31,7 @@ ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecord newLabels.insert(newLabels.end(), labels.begin(), labels.end()); ReentrantMetricsRecordRef ptr = std::make_shared(); - ptr->Init(newLabels, mMetricKeys); + ptr->Init(mDefaultCategory, newLabels, dynamicLabels, mMetricKeys); mReentrantMetricsRecordRefsMap.emplace(key, ptr); if (mSizeGauge != nullptr) { diff --git a/core/monitor/PluginMetricManager.h b/core/monitor/PluginMetricManager.h index db043fceda..dfa0a23ecb 100644 --- a/core/monitor/PluginMetricManager.h +++ b/core/monitor/PluginMetricManager.h @@ -23,10 +23,15 @@ namespace logtail { class PluginMetricManager { public: - PluginMetricManager(const MetricLabelsPtr defaultLabels, std::unordered_map metricKeys) - : mDefaultLabels(defaultLabels->begin(), defaultLabels->end()), mMetricKeys(metricKeys) {} - - ReentrantMetricsRecordRef GetOrCreateReentrantMetricsRecordRef(MetricLabels labels); + PluginMetricManager(const MetricLabelsPtr defaultLabels, + std::unordered_map metricKeys, + std::string category = MetricCategory::METRIC_CATEGORY_UNKNOWN) + : mDefaultLabels(defaultLabels->begin(), defaultLabels->end()), + mMetricKeys(metricKeys), + mDefaultCategory(category) {} + + ReentrantMetricsRecordRef GetOrCreateReentrantMetricsRecordRef(MetricLabels labels, + DynamicMetricLabels dynamicLabels = {}); void ReleaseReentrantMetricsRecordRef(MetricLabels labels); void RegisterSizeGauge(IntGaugePtr ptr) { mSizeGauge = ptr; } @@ -36,6 +41,7 @@ class PluginMetricManager { MetricLabels mDefaultLabels; std::unordered_map mMetricKeys; + std::string mDefaultCategory; std::unordered_map mReentrantMetricsRecordRefsMap; mutable std::mutex mutex; diff --git a/core/monitor/metric_constants/AgentMetrics.cpp b/core/monitor/metric_constants/AgentMetrics.cpp index f2e49630ac..ad17130e6c 100644 --- a/core/monitor/metric_constants/AgentMetrics.cpp +++ b/core/monitor/metric_constants/AgentMetrics.cpp @@ -19,7 +19,6 @@ using namespace std; namespace logtail { // label keys -const string METRIC_LABEL_KEY_METRIC_CATEGORY = "metric_category"; const string METRIC_LABEL_KEY_ALIUIDS = "aliuids"; const string METRIC_LABEL_KEY_INSTANCE_ID = "instance_id"; const string METRIC_LABEL_KEY_START_TIME = "start_time"; @@ -30,16 +29,13 @@ const string METRIC_LABEL_KEY_USER_DEFINED_ID = "user_defined_id"; const string METRIC_LABEL_KEY_UUID = "uuid"; const string METRIC_LABEL_KEY_VERSION = "version"; -// label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_AGENT = "agent"; - // metric keys -const string METRIC_AGENT_CPU = "agent_cpu_percent"; -const string METRIC_AGENT_GO_ROUTINES_TOTAL = "agent_go_routines_total"; -const string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "agent_instance_config_total"; // Not Implemented -const string METRIC_AGENT_MEMORY = "agent_memory_used_mb"; -const string METRIC_AGENT_MEMORY_GO = "agent_go_memory_used_mb"; -const string METRIC_AGENT_OPEN_FD_TOTAL = "agent_open_fd_total"; -const string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "agent_pipeline_config_total"; +const string METRIC_AGENT_CPU = "cpu"; +const string METRIC_AGENT_GO_ROUTINES_TOTAL = "go_routines_total"; +const string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "instance_config_total"; // Not Implemented +const string METRIC_AGENT_MEMORY = "memory_used_mb"; +const string METRIC_AGENT_MEMORY_GO = "go_memory_used_mb"; +const string METRIC_AGENT_OPEN_FD_TOTAL = "open_fd_total"; +const string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "pipeline_config_total"; } // namespace logtail diff --git a/core/monitor/metric_constants/ComponentMetrics.cpp b/core/monitor/metric_constants/ComponentMetrics.cpp index cbbb8ed40b..358a7a6a36 100644 --- a/core/monitor/metric_constants/ComponentMetrics.cpp +++ b/core/monitor/metric_constants/ComponentMetrics.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "MetricCommonConstants.h" #include "MetricConstants.h" using namespace std; @@ -34,7 +35,6 @@ const string METRIC_LABEL_KEY_EXACTLY_ONCE_ENABLED = "exactly_once_enabled"; const string METRIC_LABEL_KEY_GROUP_BATCH_ENABLED = "group_batch_enabled"; // label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT = "component"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER = "batcher"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR = "compressor"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE = "process_queue"; @@ -43,46 +43,42 @@ const string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE = "sender_queue"; const string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER = "serializer"; // metric keys -const string METRIC_COMPONENT_IN_EVENTS_TOTAL = "component_in_events_total"; -const string METRIC_COMPONENT_IN_SIZE_BYTES = "component_in_size_bytes"; -const string METRIC_COMPONENT_IN_ITEMS_TOTAL = "component_in_items_total"; -const string METRIC_COMPONENT_OUT_EVENTS_TOTAL = "component_out_events_total"; -const string METRIC_COMPONENT_OUT_ITEMS_TOTAL = "component_out_items_total"; -const string METRIC_COMPONENT_OUT_SIZE_BYTES = "component_out_size_bytes"; -const string METRIC_COMPONENT_TOTAL_DELAY_MS = "component_total_delay_ms"; -const string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = "component_total_process_time_ms"; -const string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = "component_discarded_items_total"; -const string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "component_discarded_item_size_bytes"; +const string& METRIC_COMPONENT_IN_EVENTS_TOTAL = METRIC_IN_EVENTS_TOTAL; +const string& METRIC_COMPONENT_IN_SIZE_BYTES = METRIC_IN_SIZE_BYTES; +const string& METRIC_COMPONENT_IN_ITEMS_TOTAL = METRIC_IN_ITEMS_TOTAL; +const string& METRIC_COMPONENT_OUT_EVENTS_TOTAL = METRIC_OUT_EVENTS_TOTAL; +const string& METRIC_COMPONENT_OUT_ITEMS_TOTAL = METRIC_OUT_ITEMS_TOTAL; +const string& METRIC_COMPONENT_OUT_SIZE_BYTES = METRIC_OUT_SIZE_BYTES; +const string& METRIC_COMPONENT_TOTAL_DELAY_MS = METRIC_TOTAL_DELAY_MS; +const string& METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = METRIC_TOTAL_PROCESS_TIME_MS; +const string& METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = METRIC_DISCARDED_ITEMS_TOTAL; +const string& METRIC_COMPONENT_DISCARDED_SIZE_BYTES = METRIC_DISCARDED_SIZE_BYTES; /********************************************************** * batcher **********************************************************/ -const string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL = "component_event_batches_total"; -const string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_TOTAL = "component_buffered_groups_total"; -const string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_TOTAL = "component_buffered_events_total"; -const string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "component_buffered_size_bytes"; -const string METRIC_COMPONENT_BATCHER_TOTAL_ADD_TIME_MS = "component_total_add_time_ms"; +const string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL = "event_batches_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_TOTAL = "buffered_groups_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_TOTAL = "buffered_events_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "buffered_size_bytes"; +const string METRIC_COMPONENT_BATCHER_TOTAL_ADD_TIME_MS = "total_add_time_ms"; /********************************************************** * queue **********************************************************/ -const string METRIC_COMPONENT_QUEUE_SIZE = "component_queue_size"; -const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "component_queue_size_bytes"; -const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "component_valid_to_push_status"; -const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "component_extra_buffer_size"; -const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "component_extra_buffer_size_bytes"; -const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = "component_discarded_events_total"; +const string METRIC_COMPONENT_QUEUE_SIZE = "queue_size"; +const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "queue_size_bytes"; +const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "valid_to_push_status"; +const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "extra_buffer_size"; +const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "extra_buffer_size_bytes"; +const string& METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = METRIC_DISCARDED_EVENTS_TOTAL; -const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "component_fetched_items_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL = "component_fetch_times_total"; -const string METRIC_COMPONENT_QUEUE_VALID_FETCH_TIMES_TOTAL = "component_valid_fetch_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_REGION_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_region_limiter_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_PROJECT_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_project_limiter_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_LOGSTORE_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_logstore_limiter_times_total"; -const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL - = "component_fetch_rejected_by_rate_limiter_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL = "fetched_items_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL = "fetch_times_total"; +const string METRIC_COMPONENT_QUEUE_VALID_FETCH_TIMES_TOTAL = "valid_fetch_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_REGION_LIMITER_TIMES_TOTAL = "region_reject_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_PROJECT_LIMITER_TIMES_TOTAL = "project_reject_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_LOGSTORE_LIMITER_TIMES_TOTAL = "logstore_reject_times_total"; +const string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL = "rate_reject_times_total"; } // namespace logtail diff --git a/core/monitor/metric_constants/MetricCommonConstants.cpp b/core/monitor/metric_constants/MetricCommonConstants.cpp new file mode 100644 index 0000000000..a2a1e3542d --- /dev/null +++ b/core/monitor/metric_constants/MetricCommonConstants.cpp @@ -0,0 +1,35 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "MetricCommonConstants.h" + +using namespace std; + +namespace logtail { + +const string METRIC_DISCARDED_EVENTS_TOTAL = "discarded_events_total"; +const string METRIC_DISCARDED_ITEMS_TOTAL = "discarded_items_total"; +const string METRIC_DISCARDED_SIZE_BYTES = "discarded_size_bytes"; +const string METRIC_IN_EVENTS_TOTAL = "in_events_total"; +const string METRIC_IN_EVENT_GROUPS_TOTAL = "in_event_groups_total"; +const string METRIC_IN_ITEMS_TOTAL = "in_items_total"; +const string METRIC_IN_SIZE_BYTES = "in_size_bytes"; +const string METRIC_OUT_EVENTS_TOTAL = "out_events_total"; +const string METRIC_OUT_EVENT_GROUPS_TOTAL = "out_event_groups_total"; +const string METRIC_OUT_ITEMS_TOTAL = "out_items_total"; +const string METRIC_OUT_SIZE_BYTES = "out_size_bytes"; +const string METRIC_TOTAL_DELAY_MS = "total_delay_ms"; +const string METRIC_TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; + +} \ No newline at end of file diff --git a/core/monitor/metric_constants/MetricCommonConstants.h b/core/monitor/metric_constants/MetricCommonConstants.h new file mode 100644 index 0000000000..4a78b4e9d1 --- /dev/null +++ b/core/monitor/metric_constants/MetricCommonConstants.h @@ -0,0 +1,36 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace logtail { + +extern const std::string METRIC_DISCARDED_EVENTS_TOTAL; +extern const std::string METRIC_DISCARDED_ITEMS_TOTAL; +extern const std::string METRIC_DISCARDED_SIZE_BYTES; +extern const std::string METRIC_IN_EVENTS_TOTAL; +extern const std::string METRIC_IN_EVENT_GROUPS_TOTAL; +extern const std::string METRIC_IN_ITEMS_TOTAL; +extern const std::string METRIC_IN_SIZE_BYTES; +extern const std::string METRIC_OUT_EVENTS_TOTAL; +extern const std::string METRIC_OUT_EVENT_GROUPS_TOTAL; +extern const std::string METRIC_OUT_ITEMS_TOTAL; +extern const std::string METRIC_OUT_SIZE_BYTES; +extern const std::string METRIC_TOTAL_DELAY_MS; +extern const std::string METRIC_TOTAL_PROCESS_TIME_MS; + +} \ No newline at end of file diff --git a/core/monitor/metric_constants/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h index 84740a09b9..73a62d30bf 100644 --- a/core/monitor/metric_constants/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -19,16 +19,6 @@ namespace logtail { -// label keys -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY; - -// label values -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_AGENT; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_PIPELINE; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_PLUGIN; -extern const std::string METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER; - ////////////////////////////////////////////////////////////////////////// // agent ////////////////////////////////////////////////////////////////////////// @@ -81,14 +71,14 @@ extern const std::string METRIC_LABEL_KEY_PLUGIN_ID; extern const std::string METRIC_LABEL_KEY_PLUGIN_TYPE; // metric keys -extern const std::string METRIC_PLUGIN_IN_EVENTS_TOTAL; -extern const std::string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_PLUGIN_IN_SIZE_BYTES; -extern const std::string METRIC_PLUGIN_OUT_EVENTS_TOTAL; -extern const std::string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_PLUGIN_OUT_SIZE_BYTES; -extern const std::string METRIC_PLUGIN_TOTAL_DELAY_MS; -extern const std::string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS; +extern const std::string& METRIC_PLUGIN_IN_EVENTS_TOTAL; +extern const std::string& METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL; +extern const std::string& METRIC_PLUGIN_IN_SIZE_BYTES; +extern const std::string& METRIC_PLUGIN_OUT_EVENTS_TOTAL; +extern const std::string& METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL; +extern const std::string& METRIC_PLUGIN_OUT_SIZE_BYTES; +extern const std::string& METRIC_PLUGIN_TOTAL_DELAY_MS; +extern const std::string& METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS; /********************************************************** * input_file @@ -159,7 +149,7 @@ extern const std::string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL; /********************************************************** * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ -extern const std::string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL; +extern const std::string& METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL; @@ -230,16 +220,16 @@ extern const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE; extern const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER; // metric keys -extern const std::string METRIC_COMPONENT_IN_EVENTS_TOTAL; -extern const std::string METRIC_COMPONENT_IN_SIZE_BYTES; -extern const std::string METRIC_COMPONENT_IN_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_OUT_EVENTS_TOTAL; -extern const std::string METRIC_COMPONENT_OUT_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_OUT_SIZE_BYTES; -extern const std::string METRIC_COMPONENT_TOTAL_DELAY_MS; -extern const std::string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS; -extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES; +extern const std::string& METRIC_COMPONENT_IN_EVENTS_TOTAL; +extern const std::string& METRIC_COMPONENT_IN_SIZE_BYTES; +extern const std::string& METRIC_COMPONENT_IN_ITEMS_TOTAL; +extern const std::string& METRIC_COMPONENT_OUT_EVENTS_TOTAL; +extern const std::string& METRIC_COMPONENT_OUT_ITEMS_TOTAL; +extern const std::string& METRIC_COMPONENT_OUT_SIZE_BYTES; +extern const std::string& METRIC_COMPONENT_TOTAL_DELAY_MS; +extern const std::string& METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS; +extern const std::string& METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL; +extern const std::string& METRIC_COMPONENT_DISCARDED_SIZE_BYTES; /********************************************************** * batcher @@ -258,7 +248,7 @@ extern const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES; extern const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG; extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE; extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES; -extern const std::string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL; +extern const std::string& METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL; extern const std::string METRIC_COMPONENT_QUEUE_FETCHED_ITEMS_TOTAL; extern const std::string METRIC_COMPONENT_QUEUE_FETCH_TIMES_TOTAL; @@ -274,6 +264,7 @@ extern const std::string METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_T // label keys extern const std::string METRIC_LABEL_KEY_RUNNER_NAME; +extern const std::string METRIC_LABEL_KEY_THREAD_NO; // label values extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER; @@ -284,22 +275,26 @@ extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS; extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER; // metric keys -extern const std::string METRIC_RUNNER_IN_EVENTS_TOTAL; -extern const std::string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_RUNNER_IN_SIZE_BYTES; -extern const std::string METRIC_RUNNER_IN_ITEMS_TOTAL; +extern const std::string& METRIC_RUNNER_IN_EVENTS_TOTAL; +extern const std::string& METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL; +extern const std::string& METRIC_RUNNER_IN_SIZE_BYTES; +extern const std::string& METRIC_RUNNER_IN_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_LAST_RUN_TIME; -extern const std::string METRIC_RUNNER_OUT_ITEMS_TOTAL; -extern const std::string METRIC_RUNNER_TOTAL_DELAY_MS; +extern const std::string& METRIC_RUNNER_OUT_ITEMS_TOTAL; +extern const std::string& METRIC_RUNNER_TOTAL_DELAY_MS; +extern const std::string METRIC_RUNNER_CLIENT_REGISTER_STATE; +extern const std::string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL; +extern const std::string METRIC_RUNNER_JOBS_TOTAL; + +/********************************************************** + * all sinks + **********************************************************/ extern const std::string METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS; extern const std::string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS; extern const std::string METRIC_RUNNER_SINK_SENDING_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_SINK_SEND_CONCURRENCY; -extern const std::string METRIC_RUNNER_CLIENT_REGISTER_STATE; -extern const std::string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL; -extern const std::string METRIC_RUNNER_JOBS_TOTAL; /********************************************************** * flusher runner diff --git a/core/monitor/metric_constants/PipelineMetrics.cpp b/core/monitor/metric_constants/PipelineMetrics.cpp index 85fd7f3e8f..b82904101c 100644 --- a/core/monitor/metric_constants/PipelineMetrics.cpp +++ b/core/monitor/metric_constants/PipelineMetrics.cpp @@ -23,18 +23,15 @@ const string METRIC_LABEL_KEY_LOGSTORE = "logstore"; const string METRIC_LABEL_KEY_PIPELINE_NAME = "pipeline_name"; const string METRIC_LABEL_KEY_REGION = "region"; -// label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_PIPELINE = "pipeline"; - // metric keys -const string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "pipeline_processors_in_events_total"; -const string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "pipeline_processors_in_event_groups_total"; -const string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES = "pipeline_processors_in_size_bytes"; -const string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS = "pipeline_processors_total_process_time_ms"; -const string METRIC_PIPELINE_FLUSHERS_IN_EVENTS_TOTAL = "pipeline_flushers_in_events_total"; -const string METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL = "pipeline_flushers_in_event_groups_total"; -const string METRIC_PIPELINE_FLUSHERS_IN_SIZE_BYTES = "pipeline_flushers_in_size_bytes"; -const string METRIC_PIPELINE_FLUSHERS_TOTAL_PACKAGE_TIME_MS = "pipeline_flushers_total_package_time_ms"; -const string METRIC_PIPELINE_START_TIME = "pipeline_start_time"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "processor_in_events_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "processor_in_event_groups_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES = "processor_in_size_bytes"; +const string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS = "processor_total_process_time_ms"; +const string METRIC_PIPELINE_FLUSHERS_IN_EVENTS_TOTAL = "flusher_in_events_total"; +const string METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL = "flusher_in_event_groups_total"; +const string METRIC_PIPELINE_FLUSHERS_IN_SIZE_BYTES = "flusher_in_size_bytes"; +const string METRIC_PIPELINE_FLUSHERS_TOTAL_PACKAGE_TIME_MS = "flusher_total_package_time_ms"; +const string METRIC_PIPELINE_START_TIME = "start_time"; } // namespace logtail diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index 448a551cae..329e487539 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "MetricCommonConstants.h" #include "MetricConstants.h" using namespace std; @@ -22,18 +23,15 @@ namespace logtail { const string METRIC_LABEL_KEY_PLUGIN_ID = "plugin_id"; const string METRIC_LABEL_KEY_PLUGIN_TYPE = "plugin_type"; -// label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_PLUGIN = "plugin"; - // metric keys -const string METRIC_PLUGIN_IN_EVENTS_TOTAL = "plugin_in_events_total"; -const string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = "plugin_in_event_groups_total"; -const string METRIC_PLUGIN_IN_SIZE_BYTES = "plugin_in_size_bytes"; -const string METRIC_PLUGIN_OUT_EVENTS_TOTAL = "plugin_out_events_total"; -const string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "plugin_out_event_groups_total"; -const string METRIC_PLUGIN_OUT_SIZE_BYTES = "plugin_out_size_bytes"; -const string METRIC_PLUGIN_TOTAL_DELAY_MS = "plugin_total_delay_ms"; -const string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = "plugin_total_process_time_ms"; +const string& METRIC_PLUGIN_IN_EVENTS_TOTAL = METRIC_IN_EVENTS_TOTAL; +const string& METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = METRIC_IN_EVENT_GROUPS_TOTAL; +const string& METRIC_PLUGIN_IN_SIZE_BYTES = METRIC_IN_SIZE_BYTES; +const string& METRIC_PLUGIN_OUT_EVENTS_TOTAL = METRIC_OUT_EVENTS_TOTAL; +const string& METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = METRIC_OUT_EVENT_GROUPS_TOTAL; +const string& METRIC_PLUGIN_OUT_SIZE_BYTES = METRIC_OUT_SIZE_BYTES; +const string& METRIC_PLUGIN_TOTAL_DELAY_MS = METRIC_TOTAL_DELAY_MS; +const string& METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = METRIC_TOTAL_PROCESS_TIME_MS; /********************************************************** * input_file @@ -43,9 +41,9 @@ const string METRIC_LABEL_KEY_FILE_DEV = "file_dev"; const string METRIC_LABEL_KEY_FILE_INODE = "file_inode"; const string METRIC_LABEL_KEY_FILE_NAME = "file_name"; -const string METRIC_PLUGIN_MONITOR_FILE_TOTAL = "plugin_monitor_file_total"; -const string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "plugin_source_read_offset_bytes"; -const string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "plugin_source_size_bytes"; +const string METRIC_PLUGIN_MONITOR_FILE_TOTAL = "monitor_file_total"; +const string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "read_offset_bytes"; +const string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "size_bytes"; /********************************************************** * input_prometheus @@ -57,11 +55,11 @@ const std::string METRIC_LABEL_KEY_SERVICE_PORT = "service_port"; const std::string METRIC_LABEL_KEY_STATUS = "status"; const std::string METRIC_LABEL_KEY_INSTANCE = "instance"; -const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS = "plugin_prom_subscribe_targets"; -const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL = "plugin_prom_subscribe_total"; -const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TIME_MS = "plugin_prom_subscribe_time_ms"; -const std::string METRIC_PLUGIN_PROM_SCRAPE_TIME_MS = "plugin_prom_scrape_time_ms"; -const std::string METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL = "plugin_prom_scrape_delay_total"; +const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS = "prom_subscribe_targets"; +const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL = "prom_subscribe_total"; +const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TIME_MS = "prom_subscribe_time_ms"; +const std::string METRIC_PLUGIN_PROM_SCRAPE_TIME_MS = "prom_scrape_time_ms"; +const std::string METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL = "prom_scrape_delay_total"; /********************************************************** * input_ebpf @@ -92,69 +90,68 @@ const string METRIC_LABEL_VALUE_PLUGIN_TYPE_FILE_SECURITY = "file_security"; const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_OBSERVER = "process_observer"; const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_SECURITY = "process_security"; -const string METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL = "plugin_ebpf_loss_kernel_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM = "plugin_network_observer_conntracker_num"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL = "plugin_network_observer_worker_handle_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL = "plugin_network_observer_parse_records_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL = "plugin_network_observer_aggregate_events_total"; -const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM = "plugin_network_observer_aggregate_key_num"; -const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM = "plugin_process_cache_entries_num"; -const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "plugin_process_cache_miss_total"; +const string METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL = "ebpf_loss_kernel_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM = "conntracker_num"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL = "handle_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL = "parse_records_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL = "aggregate_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM = "aggregate_key_num"; +const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM = "process_cache_entries_num"; +const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "process_cache_miss_total"; /********************************************************** * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ -const string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = "plugin_discarded_events_total"; -const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "plugin_out_failed_events_total"; -const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "plugin_out_key_not_found_events_total"; -const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "plugin_out_successful_events_total"; +const string& METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = METRIC_DISCARDED_EVENTS_TOTAL; +const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "out_failed_events_total"; +const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "out_key_not_found_events_total"; +const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "out_successful_events_total"; /********************************************************** * processor_parse_apsara_native * processor_parse_timestamp_native **********************************************************/ -const string METRIC_PLUGIN_HISTORY_FAILURE_TOTAL = "plugin_history_failure_total"; +const string METRIC_PLUGIN_HISTORY_FAILURE_TOTAL = "history_failure_total"; /********************************************************** * processor_split_multiline_log_string_native **********************************************************/ -const string METRIC_PLUGIN_MATCHED_EVENTS_TOTAL = "plugin_matched_events_total"; -const string METRIC_PLUGIN_MATCHED_LINES_TOTAL = "plugin_matched_lines_total"; -const string METRIC_PLUGIN_UNMATCHED_LINES_TOTAL = "plugin_unmatched_lines_total"; +const string METRIC_PLUGIN_MATCHED_EVENTS_TOTAL = "matched_events_total"; +const string METRIC_PLUGIN_MATCHED_LINES_TOTAL = "matched_lines_total"; +const string METRIC_PLUGIN_UNMATCHED_LINES_TOTAL = "unmatched_lines_total"; /********************************************************** * processor_merge_multiline_log_native **********************************************************/ -const string METRIC_PLUGIN_MERGED_EVENTS_TOTAL = "plugin_merged_events_total"; -const string METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL = "plugin_unmatched_events_total"; +const string METRIC_PLUGIN_MERGED_EVENTS_TOTAL = "merged_events_total"; +const string METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL = "unmatched_events_total"; /********************************************************** * processor_parse_container_log_native **********************************************************/ -const string METRIC_PLUGIN_PARSE_STDERR_TOTAL = "plugin_parse_stderr_total"; -const string METRIC_PLUGIN_PARSE_STDOUT_TOTAL = "plugin_parse_stdout_total"; +const string METRIC_PLUGIN_PARSE_STDERR_TOTAL = "parse_stderr_total"; +const string METRIC_PLUGIN_PARSE_STDOUT_TOTAL = "parse_stdout_total"; /********************************************************** * all flusher （所有发送插件通用指标） **********************************************************/ -const string METRIC_PLUGIN_FLUSHER_TOTAL_PACKAGE_TIME_MS = "plugin_flusher_total_package_time_ms"; -const string METRIC_PLUGIN_FLUSHER_OUT_EVENT_GROUPS_TOTAL = "plugin_flusher_send_total"; -const string METRIC_PLUGIN_FLUSHER_SEND_DONE_TOTAL = "plugin_flusher_send_done_total"; -const string METRIC_PLUGIN_FLUSHER_SUCCESS_TOTAL = "plugin_flusher_success_total"; -const string METRIC_PLUGIN_FLUSHER_NETWORK_ERROR_TOTAL = "plugin_flusher_network_error_total"; -const string METRIC_PLUGIN_FLUSHER_SERVER_ERROR_TOTAL = "plugin_flusher_server_error_total"; -const string METRIC_PLUGIN_FLUSHER_UNAUTH_ERROR_TOTAL = "plugin_flusher_unauth_error_total"; -const string METRIC_PLUGIN_FLUSHER_PARAMS_ERROR_TOTAL = "plugin_flusher_params_error_total"; -const string METRIC_PLUGIN_FLUSHER_OTHER_ERROR_TOTAL = "plugin_flusher_other_error_total"; +const string METRIC_PLUGIN_FLUSHER_TOTAL_PACKAGE_TIME_MS = "total_package_time_ms"; +const string METRIC_PLUGIN_FLUSHER_OUT_EVENT_GROUPS_TOTAL = "send_total"; +const string METRIC_PLUGIN_FLUSHER_SEND_DONE_TOTAL = "send_done_total"; +const string METRIC_PLUGIN_FLUSHER_SUCCESS_TOTAL = "success_total"; +const string METRIC_PLUGIN_FLUSHER_NETWORK_ERROR_TOTAL = "network_error_total"; +const string METRIC_PLUGIN_FLUSHER_SERVER_ERROR_TOTAL = "server_error_total"; +const string METRIC_PLUGIN_FLUSHER_UNAUTH_ERROR_TOTAL = "unauth_error_total"; +const string METRIC_PLUGIN_FLUSHER_PARAMS_ERROR_TOTAL = "params_error_total"; +const string METRIC_PLUGIN_FLUSHER_OTHER_ERROR_TOTAL = "other_error_total"; /********************************************************** * flusher_sls **********************************************************/ -const string METRIC_PLUGIN_FLUSHER_SLS_SHARD_WRITE_QUOTA_ERROR_TOTAL - = "plugin_flusher_sls_shard_write_quota_error_total"; -const string METRIC_PLUGIN_FLUSHER_SLS_PROJECT_QUOTA_ERROR_TOTAL = "plugin_flusher_sls_project_quota_error_total"; -const string METRIC_PLUGIN_FLUSHER_SLS_SEQUENCE_ID_ERROR_TOTAL = "plugin_flusher_sls_sequence_id_error_total"; -const string METRIC_PLUGIN_FLUSHER_SLS_REQUEST_EXPRIRED_ERROR_TOTAL = "plugin_flusher_sls_request_exprired_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_SHARD_WRITE_QUOTA_ERROR_TOTAL = "shard_write_quota_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_PROJECT_QUOTA_ERROR_TOTAL = "project_quota_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_SEQUENCE_ID_ERROR_TOTAL = "sequence_id_error_total"; +const string METRIC_PLUGIN_FLUSHER_SLS_REQUEST_EXPRIRED_ERROR_TOTAL = "request_exprired_error_total"; } // namespace logtail \ No newline at end of file diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index 8fb8519167..c1aead8643 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "MetricCommonConstants.h" #include "MetricConstants.h" using namespace std; @@ -20,9 +21,9 @@ namespace logtail { // label keys const string METRIC_LABEL_KEY_RUNNER_NAME = "runner_name"; +const string METRIC_LABEL_KEY_THREAD_NO = "thread_no"; // label values -const string METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER = "runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER = "file_server"; const string METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER = "flusher_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK = "http_sink"; @@ -31,49 +32,48 @@ const string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS = "prometheus_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER = "ebpf_server"; // metric keys -const string METRIC_RUNNER_IN_EVENTS_TOTAL = "runner_in_events_total"; -const string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = "runner_in_event_groups_total"; -const string METRIC_RUNNER_IN_SIZE_BYTES = "runner_in_size_bytes"; -const string METRIC_RUNNER_IN_ITEMS_TOTAL = "runner_in_items_total"; -const string METRIC_RUNNER_LAST_RUN_TIME = "runner_last_run_time"; -const string METRIC_RUNNER_OUT_ITEMS_TOTAL = "runner_out_items_total"; -const string METRIC_RUNNER_TOTAL_DELAY_MS = "runner_total_delay_ms"; -const string METRIC_RUNNER_CLIENT_REGISTER_STATE = "runner_client_register_state"; -const string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL = "runner_client_register_retry_total"; -const string METRIC_RUNNER_JOBS_TOTAL = "runner_jobs_total"; +const string& METRIC_RUNNER_IN_EVENTS_TOTAL = METRIC_IN_EVENTS_TOTAL; +const string& METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = METRIC_IN_EVENT_GROUPS_TOTAL; +const string& METRIC_RUNNER_IN_SIZE_BYTES = METRIC_IN_SIZE_BYTES; +const string& METRIC_RUNNER_IN_ITEMS_TOTAL = METRIC_IN_ITEMS_TOTAL; +const string METRIC_RUNNER_LAST_RUN_TIME = "last_run_time"; +const string& METRIC_RUNNER_OUT_ITEMS_TOTAL = METRIC_OUT_ITEMS_TOTAL; +const string& METRIC_RUNNER_TOTAL_DELAY_MS = METRIC_TOTAL_DELAY_MS; +const string METRIC_RUNNER_CLIENT_REGISTER_STATE = "client_register_state"; +const string METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL = "client_register_retry_total"; +const string METRIC_RUNNER_JOBS_TOTAL = "jobs_total"; /********************************************************** * all sinks **********************************************************/ -const string METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL = "runner_out_successful_items_total"; -const string METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL = "runner_out_failed_items_total"; -const string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS = "runner_successful_item_total_response_time_ms"; -const string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS = "runner_failed_item_total_response_time_ms"; -const string METRIC_RUNNER_SINK_SENDING_ITEMS_TOTAL = "runner_sending_items_total"; -const string METRIC_RUNNER_SINK_SEND_CONCURRENCY = "runner_send_concurrency"; +const string METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL = "out_successful_items_total"; +const string METRIC_RUNNER_SINK_OUT_FAILED_ITEMS_TOTAL = "out_failed_items_total"; +const string METRIC_RUNNER_SINK_SUCCESSFUL_ITEM_TOTAL_RESPONSE_TIME_MS = "successful_response_time_ms"; +const string METRIC_RUNNER_SINK_FAILED_ITEM_TOTAL_RESPONSE_TIME_MS = "failed_response_time_ms"; +const string METRIC_RUNNER_SINK_SENDING_ITEMS_TOTAL = "sending_items_total"; +const string METRIC_RUNNER_SINK_SEND_CONCURRENCY = "send_concurrency"; /********************************************************** * flusher runner **********************************************************/ -const string METRIC_RUNNER_FLUSHER_IN_RAW_SIZE_BYTES = "runner_in_raw_size_bytes"; -const string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL = "runner_waiting_items_total"; +const string METRIC_RUNNER_FLUSHER_IN_RAW_SIZE_BYTES = "in_raw_size_bytes"; +const string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL = "waiting_items_total"; /********************************************************** * file server **********************************************************/ -const string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL = "runner_watched_dirs_total"; -const string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL = "runner_active_readers_total"; -const string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG - = "runner_enable_file_included_by_multi_configs"; -const string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "runner_polling_modify_cache_size"; -const string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "runner_polling_dir_cache_size"; -const string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "runner_polling_file_cache_size"; +const string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL = "watched_dirs_total"; +const string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL = "active_readers_total"; +const string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG = "enable_multi_configs"; +const string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "polling_modify_cache_size"; +const string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "polling_dir_cache_size"; +const string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "polling_file_cache_size"; /********************************************************** * ebpf server **********************************************************/ -const string METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL = "runner_start_plugin_total"; -const string METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL = "runner_stop_plugin_total"; -const string METRIC_RUNNER_EBPF_SUSPEND_PLUGIN_TOTAL = "runner_suspend_plugin_total"; +const string METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL = "start_plugin_total"; +const string METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL = "stop_plugin_total"; +const string METRIC_RUNNER_EBPF_SUSPEND_PLUGIN_TOTAL = "suspend_plugin_total"; } // namespace logtail \ No newline at end of file diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 40dec54bd6..d80ebc5064 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -318,9 +318,8 @@ bool Pipeline::Init(PipelineConfig&& config) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, - {METRIC_LABEL_KEY_PIPELINE_NAME, mName}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_PIPELINE}}); + MetricCategory::METRIC_CATEGORY_PIPELINE, + {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mName}}); mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); mProcessorsInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL); mProcessorsInGroupsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL); diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index aeca80b600..2b1898fd55 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -104,14 +104,14 @@ class Batcher { {METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, flusher->GetPluginID()}}; if (enableGroupBatch) { labels.emplace_back(METRIC_LABEL_KEY_GROUP_BATCH_ENABLED, "true"); } else { labels.emplace_back(METRIC_LABEL_KEY_GROUP_BATCH_ENABLED, "false"); } - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_COMPONENT, std::move(labels)); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_EVENTS_TOTAL); diff --git a/core/pipeline/plugin/interface/Plugin.h b/core/pipeline/plugin/interface/Plugin.h index 087a44cd41..1dbd872756 100644 --- a/core/pipeline/plugin/interface/Plugin.h +++ b/core/pipeline/plugin/interface/Plugin.h @@ -39,9 +39,9 @@ class Plugin { void SetMetricsRecordRef(const std::string& name, const std::string& id) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_PLUGIN, {{METRIC_LABEL_KEY_PROJECT, mContext->GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mContext->GetConfigName()}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_PLUGIN}, {METRIC_LABEL_KEY_PLUGIN_TYPE, name}, {METRIC_LABEL_KEY_PLUGIN_ID, id}}); } diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index e7cf3a37a1..b0ea6d0e47 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -28,10 +28,10 @@ class QueueInterface { public: QueueInterface(QueueKey key, size_t cap, const PipelineContext& ctx) : mKey(key), mCapacity(cap) { WriteMetrics::GetInstance()->CreateMetricsRecordRef(mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_COMPONENT, { {METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, }); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index 08abd08cca..521f4b0404 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -37,10 +37,10 @@ bool Router::Init(std::vector> configs, const P WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_COMPONENT, {{METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, - {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}}); + {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER}}); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); return true; diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index a9a99c737e..27cc2e0847 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -49,10 +49,10 @@ class Serializer { Serializer(Flusher* f) : mFlusher(f) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_COMPONENT, {{METRIC_LABEL_KEY_PROJECT, f->GetContext().GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, f->GetContext().GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_COMPONENT}, {METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, f->GetPluginID()}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); @@ -60,7 +60,7 @@ class Serializer { mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_SIZE_BYTES); mTotalProcessMs = mMetricsRecordRef.CreateTimeCounter(METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS); mDiscardedItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL); - mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_SIZE_BYTES); } virtual ~Serializer() = default; diff --git a/core/plugin/input/InputContainerStdio.cpp b/core/plugin/input/InputContainerStdio.cpp index 46a018e067..6c3f5d49a4 100644 --- a/core/plugin/input/InputContainerStdio.cpp +++ b/core/plugin/input/InputContainerStdio.cpp @@ -69,7 +69,8 @@ bool InputContainerStdio::Init(const Json::Value& config, Json::Value& optionalG if (!mContainerDiscovery.Init(config, *mContext, sName)) { return false; } - mContainerDiscovery.GenerateContainerMetaFetchingGoPipeline(optionalGoPipeline, nullptr, mContext->GetPipeline().GenNextPluginMeta(false)); + mContainerDiscovery.GenerateContainerMetaFetchingGoPipeline( + optionalGoPipeline, nullptr, mContext->GetPipeline().GenNextPluginMeta(false)); if (!mFileReader.Init(config, *mContext, sName)) { return false; @@ -167,8 +168,8 @@ bool InputContainerStdio::Init(const Json::Value& config, Json::Value& optionalG {METRIC_PLUGIN_SOURCE_SIZE_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, {METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, }; - mPluginMetricManager - = std::make_shared(GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys); + mPluginMetricManager = std::make_shared( + GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); // Register a Gauge metric to record PluginMetricManager‘s map size mMonitorFileTotal = GetMetricsRecordRef().CreateIntGauge(METRIC_PLUGIN_MONITOR_FILE_TOTAL); mPluginMetricManager->RegisterSizeGauge(mMonitorFileTotal); diff --git a/core/plugin/input/InputFile.cpp b/core/plugin/input/InputFile.cpp index c5b0039a19..b8186797b8 100644 --- a/core/plugin/input/InputFile.cpp +++ b/core/plugin/input/InputFile.cpp @@ -165,8 +165,8 @@ bool InputFile::Init(const Json::Value& config, Json::Value& optionalGoPipeline) {METRIC_PLUGIN_SOURCE_SIZE_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, {METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES, MetricType::METRIC_TYPE_INT_GAUGE}, }; - mPluginMetricManager - = std::make_shared(GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys); + mPluginMetricManager = std::make_shared( + GetMetricsRecordRef()->GetLabels(), inputFileMetricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); mPluginMetricManager->RegisterSizeGauge(mMonitorFileTotal); return CreateInnerProcessors(); diff --git a/core/plugin/input/InputFileSecurity.cpp b/core/plugin/input/InputFileSecurity.cpp index 3510833830..5c15536721 100644 --- a/core/plugin/input/InputFileSecurity.cpp +++ b/core/plugin/input/InputFileSecurity.cpp @@ -43,7 +43,7 @@ bool InputFileSecurity::Init(const Json::Value& config, Json::Value& optionalGoP {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::FILE, config, mContext, sName); } diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index e096e585f9..fea0b459fc 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -36,7 +36,7 @@ class InputFileSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputNetworkObserver.cpp b/core/plugin/input/InputNetworkObserver.cpp index c2b997d54f..ce9c4218ff 100644 --- a/core/plugin/input/InputNetworkObserver.cpp +++ b/core/plugin/input/InputNetworkObserver.cpp @@ -49,7 +49,7 @@ bool InputNetworkObserver::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return ebpf::InitObserverNetworkOption(config, mNetworkOption, mContext, sName); } diff --git a/core/plugin/input/InputNetworkObserver.h b/core/plugin/input/InputNetworkObserver.h index 7cc3d98ca6..7f204a2c90 100644 --- a/core/plugin/input/InputNetworkObserver.h +++ b/core/plugin/input/InputNetworkObserver.h @@ -36,7 +36,7 @@ class InputNetworkObserver : public Input { bool SupportAck() const override { return false; } nami::ObserverNetworkOption mNetworkOption; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputNetworkSecurity.cpp b/core/plugin/input/InputNetworkSecurity.cpp index 409d40f77c..ccadcc26ca 100644 --- a/core/plugin/input/InputNetworkSecurity.cpp +++ b/core/plugin/input/InputNetworkSecurity.cpp @@ -45,7 +45,7 @@ bool InputNetworkSecurity::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::NETWORK, config, mContext, sName); } diff --git a/core/plugin/input/InputNetworkSecurity.h b/core/plugin/input/InputNetworkSecurity.h index 20e3025246..cda3a7c170 100644 --- a/core/plugin/input/InputNetworkSecurity.h +++ b/core/plugin/input/InputNetworkSecurity.h @@ -35,7 +35,7 @@ class InputNetworkSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputProcessSecurity.cpp b/core/plugin/input/InputProcessSecurity.cpp index cc11afa854..3baa6cbed8 100644 --- a/core/plugin/input/InputProcessSecurity.cpp +++ b/core/plugin/input/InputProcessSecurity.cpp @@ -42,7 +42,7 @@ bool InputProcessSecurity::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::PROCESS, config, mContext, sName); } diff --git a/core/plugin/input/InputProcessSecurity.h b/core/plugin/input/InputProcessSecurity.h index b45fec9b5c..d26d7a95e3 100644 --- a/core/plugin/input/InputProcessSecurity.h +++ b/core/plugin/input/InputProcessSecurity.h @@ -35,7 +35,7 @@ class InputProcessSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; - std::shared_ptr mPluginMgr; + PluginMetricManagerPtr mPluginMgr; }; } // namespace logtail diff --git a/core/prometheus/PromSelfMonitor.cpp b/core/prometheus/PromSelfMonitor.cpp index 39c1473014..5852fc45ae 100644 --- a/core/prometheus/PromSelfMonitor.cpp +++ b/core/prometheus/PromSelfMonitor.cpp @@ -13,7 +13,7 @@ namespace logtail { void PromSelfMonitorUnsafe::InitMetricManager(const std::unordered_map& metricKeys, const MetricLabels& labels) { auto metricLabels = std::make_shared(labels); - mPluginMetricManagerPtr = std::make_shared(metricLabels, metricKeys); + mPluginMetricManagerPtr = std::make_shared(metricLabels, metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); } void PromSelfMonitorUnsafe::AddCounter(const std::string& metricName, uint64_t statusCode, uint64_t val) { diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index c033b5c2d1..59cc58d4d5 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -53,7 +53,6 @@ PrometheusInputRunner::PrometheusInputRunner() // self monitor MetricLabels labels; - labels.emplace_back(METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER); labels.emplace_back(METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS); labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId()); labels.emplace_back(METRIC_LABEL_KEY_POD_NAME, mPodName); @@ -64,7 +63,7 @@ PrometheusInputRunner::PrometheusInputRunner() dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); }); WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_RUNNER, std::move(labels), std::move(dynamicLabels)); mPromRegisterState = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_CLIENT_REGISTER_STATE); mPromJobNum = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_JOBS_TOTAL); diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index 33ef0886ad..0908cb6ea7 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -221,7 +221,7 @@ void ScrapeScheduler::InitSelfMonitor(const MetricLabels& defaultLabels) { mSelfMonitor->InitMetricManager(sScrapeMetricKeys, labels); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE, std::move(labels)); mPromDelayTotal = mMetricsRecordRef.CreateCounter(METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL); mPluginTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PLUGIN_TOTAL_DELAY_MS); } diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index 96de4f8ee6..d44948e469 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -334,7 +334,7 @@ void TargetSubscriberScheduler::InitSelfMonitor(const MetricLabels& defaultLabel mSelfMonitor = std::make_shared(); mSelfMonitor->InitMetricManager(sSubscriberMetricKeys, mDefaultLabels); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(mDefaultLabels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE, std::move(mDefaultLabels)); mPromSubscriberTargets = mMetricsRecordRef.CreateIntGauge(METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS); mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PLUGIN_TOTAL_DELAY_MS); } diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 095241c6ef..c98acd7bf2 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -45,8 +45,8 @@ bool FlusherRunner::Init() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}); + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_OUT_ITEMS_TOTAL); diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index 98b604747a..7a0963d542 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -89,9 +89,9 @@ void ProcessorRunner::Run(uint32_t threadNo) { // thread local metrics should be initialized in each thread WriteMetrics::GetInstance()->PrepareMetricsRecordRef( sMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_RUNNER, {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}, - {"thread_no", ToString(threadNo)}}); + {METRIC_LABEL_KEY_THREAD_NO, ToString(threadNo)}}); sInGroupsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL); sInEventsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENTS_TOTAL); sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); @@ -138,7 +138,7 @@ void ProcessorRunner::Run(uint32_t threadNo) { pipeline->Process(eventGroupList, item->mInputIndex); if (pipeline->IsFlushingThroughGoPipeline()) { - // TODO: + // TODO: // 1. allow all event types to be sent to Go pipelines // 2. use event group protobuf instead if (isLog) { diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index 2969ee5bf9..4429ce40ec 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -40,8 +40,8 @@ bool HttpSink::Init() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK}, - {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}); + MetricCategory::METRIC_CATEGORY_RUNNER, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); mOutSuccessfulItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL); diff --git a/core/unittest/batch/BatcherUnittest.cpp b/core/unittest/batch/BatcherUnittest.cpp index 5e105d6f90..20ae0e11fc 100644 --- a/core/unittest/batch/BatcherUnittest.cpp +++ b/core/unittest/batch/BatcherUnittest.cpp @@ -614,7 +614,7 @@ void BatcherUnittest::TestMetric() { vector res; batch.Add(std::move(g), res); - APSARA_TEST_EQUAL(6U, batch.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(5U, batch.mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, diff --git a/core/unittest/compression/CompressorFactoryUnittest.cpp b/core/unittest/compression/CompressorFactoryUnittest.cpp index 3a9aefb8eb..e9244c3904 100644 --- a/core/unittest/compression/CompressorFactoryUnittest.cpp +++ b/core/unittest/compression/CompressorFactoryUnittest.cpp @@ -95,7 +95,7 @@ void CompressorFactoryUnittest::TestCompressTypeToString() { void CompressorFactoryUnittest::TestMetric() { auto compressor = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); - APSARA_TEST_EQUAL(5U, compressor->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(4U, compressor->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR)); diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 9a1c680f51..30d5822e47 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -54,7 +54,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); APSARA_TEST_EQUAL(fileMetric->GetLabels()->size(), 3); @@ -92,7 +92,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric2; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter2 = fileMetric2.CreateCounter("filed2"); fileCounter2->Add(222UL); } @@ -103,7 +103,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("logstore", "logstore1")); labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric3; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter3 = fileMetric3.CreateCounter("filed3"); fileCounter3->Add(333UL); } @@ -144,7 +144,7 @@ void createMetrics(int count) { labels.emplace_back(std::make_pair("count", std::to_string(count))); labels.emplace_back(std::make_pair("region", "cn-beijing")); MetricsRecordRef fileMetric; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric.CreateCounter("filed1"); fileCounter->Add(111UL); } @@ -212,7 +212,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test1")); labels.emplace_back(std::make_pair("logstore", "test1")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric1->CreateCounter("filed1"); fileCounter->Add(111UL); @@ -221,7 +221,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test2")); labels.emplace_back(std::make_pair("logstore", "test2")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric2->CreateCounter("filed1"); fileCounter->Add(111UL); } @@ -231,7 +231,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test3")); labels.emplace_back(std::make_pair("logstore", "test3")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric3->CreateCounter("filed1"); fileCounter->Add(111UL); } diff --git a/core/unittest/monitor/PluginMetricManagerUnittest.cpp b/core/unittest/monitor/PluginMetricManagerUnittest.cpp index 2e0db37022..41ef503c75 100644 --- a/core/unittest/monitor/PluginMetricManagerUnittest.cpp +++ b/core/unittest/monitor/PluginMetricManagerUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/metric_constants/MetricConstants.h" #include "monitor/PluginMetricManager.h" +#include "monitor/metric_constants/MetricConstants.h" #include "unittest/Unittest.h" namespace logtail { @@ -28,11 +28,12 @@ class PluginMetricManagerUnittest : public ::testing::Test { defaultLabels->emplace_back(METRIC_LABEL_KEY_PIPELINE_NAME, "default_config"); defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_TYPE, "default_plugin"); defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_ID, "default_id"); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(*defaultLabels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(*defaultLabels)); std::unordered_map metricKeys; metricKeys.emplace("default_counter", MetricType::METRIC_TYPE_COUNTER); metricKeys.emplace("default_gauge", MetricType::METRIC_TYPE_INT_GAUGE); - pluginMetricManager = std::make_shared(mMetricsRecordRef->GetLabels(), metricKeys); + pluginMetricManager = std::make_shared( + mMetricsRecordRef->GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); } void TearDown() {} diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 396e84627c..d8a2581a62 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -124,7 +124,7 @@ void PipelineUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL(QueueKeyManager::GetInstance()->GetKey("test_config-flusher_sls-test_project#test_logstore"), pipeline->GetContext().GetLogstoreKey()); APSARA_TEST_EQUAL(0, pipeline->mInProcessCnt.load()); - APSARA_TEST_EQUAL(3U, pipeline->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(2U, pipeline->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, configName)); APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "test_project")); @@ -2702,7 +2702,7 @@ void PipelineUnittest::TestProcess() const { processor->Init(Json::Value(), ctx); pipeline.mProcessorLine.emplace_back(std::move(processor)); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, {}); pipeline.mProcessorsInEventsTotal = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL); pipeline.mProcessorsInGroupsTotal @@ -2750,7 +2750,7 @@ void PipelineUnittest::TestSend() const { configs.emplace_back(1, nullptr); pipeline.mRouter.Init(configs, ctx); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, {}); pipeline.mFlushersInGroupsTotal = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL); pipeline.mFlushersInEventsTotal @@ -2816,7 +2816,7 @@ void PipelineUnittest::TestSend() const { configs.emplace_back(configJson.size(), nullptr); pipeline.mRouter.Init(configs, ctx); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, {}); pipeline.mFlushersInGroupsTotal = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_FLUSHERS_IN_EVENT_GROUPS_TOTAL); pipeline.mFlushersInEventsTotal diff --git a/core/unittest/prometheus/PromSelfMonitorUnittest.cpp b/core/unittest/prometheus/PromSelfMonitorUnittest.cpp index 45501ca356..b262b7d8ec 100644 --- a/core/unittest/prometheus/PromSelfMonitorUnittest.cpp +++ b/core/unittest/prometheus/PromSelfMonitorUnittest.cpp @@ -23,7 +23,7 @@ void PromSelfMonitorUnittest::TestCounterAdd() { // check result auto metric = selfMonitor->mPromStatusMap["2XX"]->GetCounter(METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL); - APSARA_TEST_EQUAL("plugin_prom_subscribe_total", metric->GetName()); + APSARA_TEST_EQUAL("prom_subscribe_total", metric->GetName()); APSARA_TEST_EQUAL(999ULL, metric->GetValue()); selfMonitor->AddCounter(METRIC_PLUGIN_PROM_SUBSCRIBE_TOTAL, 200); APSARA_TEST_EQUAL(1000ULL, metric->GetValue()); @@ -41,7 +41,7 @@ void PromSelfMonitorUnittest::TestIntGaugeSet() { // check result auto metric = selfMonitor->mPromStatusMap["2XX"]->GetIntGauge(METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS); - APSARA_TEST_EQUAL("plugin_prom_subscribe_targets", metric->GetName()); + APSARA_TEST_EQUAL("prom_subscribe_targets", metric->GetName()); APSARA_TEST_EQUAL(999ULL, metric->GetValue()); selfMonitor->SetIntGauge(METRIC_PLUGIN_PROM_SUBSCRIBE_TARGETS, 200, 0); APSARA_TEST_EQUAL(0ULL, metric->GetValue()); diff --git a/core/unittest/queue/BoundedProcessQueueUnittest.cpp b/core/unittest/queue/BoundedProcessQueueUnittest.cpp index cb14522036..08a178b207 100644 --- a/core/unittest/queue/BoundedProcessQueueUnittest.cpp +++ b/core/unittest/queue/BoundedProcessQueueUnittest.cpp @@ -118,7 +118,7 @@ void BoundedProcessQueueUnittest::TestPop() { } void BoundedProcessQueueUnittest::TestMetric() { - APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index 3708fdc309..cd80c823ad 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -151,7 +151,7 @@ void CircularProcessQueueUnittest::TestReset() { } void CircularProcessQueueUnittest::TestMetric() { - APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); diff --git a/core/unittest/queue/SenderQueueUnittest.cpp b/core/unittest/queue/SenderQueueUnittest.cpp index b31c68edda..78cfa287d6 100644 --- a/core/unittest/queue/SenderQueueUnittest.cpp +++ b/core/unittest/queue/SenderQueueUnittest.cpp @@ -174,7 +174,7 @@ void SenderQueueUnittest::TestGetAvailableItems() { } void SenderQueueUnittest::TestMetric() { - APSARA_TEST_EQUAL(6U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE)); diff --git a/core/unittest/route/RouterUnittest.cpp b/core/unittest/route/RouterUnittest.cpp index 4d7ae94531..edc78c7a0b 100644 --- a/core/unittest/route/RouterUnittest.cpp +++ b/core/unittest/route/RouterUnittest.cpp @@ -169,7 +169,7 @@ void RouterUnittest::TestMetric() { Router router; router.Init(configs, ctx); - APSARA_TEST_EQUAL(4U, router.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_EQUAL(3U, router.mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE( diff --git a/pkg/helper/k8smeta/k8s_meta_manager.go b/pkg/helper/k8smeta/k8s_meta_manager.go index 017e57782e..1432b1561e 100644 --- a/pkg/helper/k8smeta/k8s_meta_manager.go +++ b/pkg/helper/k8smeta/k8s_meta_manager.go @@ -226,18 +226,18 @@ func GetMetaManagerMetrics() []map[string]string { manager.metricRecord.Labels = []pipeline.Label{ { Key: helper.MetricLabelKeyMetricCategory, - Value: helper.MetricLabelKeyMetricCategoryRunner, + Value: helper.MetricLabelValueMetricCategoryRunner, }, { - Key: "cluster_id", + Key: helper.MetricLabelKeyClusterID, Value: *flags.ClusterID, }, { - Key: "runner_name", - Value: "k8s_meta_manager", + Key: helper.MetricLabelKeyRunnerName, + Value: helper.MetricLabelValueRunnerNameK8sMeta, }, { - Key: "project", + Key: helper.MetricLabelKeyProject, Value: strings.Join(projectName, " "), }, } diff --git a/pkg/helper/self_metrics_agent_constants.go b/pkg/helper/self_metrics_agent_constants.go index 52ed601c2e..e236e748da 100644 --- a/pkg/helper/self_metrics_agent_constants.go +++ b/pkg/helper/self_metrics_agent_constants.go @@ -20,6 +20,6 @@ package helper // metric keys const ( - MetricAgentMemoryGo = "agent_go_memory_used_mb" - MetricAgentGoRoutinesTotal = "agent_go_routines_total" + MetricAgentMemoryGo = "go_memory_used_mb" + MetricAgentGoRoutinesTotal = "go_routines_total" ) diff --git a/pkg/helper/self_metrics_plugin_constants.go b/pkg/helper/self_metrics_plugin_constants.go index e7638b215f..4233904edf 100644 --- a/pkg/helper/self_metrics_plugin_constants.go +++ b/pkg/helper/self_metrics_plugin_constants.go @@ -32,33 +32,33 @@ const ( // label values const ( - MetricLabelKeyMetricCategoryPlugin = "plugin" + MetricLabelValueMetricCategoryPlugin = "plugin" ) // metric keys const ( - MetricPluginInEventsTotal = "plugin_in_events_total" - MetricPluginInEventGroupsTotal = "plugin_in_event_groups_total" - MetricPluginInSizeBytes = "plugin_in_size_bytes" - MetricPluginOutEventsTotal = "plugin_out_events_total" - MetricPluginOutEventGroupsTotal = "plugin_out_event_groups_total" - MetricPluginOutSizeBytes = "plugin_out_size_bytes" - MetricPluginTotalDelayMs = "plugin_total_delay_ms" - MetricPluginTotalProcessTimeMs = "plugin_total_process_time_ms" + MetricPluginInEventsTotal = "in_events_total" + MetricPluginInEventGroupsTotal = "in_event_groups_total" + MetricPluginInSizeBytes = "in_size_bytes" + MetricPluginOutEventsTotal = "out_events_total" + MetricPluginOutEventGroupsTotal = "out_event_groups_total" + MetricPluginOutSizeBytes = "out_size_bytes" + MetricPluginTotalDelayMs = "total_delay_ms" + MetricPluginTotalProcessTimeMs = "total_process_time_ms" ) /********************************************************** * input_canal **********************************************************/ const ( - MetricPluginBinlogRotate = "plugin_binlog_rotate" - MetricPluginBinlogSync = "plugin_binlog_sync" - MetricPluginBinlogDdl = "plugin_binlog_ddl" - MetricPluginBinlogRow = "plugin_binlog_row" - MetricPluginBinlogXgid = "plugin_binlog_xgid" - MetricPluginBinlogCheckpoint = "plugin_binlog_checkpoint" - MetricPluginBinlogFilename = "plugin_binlog_filename" - MetricPluginBinlogGtid = "plugin_binlog_gtid" + MetricPluginBinlogRotate = "binlog_rotate" + MetricPluginBinlogSync = "binlog_sync" + MetricPluginBinlogDdl = "binlog_ddl" + MetricPluginBinlogRow = "binlog_row" + MetricPluginBinlogXgid = "binlog_xgid" + MetricPluginBinlogCheckpoint = "binlog_checkpoint" + MetricPluginBinlogFilename = "binlog_filename" + MetricPluginBinlogGtid = "binlog_gtid" ) /********************************************************** @@ -66,10 +66,10 @@ const ( * service_docker_stdout_v2 **********************************************************/ const ( - MetricPluginContainerTotal = "plugin_container_total" - MetricPluginAddContainerTotal = "plugin_add_container_total" - MetricPluginRemoveContainerTotal = "plugin_remove_container_total" - MetricPluginUpdateContainerTotal = "plugin_update_container_total" + MetricPluginContainerTotal = "container_total" + MetricPluginAddContainerTotal = "add_container_total" + MetricPluginRemoveContainerTotal = "remove_container_total" + MetricPluginUpdateContainerTotal = "update_container_total" ) /********************************************************** @@ -77,26 +77,26 @@ const ( * service_rdb **********************************************************/ const ( - MetricPluginCollectAvgCostTimeMs = "plugin_collect_avg_cost_time_ms" - MetricPluginCollectTotal = "plugin_collect_total" + MetricPluginCollectAvgCostTimeMs = "collect_avg_cost_time_ms" + MetricPluginCollectTotal = "collect_total" ) /********************************************************** * service_k8s_meta **********************************************************/ const ( - MetricCollectEntityTotal = "plugin_collect_entity_total" - MetricCollectLinkTotal = "plugin_collect_link_total" + MetricCollectEntityTotal = "collect_entity_total" + MetricCollectLinkTotal = "collect_link_total" ) /********************************************************** * all processor（所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） **********************************************************/ const ( - MetricPluginDiscardedEventsTotal = "plugin_discarded_events_total" - MetricPluginOutFailedEventsTotal = "plugin_out_failed_events_total" - MetricPluginOutKeyNotFoundEventsTotal = "plugin_out_key_not_found_events_total" - MetricPluginOutSuccessfulEventsTotal = "plugin_out_successful_events_total" + MetricPluginDiscardedEventsTotal = "discarded_events_total" + MetricPluginOutFailedEventsTotal = "out_failed_events_total" + MetricPluginOutKeyNotFoundEventsTotal = "out_key_not_found_events_total" + MetricPluginOutSuccessfulEventsTotal = "out_successful_events_total" ) /********************************************************** @@ -105,12 +105,12 @@ const ( * processor_string_replace **********************************************************/ const ( - PluginPairsPerLogTotal = "plugin_pairs_per_log_total" + PluginPairsPerLogTotal = "pairs_per_log_total" ) func GetPluginCommonLabels(context pipeline.Context, pluginMeta *pipeline.PluginMeta) []pipeline.LabelPair { labels := make([]pipeline.LabelPair, 0) - labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyMetricCategory, Value: MetricLabelKeyMetricCategoryPlugin}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyMetricCategory, Value: MetricLabelValueMetricCategoryPlugin}) labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyProject, Value: context.GetProject()}) labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyLogstore, Value: context.GetLogstore()}) labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyPipelineName, Value: context.GetConfigName()}) diff --git a/pkg/helper/self_metrics_runner_constants.go b/pkg/helper/self_metrics_runner_constants.go index 317ac7f23d..fedcb15899 100644 --- a/pkg/helper/self_metrics_runner_constants.go +++ b/pkg/helper/self_metrics_runner_constants.go @@ -18,21 +18,38 @@ package helper // runner ////////////////////////////////////////////////////////////////////////// +// lebel keys +const ( + MetricLabelKeyRunnerName = "runner_name" +) + // label values const ( - MetricLabelKeyMetricCategoryRunner = "runner" + MetricLabelValueMetricCategoryRunner = "runner" ) /********************************************************** * k8s meta **********************************************************/ + +// label keys +const ( + MetricLabelKeyClusterID = "cluster_id" +) + +// label values +const ( + MetricLabelValueRunnerNameK8sMeta = "k8s_meta" +) + +// metric keys const ( - MetricRunnerK8sMetaAddEventTotal = "runner_k8s_meta_add_event_total" - MetricRunnerK8sMetaUpdateEventTotal = "runner_k8s_meta_update_event_total" - MetricRunnerK8sMetaDeleteEventTotal = "runner_k8s_meta_delete_event_total" - MetricRunnerK8sMetaCacheSize = "runner_k8s_meta_cache_size" - MetricRunnerK8sMetaQueueSize = "runner_k8s_meta_queue_size" - MetricRunnerK8sMetaHTTPRequestTotal = "runner_k8s_meta_http_request_total" - MetricRunnerK8sMetaHTTPAvgDelayMs = "runner_k8s_meta_avg_delay_ms" - MetricRunnerK8sMetaHTTPMaxDelayMs = "runner_k8s_meta_max_delay_ms" + MetricRunnerK8sMetaAddEventTotal = "add_event_total" + MetricRunnerK8sMetaUpdateEventTotal = "update_event_total" + MetricRunnerK8sMetaDeleteEventTotal = "delete_event_total" + MetricRunnerK8sMetaCacheSize = "cache_size" + MetricRunnerK8sMetaQueueSize = "queue_size" + MetricRunnerK8sMetaHTTPRequestTotal = "http_request_total" + MetricRunnerK8sMetaHTTPAvgDelayMs = "avg_delay_ms" + MetricRunnerK8sMetaHTTPMaxDelayMs = "max_delay_ms" )