diff --git a/core/ebpf/SelfMonitor.cpp b/core/ebpf/SelfMonitor.cpp new file mode 100644 index 0000000000..ad49fce33c --- /dev/null +++ b/core/ebpf/SelfMonitor.cpp @@ -0,0 +1,251 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ebpf/SelfMonitor.h" +#include "logger/Logger.h" + +namespace logtail { +namespace ebpf { + +void BaseBPFMonitor::HandleStatistic(nami::eBPFStatistics& stats) { + if (!stats.updated_) return; + UpdateMetricInner(stats); +} + +void BaseBPFMonitor::InitMetric() { + if (mMetricInited) return; + mMetricInited = true; + InitMetricInner(); +} + +void BaseBPFMonitor::ReleaseMetric() { + if (!mMetricInited) return; + for (auto& item : mRefAndLabels) { + auto labels = item.second; + if (mPluginMetricMgr) { + mPluginMetricMgr->ReleaseReentrantMetricsRecordRef(labels); + } + } + + mMetricInited = false; +} + +void BaseBPFMonitor::InitMetricInner() { + // init base metrics, only plugin relative + // poll kernel events + MetricLabels pollKernelEventsLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_POLL_KERNEL} + }; + auto ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(pollKernelEventsLabels); + mRecvKernelEventsTotal = ref->GetCounter(METRIC_PLUGIN_IN_EVENTS_TOTAL); + // loss kernel events + mLossKernelEventsTotal = ref->GetCounter(METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL); + mProcessCacheEntitiesNum = ref->GetIntGauge(METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM); + mProcessCacheMissTotal = ref->GetCounter(METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, pollKernelEventsLabels)); + + // push logs/spans/metrics + MetricLabels pushLogsLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_REPORT_TO_LC}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_LOG} + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(pushLogsLabels); + mPushEventsTotal = ref->GetCounter(METRIC_PLUGIN_IN_EVENTS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, pushLogsLabels)); + + MetricLabels pushMetricsLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_REPORT_TO_LC}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_METRIC} + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(pushMetricsLabels); + mPushMetricsTotal = ref->GetCounter(METRIC_PLUGIN_IN_EVENTS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, pushMetricsLabels)); + + MetricLabels pushSpansLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_REPORT_TO_LC}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_TRACE} + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(pushSpansLabels); + mPushSpansTotal = ref->GetCounter(METRIC_PLUGIN_IN_EVENTS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, pushSpansLabels)); +} + +void BaseBPFMonitor::UpdateMetricInner(nami::eBPFStatistics& currStat) { + if (!currStat.updated_) return; + mProcessCacheEntitiesNum->Set(currStat.process_cache_entities_num_); + mRecvKernelEventsTotal->Add(currStat.recv_kernel_events_total_); + mLossKernelEventsTotal->Add(currStat.loss_kernel_events_total_); + mProcessCacheMissTotal->Add(currStat.miss_process_cache_total_); + mPushMetricsTotal->Add(currStat.push_metrics_total_); + mPushSpansTotal->Add(currStat.push_spans_total_); + mPushEventsTotal->Add(currStat.push_events_total_); +} + +/////////////////////////// NetworkObserverSelfMonitor /////////////////////////// +void NetworkObserverSelfMonitor::InitMetric() { + if (mMetricInited) return; + mMetricInited = true; + + InitMetricInner(); + + // use default labels ... + MetricLabels recvEventLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER}, + }; + auto ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(recvEventLabels); + + mConnTrackerNum = ref->GetIntGauge(METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM); + mAggMapEntitiesNum = ref->GetIntGauge(METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM); + mRefAndLabels.emplace_back(std::make_pair<>(ref, recvEventLabels)); + + // event type relative labels ... + MetricLabels eventTypeLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_CONN_STATS} + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(eventTypeLabels); + mRecvConnStatsTotal = ref->GetCounter(METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, eventTypeLabels)); + + eventTypeLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_CTRL_EVENT} + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(eventTypeLabels); + mRecvCtrlEventsTotal = ref->GetCounter(METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, eventTypeLabels)); + + // + MetricLabels eventTypeAndProtocolLbales = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_DATA_EVENT}, + {METRIC_LABEL_KEY_PARSER_PROTOCOL, METRIC_LABEL_VALUE_PARSER_PROTOCOL_HTTP}, + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(eventTypeAndProtocolLbales); + mRecvHTTPDataEventsTotal = ref->GetCounter(METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, eventTypeAndProtocolLbales)); + + // protocol relative labels ... + MetricLabels httpSuccessLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_DATA_EVENT}, + {METRIC_LABEL_KEY_PARSER_PROTOCOL, METRIC_LABEL_VALUE_PARSER_PROTOCOL_HTTP}, + {METRIC_LABEL_KEY_PARSE_STATUS, METRIC_LABEL_VALUE_PARSE_STATUS_SUCCESS} + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(httpSuccessLabels); + mParseHTTPEventsSuccessTotal = ref->GetCounter(METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, httpSuccessLabels)); + + MetricLabels httpFailLabels = { + {METRIC_LABEL_KEY_RECV_EVENT_STAGE, METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER}, + {METRIC_LABEL_KEY_EVENT_TYPE, METRIC_LABEL_VALUE_EVENT_TYPE_DATA_EVENT}, + {METRIC_LABEL_KEY_PARSER_PROTOCOL, METRIC_LABEL_VALUE_PARSER_PROTOCOL_HTTP}, + {METRIC_LABEL_KEY_PARSE_STATUS, METRIC_LABEL_VALUE_PARSE_STATUS_FAILED} + }; + ref = mPluginMetricMgr->GetOrCreateReentrantMetricsRecordRef(httpFailLabels); + mParseHTTPEventsFailTotal = ref->GetCounter(METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL); + mRefAndLabels.emplace_back(std::make_pair<>(ref, httpFailLabels)); +} + +void NetworkObserverSelfMonitor::HandleStatistic(nami::eBPFStatistics& stats) { + if (!stats.updated_) return; + UpdateMetricInner(stats); + // recv kernel events metric + assert(stats.plugin_type_ == nami::PluginType::NETWORK_OBSERVE); + nami::NetworkObserverStatistics* currNetworkStatsPtr = static_cast(&stats); + + mRecvConnStatsTotal->Add(currNetworkStatsPtr->recv_conn_stat_events_total_); + mRecvCtrlEventsTotal->Add(currNetworkStatsPtr->recv_ctrl_events_total_); + mRecvHTTPDataEventsTotal->Add(currNetworkStatsPtr->recv_http_data_events_total_); + + // cache relative metric + mConnTrackerNum->Set(currNetworkStatsPtr->conntracker_num_); + + mParseHTTPEventsSuccessTotal->Add(currNetworkStatsPtr->parse_http_records_success_total_); + mParseHTTPEventsFailTotal->Add(currNetworkStatsPtr->parse_http_records_failed_total_); + mAggMapEntitiesNum->Set(currNetworkStatsPtr->agg_map_entities_num_); +} + +eBPFSelfMonitorMgr::eBPFSelfMonitorMgr() : mSelfMonitors({}), mInited({}) {} + +void eBPFSelfMonitorMgr::Init(const nami::PluginType type, std::shared_ptr mgr, const std::string& name, const std::string& logstore) { + if (mInited[int(type)]) return; + + WriteLock lk(mLock); + + // double check + if (mInited[int(type)]) return; + + switch (type) + { + case nami::PluginType::NETWORK_OBSERVE: { + mSelfMonitors[int(type)] = std::make_unique(name, mgr); + break; + } + case nami::PluginType::NETWORK_SECURITY: { + mSelfMonitors[int(type)] = std::make_unique(name, mgr); + break; + } + case nami::PluginType::FILE_SECURITY: { + mSelfMonitors[int(type)] = std::make_unique(name, mgr); + break; + } + case nami::PluginType::PROCESS_SECURITY: { + mSelfMonitors[int(type)] = std::make_unique(name, mgr); + break; + } + default: + break; + } + mSelfMonitors[int(type)]->InitMetric(); + mInited[int(type)] = true; +} + +void eBPFSelfMonitorMgr::Release(const nami::PluginType type) { + if (!mInited[int(type)]) return; + + WriteLock lk(mLock); + // double check + if (!mInited[int(type)]) return; + if (mSelfMonitors[int(type)]) { + mSelfMonitors[int(type)]->ReleaseMetric(); + mSelfMonitors[int(type)].reset(); + } + mInited[int(type)] = false; +} + +void eBPFSelfMonitorMgr::Suspend(const nami::PluginType type) { + if (!mInited[int(type)]) { + return; + } + mInited[int(type)] = false; +} + +void eBPFSelfMonitorMgr::HandleStatistic(std::vector&& stats) { + for (auto& stat : stats) { + if (!stat.updated_) { + continue; + } + auto type = stat.plugin_type_; + { + ReadLock lk(mLock); + if (mInited[int(type)] && mSelfMonitors[int(type)]) { + mSelfMonitors[int(type)]->HandleStatistic(stat); + } + } + } +} + +} +} diff --git a/core/ebpf/SelfMonitor.h b/core/ebpf/SelfMonitor.h new file mode 100644 index 0000000000..4d59934266 --- /dev/null +++ b/core/ebpf/SelfMonitor.h @@ -0,0 +1,141 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "ebpf/include/export.h" +#include "monitor/PluginMetricManager.h" +#include "common/Lock.h" +#include "monitor/LoongCollectorMetricTypes.h" +#include "monitor/metric_constants/MetricConstants.h" + +namespace logtail { +namespace ebpf { + +class BaseBPFMonitor { +public: + virtual void HandleStatistic(nami::eBPFStatistics& stats); + virtual void InitMetric(); + virtual void ReleaseMetric(); + virtual ~BaseBPFMonitor() = default; +protected: + BaseBPFMonitor(const std::string& name, PluginMetricManagerPtr mgr) + : mPipelineName(name), mPluginMetricMgr(mgr) {} + + // attention: not thread safe!! + void InitMetricInner(); + + // attention: not thread safe!! + void UpdateMetricInner(nami::eBPFStatistics& currStat); + + std::string mPipelineName; + PluginMetricManagerPtr mPluginMetricMgr; + // MetricsRecordRef& mRef; + std::vector> mRefAndLabels; + + std::atomic_bool mMetricInited = false; + + CounterPtr mRecvKernelEventsTotal; + CounterPtr mLossKernelEventsTotal; + CounterPtr mPushEventsTotal; + CounterPtr mPushSpansTotal; + CounterPtr mPushMetricsTotal; + IntGaugePtr mProcessCacheEntitiesNum; + CounterPtr mProcessCacheMissTotal; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class eBPFServerUnittest; +#endif +}; + +class NetworkObserverSelfMonitor : public BaseBPFMonitor { +public: + NetworkObserverSelfMonitor(const std::string& name, PluginMetricManagerPtr mgr/**/) + : BaseBPFMonitor(name, mgr) {} + + void InitMetric() override; + + void HandleStatistic(nami::eBPFStatistics& stats) override; + +private: + + // recv kernel events metric + CounterPtr mRecvConnStatsTotal; + CounterPtr mRecvCtrlEventsTotal; + CounterPtr mRecvHTTPDataEventsTotal; + + // cache relative metric + IntGaugePtr mConnTrackerNum; + + // protocol parsing metric + CounterPtr mParseHTTPEventsSuccessTotal; + CounterPtr mParseHTTPEventsFailTotal; + + // aggregation relative metric + IntGaugePtr mAggMapEntitiesNum; +#ifdef APSARA_UNIT_TEST_MAIN + friend class eBPFServerUnittest; +#endif +}; + +class NetworkSecuritySelfMonitor : public BaseBPFMonitor { +public: + NetworkSecuritySelfMonitor(const std::string& name, PluginMetricManagerPtr mgr) + : BaseBPFMonitor(name, mgr) {} + + void HandleStatistic(nami::eBPFStatistics& stats) override { + if (!stats.updated_) return; + UpdateMetricInner(stats); + } +}; + +class ProcessSecuritySelfMonitor : public BaseBPFMonitor { +public: + ProcessSecuritySelfMonitor(const std::string& name, PluginMetricManagerPtr mgr) + : BaseBPFMonitor(name, mgr) {} +}; + +class FileSecuritySelfMonitor : public BaseBPFMonitor { +public: + FileSecuritySelfMonitor(const std::string& name, PluginMetricManagerPtr mgr) + : BaseBPFMonitor(name, mgr) {} +}; + +/** + * eBPFSelfMonitorMgr is only used to manage the self-monitoring data in libnetwork_observer.so, updating the statistics through callbacks. + */ +class eBPFSelfMonitorMgr { +public: + eBPFSelfMonitorMgr(); + void Init(const nami::PluginType type, std::shared_ptr mgr, const std::string& name, const std::string& project); + void Release(const nami::PluginType type); + void Suspend(const nami::PluginType type); + void HandleStatistic(std::vector&& stats); +private: + // `mLock` is used to protect mSelfMonitors + ReadWriteLock mLock; + std::array, int(nami::PluginType::MAX)> mSelfMonitors; + std::array mInited; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class eBPFServerUnittest; +#endif +}; + +} // ebpf +} // logtail \ No newline at end of file diff --git a/core/ebpf/SourceManager.cpp b/core/ebpf/SourceManager.cpp index 292a9f6352..93aff008bf 100644 --- a/core/ebpf/SourceManager.cpp +++ b/core/ebpf/SourceManager.cpp @@ -167,19 +167,15 @@ bool SourceManager::CheckPluginRunning(nami::PluginType plugin_type) { return mRunning[int(plugin_type)]; } -bool SourceManager::StartPlugin(nami::PluginType plugin_type, - std::variant config) { +bool SourceManager::StartPlugin(nami::PluginType plugin_type, std::unique_ptr conf) { if (CheckPluginRunning(plugin_type)) { // plugin update ... - return UpdatePlugin(plugin_type, std::move(config)); + return UpdatePlugin(plugin_type, std::move(conf)); } // plugin not started ... LOG_INFO(sLogger, ("begin to start plugin, type", int(plugin_type))); - auto conf = std::make_unique(); - conf->plugin_type_ = plugin_type; conf->type = UpdataType::SECURE_UPDATE_TYPE_ENABLE_PROBE; - conf->config_ = config; FillCommonConf(conf); #ifdef APSARA_UNIT_TEST_MAIN mConfig = std::move(conf); @@ -197,17 +193,13 @@ bool SourceManager::StartPlugin(nami::PluginType plugin_type, return !res; } -bool SourceManager::UpdatePlugin(nami::PluginType plugin_type, - std::variant config) { +bool SourceManager::UpdatePlugin(nami::PluginType plugin_type, std::unique_ptr conf) { if (!CheckPluginRunning(plugin_type)) { LOG_ERROR(sLogger, ("plugin not started, type", int(plugin_type))); return false; } - auto conf = std::make_unique(); - conf->plugin_type_ = plugin_type; conf->type = UpdataType::SECURE_UPDATE_TYPE_CONFIG_CHAGE; - conf->config_ = config; FillCommonConf(conf); #ifdef APSARA_UNIT_TEST_MAIN mConfig = std::move(conf); diff --git a/core/ebpf/SourceManager.h b/core/ebpf/SourceManager.h index 34e895048e..89b12e43e7 100644 --- a/core/ebpf/SourceManager.h +++ b/core/ebpf/SourceManager.h @@ -47,8 +47,7 @@ class SourceManager { void Init(); - bool StartPlugin(nami::PluginType plugin_type, - std::variant config); + bool StartPlugin(nami::PluginType plugin_type, std::unique_ptr conf); bool StopPlugin(nami::PluginType plugin_type); @@ -65,8 +64,7 @@ class SourceManager { void FillCommonConf(std::unique_ptr& conf); bool LoadDynamicLib(const std::string& lib_name); bool DynamicLibSuccess(); - bool UpdatePlugin(nami::PluginType plugin_type, - std::variant config); + bool UpdatePlugin(nami::PluginType plugin_type, std::unique_ptr conf); enum class ebpf_func { EBPF_INIT, diff --git a/core/ebpf/eBPFServer.cpp b/core/ebpf/eBPFServer.cpp index c35d190f20..874a29ec5a 100644 --- a/core/ebpf/eBPFServer.cpp +++ b/core/ebpf/eBPFServer.cpp @@ -25,6 +25,8 @@ #include "ebpf/include/export.h" #include "common/LogtailCommonFlags.h" #include "common/MachineInfoUtil.h" +#include "monitor/PluginMetricManager.h" +#include "common/Lock.h" DEFINE_FLAG_INT64(kernel_min_version_for_ebpf, "the minimum kernel version that supported eBPF normal running, 4.19.0.0 -> 4019000000", @@ -144,6 +146,20 @@ void eBPFServer::Init() { return; } mInited = true; + // check env + + mMonitorMgr = std::make_unique(); + DynamicMetricLabels dynamicLabels; + dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); }); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mRef, + {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER}, + {METRIC_LABEL_KEY_METRIC_CATEGORY, METRIC_LABEL_KEY_METRIC_CATEGORY_RUNNER}}, + std::move(dynamicLabels)); + + mStartPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL); + mStopPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL); + mSuspendPluginTotal = mRef.CreateCounter(METRIC_RUNNER_EBPF_SUSPEND_PLUGIN_TOTAL); + mSourceManager = std::make_unique(); mSourceManager->Init(); // ebpf config @@ -170,8 +186,8 @@ void eBPFServer::Stop() { mSourceManager->StopAll(); // destroy source manager mSourceManager.reset(); - for (std::size_t i = 0; i < mLoadedPipeline.size(); i ++) { - UpdatePipelineName(static_cast(i), ""); + for (int i = 0; i < int(nami::PluginType::MAX); i ++) { + UpdatePipelineName(static_cast(i), "", ""); } // UpdateContext must after than StopPlugin @@ -186,7 +202,7 @@ void eBPFServer::Stop() { bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options) { + const std::variant options, std::shared_ptr mgr) { std::string prev_pipeline_name = CheckLoadedPipelineName(type); if (prev_pipeline_name.size() && prev_pipeline_name != pipeline_name) { @@ -195,11 +211,17 @@ bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t return false; } - UpdatePipelineName(type, pipeline_name); + UpdatePipelineName(type, pipeline_name, ctx->GetProjectName()); + + // init self monitor + mMonitorMgr->Init(type, mgr, pipeline_name, ctx->GetProjectName()); // step1: convert options to export type std::variant config; bool ret = false; + auto eBPFConfig = std::make_unique(); + eBPFConfig->plugin_type_ = type; + eBPFConfig->stats_handler_ = [this](auto stats){ return mMonitorMgr->HandleStatistic(std::move(stats)); }; // call update function // step2: call init function switch(type) { @@ -211,7 +233,8 @@ bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t config = std::move(pconfig); // UpdateContext must ahead of StartPlugin mProcessSecureCB->UpdateContext(ctx, ctx->GetProcessQueueKey(), plugin_index); - ret = mSourceManager->StartPlugin(type, config); + eBPFConfig->config_ = config; + ret = mSourceManager->StartPlugin(type, std::move(eBPFConfig)); break; } @@ -221,21 +244,25 @@ bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t if (opts->mEnableMetric) { nconfig.enable_metric_ = true; nconfig.measure_cb_ = [this](auto events, auto ts) { return mMeterCB->handle(std::move(events), ts); }; + nconfig.enable_metric_ = true; mMeterCB->UpdateContext(ctx, ctx->GetProcessQueueKey(), plugin_index); } if (opts->mEnableSpan) { nconfig.enable_span_ = true; nconfig.span_cb_ = [this](auto events) { return mSpanCB->handle(std::move(events)); }; + nconfig.enable_span_ = true; mSpanCB->UpdateContext(ctx, ctx->GetProcessQueueKey(), plugin_index); } if (opts->mEnableLog) { nconfig.enable_event_ = true; nconfig.event_cb_ = [this](auto events) { return mEventCB->handle(std::move(events)); }; + nconfig.enable_event_ = true; mEventCB->UpdateContext(ctx, ctx->GetProcessQueueKey(), plugin_index); } config = std::move(nconfig); - ret = mSourceManager->StartPlugin(type, config); + eBPFConfig->config_ = config; + ret = mSourceManager->StartPlugin(type, std::move(eBPFConfig)); break; } @@ -245,9 +272,10 @@ bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t SecurityOptions* opts = std::get(options); nconfig.options_ = opts->mOptionList; config = std::move(nconfig); + eBPFConfig->config_ = config; // UpdateContext must ahead of StartPlugin mNetworkSecureCB->UpdateContext(ctx, ctx->GetProcessQueueKey(), plugin_index); - ret = mSourceManager->StartPlugin(type, config); + ret = mSourceManager->StartPlugin(type, std::move(eBPFConfig)); break; } @@ -257,9 +285,10 @@ bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t SecurityOptions* opts = std::get(options); fconfig.options_ = opts->mOptionList; config = std::move(fconfig); + eBPFConfig->config_ = config; // UpdateContext must ahead of StartPlugin mFileSecureCB->UpdateContext(ctx, ctx->GetProcessQueueKey(), plugin_index); - ret = mSourceManager->StartPlugin(type, config); + ret = mSourceManager->StartPlugin(type, std::move(eBPFConfig)); break; } default: @@ -267,6 +296,10 @@ bool eBPFServer::StartPluginInternal(const std::string& pipeline_name, uint32_t return false; } + if (ret) { + mStartPluginTotal->Add(1); + } + return ret; } @@ -281,11 +314,11 @@ bool eBPFServer::HasRegisteredPlugins() const { bool eBPFServer::EnablePlugin(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const PipelineContext* ctx, - const std::variant options) { + const std::variant options, std::shared_ptr mgr) { if (!IsSupportedEnv(type)) { return false; } - return StartPluginInternal(pipeline_name, plugin_index, type, ctx, options); + return StartPluginInternal(pipeline_name, plugin_index, type, ctx, options, mgr); } bool eBPFServer::DisablePlugin(const std::string& pipeline_name, nami::PluginType type) { @@ -294,14 +327,17 @@ bool eBPFServer::DisablePlugin(const std::string& pipeline_name, nami::PluginTyp } std::string prev_pipeline = CheckLoadedPipelineName(type); if (prev_pipeline == pipeline_name) { - UpdatePipelineName(type, ""); + UpdatePipelineName(type, "", ""); } else { LOG_WARNING(sLogger, ("prev pipeline", prev_pipeline)("curr pipeline", pipeline_name)); return true; } bool ret = mSourceManager->StopPlugin(type); // UpdateContext must after than StopPlugin - if (ret) UpdateCBContext(type, nullptr, -1, -1); + if (ret) { + UpdateCBContext(type, nullptr, -1, -1); + mStopPluginTotal->Add(1); + } return ret; } @@ -310,9 +346,22 @@ std::string eBPFServer::CheckLoadedPipelineName(nami::PluginType type) { return mLoadedPipeline[int(type)]; } -void eBPFServer::UpdatePipelineName(nami::PluginType type, const std::string& name) { +std::string eBPFServer::GetAllProjects() { + std::lock_guard lk(mMtx); + std::string res; + for (int i = 0; i < int(nami::PluginType::MAX); i ++) { + if (mPluginProject[i] != "") { + res += mPluginProject[i]; + res += " "; + } + } + return res; +} + +void eBPFServer::UpdatePipelineName(nami::PluginType type, const std::string& name, const std::string& project) { std::lock_guard lk(mMtx); mLoadedPipeline[int(type)] = name; + mPluginProject[int(type)] = project; return; } @@ -322,7 +371,10 @@ bool eBPFServer::SuspendPlugin(const std::string& pipeline_name, nami::PluginTyp } // mark plugin status is update bool ret = mSourceManager->SuspendPlugin(type); - if (ret) UpdateCBContext(type, nullptr, -1, -1); + if (ret) { + UpdateCBContext(type, nullptr, -1, -1); + mSuspendPluginTotal->Add(1); + } return ret; } diff --git a/core/ebpf/eBPFServer.h b/core/ebpf/eBPFServer.h index 9493a3cedb..4887080a79 100644 --- a/core/ebpf/eBPFServer.h +++ b/core/ebpf/eBPFServer.h @@ -29,6 +29,8 @@ #include "ebpf/handler/AbstractHandler.h" #include "ebpf/handler/ObserveHandler.h" #include "ebpf/handler/SecurityHandler.h" +#include "monitor/LoongCollectorMetricTypes.h" +#include "ebpf/SelfMonitor.h" namespace logtail { namespace ebpf { @@ -65,12 +67,12 @@ class eBPFServer : public InputRunner { std::string CheckLoadedPipelineName(nami::PluginType type); - void UpdatePipelineName(nami::PluginType type, const std::string& name); + void UpdatePipelineName(nami::PluginType type, const std::string& name, const std::string& project); bool EnablePlugin(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options); + const std::variant options, std::shared_ptr mgr); bool DisablePlugin(const std::string& pipeline_name, nami::PluginType type); @@ -80,11 +82,13 @@ class eBPFServer : public InputRunner { bool IsSupportedEnv(nami::PluginType type); + std::string GetAllProjects(); + private: bool StartPluginInternal(const std::string& pipeline_name, uint32_t plugin_index, nami::PluginType type, const logtail::PipelineContext* ctx, - const std::variant options); + const std::variant options, std::shared_ptr mgr); eBPFServer() = default; ~eBPFServer() = default; @@ -101,11 +105,18 @@ class eBPFServer : public InputRunner { mutable std::mutex mMtx; std::array mLoadedPipeline = {}; + std::array mPluginProject = {}; eBPFAdminConfig mAdminConfig; volatile bool mInited = false; EnvManager mEnvMgr; + std::unique_ptr mMonitorMgr; + MetricsRecordRef mRef; + + CounterPtr mStartPluginTotal; + CounterPtr mStopPluginTotal; + CounterPtr mSuspendPluginTotal; #ifdef APSARA_UNIT_TEST_MAIN friend class eBPFServerUnittest; diff --git a/core/ebpf/handler/AbstractHandler.h b/core/ebpf/handler/AbstractHandler.h index 251b361631..0df2b539df 100644 --- a/core/ebpf/handler/AbstractHandler.h +++ b/core/ebpf/handler/AbstractHandler.h @@ -17,6 +17,8 @@ #include #include "pipeline/PipelineContext.h" +#include "monitor/LoongCollectorMetricTypes.h" +#include "monitor/LogtailMetric.h" namespace logtail{ namespace ebpf { @@ -24,7 +26,8 @@ namespace ebpf { class AbstractHandler { public: AbstractHandler() {} - AbstractHandler(const logtail::PipelineContext* ctx, logtail::QueueKey key, uint32_t idx) : mCtx(ctx), mQueueKey(key), mPluginIdx(idx) {} + AbstractHandler(const logtail::PipelineContext* ctx, logtail::QueueKey key, uint32_t idx) + : mCtx(ctx), mQueueKey(key), mPluginIdx(idx) {} void UpdateContext(const logtail::PipelineContext* ctx, logtail::QueueKey key, uint32_t index) { mCtx = ctx; mQueueKey = key; diff --git a/core/ebpf/handler/ObserveHandler.cpp b/core/ebpf/handler/ObserveHandler.cpp index b2bec42e57..2165cff094 100644 --- a/core/ebpf/handler/ObserveHandler.cpp +++ b/core/ebpf/handler/ObserveHandler.cpp @@ -245,7 +245,6 @@ void ArmsMeterHandler::handle(std::vector&)>; + struct eBPFConfig { PluginType plugin_type_; UpdataType type = UpdataType::SECURE_UPDATE_TYPE_ENABLE_PROBE; @@ -300,6 +331,7 @@ struct eBPFConfig { std::string host_path_prefix_; // specific config std::variant config_; + NamiStatisticsHandler stats_handler_; bool operator==(const eBPFConfig& other) const { return plugin_type_ == other.plugin_type_ && type == other.type && diff --git a/core/monitor/metric_constants/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h index 46da86ceba..63d48b9ca8 100644 --- a/core/monitor/metric_constants/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -118,6 +118,44 @@ extern const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TIME_MS; extern const std::string METRIC_PLUGIN_PROM_SCRAPE_TIME_MS; extern const std::string METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL; +/********************************************************** + * input_ebpf + * input_network_observer + **********************************************************/ + +extern const std::string METRIC_LABEL_KEY_RECV_EVENT_STAGE; +extern const std::string METRIC_LABEL_KEY_EVENT_TYPE; +extern const std::string METRIC_LABEL_KEY_PARSER_PROTOCOL; +extern const std::string METRIC_LABEL_KEY_PARSE_STATUS; + +extern const std::string METRIC_LABEL_VALUE_RECV_EVENT_STAGE_POLL_KERNEL; +extern const std::string METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER; +extern const std::string METRIC_LABEL_VALUE_RECV_EVENT_STAGE_REPORT_TO_LC; +extern const std::string METRIC_LABEL_VALUE_EVENT_TYPE_CONN_STATS; +extern const std::string METRIC_LABEL_VALUE_EVENT_TYPE_DATA_EVENT; +extern const std::string METRIC_LABEL_VALUE_EVENT_TYPE_CTRL_EVENT; +extern const std::string METRIC_LABEL_VALUE_EVENT_TYPE_LOG; +extern const std::string METRIC_LABEL_VALUE_EVENT_TYPE_METRIC; +extern const std::string METRIC_LABEL_VALUE_EVENT_TYPE_TRACE; +extern const std::string METRIC_LABEL_VALUE_PARSER_PROTOCOL_HTTP; +extern const std::string METRIC_LABEL_VALUE_PARSE_STATUS_SUCCESS; +extern const std::string METRIC_LABEL_VALUE_PARSE_STATUS_FAILED; +extern const std::string METRIC_LABEL_VALUE_PLUGIN_TYPE_NETWORK_OBSERVER; +extern const std::string METRIC_LABEL_VALUE_PLUGIN_TYPE_NETWORK_SECURITY; +extern const std::string METRIC_LABEL_VALUE_PLUGIN_TYPE_FILE_OBSERVER; +extern const std::string METRIC_LABEL_VALUE_PLUGIN_TYPE_FILE_SECURITY; +extern const std::string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_OBSERVER; +extern const std::string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_SECURITY; + +extern const std::string METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL; +extern const std::string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM; +extern const std::string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL; +extern const std::string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL; +extern const std::string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL; +extern const std::string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM; +extern const std::string METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM; +extern const std::string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL; + /********************************************************** * all processor (所有解析类的处理插件通用指标。Todo:目前统计还不全、不准确) **********************************************************/ @@ -241,6 +279,7 @@ extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER; extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK; extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR; extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS; +extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER; // metric keys extern const std::string METRIC_RUNNER_IN_EVENTS_TOTAL; @@ -276,4 +315,11 @@ extern const std::string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE; extern const std::string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE; extern const std::string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE; +/********************************************************** + * ebpf server + **********************************************************/ +extern const std::string METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL; +extern const std::string METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL; +extern const std::string METRIC_RUNNER_EBPF_SUSPEND_PLUGIN_TOTAL; + } // namespace logtail diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index 1a7923531d..cc4542adeb 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -63,6 +63,44 @@ const std::string METRIC_PLUGIN_PROM_SUBSCRIBE_TIME_MS = "plugin_prom_subscribe_ const std::string METRIC_PLUGIN_PROM_SCRAPE_TIME_MS = "plugin_prom_scrape_time_ms"; const std::string METRIC_PLUGIN_PROM_SCRAPE_DELAY_TOTAL = "plugin_prom_scrape_delay_total"; +/********************************************************** + * input_ebpf + * input_network_observer + **********************************************************/ + +const string METRIC_LABEL_KEY_RECV_EVENT_STAGE = "recv_event_stage"; +const string METRIC_LABEL_KEY_EVENT_TYPE = "event_type"; +const string METRIC_LABEL_KEY_PARSER_PROTOCOL = "parser_protocol"; +const string METRIC_LABEL_KEY_PARSE_STATUS = "parser_status"; + +const string METRIC_LABEL_VALUE_RECV_EVENT_STAGE_POLL_KERNEL = "poll_kernel"; +const string METRIC_LABEL_VALUE_RECV_EVENT_STAGE_AFTER_PERF_WORKER = "after_perf_worker"; +const string METRIC_LABEL_VALUE_RECV_EVENT_STAGE_REPORT_TO_LC = "report_to_lc"; +const string METRIC_LABEL_VALUE_EVENT_TYPE_CONN_STATS = "conn_stats"; +const string METRIC_LABEL_VALUE_EVENT_TYPE_DATA_EVENT = "data_event"; +const string METRIC_LABEL_VALUE_EVENT_TYPE_CTRL_EVENT = "ctrl_event"; +const string METRIC_LABEL_VALUE_EVENT_TYPE_LOG = "log"; +const string METRIC_LABEL_VALUE_EVENT_TYPE_METRIC = "metric"; +const string METRIC_LABEL_VALUE_EVENT_TYPE_TRACE = "trace"; +const string METRIC_LABEL_VALUE_PARSER_PROTOCOL_HTTP = "http"; +const string METRIC_LABEL_VALUE_PARSE_STATUS_SUCCESS = "success"; +const string METRIC_LABEL_VALUE_PARSE_STATUS_FAILED = "failed"; +const string METRIC_LABEL_VALUE_PLUGIN_TYPE_NETWORK_OBSERVER = "network_observer"; +const string METRIC_LABEL_VALUE_PLUGIN_TYPE_NETWORK_SECURITY = "network_security"; +const string METRIC_LABEL_VALUE_PLUGIN_TYPE_FILE_OBSERVER = "file_observer"; +const string METRIC_LABEL_VALUE_PLUGIN_TYPE_FILE_SECURITY = "file_security"; +const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_OBSERVER = "process_observer"; +const string METRIC_LABEL_VALUE_PLUGIN_TYPE_PROCESS_SECURITY = "process_security"; + +const string METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL = "plugin_ebpf_loss_kernel_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM = "plugin_network_observer_conntracker_num"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL = "plugin_network_observer_worker_handle_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL = "plugin_network_observer_parse_records_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_EVENTS_TOTAL = "plugin_network_observer_aggregate_events_total"; +const string METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM = "plugin_network_observer_aggregate_key_num"; +const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM = "plugin_process_cache_entries_num"; +const string METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL = "plugin_process_cache_miss_total"; + /********************************************************** * all processor (所有解析类的处理插件通用指标。Todo:目前统计还不全、不准确) **********************************************************/ diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index 3c807c9299..8fb8519167 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -28,6 +28,7 @@ const string METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER = "flusher_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK = "http_sink"; const string METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR = "processor_runner"; const string METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS = "prometheus_runner"; +const string METRIC_LABEL_VALUE_RUNNER_NAME_EBPF_SERVER = "ebpf_server"; // metric keys const string METRIC_RUNNER_IN_EVENTS_TOTAL = "runner_in_events_total"; @@ -68,4 +69,11 @@ const string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "runner_polling_modi const string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "runner_polling_dir_cache_size"; const string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "runner_polling_file_cache_size"; +/********************************************************** + * ebpf server + **********************************************************/ +const string METRIC_RUNNER_EBPF_START_PLUGIN_TOTAL = "runner_start_plugin_total"; +const string METRIC_RUNNER_EBPF_STOP_PLUGIN_TOTAL = "runner_stop_plugin_total"; +const string METRIC_RUNNER_EBPF_SUSPEND_PLUGIN_TOTAL = "runner_suspend_plugin_total"; + } // namespace logtail \ No newline at end of file diff --git a/core/plugin/input/InputFileSecurity.cpp b/core/plugin/input/InputFileSecurity.cpp index 833e1835ed..3510833830 100644 --- a/core/plugin/input/InputFileSecurity.cpp +++ b/core/plugin/input/InputFileSecurity.cpp @@ -36,11 +36,19 @@ bool InputFileSecurity::Init(const Json::Value& config, Json::Value& optionalGoP LOG_WARNING(sLogger, ("pipeline already loaded", "FILE_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); return false; } + static const std::unordered_map metricKeys = { + {METRIC_PLUGIN_IN_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM, MetricType::METRIC_TYPE_INT_GAUGE}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + }; + + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); return mSecurityOptions.Init(ebpf::SecurityProbeType::FILE, config, mContext, sName); } bool InputFileSecurity::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::FILE_SECURITY, mContext, &mSecurityOptions); + return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::FILE_SECURITY, mContext, &mSecurityOptions, mPluginMgr); } bool InputFileSecurity::Stop(bool isPipelineRemoving) { diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index a4e7aa553a..e096e585f9 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -21,6 +21,7 @@ #include "ebpf/config.h" #include "pipeline/plugin/interface/Input.h" #include "ebpf/eBPFServer.h" +#include "monitor/PluginMetricManager.h" namespace logtail { @@ -35,6 +36,7 @@ class InputFileSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; + std::shared_ptr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputNetworkObserver.cpp b/core/plugin/input/InputNetworkObserver.cpp index f497fb759b..c2b997d54f 100644 --- a/core/plugin/input/InputNetworkObserver.cpp +++ b/core/plugin/input/InputNetworkObserver.cpp @@ -18,6 +18,7 @@ #include "ebpf/eBPFServer.h" #include "ebpf/config.h" #include "logger/Logger.h" +#include "monitor/PluginMetricManager.h" using namespace std; @@ -36,11 +37,24 @@ bool InputNetworkObserver::Init(const Json::Value& config, Json::Value& optional LOG_WARNING(sLogger, ("pipeline already loaded", "NETWORK_OBSERVE")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); return false; } + + static const std::unordered_map metricKeys = { + {METRIC_PLUGIN_IN_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM, MetricType::METRIC_TYPE_INT_GAUGE}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_CONNTRACKER_NUM, MetricType::METRIC_TYPE_INT_GAUGE}, + {METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_AGGREGATE_KEY_NUM, MetricType::METRIC_TYPE_INT_GAUGE}, + {METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_WORKER_HANDLE_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + }; + + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); return ebpf::InitObserverNetworkOption(config, mNetworkOption, mContext, sName); } bool InputNetworkObserver::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_OBSERVE, mContext, &mNetworkOption); + return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_OBSERVE, mContext, &mNetworkOption, mPluginMgr); } bool InputNetworkObserver::Stop(bool isPipelineRemoving) { diff --git a/core/plugin/input/InputNetworkObserver.h b/core/plugin/input/InputNetworkObserver.h index 78388c57bd..7cc3d98ca6 100644 --- a/core/plugin/input/InputNetworkObserver.h +++ b/core/plugin/input/InputNetworkObserver.h @@ -21,6 +21,7 @@ #include "ebpf/config.h" #include "pipeline/plugin/interface/Input.h" #include "ebpf/include/export.h" +#include "monitor/PluginMetricManager.h" namespace logtail { @@ -35,6 +36,7 @@ class InputNetworkObserver : public Input { bool SupportAck() const override { return false; } nami::ObserverNetworkOption mNetworkOption; + std::shared_ptr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputNetworkSecurity.cpp b/core/plugin/input/InputNetworkSecurity.cpp index 62c61abe68..409d40f77c 100644 --- a/core/plugin/input/InputNetworkSecurity.cpp +++ b/core/plugin/input/InputNetworkSecurity.cpp @@ -38,12 +38,20 @@ bool InputNetworkSecurity::Init(const Json::Value& config, Json::Value& optional LOG_WARNING(sLogger, ("pipeline already loaded", "NETWORK_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); return false; } + static const std::unordered_map metricKeys = { + {METRIC_PLUGIN_IN_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM, MetricType::METRIC_TYPE_INT_GAUGE}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + }; + + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); return mSecurityOptions.Init(ebpf::SecurityProbeType::NETWORK, config, mContext, sName); } bool InputNetworkSecurity::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_SECURITY, mContext, &mSecurityOptions); + return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_SECURITY, mContext, &mSecurityOptions, mPluginMgr); } bool InputNetworkSecurity::Stop(bool isPipelineRemoving) { diff --git a/core/plugin/input/InputNetworkSecurity.h b/core/plugin/input/InputNetworkSecurity.h index 4e7c441e7f..20e3025246 100644 --- a/core/plugin/input/InputNetworkSecurity.h +++ b/core/plugin/input/InputNetworkSecurity.h @@ -20,6 +20,7 @@ #include "ebpf/config.h" #include "pipeline/plugin/interface/Input.h" +#include "monitor/PluginMetricManager.h" namespace logtail { @@ -34,6 +35,7 @@ class InputNetworkSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; + std::shared_ptr mPluginMgr; }; } // namespace logtail diff --git a/core/plugin/input/InputProcessSecurity.cpp b/core/plugin/input/InputProcessSecurity.cpp index 74f646ae6c..cc11afa854 100644 --- a/core/plugin/input/InputProcessSecurity.cpp +++ b/core/plugin/input/InputProcessSecurity.cpp @@ -34,11 +34,20 @@ bool InputProcessSecurity::Init(const Json::Value& config, Json::Value& optional LOG_WARNING(sLogger, ("pipeline already loaded", "PROCESS_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); return false; } + + static const std::unordered_map metricKeys = { + {METRIC_PLUGIN_IN_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_LOSS_KERNEL_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_ENTRIES_NUM, MetricType::METRIC_TYPE_INT_GAUGE}, + {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, + }; + + mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys); return mSecurityOptions.Init(ebpf::SecurityProbeType::PROCESS, config, mContext, sName); } bool InputProcessSecurity::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::PROCESS_SECURITY,mContext, &mSecurityOptions); + return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::PROCESS_SECURITY,mContext, &mSecurityOptions, mPluginMgr); } bool InputProcessSecurity::Stop(bool isPipelineRemoving) { diff --git a/core/plugin/input/InputProcessSecurity.h b/core/plugin/input/InputProcessSecurity.h index afcffcb439..b45fec9b5c 100644 --- a/core/plugin/input/InputProcessSecurity.h +++ b/core/plugin/input/InputProcessSecurity.h @@ -20,7 +20,7 @@ #include "ebpf/config.h" #include "pipeline/plugin/interface/Input.h" - +#include "monitor/PluginMetricManager.h" namespace logtail { @@ -35,6 +35,7 @@ class InputProcessSecurity : public Input { bool SupportAck() const override { return false; } ebpf::SecurityOptions mSecurityOptions; + std::shared_ptr mPluginMgr; }; } // namespace logtail diff --git a/core/unittest/ebpf/eBPFServerUnittest.cpp b/core/unittest/ebpf/eBPFServerUnittest.cpp index 870f24e78f..a8c1763b91 100644 --- a/core/unittest/ebpf/eBPFServerUnittest.cpp +++ b/core/unittest/ebpf/eBPFServerUnittest.cpp @@ -15,6 +15,10 @@ #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "ebpf/config.h" +#include "plugin/input/InputNetworkObserver.h" +#include "plugin/input/InputProcessSecurity.h" +#include "plugin/input/InputNetworkSecurity.h" +#include "plugin/input/InputFileSecurity.h" DECLARE_FLAG_BOOL(logtail_mode); @@ -76,6 +80,7 @@ class eBPFServerUnittest : public testing::Test { } void InitSecurityOpts(); void InitObserverOpts(); + void HandleStats(nami::NamiStatisticsHandler cb, int plus); void GenerateBatchMeasure(nami::NamiHandleBatchMeasureFunc cb); void GenerateBatchSpan(nami::NamiHandleBatchSpanFunc cb); void GenerateBatchEvent(nami::NamiHandleBatchDataEventFn cb, SecureEventType); @@ -451,6 +456,54 @@ void eBPFServerUnittest::GenerateBatchAppEvent(nami::NamiHandleBatchEventFunc cb return; } +void eBPFServerUnittest::HandleStats(nami::NamiStatisticsHandler cb, int plus) { + std::vector stats; + + nami::eBPFStatistics ebpfStat; + ebpfStat.updated_ = true; + ebpfStat.loss_kernel_events_total_ = 1 + plus; + ebpfStat.recv_kernel_events_total_ = 1 + plus; + ebpfStat.push_events_total_ = 10 + plus; + ebpfStat.push_metrics_total_ = 11 + plus; + ebpfStat.push_spans_total_ = 12 + plus; + ebpfStat.process_cache_entities_num_ = 400 + plus; + ebpfStat.miss_process_cache_total_ = 20 + plus; + + nami::eBPFStatistics networkSecurityStat = ebpfStat; + networkSecurityStat.plugin_type_ = nami::PluginType::NETWORK_SECURITY; + nami::eBPFStatistics processSecurityStat = ebpfStat; + processSecurityStat.plugin_type_ = nami::PluginType::PROCESS_SECURITY; + nami::eBPFStatistics fileSecurityStat = ebpfStat; + fileSecurityStat.plugin_type_ = nami::PluginType::FILE_SECURITY; + + nami::NetworkObserverStatistics networkStat; + networkStat.updated_ = true; + networkStat.plugin_type_ = nami::PluginType::NETWORK_OBSERVE; + networkStat.updated_ = true; + networkStat.loss_kernel_events_total_ = 1 + plus; + networkStat.recv_kernel_events_total_ = 1 + plus; + networkStat.push_events_total_ = 10 + plus; + networkStat.push_metrics_total_ = 11 + plus; + networkStat.push_spans_total_ = 12 + plus; + networkStat.process_cache_entities_num_ = 400 + plus; + networkStat.miss_process_cache_total_ = 20 + plus; + + networkStat.agg_map_entities_num_ = 30 + plus; + networkStat.conntracker_num_ = 200 + plus; + networkStat.parse_http_records_failed_total_ = 1 + plus; + networkStat.parse_http_records_success_total_ = 100 + plus; + networkStat.recv_http_data_events_total_ = 101 + plus; + networkStat.recv_conn_stat_events_total_ = 204 + plus; + networkStat.recv_ctrl_events_total_ = 10 + plus; + + stats.emplace_back(std::move(networkStat)); + stats.emplace_back(std::move(networkSecurityStat)); + stats.emplace_back(std::move(processSecurityStat)); + stats.emplace_back(std::move(fileSecurityStat)); + std::cout << "3" << std::endl; + if (cb) cb(stats); +} + void eBPFServerUnittest::GenerateBatchSpan(nami::NamiHandleBatchSpanFunc cb) { std::vector> batch_app_spans; // agg for app level @@ -541,25 +594,43 @@ void eBPFServerUnittest::TestEnableNetworkPlugin() { } )"; std::string errorMsg; - Json::Value configJson; + Json::Value configJson, optionalGoPipeline; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); nami::ObserverNetworkOption network_option; bool res = ebpf::InitObserverNetworkOption(configJson, network_option, &ctx, "test"); EXPECT_TRUE(res); // observer_options.Init(ObserverType::NETWORK, configJson, &ctx, "test"); + auto input = new InputNetworkObserver(); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); + auto initStatus = input->Init(configJson, optionalGoPipeline); + EXPECT_TRUE(initStatus); + std::cout << "1" << std::endl; res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "test", 1, nami::PluginType::NETWORK_OBSERVE, &ctx, - &network_option); - + &network_option, input->mPluginMgr); + std::cout << "2" << std::endl; + + EXPECT_EQ(ebpf::eBPFServer::GetInstance()->mMonitorMgr->mInited[int(nami::PluginType::NETWORK_OBSERVE)], true); + auto& mgr = ebpf::eBPFServer::GetInstance()->mMonitorMgr->mSelfMonitors[int(nami::PluginType::NETWORK_OBSERVE)]; + auto mgrPtr = mgr.get(); + NetworkObserverSelfMonitor* monitor = dynamic_cast(mgrPtr); + monitor->mLossKernelEventsTotal->Add(1); + EXPECT_EQ(monitor->mProcessCacheEntitiesNum != nullptr, true); + monitor->mProcessCacheEntitiesNum->Set(10); + EXPECT_EQ(monitor->mLossKernelEventsTotal != nullptr, true); + EXPECT_EQ(monitor->mRecvCtrlEventsTotal != nullptr, true); + EXPECT_TRUE(res); auto conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); + HandleStats(conf->stats_handler_, 1); + std::cout << "3" << std::endl; auto network_conf = std::get(conf->config_); EXPECT_EQ(conf->plugin_type_, nami::PluginType::NETWORK_OBSERVE); EXPECT_EQ(conf->type, UpdataType::SECURE_UPDATE_TYPE_ENABLE_PROBE); - std::cout << "3" << std::endl; EXPECT_TRUE(network_conf.measure_cb_ != nullptr); EXPECT_TRUE(network_conf.span_cb_ != nullptr); EXPECT_TRUE(network_conf.so_.size()); @@ -580,15 +651,17 @@ void eBPFServerUnittest::TestEnableNetworkPlugin() { EXPECT_TRUE(ebpf::eBPFServer::GetInstance()->mSourceManager->mRunning[int(nami::PluginType::NETWORK_OBSERVE)]); // do update + input->SetMetricsRecordRef("test", "2"); res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "test", 8, nami::PluginType::NETWORK_OBSERVE, &ctx, - &network_option); + &network_option, input->mPluginMgr); EXPECT_TRUE(res); conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); EXPECT_EQ(conf->plugin_type_, nami::PluginType::NETWORK_OBSERVE); EXPECT_EQ(conf->type, UpdataType::SECURE_UPDATE_TYPE_CONFIG_CHAGE); + HandleStats(conf->stats_handler_, 2); GenerateBatchMeasure(network_conf.measure_cb_); GenerateBatchSpan(network_conf.span_cb_); @@ -618,16 +691,19 @@ void eBPFServerUnittest::TestEnableProcessPlugin() { )"; std::string errorMsg; - Json::Value configJson; + Json::Value configJson, optionalGoPipeline; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - std::cout << "1" << std::endl; SecurityOptions security_options; security_options.Init(SecurityProbeType::PROCESS, configJson, &ctx, "input_process_security"); + auto input = new InputProcessSecurity(); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); + input->Init(configJson, optionalGoPipeline); bool res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "test", 0, nami::PluginType::PROCESS_SECURITY, &ctx, - &security_options); + &security_options, input->mPluginMgr); EXPECT_TRUE(res); auto conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); EXPECT_EQ(conf->plugin_type_, nami::PluginType::PROCESS_SECURITY); @@ -644,12 +720,14 @@ void eBPFServerUnittest::TestEnableProcessPlugin() { EXPECT_EQ(ebpf::eBPFServer::GetInstance()->mProcessSecureCB->mPluginIdx, -1); EXPECT_TRUE(ebpf::eBPFServer::GetInstance()->mSourceManager->mRunning[int(nami::PluginType::PROCESS_SECURITY)]); + input->SetMetricsRecordRef("test", "2"); res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "test", 0, nami::PluginType::PROCESS_SECURITY, &ctx, - &security_options); + &security_options, input->mPluginMgr); EXPECT_TRUE(res); + EXPECT_TRUE(ebpf::eBPFServer::GetInstance()->mStartPluginTotal->GetValue() > 0); conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); EXPECT_EQ(conf->plugin_type_, nami::PluginType::PROCESS_SECURITY); EXPECT_EQ(conf->type, UpdataType::SECURE_UPDATE_TYPE_CONFIG_CHAGE); @@ -675,17 +753,21 @@ void eBPFServerUnittest::TestEnableNetworkSecurePlugin() { } } )"; + auto input = new InputNetworkSecurity(); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); std::string errorMsg; - Json::Value configJson; + Json::Value configJson, optionalGoPipeline;; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); SecurityOptions security_options; security_options.Init(SecurityProbeType::NETWORK, configJson, &ctx, "input_network_security"); + input->Init(configJson, optionalGoPipeline); bool res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "input_network_security", 5, nami::PluginType::NETWORK_SECURITY, &ctx, - &security_options); + &security_options, input->mPluginMgr); EXPECT_TRUE(res); auto conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); EXPECT_EQ(conf->plugin_type_, nami::PluginType::NETWORK_SECURITY); @@ -706,11 +788,13 @@ void eBPFServerUnittest::TestEnableNetworkSecurePlugin() { EXPECT_EQ(ebpf::eBPFServer::GetInstance()->mNetworkSecureCB->mPluginIdx, -1); EXPECT_TRUE(ebpf::eBPFServer::GetInstance()->mSourceManager->mRunning[int(nami::PluginType::NETWORK_SECURITY)]); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "2"); res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "input_network_security", 0, nami::PluginType::NETWORK_SECURITY, &ctx, - &security_options); + &security_options, input->mPluginMgr); EXPECT_TRUE(res); conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); EXPECT_EQ(conf->plugin_type_, nami::PluginType::NETWORK_SECURITY); @@ -740,17 +824,21 @@ void eBPFServerUnittest::TestEnableFileSecurePlugin() { } )"; + auto input = new InputFileSecurity(); + input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); + std::string errorMsg; - Json::Value configJson; + Json::Value configJson, optionalGoPipeline;; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - std::cout << "1" << std::endl; SecurityOptions security_options; security_options.Init(SecurityProbeType::FILE, configJson, &ctx, "input_file_security"); + input->Init(configJson, optionalGoPipeline); bool res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "input_file_security", 0, nami::PluginType::FILE_SECURITY, &ctx, - &security_options); + &security_options, input->mPluginMgr); EXPECT_EQ(std::get(security_options.mOptionList[0].filter_).mFilePathList.size(), 3); EXPECT_TRUE(res); auto conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); @@ -771,11 +859,12 @@ void eBPFServerUnittest::TestEnableFileSecurePlugin() { EXPECT_EQ(ebpf::eBPFServer::GetInstance()->mFileSecureCB->mPluginIdx, -1); EXPECT_TRUE(ebpf::eBPFServer::GetInstance()->mSourceManager->mRunning[int(nami::PluginType::FILE_SECURITY)]); + input->SetMetricsRecordRef("test", "2"); res = ebpf::eBPFServer::GetInstance()->EnablePlugin( "input_file_security", 0, nami::PluginType::FILE_SECURITY, &ctx, - &security_options); + &security_options, input->mPluginMgr); EXPECT_TRUE(res); conf = ebpf::eBPFServer::GetInstance()->mSourceManager->mConfig.get(); EXPECT_EQ(conf->plugin_type_, nami::PluginType::FILE_SECURITY); diff --git a/core/unittest/input/InputFileSecurityUnittest.cpp b/core/unittest/input/InputFileSecurityUnittest.cpp index 61ee2276db..5debaa7409 100644 --- a/core/unittest/input/InputFileSecurityUnittest.cpp +++ b/core/unittest/input/InputFileSecurityUnittest.cpp @@ -85,6 +85,7 @@ void InputFileSecurityUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputFileSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_file_security"); nami::SecurityFileFilter thisFilter1 = std::get(input->mSecurityOptions.mOptionList[0].filter_); @@ -108,6 +109,7 @@ void InputFileSecurityUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputFileSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_file_security"); nami::SecurityFileFilter thisFilter2 = std::get(input->mSecurityOptions.mOptionList[0].filter_); @@ -134,6 +136,7 @@ void InputFileSecurityUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputFileSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_file_security"); nami::SecurityFileFilter thisFilter = std::get(input->mSecurityOptions.mOptionList[0].filter_); @@ -155,6 +158,7 @@ void InputFileSecurityUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputFileSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_file_security"); nami::SecurityFileFilter thisFilter1 = std::get(input->mSecurityOptions.mOptionList[0].filter_); @@ -172,6 +176,7 @@ void InputFileSecurityUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputFileSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_file_security"); APSARA_TEST_EQUAL(1, input->mSecurityOptions.mOptionList.size()); // default callname @@ -198,6 +203,7 @@ void InputFileSecurityUnittest::OnSuccessfulStart() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputFileSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::FILE_SECURITY); @@ -225,6 +231,7 @@ void InputFileSecurityUnittest::OnSuccessfulStop() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputFileSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::FILE_SECURITY); diff --git a/core/unittest/input/InputNetworkObserverUnittest.cpp b/core/unittest/input/InputNetworkObserverUnittest.cpp index d6b68f25f5..a53bf43264 100644 --- a/core/unittest/input/InputNetworkObserverUnittest.cpp +++ b/core/unittest/input/InputNetworkObserverUnittest.cpp @@ -85,6 +85,7 @@ void InputNetworkObserverUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkObserver()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_network_observer"); nami::ObserverNetworkOption thisObserver = input->mNetworkOption; @@ -118,6 +119,7 @@ void InputNetworkObserverUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkObserver()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_network_observer"); nami::ObserverNetworkOption thisObserver = input->mNetworkOption; @@ -142,6 +144,7 @@ void InputNetworkObserverUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkObserver()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_FALSE(input->Init(configJson, optionalGoPipeline)); } @@ -168,6 +171,7 @@ void InputNetworkObserverUnittest::OnSuccessfulStart() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkObserver()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_OBSERVE); @@ -197,6 +201,7 @@ void InputNetworkObserverUnittest::OnSuccessfulStop() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkObserver()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_OBSERVE); diff --git a/core/unittest/input/InputNetworkSecurityUnittest.cpp b/core/unittest/input/InputNetworkSecurityUnittest.cpp index 1a36b464b4..9fc34416b6 100644 --- a/core/unittest/input/InputNetworkSecurityUnittest.cpp +++ b/core/unittest/input/InputNetworkSecurityUnittest.cpp @@ -85,6 +85,7 @@ void InputNetworkSecurityUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_network_security"); nami::SecurityNetworkFilter thisFilter1 @@ -119,6 +120,7 @@ void InputNetworkSecurityUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_network_security"); nami::SecurityNetworkFilter thisFilter1 @@ -145,6 +147,7 @@ void InputNetworkSecurityUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); nami::SecurityNetworkFilter thisFilter5 = std::get(input->mSecurityOptions.mOptionList[0].filter_); @@ -172,6 +175,7 @@ void InputNetworkSecurityUnittest::OnFailedInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); nami::SecurityNetworkFilter thisFilter6 = std::get(input->mSecurityOptions.mOptionList[0].filter_); @@ -200,6 +204,7 @@ void InputNetworkSecurityUnittest::OnSuccessfulStart() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); @@ -229,6 +234,7 @@ void InputNetworkSecurityUnittest::OnSuccessfulStop() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputNetworkSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); diff --git a/core/unittest/input/InputProcessSecurityUnittest.cpp b/core/unittest/input/InputProcessSecurityUnittest.cpp index 1658590705..c7be48b031 100644 --- a/core/unittest/input/InputProcessSecurityUnittest.cpp +++ b/core/unittest/input/InputProcessSecurityUnittest.cpp @@ -75,6 +75,7 @@ void InputProcessSecurityUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputProcessSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(input->sName, "input_process_security"); APSARA_TEST_EQUAL(input->mSecurityOptions.mOptionList[0].call_names_.size(), 5); @@ -95,6 +96,7 @@ void InputProcessSecurityUnittest::OnSuccessfulStart() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputProcessSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY); @@ -115,6 +117,7 @@ void InputProcessSecurityUnittest::OnSuccessfulStop() { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); input.reset(new InputProcessSecurity()); input->SetContext(ctx); + input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY);