diff --git a/core/pipeline/PipelineContext.h b/core/pipeline/PipelineContext.h index 2c4a7f580d..132d0ecf14 100644 --- a/core/pipeline/PipelineContext.h +++ b/core/pipeline/PipelineContext.h @@ -32,21 +32,6 @@ namespace logtail { class Pipeline; class FlusherSLS; -// for compatiblity with shennong profile -struct ProcessProfile { - int readBytes = 0; - int skipBytes = 0; - int feedLines = 0; - int splitLines = 0; - int parseFailures = 0; - int regexMatchFailures = 0; - int parseTimeFailures = 0; - int historyFailures = 0; - int logGroupSize = 0; - - void Reset() { memset(this, 0, sizeof(ProcessProfile)); } -}; - class PipelineContext { public: PipelineContext() {} @@ -89,7 +74,6 @@ class PipelineContext { bool IsFlushingThroughGoPipeline() const { return mIsFlushingThroughGoPipeline; } void SetIsFlushingThroughGoPipelineFlag(bool flag) { mIsFlushingThroughGoPipeline = flag; } - ProcessProfile& GetProcessProfile() const { return mProcessProfile; } const Logger::logger& GetLogger() const { return mLogger; } AlarmManager& GetAlarm() const { return *mAlarm; }; @@ -111,7 +95,6 @@ class PipelineContext { bool mHasNativeProcessors = false; bool mIsFlushingThroughGoPipeline = false; - mutable ProcessProfile mProcessProfile; Logger::logger mLogger = sLogger; AlarmManager* mAlarm = AlarmManager::GetInstance(); }; diff --git a/core/pipeline/plugin/instance/ProcessorInstance.h b/core/pipeline/plugin/instance/ProcessorInstance.h index b7b5460c1c..298c99a78a 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.h +++ b/core/pipeline/plugin/instance/ProcessorInstance.h @@ -55,6 +55,7 @@ class ProcessorInstance : public PluginInstance { friend class ProcessorParseDelimiterNativeUnittest; friend class ProcessorFilterNativeUnittest; friend class ProcessorDesensitizeNativeUnittest; + friend class ProcessorSplitLogStringNativeUnittest; friend class InputFileUnittest; friend class InputPrometheusUnittest; friend class PipelineUnittest; diff --git a/core/plugin/processor/ProcessorParseApsaraNative.cpp b/core/plugin/processor/ProcessorParseApsaraNative.cpp index 43d3854266..493ee354cf 100644 --- a/core/plugin/processor/ProcessorParseApsaraNative.cpp +++ b/core/plugin/processor/ProcessorParseApsaraNative.cpp @@ -74,10 +74,6 @@ bool ProcessorParseApsaraNative::Init(const Json::Value& config) { return false; } - mLogGroupSize = &(GetContext().GetProcessProfile().logGroupSize); - mParseFailures = &(GetContext().GetProcessProfile().parseFailures); - mHistoryFailures = &(GetContext().GetProcessProfile().historyFailures); - mDiscardedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL); mOutFailedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL); mOutKeyNotFoundEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL); @@ -159,7 +155,6 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, GetContext().GetLogstoreName(), GetContext().GetRegion()); mOutFailedEventsTotal->Add(1); - ++(*mParseFailures); sourceEvent.DelContent(mSourceKey); if (mCommonParserOptions.ShouldAddSourceContent(false)) { AddLog(mCommonParserOptions.mRenamedSourceKey, buffer, sourceEvent, false); @@ -195,7 +190,6 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, GetContext().GetLogstoreName(), GetContext().GetRegion()); } - ++(*mHistoryFailures); mHistoryFailureTotal->Add(1); mDiscardedEventsTotal->Add(1); return false; @@ -473,7 +467,6 @@ void ProcessorParseApsaraNative::AddLog(const StringView& key, return; } targetEvent.AppendContentNoCopy(key, value); - *mLogGroupSize += key.size() + value.size() + 5; } bool ProcessorParseApsaraNative::IsSupportedEvent(const PipelineEventPtr& e) const { diff --git a/core/plugin/processor/ProcessorParseApsaraNative.h b/core/plugin/processor/ProcessorParseApsaraNative.h index 4bd77ee8e1..77ddae3a5a 100644 --- a/core/plugin/processor/ProcessorParseApsaraNative.h +++ b/core/plugin/processor/ProcessorParseApsaraNative.h @@ -51,10 +51,6 @@ class ProcessorParseApsaraNative : public Processor { int32_t mLogTimeZoneOffsetSecond = 0; - int* mLogGroupSize = nullptr; - int* mParseFailures = nullptr; - int* mHistoryFailures = nullptr; - CounterPtr mDiscardedEventsTotal; CounterPtr mOutFailedEventsTotal; CounterPtr mOutKeyNotFoundEventsTotal; diff --git a/core/plugin/processor/ProcessorParseDelimiterNative.cpp b/core/plugin/processor/ProcessorParseDelimiterNative.cpp index ff674cace5..61abbfd572 100644 --- a/core/plugin/processor/ProcessorParseDelimiterNative.cpp +++ b/core/plugin/processor/ProcessorParseDelimiterNative.cpp @@ -175,9 +175,6 @@ bool ProcessorParseDelimiterNative::Init(const Json::Value& config) { return false; } - mParseFailures = &(GetContext().GetProcessProfile().parseFailures); - mLogGroupSize = &(GetContext().GetProcessProfile().logGroupSize); - mDiscardedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL); mOutFailedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL); mOutKeyNotFoundEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL); @@ -297,7 +294,6 @@ bool ProcessorParseDelimiterNative::ProcessEvent(const StringView& logPath, Pipe GetContext().GetProjectName(), GetContext().GetLogstoreName(), GetContext().GetRegion()); - ++(*mParseFailures); parseSuccess = false; } } else { @@ -307,7 +303,6 @@ bool ProcessorParseDelimiterNative::ProcessEvent(const StringView& logPath, Pipe GetContext().GetProjectName(), GetContext().GetLogstoreName(), GetContext().GetRegion()); - ++(*mParseFailures); parseSuccess = false; } } else { @@ -319,7 +314,6 @@ bool ProcessorParseDelimiterNative::ProcessEvent(const StringView& logPath, Pipe LOG_WARNING(sLogger, ("parse delimiter log fail", "no column keys defined")("project", GetContext().GetProjectName())( "logstore", GetContext().GetLogstoreName())("file", logPath)); - ++(*mParseFailures); parseSuccess = false; } @@ -417,7 +411,6 @@ void ProcessorParseDelimiterNative::AddLog(const StringView& key, return; } targetEvent.SetContentNoCopy(key, value); - *mLogGroupSize += key.size() + value.size() + 5; } bool ProcessorParseDelimiterNative::IsSupportedEvent(const PipelineEventPtr& e) const { diff --git a/core/plugin/processor/ProcessorParseDelimiterNative.h b/core/plugin/processor/ProcessorParseDelimiterNative.h index cebed7a582..741c3b6931 100644 --- a/core/plugin/processor/ProcessorParseDelimiterNative.h +++ b/core/plugin/processor/ProcessorParseDelimiterNative.h @@ -74,9 +74,6 @@ class ProcessorParseDelimiterNative : public Processor { bool mSourceKeyOverwritten = false; std::unique_ptr mDelimiterModeFsmParserPtr; - int* mLogGroupSize = nullptr; - int* mParseFailures = nullptr; - CounterPtr mDiscardedEventsTotal; CounterPtr mOutFailedEventsTotal; CounterPtr mOutKeyNotFoundEventsTotal; diff --git a/core/plugin/processor/ProcessorParseJsonNative.cpp b/core/plugin/processor/ProcessorParseJsonNative.cpp index e7f65a998e..924ff51bff 100644 --- a/core/plugin/processor/ProcessorParseJsonNative.cpp +++ b/core/plugin/processor/ProcessorParseJsonNative.cpp @@ -47,9 +47,6 @@ bool ProcessorParseJsonNative::Init(const Json::Value& config) { return false; } - mParseFailures = &(GetContext().GetProcessProfile().parseFailures); - mLogGroupSize = &(GetContext().GetProcessProfile().logGroupSize); - mDiscardedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL); mOutFailedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL); mOutKeyNotFoundEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL); @@ -135,7 +132,6 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent, GetContext().GetLogstoreName(), GetContext().GetRegion()); } - ++(*mParseFailures); mOutFailedEventsTotal->Add(1); parseSuccess = false; } else if (!doc.IsObject()) { @@ -149,7 +145,6 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent, GetContext().GetLogstoreName(), GetContext().GetRegion()); } - ++(*mParseFailures); mOutFailedEventsTotal->Add(1); parseSuccess = false; } @@ -209,7 +204,6 @@ void ProcessorParseJsonNative::AddLog(const StringView& key, return; } targetEvent.SetContentNoCopy(key, value); - *mLogGroupSize += key.size() + value.size() + 5; } bool ProcessorParseJsonNative::IsSupportedEvent(const PipelineEventPtr& e) const { diff --git a/core/plugin/processor/ProcessorParseJsonNative.h b/core/plugin/processor/ProcessorParseJsonNative.h index b9d1902696..7382079585 100644 --- a/core/plugin/processor/ProcessorParseJsonNative.h +++ b/core/plugin/processor/ProcessorParseJsonNative.h @@ -44,9 +44,6 @@ class ProcessorParseJsonNative : public Processor { bool ProcessEvent(const StringView& logPath, PipelineEventPtr& e); static std::string RapidjsonValueToString(const rapidjson::Value& value); - int* mParseFailures = nullptr; - int* mLogGroupSize = nullptr; - CounterPtr mDiscardedEventsTotal; CounterPtr mOutFailedEventsTotal; CounterPtr mOutKeyNotFoundEventsTotal; diff --git a/core/plugin/processor/ProcessorParseRegexNative.cpp b/core/plugin/processor/ProcessorParseRegexNative.cpp index f04d8ad0dd..a543632027 100644 --- a/core/plugin/processor/ProcessorParseRegexNative.cpp +++ b/core/plugin/processor/ProcessorParseRegexNative.cpp @@ -92,10 +92,6 @@ bool ProcessorParseRegexNative::Init(const Json::Value& config) { return false; } - mParseFailures = &(GetContext().GetProcessProfile().parseFailures); - mRegexMatchFailures = &(GetContext().GetProcessProfile().regexMatchFailures); - mLogGroupSize = &(GetContext().GetProcessProfile().logGroupSize); - mDiscardedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL); mOutFailedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL); mOutKeyNotFoundEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL); @@ -178,7 +174,6 @@ void ProcessorParseRegexNative::AddLog(const StringView& key, return; } targetEvent.SetContentNoCopy(key, value); - *mLogGroupSize += key.size() + value.size() + 5; } bool ProcessorParseRegexNative::RegexLogLineParser(LogEvent& sourceEvent, @@ -218,8 +213,6 @@ bool ProcessorParseRegexNative::RegexLogLineParser(LogEvent& sourceEvent, GetContext().GetRegion()); } } - ++(*mRegexMatchFailures); - ++(*mParseFailures); mOutFailedEventsTotal->Add(1); parseSuccess = false; } else if (what.size() <= keys.size()) { @@ -237,8 +230,6 @@ bool ProcessorParseRegexNative::RegexLogLineParser(LogEvent& sourceEvent, GetContext().GetLogstoreName(), GetContext().GetRegion()); } - ++(*mRegexMatchFailures); - ++(*mParseFailures); parseSuccess = false; } if (!parseSuccess) { diff --git a/core/plugin/processor/ProcessorParseRegexNative.h b/core/plugin/processor/ProcessorParseRegexNative.h index c2a8fc8d88..087a088f6c 100644 --- a/core/plugin/processor/ProcessorParseRegexNative.h +++ b/core/plugin/processor/ProcessorParseRegexNative.h @@ -59,10 +59,6 @@ class ProcessorParseRegexNative : public Processor { bool mIsWholeLineMode = false; boost::regex mReg; - int* mParseFailures = nullptr; - int* mRegexMatchFailures = nullptr; - int* mLogGroupSize = nullptr; - CounterPtr mDiscardedEventsTotal; CounterPtr mOutFailedEventsTotal; CounterPtr mOutKeyNotFoundEventsTotal; diff --git a/core/plugin/processor/ProcessorParseTimestampNative.cpp b/core/plugin/processor/ProcessorParseTimestampNative.cpp index 2bc5292398..d7bfdbad3b 100644 --- a/core/plugin/processor/ProcessorParseTimestampNative.cpp +++ b/core/plugin/processor/ProcessorParseTimestampNative.cpp @@ -87,9 +87,6 @@ bool ProcessorParseTimestampNative::Init(const Json::Value& config) { mContext->GetRegion()); } - mParseTimeFailures = &(GetContext().GetProcessProfile().parseTimeFailures); - mHistoryFailures = &(GetContext().GetProcessProfile().historyFailures); - mDiscardedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL); mOutFailedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL); mOutKeyNotFoundEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL); @@ -164,7 +161,6 @@ bool ProcessorParseTimestampNative::ProcessEvent(StringView logPath, GetContext().GetLogstoreName(), GetContext().GetRegion()); } - ++(*mHistoryFailures); mHistoryFailureTotal->Add(1); mDiscardedEventsTotal->Add(1); return false; @@ -221,7 +217,6 @@ bool ProcessorParseTimestampNative::ParseLogTime(const StringView& curTimeStr, / GetContext().GetLogstoreName(), GetContext().GetRegion()); } - ++(*mParseTimeFailures); return false; } diff --git a/core/plugin/processor/ProcessorParseTimestampNative.h b/core/plugin/processor/ProcessorParseTimestampNative.h index 0f7c9ea6f0..ea1a508b15 100644 --- a/core/plugin/processor/ProcessorParseTimestampNative.h +++ b/core/plugin/processor/ProcessorParseTimestampNative.h @@ -53,9 +53,6 @@ class ProcessorParseTimestampNative : public Processor { int32_t mLogTimeZoneOffsetSecond = 0; - int* mParseTimeFailures = nullptr; - int* mHistoryFailures = nullptr; - CounterPtr mDiscardedEventsTotal; CounterPtr mOutFailedEventsTotal; CounterPtr mOutKeyNotFoundEventsTotal; diff --git a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp index 64ccd57cc1..e041c48971 100644 --- a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp +++ b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp @@ -73,8 +73,6 @@ bool ProcessorMergeMultilineLogNative::Init(const Json::Value& config) { mContext->GetRegion()); } - mSplitLines = &(GetContext().GetProcessProfile().splitLines); - mMergedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MERGED_EVENTS_TOTAL); mUnmatchedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL); @@ -93,7 +91,6 @@ void ProcessorMergeMultilineLogNative::Process(PipelineEventGroup& logGroup) { MergeLogsByFlag(logGroup); logGroup.DelMetadata(EventGroupMetaKey::HAS_PART_LOG); } - *mSplitLines = logGroup.GetEvents().size(); } bool ProcessorMergeMultilineLogNative::IsSupportedEvent(const PipelineEventPtr& e) const { diff --git a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.h b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.h index fe1334dde2..5a6c6d900b 100644 --- a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.h +++ b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.h @@ -55,8 +55,6 @@ class ProcessorMergeMultilineLogNative : public Processor { CounterPtr mUnmatchedEventsTotal; // 未成功合并的日志条数 // CounterPtr mProcUnmatchedEventsBytes; // 未成功合并的日志字节数 - int* mSplitLines = nullptr; - #ifdef APSARA_UNIT_TEST_MAIN friend class ProcessorMergeMultilineLogNativeUnittest; friend class ProcessorMergeMultilineLogDisacardUnmatchUnittest; diff --git a/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp b/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp index 8f8424050a..681af4cfce 100644 --- a/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp +++ b/core/plugin/processor/inner/ProcessorSplitLogStringNative.cpp @@ -81,8 +81,6 @@ bool ProcessorSplitLogStringNative::Init(const Json::Value& config) { mContext->GetRegion()); } - mSplitLines = &(GetContext().GetProcessProfile().splitLines); - return true; } @@ -94,7 +92,6 @@ void ProcessorSplitLogStringNative::Process(PipelineEventGroup& logGroup) { for (PipelineEventPtr& e : logGroup.MutableEvents()) { ProcessEvent(logGroup, std::move(e), newEvents); } - *mSplitLines = newEvents.size(); logGroup.SwapEvents(newEvents); } diff --git a/core/plugin/processor/inner/ProcessorSplitLogStringNative.h b/core/plugin/processor/inner/ProcessorSplitLogStringNative.h index 0bec830b23..58057b3f8c 100644 --- a/core/plugin/processor/inner/ProcessorSplitLogStringNative.h +++ b/core/plugin/processor/inner/ProcessorSplitLogStringNative.h @@ -44,8 +44,6 @@ class ProcessorSplitLogStringNative : public Processor { void ProcessEvent(PipelineEventGroup& logGroup, PipelineEventPtr&& e, EventsContainer& newEvents); StringView GetNextLine(StringView log, size_t begin); - int* mSplitLines = nullptr; - #ifdef APSARA_UNIT_TEST_MAIN friend class ProcessorRegexStringNativeUnittest; friend class ProcessorParseDelimiterNativeUnittest; diff --git a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp index 9afb6b57dd..3ced98912a 100644 --- a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp +++ b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp @@ -81,8 +81,6 @@ bool ProcessorSplitMultilineLogStringNative::Init(const Json::Value& config) { mMatchedLinesTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MATCHED_LINES_TOTAL); mUnmatchedLinesTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_UNMATCHED_LINES_TOTAL); - mSplitLines = &(mContext->GetProcessProfile().splitLines); - return true; } @@ -105,7 +103,6 @@ void ProcessorSplitMultilineLogStringNative::Process(PipelineEventGroup& logGrou } mMatchedLinesTotal->Add(inputLines - unmatchLines); mUnmatchedLinesTotal->Add(unmatchLines); - *mSplitLines = newEvents.size(); logGroup.SwapEvents(newEvents); } diff --git a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h index f32de10618..4a9d25cb57 100644 --- a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h +++ b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h @@ -65,8 +65,6 @@ class ProcessorSplitMultilineLogStringNative : public Processor { int* unmatchLines); StringView GetNextLine(StringView log, size_t begin); - int* mSplitLines = nullptr; - CounterPtr mMatchedEventsTotal; CounterPtr mMatchedLinesTotal; CounterPtr mUnmatchedLinesTotal; diff --git a/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp b/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp index 7ceeb708dc..26ac2f3892 100644 --- a/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp @@ -19,8 +19,8 @@ #include "models/LogEvent.h" #include "models/StringView.h" #include "pipeline/plugin/instance/ProcessorInstance.h" -#include "plugin/processor/inner/ProcessorMergeMultilineLogNative.h" #include "plugin/processor/ProcessorParseApsaraNative.h" +#include "plugin/processor/inner/ProcessorMergeMultilineLogNative.h" #include "plugin/processor/inner/ProcessorSplitLogStringNative.h" #include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" #include "unittest/Unittest.h" @@ -63,7 +63,7 @@ UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestProcessEventMicrosecondUn UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestApsaraEasyReadLogTimeParser); UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestApsaraLogLineParser); -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -785,7 +785,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessWholeLinePart() { })"; APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); // check observablity - APSARA_TEST_EQUAL_FATAL(1, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(uint64_t(3), processorInstance.mInEventsTotal->GetValue()); // only timestamp failed, so output is 2 APSARA_TEST_EQUAL_FATAL(uint64_t(2), processorInstance.mOutEventsTotal->GetValue()); @@ -961,8 +960,7 @@ void ProcessorParseApsaraNativeUnittest::TestAddLog() { char value[] = "value"; processor.AddLog(key, value, *logEvent); // check observability - APSARA_TEST_EQUAL_FATAL(int(strlen(key) + strlen(value) + 5), - processor.GetContext().GetProcessProfile().logGroupSize); + // Todo: add metrics for check result } void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { @@ -1080,7 +1078,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); // check observablity int count = 5; - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mInEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mOutEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mDiscardedEventsTotal->GetValue()); @@ -1157,7 +1154,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventDiscardUnmatch() { APSARA_TEST_STREQ_FATAL("null", CompactJson(outJson).c_str()); // check observablity int count = 5; - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mInEventsTotal->GetValue()); // discard unmatch, so output is 0 APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventsTotal->GetValue()); @@ -1275,7 +1271,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventMicrosecondUnmatch() { std::string outJson = eventGroupList[0].ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); // check observablity - APSARA_TEST_EQUAL_FATAL(1, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(uint64_t(4), processorInstance.mInEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(4), processorInstance.mOutEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mDiscardedEventsTotal->GetValue()); diff --git a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp index 51018f1de3..56af50c91e 100644 --- a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp @@ -65,7 +65,7 @@ UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestAllowingShortenedField UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestExtend); UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestEmpty); -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -2199,8 +2199,7 @@ void ProcessorParseDelimiterNativeUnittest::TestAddLog() { char value[] = "value"; processor.AddLog(key, value, *logEvent); // check observability - APSARA_TEST_EQUAL_FATAL(int(strlen(key) + strlen(value) + 5), - processor.GetContext().GetProcessProfile().logGroupSize); + // Todo: add metrics for check result } void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { @@ -2334,7 +2333,6 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); // check observablity int count = 5; - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mInEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mOutEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mDiscardedEventsTotal->GetValue()); @@ -2422,7 +2420,6 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { APSARA_TEST_STREQ_FATAL("null", CompactJson(outJson).c_str()); // check observablity int count = 5; - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mInEventsTotal->GetValue()); // discard unmatch, so output is 0 APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventsTotal->GetValue()); diff --git a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp index 2b5a6a91bd..da2248c4ff 100644 --- a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp @@ -58,7 +58,7 @@ UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonRaw); UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestMultipleLines); -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -257,8 +257,7 @@ void ProcessorParseJsonNativeUnittest::TestAddLog() { char value[] = "value"; processor.AddLog(key, value, *logEvent); // check observability - APSARA_TEST_EQUAL_FATAL(int(strlen(key) + strlen(value) + 5), - processor.GetContext().GetProcessProfile().logGroupSize); + // Todo: add metrics for check result } void ProcessorParseJsonNativeUnittest::TestProcessJsonEscapedNullByte() { @@ -617,8 +616,6 @@ void ProcessorParseJsonNativeUnittest::TestProcessEventKeepUnmatch() { int count = 1; // check observablity - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mInEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mOutEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor.mDiscardedEventsTotal->GetValue()); @@ -683,8 +680,6 @@ void ProcessorParseJsonNativeUnittest::TestProcessEventDiscardUnmatch() { int count = 1; // check observablity - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(uint64_t(count), processorInstance.mInEventsTotal->GetValue()); // discard unmatch, so output is 0 APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventsTotal->GetValue()); diff --git a/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp b/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp index f7cbee3870..d64b0215fa 100644 --- a/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp @@ -43,7 +43,7 @@ class ProcessorParseRegexNativeUnittest : public ::testing::Test { PipelineContext ctx; }; -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -398,7 +398,7 @@ void ProcessorParseRegexNativeUnittest::TestAddLog() { char value[] = "value"; processor.AddLog(key, value, *logEvent); // check observability - APSARA_TEST_EQUAL_FATAL(strlen(key) + strlen(value) + 5, processor.GetContext().GetProcessProfile().logGroupSize); + // Todo: add metrics for check result } void ProcessorParseRegexNativeUnittest::TestProcessEventKeepUnmatch() { @@ -475,9 +475,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessEventKeepUnmatch() { int count = 5; // check observablity - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().regexMatchFailures); - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mInEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(count, processorInstance.mOutEventsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(0, processor.mDiscardedEventsTotal->GetValue()); @@ -559,8 +556,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessEventDiscardUnmatch() { int count = 5; // check observablity - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().regexMatchFailures); - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(count, processorInstance.mInEventsTotal->GetValue()); // discard unmatch, so output is 0 APSARA_TEST_EQUAL_FATAL(0, processorInstance.mOutEventsTotal->GetValue()); @@ -646,9 +641,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessEventKeyCountUnmatch() { int count = 5; // check observablity - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().regexMatchFailures); - APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); - APSARA_TEST_EQUAL_FATAL(count, processorInstance.mInEventsTotal->GetValue()); // discard unmatch, so output is 0 APSARA_TEST_EQUAL_FATAL(0, processorInstance.mOutEventsTotal->GetValue()); diff --git a/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp b/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp index c8dd9c788e..f50d573f1c 100644 --- a/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp @@ -397,7 +397,6 @@ void ProcessorParseTimestampNativeUnittest::TestProcessRegularFormat() { })"; APSARA_TEST_EQUAL_FATAL(CompactJson(expectJsonSs.str()), CompactJson(outJson)); // check observablity - APSARA_TEST_EQUAL_FATAL(0, processor.GetContext().GetProcessProfile().historyFailures); APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mInEventsTotal->GetValue()); // discard history, so output is 0 APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mOutEventsTotal->GetValue()); @@ -486,7 +485,6 @@ void ProcessorParseTimestampNativeUnittest::TestProcessNoYearFormat() { })"; APSARA_TEST_EQUAL_FATAL(CompactJson(expectJsonSs.str()), CompactJson(outJson)); // check observablity - APSARA_TEST_EQUAL_FATAL(0, processor.GetContext().GetProcessProfile().historyFailures); APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mInEventsTotal->GetValue()); // discard history, so output is 0 APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mOutEventsTotal->GetValue()); @@ -548,7 +546,6 @@ void ProcessorParseTimestampNativeUnittest::TestProcessRegularFormatFailed() { std::string outJson = eventGroupList[0].ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(inJson).c_str(), CompactJson(outJson).c_str()); // check observablity - APSARA_TEST_EQUAL_FATAL(0, processor.GetContext().GetProcessProfile().historyFailures); APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mInEventsTotal->GetValue()); // discard history, so output is 0 APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mOutEventsTotal->GetValue()); @@ -603,8 +600,6 @@ void ProcessorParseTimestampNativeUnittest::TestProcessHistoryDiscard() { processorInstance.Process(eventGroupList); // check observablity - std::string outJson = eventGroupList[0].ToJsonString(); - APSARA_TEST_EQUAL_FATAL(2, processor.GetContext().GetProcessProfile().historyFailures); APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mInEventsTotal->GetValue()); // discard history, so output is 0 APSARA_TEST_EQUAL_FATAL(0UL, processorInstance.mOutEventsTotal->GetValue()); diff --git a/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp b/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp index a762cc9290..d48ec7b4a4 100644 --- a/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp @@ -84,11 +84,12 @@ void ProcessorSplitLogStringNativeUnittest::TestProcessJson() { })"; APSARA_TEST_TRUE_FATAL(eventGroup.FromJsonString(inJson.str())); // run function - ProcessorSplitLogStringNative processor; - processor.SetContext(mContext); - - APSARA_TEST_TRUE_FATAL(processor.Init(config)); - processor.Process(eventGroup); + ProcessorSplitLogStringNative& processor = *(new ProcessorSplitLogStringNative); + ProcessorInstance processorInstance(&processor, getPluginMeta()); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); + std::vector logGroupList; + logGroupList.emplace_back(std::move(eventGroup)); + processorInstance.Process(logGroupList); // judge result std::stringstream expectJson; expectJson << R"({ @@ -124,10 +125,10 @@ void ProcessorSplitLogStringNativeUnittest::TestProcessJson() { } ] })"; - std::string outJson = eventGroup.ToJsonString(true); + std::string outJson = logGroupList[0].ToJsonString(true); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); // check observability - APSARA_TEST_EQUAL_FATAL(2, processor.GetContext().GetProcessProfile().splitLines); + APSARA_TEST_EQUAL_FATAL(2, processorInstance.mOutEventsTotal->GetValue()); } void ProcessorSplitLogStringNativeUnittest::TestProcessCommon() { @@ -166,10 +167,12 @@ void ProcessorSplitLogStringNativeUnittest::TestProcessCommon() { })"; eventGroup.FromJsonString(inJson); // run function - ProcessorSplitLogStringNative processor; - processor.SetContext(mContext); - APSARA_TEST_TRUE_FATAL(processor.Init(config)); - processor.Process(eventGroup); + ProcessorSplitLogStringNative& processor = *(new ProcessorSplitLogStringNative); + ProcessorInstance processorInstance(&processor, getPluginMeta()); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); + std::vector logGroupList; + logGroupList.emplace_back(std::move(eventGroup)); + processorInstance.Process(logGroupList); // judge result std::string expectJson = R"({ "events" : @@ -220,10 +223,10 @@ void ProcessorSplitLogStringNativeUnittest::TestProcessCommon() { } ] })"; - std::string outJson = eventGroup.ToJsonString(true); + std::string outJson = logGroupList[0].ToJsonString(true); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); // check observability - APSARA_TEST_EQUAL_FATAL(4, processor.GetContext().GetProcessProfile().splitLines); + APSARA_TEST_EQUAL_FATAL(4, processorInstance.mOutEventsTotal->GetValue()); } void ProcessorSplitLogStringNativeUnittest::TestEnableRawContent() {