From 507ae55a26db06a4719c4eac1b2ae582a1b4b021 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 31 Dec 2024 08:45:30 +0000 Subject: [PATCH 1/8] reformat header file and code --- .clang-format | 13 + core/app_config/AppConfig.cpp | 11 +- core/app_config/AppConfig.h | 221 +- core/application/Application.cpp | 2 +- core/application/Application.h | 5 +- core/checkpoint/AdhocCheckpointManager.cpp | 7 +- core/checkpoint/AdhocCheckpointManager.h | 1 + core/checkpoint/AdhocFileCheckpoint.cpp | 3 +- core/checkpoint/AdhocFileCheckpoint.h | 3 +- core/checkpoint/AdhocJobCheckpoint.cpp | 4 +- core/checkpoint/AdhocJobCheckpoint.h | 1 + core/checkpoint/CheckPointManager.h | 7 +- core/checkpoint/CheckpointManagerV2.cpp | 73 +- core/checkpoint/CheckpointManagerV2.h | 12 +- core/checkpoint/RangeCheckpoint.cpp | 1 + core/checkpoint/RangeCheckpoint.h | 3 +- core/common/BoostRegexValidator.cpp | 14 +- core/common/CircularBuffer.h | 11 +- core/common/CompressTools.cpp | 2 +- core/common/DNSCache.h | 1 + core/common/DevInode.cpp | 3 +- core/common/DynamicLibHelper.cpp | 5 +- core/common/DynamicLibHelper.h | 13 +- core/common/EncodingConverter.cpp | 1 + core/common/EncodingConverter.h | 6 +- core/common/EncodingUtil.cpp | 2 +- core/common/EnvUtil.h | 2 +- core/common/ErrorUtil.h | 3 +- core/common/FileEncryption.cpp | 6 +- core/common/FileEncryption.h | 3 +- core/common/FileInfo.h | 3 +- core/common/FileSystemUtil.cpp | 517 ++--- core/common/FileSystemUtil.h | 204 +- core/common/Flags.h | 2 +- core/common/HashUtil.cpp | 2 +- core/common/HashUtil.h | 1 + core/common/JsonUtil.cpp | 30 +- core/common/JsonUtil.h | 3 +- core/common/LRUCache.h | 264 +-- core/common/Lock.h | 10 +- core/common/LogFileOperator.h | 7 +- core/common/MachineInfoUtil.cpp | 7 +- core/common/MachineInfoUtil.h | 7 +- core/common/ParamExtractor.h | 5 +- core/common/RandomUtil.cpp | 33 +- core/common/RuntimeUtil.cpp | 17 +- core/common/SafeQueue.h | 4 +- core/common/Semaphore.h | 2 +- core/common/SlidingWindowCounter.h | 4 +- core/common/SplitedFilePath.h | 1 + core/common/StringTools.cpp | 6 +- core/common/StringTools.h | 5 +- core/common/Strptime.cpp | 85 +- core/common/Thread.h | 4 +- core/common/ThreadPool.h | 6 +- core/common/TimeUtil.cpp | 4 +- core/common/TimeUtil.h | 3 +- core/common/WaitObject.h | 7 +- core/common/YamlUtil.cpp | 22 +- core/common/YamlUtil.h | 2 +- core/common/compression/Compressor.h | 6 +- core/common/compression/CompressorFactory.h | 6 +- core/common/compression/LZ4Compressor.cpp | 2 +- core/common/compression/LZ4Compressor.h | 2 +- core/common/compression/ZstdCompressor.cpp | 5 +- core/common/compression/ZstdCompressor.h | 2 +- core/common/http/AsynCurlRunner.cpp | 4 +- core/common/http/AsynCurlRunner.h | 4 +- core/common/http/Constant.cpp | 2 +- core/common/http/Curl.cpp | 1 + core/common/http/Curl.h | 5 +- core/common/http/HttpRequest.h | 1 + core/common/http/HttpResponse.h | 5 +- core/config/ConfigUtil.h | 4 +- core/config/InstanceConfig.h | 4 +- core/config/InstanceConfigManager.cpp | 14 +- core/config/PipelineConfig.cpp | 2 + core/config/PipelineConfig.h | 5 +- core/config/TaskConfig.h | 5 +- .../common_provider/CommonConfigProvider.cpp | 4 +- .../common_provider/CommonConfigProvider.h | 3 +- .../LegacyCommonConfigProvider.cpp | 4 +- .../LegacyCommonConfigProvider.h | 3 +- core/config/feedbacker/ConfigFeedbackable.h | 1 + core/config/watcher/InstanceConfigWatcher.h | 4 +- core/config/watcher/PipelineConfigWatcher.cpp | 6 +- core/config/watcher/PipelineConfigWatcher.h | 4 +- core/constants/SpanConstants.cpp | 42 +- core/constants/SpanConstants.h | 42 +- core/constants/TagConstants.cpp | 74 +- core/constants/TagConstants.h | 37 +- .../ConfigContainerInfoUpdateCmd.h | 4 +- .../ContainerDiscoveryOptions.cpp | 5 +- .../ContainerDiscoveryOptions.h | 4 +- core/ebpf/Config.h | 4 +- core/ebpf/SourceManager.cpp | 3 +- core/ebpf/SourceManager.h | 4 +- core/ebpf/eBPFServer.cpp | 3 +- core/ebpf/include/export.h | 4 +- core/file_server/AdhocFileManager.cpp | 11 +- core/file_server/AdhocFileManager.h | 4 +- core/file_server/ConfigManager.cpp | 5 +- core/file_server/ConfigManager.h | 3 +- core/file_server/ContainerInfo.h | 5 +- core/file_server/FileDiscoveryOptions.h | 5 +- core/file_server/MultilineOptions.h | 4 +- core/file_server/event/BlockEventManager.h | 1 + core/file_server/event/Event.h | 8 +- core/file_server/event/EventQueue.cpp | 5 +- core/file_server/event/EventQueue.h | 3 +- .../event_handler/HistoryFileImporter.h | 3 +- core/file_server/event_handler/LogInput.cpp | 2 +- .../event_listener/EventListener_Linux.cpp | 8 +- .../event_listener/EventListener_Linux.h | 1 + core/file_server/polling/PollingCache.cpp | 1 + core/file_server/polling/PollingCache.h | 10 +- core/file_server/polling/PollingDirFile.h | 4 +- .../file_server/polling/PollingEventQueue.cpp | 3 +- core/file_server/polling/PollingEventQueue.h | 2 +- core/file_server/polling/PollingModify.h | 2 +- core/file_server/reader/FileReaderOptions.h | 5 +- core/file_server/reader/LogFileReader.cpp | 41 +- core/file_server/reader/LogFileReader.h | 9 +- core/go_pipeline/LogtailPlugin.cpp | 2 +- core/go_pipeline/LogtailPlugin.h | 5 +- core/go_pipeline/LogtailPluginAdapter.cpp | 1 + core/logger/Logger.cpp | 12 +- core/logger/Logger.h | 5 +- core/logtail.cpp | 2 +- core/metadata/K8sMetadata.cpp | 1 + core/metadata/K8sMetadata.h | 132 +- core/metadata/LabelingK8sMetadata.cpp | 20 +- core/metadata/LabelingK8sMetadata.h | 34 +- core/models/EventPool.h | 3 +- core/models/LogEvent.cpp | 2 +- core/models/MetricEvent.h | 2 +- core/models/MetricValue.h | 4 +- core/models/PipelineEvent.h | 7 +- core/models/PipelineEventGroup.h | 2 +- core/models/PipelineEventPtr.h | 2 +- core/models/SpanEvent.cpp | 34 +- core/models/SpanEvent.h | 99 +- core/models/StringView.h | 2 +- core/monitor/AlarmManager.cpp | 3 +- core/monitor/Monitor.cpp | 4 +- core/monitor/Monitor.h | 4 +- .../MetricCommonConstants.cpp | 2 +- .../metric_constants/MetricCommonConstants.h | 2 +- core/monitor/metric_models/MetricRecord.cpp | 2 +- core/monitor/metric_models/MetricRecord.h | 2 +- core/monitor/metric_models/MetricTypes.h | 5 +- .../metric_models/ReentrantMetricsRecord.cpp | 3 +- core/monitor/profile_sender/ProfileSender.cpp | 8 +- core/monitor/profile_sender/ProfileSender.h | 2 +- core/pipeline/GlobalConfig.cpp | 2 +- core/pipeline/GlobalConfig.h | 5 +- core/pipeline/Pipeline.cpp | 1 + core/pipeline/Pipeline.h | 4 +- core/pipeline/PipelineContext.cpp | 2 +- core/pipeline/PipelineContext.h | 7 +- core/pipeline/PipelineManager.cpp | 8 +- core/pipeline/batch/BatchItem.h | 3 +- core/pipeline/batch/BatchStatus.h | 4 +- core/pipeline/batch/Batcher.h | 13 +- core/pipeline/batch/FlushStrategy.cpp | 2 +- core/pipeline/batch/FlushStrategy.h | 5 +- core/pipeline/batch/TimeoutFlushManager.h | 1 + core/pipeline/limiter/ConcurrencyLimiter.cpp | 27 +- core/pipeline/limiter/ConcurrencyLimiter.h | 4 +- core/pipeline/plugin/PluginRegistry.cpp | 13 +- core/pipeline/plugin/creator/PluginCreator.h | 2 +- .../plugin/instance/FlusherInstance.cpp | 5 +- .../plugin/instance/FlusherInstance.h | 4 +- core/pipeline/plugin/instance/InputInstance.h | 12 +- .../pipeline/plugin/instance/PluginInstance.h | 3 +- .../plugin/instance/ProcessorInstance.cpp | 4 +- .../plugin/instance/ProcessorInstance.h | 4 +- core/pipeline/plugin/interface/Flusher.h | 5 +- core/pipeline/plugin/interface/HttpFlusher.h | 4 +- core/pipeline/plugin/interface/Input.h | 4 +- core/pipeline/plugin/interface/Processor.h | 2 +- core/pipeline/queue/BoundedProcessQueue.h | 1 + .../queue/BoundedSenderQueueInterface.cpp | 9 +- .../queue/BoundedSenderQueueInterface.h | 15 +- core/pipeline/queue/CircularProcessQueue.h | 1 + core/pipeline/queue/ExactlyOnceQueueManager.h | 1 + core/pipeline/queue/ProcessQueueInterface.h | 1 + core/pipeline/queue/ProcessQueueItem.h | 1 + core/pipeline/queue/ProcessQueueManager.h | 3 +- core/pipeline/queue/SenderQueueItem.h | 3 +- core/pipeline/route/Condition.h | 4 +- core/pipeline/route/Router.h | 4 +- core/pipeline/serializer/SLSSerializer.cpp | 4 +- core/pipeline/serializer/SLSSerializer.h | 6 +- core/pipeline/serializer/Serializer.h | 1 + .../flusher/blackhole/FlusherBlackHole.h | 2 +- core/plugin/flusher/file/FlusherFile.cpp | 6 +- core/plugin/flusher/file/FlusherFile.h | 4 +- core/plugin/flusher/sls/DiskBufferWriter.cpp | 9 +- core/plugin/flusher/sls/DiskBufferWriter.h | 16 +- core/plugin/flusher/sls/Exception.h | 142 +- core/plugin/flusher/sls/FlusherSLS.cpp | 18 +- core/plugin/flusher/sls/FlusherSLS.h | 14 +- core/plugin/flusher/sls/PackIdManager.h | 1 + core/plugin/flusher/sls/SLSClientManager.cpp | 4 +- core/plugin/flusher/sls/SLSClientManager.h | 3 +- core/plugin/flusher/sls/SLSResponse.cpp | 4 +- core/plugin/flusher/sls/SLSResponse.h | 1 + core/plugin/flusher/sls/SLSUtil.h | 1 + core/plugin/input/InputContainerStdio.h | 2 +- core/plugin/input/InputFile.h | 2 +- core/plugin/input/InputFileSecurity.cpp | 15 +- core/plugin/input/InputInternalMetrics.h | 1 + core/plugin/input/InputNetworkObserver.cpp | 13 +- core/plugin/input/InputNetworkSecurity.cpp | 18 +- core/plugin/input/InputProcessSecurity.cpp | 18 +- core/plugin/input/InputPrometheus.cpp | 9 +- core/plugin/input/InputPrometheus.h | 1 + core/plugin/input/InputStaticFile.cpp | 4 +- core/plugin/input/InputStaticFile.h | 6 +- core/plugin/processor/CommonParserOptions.cpp | 2 +- core/plugin/processor/CommonParserOptions.h | 4 +- .../processor/ProcessorDesensitizeNative.cpp | 4 +- .../processor/ProcessorDesensitizeNative.h | 2 +- .../processor/ProcessorFilterNative.cpp | 4 +- core/plugin/processor/ProcessorFilterNative.h | 7 +- .../ProcessorParseDelimiterNative.cpp | 2 +- .../processor/ProcessorParseJsonNative.cpp | 57 +- .../processor/ProcessorParseJsonNative.h | 8 +- .../processor/ProcessorParseRegexNative.h | 4 +- .../processor/ProcessorParseTimestampNative.h | 2 +- core/plugin/processor/ProcessorSPL.cpp | 4 - core/plugin/processor/ProcessorSPL.h | 1 + .../ProcessorMergeMultilineLogNative.cpp | 6 +- .../ProcessorParseContainerLogNative.cpp | 5 - .../inner/ProcessorPromParseMetricNative.cpp | 2 +- .../ProcessorPromRelabelMetricNative.cpp | 4 +- .../inner/ProcessorPromRelabelMetricNative.h | 20 +- .../inner/ProcessorSplitLogStringNative.h | 1 + ...ProcessorSplitMultilineLogStringNative.cpp | 3 +- .../ProcessorSplitMultilineLogStringNative.h | 1 + core/prometheus/Constants.h | 1 + core/prometheus/PromSelfMonitor.cpp | 7 +- core/prometheus/PrometheusInputRunner.cpp | 1 + core/prometheus/PrometheusInputRunner.h | 1 + core/prometheus/Utils.cpp | 50 +- core/prometheus/Utils.h | 6 +- core/prometheus/async/PromHttpRequest.cpp | 1 + core/prometheus/async/PromHttpRequest.h | 1 + core/prometheus/component/StreamScraper.cpp | 1 + core/prometheus/labels/Labels.cpp | 4 +- core/prometheus/labels/Labels.h | 1 + core/prometheus/labels/Relabel.cpp | 13 +- core/prometheus/labels/Relabel.h | 6 +- core/prometheus/labels/TextParser.cpp | 4 +- core/prometheus/schedulers/ScrapeConfig.cpp | 6 +- core/prometheus/schedulers/ScrapeConfig.h | 5 +- .../prometheus/schedulers/ScrapeScheduler.cpp | 1 + .../schedulers/TargetSubscriberScheduler.cpp | 1 + .../schedulers/TargetSubscriberScheduler.h | 5 +- core/protobuf/models/ProtocolConversion.cpp | 200 +- core/protobuf/models/ProtocolConversion.h | 12 +- core/protobuf/sls/LogGroupSerializer.h | 3 +- core/runner/FlusherRunner.cpp | 5 +- core/runner/FlusherRunner.h | 3 +- core/runner/ProcessorRunner.cpp | 3 +- core/runner/ProcessorRunner.h | 3 +- core/runner/sink/http/HttpSink.h | 3 +- core/runner/sink/http/HttpSinkRequest.h | 6 +- core/task_pipeline/Task.h | 4 +- core/task_pipeline/TaskPipeline.h | 8 +- core/task_pipeline/TaskPipelineManager.h | 4 +- core/unittest/Unittest.h | 10 +- core/unittest/UnittestHelper.h | 6 +- .../unittest/app_config/AppConfigUnittest.cpp | 8 +- .../app_config/AppConfigUnittestLegal.cpp | 7 +- .../AdhocCheckpointManagerUnittest.cpp | 8 +- .../checkpoint/CheckpointManagerUnittest.cpp | 2 +- .../CheckpointManagerV2Unittest.cpp | 8 +- core/unittest/common/DevInodeUnittest.h | 5 +- .../common/EncodingConverterUnittest.cpp | 2 +- core/unittest/common/FileSystemUtilUnittest.h | 27 +- core/unittest/common/LoadParameterUnittest.h | 4 +- core/unittest/common/LockUnittest.h | 5 +- .../common/LogFileOperatorUnittest.cpp | 8 +- .../common/MachineInfoUtilUnittest.cpp | 14 +- core/unittest/common/RandomUtilUnittest.h | 7 +- core/unittest/common/SimpleUtilsUnittest.cpp | 4 +- .../common/SlidingWindowCounterUnittest.cpp | 2 +- core/unittest/common/StringPieceUnittest.cpp | 6 +- core/unittest/common/StringToolsUnittest.cpp | 4 +- core/unittest/common/TimeUtilUnittest.h | 41 +- core/unittest/common/YamlUtilUnittest.cpp | 3 +- .../compression/CompressorFactoryUnittest.cpp | 5 +- .../compression/CompressorUnittest.cpp | 2 +- .../config/CommonConfigProviderUnittest.cpp | 8 +- .../config/ConfigContainerUnittest.cpp | 2 +- core/unittest/config/ConfigMatchUnittest.cpp | 45 +- core/unittest/config/ConfigUpdateUnittest.cpp | 6 +- .../unittest/config/ConfigUpdatorUnittest.cpp | 5 +- .../unittest/config/ConfigWatcherUnittest.cpp | 6 +- .../config/PipelineConfigUnittest.cpp | 4 +- .../config/PipelineConfigWatcherUnittest.cpp | 180 +- core/unittest/config/TaskConfigUnittest.cpp | 4 +- .../ContainerDiscoveryOptionsUnittest.cpp | 2 +- .../controller/EventDispatcherDirUnittest.cpp | 8 +- core/unittest/ebpf/eBPFServerUnittest.cpp | 4 +- core/unittest/event/EventUnittest.cpp | 8 +- .../CreateModifyHandlerUnittest.cpp | 8 +- .../event_handler/LogInputUnittest.cpp | 12 +- .../FileDiscoveryOptionsUnittest.cpp | 2 +- .../file_source/MultilineOptionsUnittest.cpp | 4 +- core/unittest/flusher/FlusherSLSUnittest.cpp | 25 +- .../input/InputContainerStdioUnittest.cpp | 6 +- .../input/InputFileSecurityUnittest.cpp | 4 +- core/unittest/input/InputFileUnittest.cpp | 4 +- .../input/InputInternalMetricsUnittest.cpp | 8 +- .../input/InputNetworkObserverUnittest.cpp | 2 +- .../input/InputNetworkSecurityUnittest.cpp | 8 +- .../input/InputProcessSecurityUnittest.cpp | 8 +- .../input/InputPrometheusUnittest.cpp | 8 +- core/unittest/logger/logger_unittest.cpp | 1 + .../unittest/metadata/K8sMetadataUnittest.cpp | 28 +- core/unittest/models/EventGroupBenchmark.cpp | 1 + core/unittest/models/EventPoolUnittest.cpp | 1 - core/unittest/models/MetricEventUnittest.cpp | 6 +- .../monitor/MetricManagerUnittest.cpp | 31 +- .../monitor/PluginMetricManagerUnittest.cpp | 5 +- .../pipeline/ConcurrencyLimiterUnittest.cpp | 9 +- .../pipeline/GlobalConfigUnittest.cpp | 2 +- core/unittest/pipeline/HttpSinkMock.h | 2 +- core/unittest/pipeline/PipelineUnittest.cpp | 4 +- .../pipeline/PipelineUpdateUnittest.cpp | 9 +- core/unittest/plugin/PluginMock.h | 5 +- .../plugin/PluginRegistryUnittest.cpp | 3 +- .../plugin/ProcessorInstanceUnittest.cpp | 9 +- .../plugin/StaticFlusherCreatorUnittest.cpp | 2 +- .../plugin/StaticInputCreatorUnittest.cpp | 2 +- .../plugin/StaticProcessorCreatorUnittest.cpp | 2 +- .../PollingPreservedDirDepthUnittest.cpp | 5 +- core/unittest/polling/PollingUnittest.cpp | 56 +- .../processor/BoostRegexBenchmark.cpp | 4 +- .../processor/ParseContainerLogBenchmark.cpp | 1 + .../ProcessorDesensitizeNativeUnittest.cpp | 2 +- .../ProcessorFilterNativeUnittest.cpp | 5 +- ...ocessorMergeMultilineLogNativeUnittest.cpp | 622 +++-- ...ocessorParseContainerLogNativeUnittest.cpp | 635 +++--- .../ProcessorParseTimestampNativeUnittest.cpp | 9 +- .../ProcessorSplitLogStringNativeUnittest.cpp | 5 +- core/unittest/prometheus/LabelsUnittest.cpp | 3 +- .../PrometheusInputRunnerUnittest.cpp | 4 +- core/unittest/prometheus/RelabelUnittest.cpp | 6 +- .../prometheus/ScrapeConfigUnittest.cpp | 4 +- .../prometheus/ScrapeSchedulerUnittest.cpp | 8 +- .../TargetSubscriberSchedulerUnittest.cpp | 10 +- core/unittest/provider/ProviderUnittest.cpp | 3 +- .../queue/BoundedProcessQueueUnittest.cpp | 3 +- .../queue/CircularProcessQueueUnittest.cpp | 3 +- .../queue/ExactlyOnceSenderQueueUnittest.cpp | 6 +- .../queue/SenderQueueManagerUnittest.cpp | 22 +- core/unittest/queue/SenderQueueUnittest.cpp | 3 +- core/unittest/reader/DeletedFileUnittest.cpp | 2 +- .../reader/FileReaderOptionsUnittest.cpp | 10 +- core/unittest/reader/ForceReadUnittest.cpp | 9 +- .../reader/GetLastLineDataUnittest.cpp | 1992 ++++++++--------- .../unittest/reader/LogFileReaderUnittest.cpp | 2 +- core/unittest/reader/SourceBufferUnittest.cpp | 6 +- .../unittest/sender/FlusherRunnerUnittest.cpp | 6 +- core/unittest/sender/SenderUnittest.cpp | 6 +- .../serializer/SLSSerializerUnittest.cpp | 620 +++-- .../serializer/SerializerUnittest.cpp | 2 +- core/unittest/spl/SplBenchmark.cpp | 291 ++- core/unittest/spl/SplUnittest.cpp | 86 +- .../task_pipeline/TaskPipelineUnittest.cpp | 2 +- 374 files changed, 4644 insertions(+), 4123 deletions(-) diff --git a/.clang-format b/.clang-format index 02599e9017..f911ab3561 100644 --- a/.clang-format +++ b/.clang-format @@ -67,3 +67,16 @@ SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^' + Priority: 2 + - Regex: '^ -#include #include #include #include +#include "boost/filesystem.hpp" +#include "json/value.h" + #include "RuntimeUtil.h" #include "common/EnvUtil.h" #include "common/FileSystemUtil.h" @@ -30,7 +32,6 @@ #include "config/watcher/InstanceConfigWatcher.h" #include "file_server/ConfigManager.h" #include "file_server/reader/LogFileReader.h" -#include "json/value.h" #include "logger/Logger.h" #include "monitor/AlarmManager.h" #include "monitor/Monitor.h" @@ -200,7 +201,7 @@ const double GLOBAL_CONCURRENCY_FREE_PERCENTAGE_FOR_ONE_REGION = 0.5; const int32_t MIN_SEND_REQUEST_CONCURRENCY = 15; // 单地域并发度最大值 const int32_t MAX_SEND_REQUEST_CONCURRENCY = 80; -// 并发度统计数量&&时间间隔 +// 并发度统计数量&&时间间隔 const uint32_t CONCURRENCY_STATISTIC_THRESHOLD = 10; const uint32_t CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS = 3; // 并发度不回退百分比阈值 @@ -1161,10 +1162,10 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) { LOG_INFO(sLogger, ("bind_interface", mBindInterface)); } - // mSendRequestConcurrency was limited + // mSendRequestConcurrency was limited if (mSendRequestConcurrency < MIN_SEND_REQUEST_CONCURRENCY) { mSendRequestConcurrency = MIN_SEND_REQUEST_CONCURRENCY; - } + } if (mSendRequestConcurrency > MAX_SEND_REQUEST_CONCURRENCY) { mSendRequestConcurrency = MAX_SEND_REQUEST_CONCURRENCY; } diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 4c555f5b8b..1082b6e11e 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -16,8 +16,6 @@ #pragma once -#include - #include #include #include @@ -25,6 +23,8 @@ #include #include +#include "json/json.h" + #include "InstanceConfig.h" #include "protobuf/sls/sls_logs.pb.h" @@ -35,9 +35,9 @@ extern const double GLOBAL_CONCURRENCY_FREE_PERCENTAGE_FOR_ONE_REGION; extern const int32_t MIN_SEND_REQUEST_CONCURRENCY; extern const int32_t MAX_SEND_REQUEST_CONCURRENCY; extern const uint32_t CONCURRENCY_STATISTIC_THRESHOLD; -extern const uint32_t CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS; -extern const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE; -extern const uint32_t SLOW_FALL_BACK_FAIL_PERCENTAGE; +extern const uint32_t CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS; +extern const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE; +extern const uint32_t SLOW_FALL_BACK_FAIL_PERCENTAGE; void CreateAgentDir(); @@ -217,7 +217,6 @@ class AppConfig { std::string mBindInterface; - // /** // * @brief Load ConfigServer, DataServer and network interface // * @@ -316,7 +315,7 @@ class AppConfig { public: AppConfig(); - ~AppConfig() {}; + ~AppConfig(){}; void LoadInstanceConfig(const std::map>&); @@ -330,9 +329,15 @@ class AppConfig { void LoadLocalInstanceConfig(); // 获取全局参数方法 - const Json::Value& GetLocalInstanceConfig() { return mLocalInstanceConfig; }; - const Json::Value& GetEnvConfig() { return mEnvConfig; }; - const Json::Value& GetRemoteInstanceConfig() { return mRemoteInstanceConfig; }; + const Json::Value& GetLocalInstanceConfig() { + return mLocalInstanceConfig; + }; + const Json::Value& GetEnvConfig() { + return mEnvConfig; + }; + const Json::Value& GetRemoteInstanceConfig() { + return mRemoteInstanceConfig; + }; template T MergeConfig(const T& defaultValue, @@ -374,26 +379,40 @@ class AppConfig { // 获取特定配置 // CPU限制参数等仅与框架相关的参数，计算逻辑可以放在AppConfig - float GetMachineCpuUsageThreshold() const { return mMachineCpuUsageThreshold; } - float GetScaledCpuUsageUpLimit() const { return mScaledCpuUsageUpLimit; } - float GetCpuUsageUpLimit() const { return mCpuUsageUpLimit; } + float GetMachineCpuUsageThreshold() const { + return mMachineCpuUsageThreshold; + } + float GetScaledCpuUsageUpLimit() const { + return mScaledCpuUsageUpLimit; + } + float GetCpuUsageUpLimit() const { + return mCpuUsageUpLimit; + } // 文件标签相关，获取从文件中来的tags - std::vector& GetFileTags() { return mFileTags.getReadBuffer(); } + std::vector& GetFileTags() { + return mFileTags.getReadBuffer(); + } // 更新从文件中来的tags void UpdateFileTags(); // Agent属性相关，获取从文件中来的attrs - std::map& GetAgentAttrs() { return mAgentAttrs.getReadBuffer(); } + std::map& GetAgentAttrs() { + return mAgentAttrs.getReadBuffer(); + } // 更新从文件中来的attrs void UpdateAgentAttrs(); // Legacy:获取各种参数 - bool NoInotify() const { return mNoInotify; } + bool NoInotify() const { + return mNoInotify; + } bool IsInInotifyBlackList(const std::string& path) const; - bool IsLogParseAlarmValid() const { return mLogParseAlarmFlag; } + bool IsLogParseAlarmValid() const { + return mLogParseAlarmFlag; + } // std::string GetDefaultRegion() const; @@ -414,45 +433,85 @@ class AppConfig { return mIlogtailConfigJson; } - bool IsAcceptMultiConfig() const { return mAcceptMultiConfigFlag; } + bool IsAcceptMultiConfig() const { + return mAcceptMultiConfigFlag; + } - void SetAcceptMultiConfig(bool flag) { mAcceptMultiConfigFlag = flag; } + void SetAcceptMultiConfig(bool flag) { + mAcceptMultiConfigFlag = flag; + } - int32_t GetMaxMultiConfigSize() const { return mMaxMultiConfigSize; } + int32_t GetMaxMultiConfigSize() const { + return mMaxMultiConfigSize; + } - void SetMaxMultiConfigSize(int32_t maxSize) { mMaxMultiConfigSize = maxSize; } + void SetMaxMultiConfigSize(int32_t maxSize) { + mMaxMultiConfigSize = maxSize; + } - const std::string& GetCheckPointFilePath() const { return mCheckPointFilePath; } + const std::string& GetCheckPointFilePath() const { + return mCheckPointFilePath; + } - bool IsInputFlowControl() const { return mInputFlowControl; } + bool IsInputFlowControl() const { + return mInputFlowControl; + } - bool IsResourceAutoScale() const { return mResourceAutoScale; } + bool IsResourceAutoScale() const { + return mResourceAutoScale; + } - int64_t GetMemUsageUpLimit() const { return mMemUsageUpLimit; } + int64_t GetMemUsageUpLimit() const { + return mMemUsageUpLimit; + } - int32_t GetMaxHoldedDataSize() const { return mMaxHoldedDataSize; } + int32_t GetMaxHoldedDataSize() const { + return mMaxHoldedDataSize; + } - uint32_t GetMaxBufferNum() const { return mMaxBufferNum; } + uint32_t GetMaxBufferNum() const { + return mMaxBufferNum; + } - int32_t GetMaxBytePerSec() const { return mMaxBytePerSec; } + int32_t GetMaxBytePerSec() const { + return mMaxBytePerSec; + } - void SetMaxBytePerSec(int32_t maxBytePerSec) { mMaxBytePerSec = maxBytePerSec; } + void SetMaxBytePerSec(int32_t maxBytePerSec) { + mMaxBytePerSec = maxBytePerSec; + } - int32_t GetBytePerSec() const { return mBytePerSec; } + int32_t GetBytePerSec() const { + return mBytePerSec; + } - int32_t GetNumOfBufferFile() const { return mNumOfBufferFile; } + int32_t GetNumOfBufferFile() const { + return mNumOfBufferFile; + } - int32_t GetLocalFileSize() const { return mLocalFileSize; } + int32_t GetLocalFileSize() const { + return mLocalFileSize; + } - const std::string& GetBufferFilePath() const { return mBufferFilePath; } + const std::string& GetBufferFilePath() const { + return mBufferFilePath; + } // 单地域并发度 - int32_t GetSendRequestConcurrency() const { return mSendRequestConcurrency; } + int32_t GetSendRequestConcurrency() const { + return mSendRequestConcurrency; + } // 全局并发度 - int32_t GetSendRequestGlobalConcurrency() const { return mSendRequestGlobalConcurrency; } + int32_t GetSendRequestGlobalConcurrency() const { + return mSendRequestGlobalConcurrency; + } - double GetGlobalConcurrencyFreePercentageForOneRegion() const { return GLOBAL_CONCURRENCY_FREE_PERCENTAGE_FOR_ONE_REGION; } + double GetGlobalConcurrencyFreePercentageForOneRegion() const { + return GLOBAL_CONCURRENCY_FREE_PERCENTAGE_FOR_ONE_REGION; + } - int32_t GetProcessThreadCount() const { return mProcessThreadCount; } + int32_t GetProcessThreadCount() const { + return mProcessThreadCount; + } // const std::string& GetMappingConfigPath() const { return mMappingConfigPath; } @@ -466,61 +525,105 @@ class AppConfig { // const std::string& GetRemoteUserYamlConfigDirPath() const { return mUserRemoteYamlConfigDirPath; } - bool IgnoreDirInodeChanged() const { return mIgnoreDirInodeChanged; } + bool IgnoreDirInodeChanged() const { + return mIgnoreDirInodeChanged; + } - void SetProcessExecutionDir(const std::string& dir) { mProcessExecutionDir = dir; } + void SetProcessExecutionDir(const std::string& dir) { + mProcessExecutionDir = dir; + } - const std::string& GetProcessExecutionDir() { return mProcessExecutionDir; } + const std::string& GetProcessExecutionDir() { + return mProcessExecutionDir; + } - void SetWorkingDir(const std::string& dir) { mWorkingDir = dir; } + void SetWorkingDir(const std::string& dir) { + mWorkingDir = dir; + } - const std::string& GetWorkingDir() const { return mWorkingDir; } + const std::string& GetWorkingDir() const { + return mWorkingDir; + } // const std::string& GetContainerMountConfigPath() const { return mContainerMountConfigPath; } - const std::string& GetConfigIP() const { return mConfigIP; } + const std::string& GetConfigIP() const { + return mConfigIP; + } // const std::string& GetCustomizedConfigIp() const { return mCustomizedConfigIP; } - const std::string& GetConfigHostName() const { return mConfigHostName; } + const std::string& GetConfigHostName() const { + return mConfigHostName; + } - int32_t GetSystemBootTime() const { return mSystemBootTime; } + int32_t GetSystemBootTime() const { + return mSystemBootTime; + } - const std::string& GetDockerFilePathConfig() const { return mDockerFilePathConfig; } + const std::string& GetDockerFilePathConfig() const { + return mDockerFilePathConfig; + } - int32_t GetDataServerPort() const { return mSendDataPort; } + int32_t GetDataServerPort() const { + return mSendDataPort; + } - bool ShennongSocketEnabled() const { return mShennongSocket; } + bool ShennongSocketEnabled() const { + return mShennongSocket; + } - const std::vector& GetEnvTags() const { return mEnvTags; } + const std::vector& GetEnvTags() const { + return mEnvTags; + } - bool IsPurageContainerMode() const { return mPurageContainerMode; } + bool IsPurageContainerMode() const { + return mPurageContainerMode; + } - int32_t GetForceQuitReadTimeout() const { return mForceQuitReadTimeout; } + int32_t GetForceQuitReadTimeout() const { + return mForceQuitReadTimeout; + } // const std::string& GetAlipayZone() const { return mAlipayZone; } // If @dirPath is not accessible, GetProcessExecutionDir will be set. void SetLoongcollectorConfDir(const std::string& dirPath); - const std::string& GetLoongcollectorConfDir() const { return mLoongcollectorConfDir; } + const std::string& GetLoongcollectorConfDir() const { + return mLoongcollectorConfDir; + } - inline bool IsHostIPReplacePolicyEnabled() const { return mEnableHostIPReplace; } + inline bool IsHostIPReplacePolicyEnabled() const { + return mEnableHostIPReplace; + } - inline bool IsResponseVerificationEnabled() const { return mEnableResponseVerification; } + inline bool IsResponseVerificationEnabled() const { + return mEnableResponseVerification; + } // EndpointAddressType GetConfigServerAddressNetType() const { return mConfigServerAddressNetType; } - inline bool EnableCheckpointSyncWrite() const { return mEnableCheckpointSyncWrite; } + inline bool EnableCheckpointSyncWrite() const { + return mEnableCheckpointSyncWrite; + } - inline bool EnableLogTimeAutoAdjust() const { return mEnableLogTimeAutoAdjust; } + inline bool EnableLogTimeAutoAdjust() const { + return mEnableLogTimeAutoAdjust; + } - inline const std::set& GetDynamicPlugins() const { return mDynamicPlugins; } + inline const std::set& GetDynamicPlugins() const { + return mDynamicPlugins; + } bool IsHostPathMatchBlacklist(const std::string& dirPath) const; - const Json::Value& GetConfig() const { return mLocalInstanceConfig; } + const Json::Value& GetConfig() const { + return mLocalInstanceConfig; + } - const std::string& GetBindInterface() const { return mBindInterface; } + const std::string& GetBindInterface() const { + return mBindInterface; + } #ifdef APSARA_UNIT_TEST_MAIN friend class SenderUnittest; diff --git a/core/application/Application.cpp b/core/application/Application.cpp index a32acb08a8..19f8065257 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -15,7 +15,7 @@ #include "application/Application.h" #ifndef LOGTAIL_NO_TC_MALLOC -#include +#include "gperftools/malloc_extension.h" #endif #include diff --git a/core/application/Application.h b/core/application/Application.h index 50fad6b229..81fb02982a 100644 --- a/core/application/Application.h +++ b/core/application/Application.h @@ -16,8 +16,9 @@ #pragma once -#include #include + +#include #include #include "common/Lock.h" @@ -42,7 +43,7 @@ class Application { std::string GetInstanceId() { return mInstanceId; } bool TryGetUUID(); - int32_t GetStartTime() {return mStartTime;} + int32_t GetStartTime() { return mStartTime; } std::string GetUUID() { mUUIDLock.lock(); std::string uuid(mUUID); diff --git a/core/checkpoint/AdhocCheckpointManager.cpp b/core/checkpoint/AdhocCheckpointManager.cpp index 9d51759b53..752dae2569 100644 --- a/core/checkpoint/AdhocCheckpointManager.cpp +++ b/core/checkpoint/AdhocCheckpointManager.cpp @@ -14,14 +14,15 @@ * limitations under the License. */ -#include "app_config/AppConfig.h" #include "AdhocCheckpointManager.h" + +#include "app_config/AppConfig.h" #include "common/FileSystemUtil.h" #include "common/Flags.h" +#include "common/HashUtil.h" +#include "common/Thread.h" #include "logger/Logger.h" #include "monitor/AlarmManager.h" -#include "common/Thread.h" -#include "common/HashUtil.h" DEFINE_FLAG_INT32(adhoc_checkpoint_dump_thread_wait_interval, "microseconds", 5 * 1000); diff --git a/core/checkpoint/AdhocCheckpointManager.h b/core/checkpoint/AdhocCheckpointManager.h index 22443b9cc0..6d8545f1a2 100644 --- a/core/checkpoint/AdhocCheckpointManager.h +++ b/core/checkpoint/AdhocCheckpointManager.h @@ -16,6 +16,7 @@ #pragma once #include + #include "AdhocJobCheckpoint.h" namespace logtail { diff --git a/core/checkpoint/AdhocFileCheckpoint.cpp b/core/checkpoint/AdhocFileCheckpoint.cpp index 87bd179290..d4ced8e568 100644 --- a/core/checkpoint/AdhocFileCheckpoint.cpp +++ b/core/checkpoint/AdhocFileCheckpoint.cpp @@ -15,6 +15,7 @@ */ #include "AdhocFileCheckpoint.h" + #include "logger/Logger.h" namespace logtail { @@ -49,4 +50,4 @@ FileReadStatus GetStatusFromString(std::string statusStr) { } } -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/checkpoint/AdhocFileCheckpoint.h b/core/checkpoint/AdhocFileCheckpoint.h index 564c199261..61b44a53ba 100644 --- a/core/checkpoint/AdhocFileCheckpoint.h +++ b/core/checkpoint/AdhocFileCheckpoint.h @@ -16,8 +16,9 @@ #pragma once #include -#include #include +#include + #include "common/DevInode.h" #include "common/StringTools.h" diff --git a/core/checkpoint/AdhocJobCheckpoint.cpp b/core/checkpoint/AdhocJobCheckpoint.cpp index dbe9430730..2996edd4c8 100644 --- a/core/checkpoint/AdhocJobCheckpoint.cpp +++ b/core/checkpoint/AdhocJobCheckpoint.cpp @@ -14,8 +14,10 @@ * limitations under the License. */ -#include #include "AdhocJobCheckpoint.h" + +#include "json/json.h" + #include "common/FileSystemUtil.h" #include "logger/Logger.h" #include "monitor/AlarmManager.h" diff --git a/core/checkpoint/AdhocJobCheckpoint.h b/core/checkpoint/AdhocJobCheckpoint.h index 8db8b7e7fa..cc47f7bb6c 100644 --- a/core/checkpoint/AdhocJobCheckpoint.h +++ b/core/checkpoint/AdhocJobCheckpoint.h @@ -16,6 +16,7 @@ #pragma once #include + #include "AdhocFileCheckpoint.h" namespace logtail { diff --git a/core/checkpoint/CheckPointManager.h b/core/checkpoint/CheckPointManager.h index 3e5a63c3cb..62721342ba 100644 --- a/core/checkpoint/CheckPointManager.h +++ b/core/checkpoint/CheckPointManager.h @@ -15,15 +15,16 @@ */ #pragma once -#include - -#include #include + #include #include #include #include +#include "boost/optional.hpp" +#include "json/json.h" + #include "common/DevInode.h" #include "common/EncodingConverter.h" #include "common/SplitedFilePath.h" diff --git a/core/checkpoint/CheckpointManagerV2.cpp b/core/checkpoint/CheckpointManagerV2.cpp index 2d24437314..d176ec9862 100644 --- a/core/checkpoint/CheckpointManagerV2.cpp +++ b/core/checkpoint/CheckpointManagerV2.cpp @@ -13,15 +13,16 @@ // limitations under the License. #include "CheckpointManagerV2.h" + #include -#include + +#include "app_config/AppConfig.h" +#include "checkpoint/CheckPointManager.h" #include "common/Flags.h" #include "common/ScopeInvoker.h" #include "common/TimeUtil.h" #include "logger/Logger.h" #include "monitor/AlarmManager.h" -#include "app_config/AppConfig.h" -#include "checkpoint/CheckPointManager.h" DEFINE_FLAG_INT32(logtail_checkpoint_check_gc_interval_sec, "60 seconds", 60); DEFINE_FLAG_INT32(logtail_checkpoint_gc_threshold_sec, "30 minutes", 30 * 60); @@ -35,46 +36,46 @@ namespace logtail { namespace detail { - std::string getDatabasePath() { - return GetExactlyOnceCheckpoint(); - } +std::string getDatabasePath() { + return GetExactlyOnceCheckpoint(); +} - // Log error locally and send alarm. - void logDatabaseError(const std::string& op, const std::string& key, const leveldb::Status& s) { - const std::string title = "error when access checkpoint database"; - std::string msg; - msg.append("op:").append(op).append(", key:").append(key).append(", status:").append(s.ToString()); - LOG_ERROR(sLogger, (title, msg)); - AlarmManager::GetInstance()->SendAlarm(CHECKPOINT_V2_ALARM, title + ", " + msg); - } +// Log error locally and send alarm. +void logDatabaseError(const std::string& op, const std::string& key, const leveldb::Status& s) { + const std::string title = "error when access checkpoint database"; + std::string msg; + msg.append("op:").append(op).append(", key:").append(key).append(", status:").append(s.ToString()); + LOG_ERROR(sLogger, (title, msg)); + AlarmManager::GetInstance()->SendAlarm(CHECKPOINT_V2_ALARM, title + ", " + msg); +} - // Range key is represented by data pointer and size to avoid copy. - // - // @return empty if failed to extract. - std::string extractPrimaryKeyFromRangeKey(const char* data, size_t size) { - const int32_t kExepectedCount = 2; - int32_t underlineCount = 0; - for (; size > 0; --size) { - if (data[size - 1] != '_') { - continue; - } - if (++underlineCount == kExepectedCount) { - break; - } +// Range key is represented by data pointer and size to avoid copy. +// +// @return empty if failed to extract. +std::string extractPrimaryKeyFromRangeKey(const char* data, size_t size) { + const int32_t kExepectedCount = 2; + int32_t underlineCount = 0; + for (; size > 0; --size) { + if (data[size - 1] != '_') { + continue; } - if (size > 2) { - return std::string(data, size - 1); + if (++underlineCount == kExepectedCount) { + break; } - return ""; } - - void makeRangeKey(std::string& key, uint32_t idx) { - key.append("_").append(std::to_string(idx)).append("_r"); + if (size > 2) { + return std::string(data, size - 1); } + return ""; +} - bool isRangeKey(const char* data, size_t len) { - return len > 0 && data[len - 1] == 'r'; - } +void makeRangeKey(std::string& key, uint32_t idx) { + key.append("_").append(std::to_string(idx)).append("_r"); +} + +bool isRangeKey(const char* data, size_t len) { + return len > 0 && data[len - 1] == 'r'; +} } // namespace detail diff --git a/core/checkpoint/CheckpointManagerV2.h b/core/checkpoint/CheckpointManagerV2.h index fe895c2a05..8a502462a4 100644 --- a/core/checkpoint/CheckpointManagerV2.h +++ b/core/checkpoint/CheckpointManagerV2.h @@ -15,15 +15,17 @@ */ #pragma once -#include -#include -#include +#include + #include #include +#include +#include +#include #include -#include -#include "protobuf/sls/checkpoint.pb.h" + #include "plugin/input/InputFile.h" +#include "protobuf/sls/checkpoint.pb.h" namespace logtail { diff --git a/core/checkpoint/RangeCheckpoint.cpp b/core/checkpoint/RangeCheckpoint.cpp index e9bc5d1988..e4cce0a530 100644 --- a/core/checkpoint/RangeCheckpoint.cpp +++ b/core/checkpoint/RangeCheckpoint.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "RangeCheckpoint.h" + #include "CheckpointManagerV2.h" namespace logtail { diff --git a/core/checkpoint/RangeCheckpoint.h b/core/checkpoint/RangeCheckpoint.h index 30880dde6f..5f524ed3f2 100644 --- a/core/checkpoint/RangeCheckpoint.h +++ b/core/checkpoint/RangeCheckpoint.h @@ -18,8 +18,9 @@ #include #include #include -#include "protobuf/sls/checkpoint.pb.h" + #include "pipeline/queue/QueueKey.h" +#include "protobuf/sls/checkpoint.pb.h" namespace logtail { diff --git a/core/common/BoostRegexValidator.cpp b/core/common/BoostRegexValidator.cpp index b3a8fd872e..6f6ad3df85 100644 --- a/core/common/BoostRegexValidator.cpp +++ b/core/common/BoostRegexValidator.cpp @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include + #include #include #include -#include -#include -#include -#include -#include -#include + +#include "boost/algorithm/string.hpp" +#include "boost/regex.hpp" +#include "json/json.h" using namespace std; diff --git a/core/common/CircularBuffer.h b/core/common/CircularBuffer.h index 81611292b6..839f1350ff 100644 --- a/core/common/CircularBuffer.h +++ b/core/common/CircularBuffer.h @@ -15,7 +15,12 @@ */ #pragma once + +#include + #include +#include + #include "MemoryBarrier.h" #include "Semaphore.h" @@ -67,7 +72,7 @@ class CircularBuffer : public AbstractCircularBuffer { if (TryPushItem(item)) { return; } - usleep(SLEEP_TIME); // microseconds + std::this_thread::sleep_for(std::chrono::microseconds(this->SLEEP_TIME)); } } @@ -99,7 +104,7 @@ class CircularBuffer : public AbstractCircularBuffer { if (TryPopItem(item)) { return; } - usleep(SLEEP_TIME); // microseconds + std::this_thread::sleep_for(std::chrono::microseconds(this->SLEEP_TIME)); } } @@ -108,7 +113,7 @@ class CircularBuffer : public AbstractCircularBuffer { private: // Because we always "Keep One Item Open", the actually size should be N + 1 static const size_t SIZE = N + 1; - static const int SLEEP_TIME = 10; // microsecond + const int SLEEP_TIME = 10; // microsecond volatile size_t mWriter; volatile size_t mReader; volatile T mData[SIZE]; diff --git a/core/common/CompressTools.cpp b/core/common/CompressTools.cpp index 4aacdfe487..2b9cb78c76 100644 --- a/core/common/CompressTools.cpp +++ b/core/common/CompressTools.cpp @@ -14,7 +14,7 @@ #include "CompressTools.h" -#include +#include "lz4/lz4.h" namespace logtail { diff --git a/core/common/DNSCache.h b/core/common/DNSCache.h index 90395ed187..33dfd3f479 100644 --- a/core/common/DNSCache.h +++ b/core/common/DNSCache.h @@ -18,6 +18,7 @@ #include #include + #include #include diff --git a/core/common/DevInode.cpp b/core/common/DevInode.cpp index 3bfce3c49b..9dc9ea2962 100644 --- a/core/common/DevInode.cpp +++ b/core/common/DevInode.cpp @@ -13,10 +13,11 @@ // limitations under the License. #include "DevInode.h" + #include #if defined(__linux__) -#include #include +#include #include #endif #include "logger/Logger.h" diff --git a/core/common/DynamicLibHelper.cpp b/core/common/DynamicLibHelper.cpp index e80c2cf33e..bb3966e9f7 100644 --- a/core/common/DynamicLibHelper.cpp +++ b/core/common/DynamicLibHelper.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "DynamicLibHelper.h" + #include "Logger.h" #if defined(__linux__) @@ -24,8 +25,8 @@ namespace logtail { namespace glibc { - glibc_setns_func g_setns_func = nullptr; - DynamicLibLoader* g_loader = nullptr; +glibc_setns_func g_setns_func = nullptr; +DynamicLibLoader* g_loader = nullptr; } // namespace glibc diff --git a/core/common/DynamicLibHelper.h b/core/common/DynamicLibHelper.h index fa5c7cc93b..edebb33b53 100644 --- a/core/common/DynamicLibHelper.h +++ b/core/common/DynamicLibHelper.h @@ -16,9 +16,10 @@ #pragma once -#include #include +#include + namespace logtail { @@ -29,11 +30,11 @@ typedef int (*glibc_setns_func)(int __fd, int __nstype); // For logtail running with centos 6, use dlopen to load some glibc functions. namespace glibc { - // setns function first appear in kernel 3 and glibc 1.14. - typedef int (*glibc_setns_func)(int __fd, int __nstype); - extern glibc_setns_func g_setns_func; - extern DynamicLibLoader* g_loader; - bool LoadGlibcFunc(); +// setns function first appear in kernel 3 and glibc 1.14. +typedef int (*glibc_setns_func)(int __fd, int __nstype); +extern glibc_setns_func g_setns_func; +extern DynamicLibLoader* g_loader; +bool LoadGlibcFunc(); } // namespace glibc diff --git a/core/common/EncodingConverter.cpp b/core/common/EncodingConverter.cpp index d295aedc69..6e6b98c67c 100644 --- a/core/common/EncodingConverter.cpp +++ b/core/common/EncodingConverter.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "EncodingConverter.h" + #include "AlarmManager.h" #include "logger/Logger.h" #if defined(__linux__) diff --git a/core/common/EncodingConverter.h b/core/common/EncodingConverter.h index 9356b805e7..1361192a91 100644 --- a/core/common/EncodingConverter.h +++ b/core/common/EncodingConverter.h @@ -17,9 +17,11 @@ #ifndef __SLS_ILOGTAIL_ENCODING_CONVERTER_H__ #define __SLS_ILOGTAIL_ENCODING_CONVERTER_H__ -#include -#include #include + +#include +#include + #include "common/memory/SourceBuffer.h" namespace logtail { diff --git a/core/common/EncodingUtil.cpp b/core/common/EncodingUtil.cpp index 32c8d836c7..ef84466f6b 100644 --- a/core/common/EncodingUtil.cpp +++ b/core/common/EncodingUtil.cpp @@ -19,7 +19,7 @@ using namespace std; namespace logtail { - + static void Base64Encoding(istream& is, ostream& os, char makeupChar, const char* alphabet) { int out[4]; int remain = 0; diff --git a/core/common/EnvUtil.h b/core/common/EnvUtil.h index 98a23d1d87..372a2beba0 100644 --- a/core/common/EnvUtil.h +++ b/core/common/EnvUtil.h @@ -22,6 +22,6 @@ namespace logtail { void SetEnv(const char* key, const char* value); void UnsetEnv(const char* key); -} +} // namespace logtail #endif diff --git a/core/common/ErrorUtil.h b/core/common/ErrorUtil.h index b859341f42..3bf536f497 100644 --- a/core/common/ErrorUtil.h +++ b/core/common/ErrorUtil.h @@ -20,10 +20,11 @@ #include #elif defined(_MSC_VER) #include + #include #elif defined(__APPLE__) -#include #include +#include #endif // Errno utility. diff --git a/core/common/FileEncryption.cpp b/core/common/FileEncryption.cpp index e6ed2c3fe4..317376cff7 100644 --- a/core/common/FileEncryption.cpp +++ b/core/common/FileEncryption.cpp @@ -13,10 +13,12 @@ // limitations under the License. #include "FileEncryption.h" -#include + #include -#include "StringTools.h" +#include + #include "FileSystemUtil.h" +#include "StringTools.h" #include "logger/Logger.h" using namespace std; diff --git a/core/common/FileEncryption.h b/core/common/FileEncryption.h index 2df8f23962..5f6a316bdf 100644 --- a/core/common/FileEncryption.h +++ b/core/common/FileEncryption.h @@ -17,9 +17,10 @@ #ifndef __OAS_SHENNONG_FILE_ENCRYPTION_H__ #define __OAS_SHENNONG_FILE_ENCRYPTION_H__ +#include #include #include -#include + #include "Flags.h" namespace logtail { diff --git a/core/common/FileInfo.h b/core/common/FileInfo.h index 923dc8e2af..f0481f5492 100644 --- a/core/common/FileInfo.h +++ b/core/common/FileInfo.h @@ -15,8 +15,9 @@ */ #pragma once -#include #include +#include + #include "common/DevInode.h" #include "common/StringTools.h" diff --git a/core/common/FileSystemUtil.cpp b/core/common/FileSystemUtil.cpp index 39ef57c067..5d994bbece 100644 --- a/core/common/FileSystemUtil.cpp +++ b/core/common/FileSystemUtil.cpp @@ -20,9 +20,10 @@ #elif defined(__linux__) #include #endif -#include #include +#include "boost/filesystem.hpp" + #include "RuntimeUtil.h" #include "StringTools.h" #include "logger/Logger.h" @@ -350,345 +351,345 @@ bool IsValidSuffix(const std::string& filename) { namespace fsutil { - Dir::Dir(const std::string& dirPath) : mDirPath(dirPath) { +Dir::Dir(const std::string& dirPath) : mDirPath(dirPath) { #if defined(__linux__) - mDir = nullptr; + mDir = nullptr; #elif defined(_MSC_VER) - mFind = INVALID_HANDLE_VALUE; + mFind = INVALID_HANDLE_VALUE; #endif - } +} - Dir::~Dir() { - Close(); - } +Dir::~Dir() { + Close(); +} #if defined(_MSC_VER) - // Invalid entry is returned if fileName starts with ., such as ., .., hidden files. - static Entry ConstructEntry(const WIN32_FIND_DATA& findData) { - std::string fileName(findData.cFileName); - if (0 == fileName.find(".")) - return Entry(); - - // NOTE: We assume there are only two types: DIR and REG_FILE, ignore symbolic. - // In fact, for symbolic path, both boost::filesystem::status and Windows stat - // will return FILE... - return Entry(fileName, - (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ? Entry::Type::DIR : Entry::Type::REG_FILE, - false); - } +// Invalid entry is returned if fileName starts with ., such as ., .., hidden files. +static Entry ConstructEntry(const WIN32_FIND_DATA& findData) { + std::string fileName(findData.cFileName); + if (0 == fileName.find(".")) + return Entry(); + + // NOTE: We assume there are only two types: DIR and REG_FILE, ignore symbolic. + // In fact, for symbolic path, both boost::filesystem::status and Windows stat + // will return FILE... + return Entry(fileName, + (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ? Entry::Type::DIR : Entry::Type::REG_FILE, + false); +} #endif - bool Dir::IsOpened() const { +bool Dir::IsOpened() const { #if defined(__linux__) - return mDir != nullptr; + return mDir != nullptr; #elif defined(_MSC_VER) - return mFind != INVALID_HANDLE_VALUE; + return mFind != INVALID_HANDLE_VALUE; #endif - } +} - bool Dir::Open() { - if (IsOpened()) - return true; - if (mDirPath.empty()) { - // Log it but don't return, let following call to generate errno. - LOG_WARNING(sLogger, ("Empty dir path", mDirPath)); - } +bool Dir::Open() { + if (IsOpened()) + return true; + if (mDirPath.empty()) { + // Log it but don't return, let following call to generate errno. + LOG_WARNING(sLogger, ("Empty dir path", mDirPath)); + } #if defined(__linux__) - DIR* dir = opendir(mDirPath.c_str()); - if (NULL == dir) { - // TODO: Record errno - return false; - } + DIR* dir = opendir(mDirPath.c_str()); + if (NULL == dir) { + // TODO: Record errno + return false; + } - mDir = dir; - return true; + mDir = dir; + return true; #elif defined(_MSC_VER) - auto findPath = PathJoin(mDirPath, "*"); - WIN32_FIND_DATA ffd; - mFind = FindFirstFile(findPath.c_str(), &ffd); - if (INVALID_HANDLE_VALUE == mFind) { - // TODO: Record errno. - return false; - } - // Cache first entry. - mCachedEntry = ConstructEntry(ffd); - return true; -#endif + auto findPath = PathJoin(mDirPath, "*"); + WIN32_FIND_DATA ffd; + mFind = FindFirstFile(findPath.c_str(), &ffd); + if (INVALID_HANDLE_VALUE == mFind) { + // TODO: Record errno. + return false; } + // Cache first entry. + mCachedEntry = ConstructEntry(ffd); + return true; +#endif +} - Entry Dir::ReadNext(bool resolveWithStat) { - if (!IsOpened()) - return Entry(); +Entry Dir::ReadNext(bool resolveWithStat) { + if (!IsOpened()) + return Entry(); #if defined(__linux__) - // Call readdir until it returns nullptr or non-dot entry. - while (true) { - struct dirent* ent = readdir(mDir); - if (nullptr == ent) - return Entry(); - - std::string fileName(ent->d_name); - if (0 == fileName.find('.')) { - continue; - } + // Call readdir until it returns nullptr or non-dot entry. + while (true) { + struct dirent* ent = readdir(mDir); + if (nullptr == ent) + return Entry(); - Entry::Type type; - bool isSymbolic = false; - switch (ent->d_type) { - case DT_FIFO: - case DT_CHR: - case DT_BLK: - case DT_SOCK: - case DT_WHT: - // Known types that should be ignored. - type = Entry::Type::UNKNOWN; - break; - case DT_DIR: - type = Entry::Type::DIR; - break; - case DT_REG: - type = Entry::Type::REG_FILE; + std::string fileName(ent->d_name); + if (0 == fileName.find('.')) { + continue; + } + + Entry::Type type; + bool isSymbolic = false; + switch (ent->d_type) { + case DT_FIFO: + case DT_CHR: + case DT_BLK: + case DT_SOCK: + case DT_WHT: + // Known types that should be ignored. + type = Entry::Type::UNKNOWN; + break; + case DT_DIR: + type = Entry::Type::DIR; + break; + case DT_REG: + type = Entry::Type::REG_FILE; + break; + + case DT_LNK: + // DT_UNKNOWN should also be resolved by stat again to against inaccurate + // meta info from file system, eg. expect DT_LNK, return DT_UNKNOWN. + // Ref: https://aone.alibaba-inc.com/issue/36979148. + case DT_UNKNOWN: + // All known types have already been enumerated at above, stat for unknown + // new types for better compatibility. + default: { + type = Entry::Type::UNKNOWN; + isSymbolic = (DT_LNK == ent->d_type); + if (!resolveWithStat) { break; + } - case DT_LNK: - // DT_UNKNOWN should also be resolved by stat again to against inaccurate - // meta info from file system, eg. expect DT_LNK, return DT_UNKNOWN. - // Ref: https://aone.alibaba-inc.com/issue/36979148. - case DT_UNKNOWN: - // All known types have already been enumerated at above, stat for unknown - // new types for better compatibility. - default: { - type = Entry::Type::UNKNOWN; - isSymbolic = (DT_LNK == ent->d_type); - if (!resolveWithStat) { - break; - } - - // Try to get target type by stat, if failed, return UNKNONW to remind - // caller that maybe the symbolic is invalid. - auto fullPath = PathJoin(mDirPath, fileName); - struct stat fileStat; - if (stat(fullPath.c_str(), &fileStat) != 0) { - LOG_WARNING(sLogger, - ("Get file info fail", - fullPath)("errno", errno)("strerror", strerror(errno))("d_type", ent->d_type)); - break; - } - if (S_ISDIR(fileStat.st_mode)) { - type = Entry::Type::DIR; - } else if (S_ISREG(fileStat.st_mode)) { - type = Entry::Type::REG_FILE; - } + // Try to get target type by stat, if failed, return UNKNONW to remind + // caller that maybe the symbolic is invalid. + auto fullPath = PathJoin(mDirPath, fileName); + struct stat fileStat; + if (stat(fullPath.c_str(), &fileStat) != 0) { + LOG_WARNING(sLogger, + ("Get file info fail", + fullPath)("errno", errno)("strerror", strerror(errno))("d_type", ent->d_type)); break; } - }; + if (S_ISDIR(fileStat.st_mode)) { + type = Entry::Type::DIR; + } else if (S_ISREG(fileStat.st_mode)) { + type = Entry::Type::REG_FILE; + } + break; + } + }; - return Entry(fileName, type, isSymbolic); - } + return Entry(fileName, type, isSymbolic); + } #elif defined(_MSC_VER) - if (mCachedEntry) { - Entry entry = mCachedEntry; - mCachedEntry = Entry(); - return entry; - } - - WIN32_FIND_DATA ffd; - while (true) { - if (0 == FindNextFile(mFind, &ffd)) { - auto err = GetLastError(); - if (err != ERROR_NO_MORE_FILES) { - LOG_WARNING(sLogger, ("Unexpected error when call FindNextFile", err)("dir path", mDirPath)); - } - return Entry(); + if (mCachedEntry) { + Entry entry = mCachedEntry; + mCachedEntry = Entry(); + return entry; + } + + WIN32_FIND_DATA ffd; + while (true) { + if (0 == FindNextFile(mFind, &ffd)) { + auto err = GetLastError(); + if (err != ERROR_NO_MORE_FILES) { + LOG_WARNING(sLogger, ("Unexpected error when call FindNextFile", err)("dir path", mDirPath)); } - auto entry = ConstructEntry(ffd); - if (entry) - return entry; + return Entry(); } -#endif + auto entry = ConstructEntry(ffd); + if (entry) + return entry; } +#endif +} - void Dir::Close() { - if (!IsOpened()) - return; +void Dir::Close() { + if (!IsOpened()) + return; #if defined(__linux__) - if (closedir(mDir) != 0) { - LOG_WARNING(sLogger, ("Close dir failed", mDirPath)("errno", errno)); - return; - } - mDir = nullptr; + if (closedir(mDir) != 0) { + LOG_WARNING(sLogger, ("Close dir failed", mDirPath)("errno", errno)); + return; + } + mDir = nullptr; #elif defined(_MSC_VER) - if (!FindClose(mFind)) { - LOG_WARNING(sLogger, ("Close dir failed", mDirPath)("errno", GetLastError())); - return; - } - mFind = INVALID_HANDLE_VALUE; -#endif + if (!FindClose(mFind)) { + LOG_WARNING(sLogger, ("Close dir failed", mDirPath)("errno", GetLastError())); + return; } + mFind = INVALID_HANDLE_VALUE; +#endif +} - PathStat::PathStat() { - } +PathStat::PathStat() { +} - PathStat::~PathStat() { - } +PathStat::~PathStat() { +} - bool PathStat::stat(const std::string& path, PathStat& ps) { - ps.mPath = path; +bool PathStat::stat(const std::string& path, PathStat& ps) { + ps.mPath = path; #if defined(__linux__) - return (0 == ::stat(path.c_str(), &(ps.mRawStat))); + return (0 == ::stat(path.c_str(), &(ps.mRawStat))); #elif defined(_MSC_VER) - // For backward performance compatibility, check the path only - // when the flag is enabled by user configuration. - if (!BOOL_FLAG(enable_root_path_collection) || path.back() != ':') { - return 0 == ::_stat64(path.c_str(), &(ps.mRawStat)); - } - // _stat64("D:") returns non-zero. - return 0 == ::_stat64((path + PATH_SEPARATOR).c_str(), &(ps.mRawStat)); -#endif + // For backward performance compatibility, check the path only + // when the flag is enabled by user configuration. + if (!BOOL_FLAG(enable_root_path_collection) || path.back() != ':') { + return 0 == ::_stat64(path.c_str(), &(ps.mRawStat)); } + // _stat64("D:") returns non-zero. + return 0 == ::_stat64((path + PATH_SEPARATOR).c_str(), &(ps.mRawStat)); +#endif +} - bool PathStat::IsDir() const { +bool PathStat::IsDir() const { #if defined(__linux__) - return S_ISDIR(mRawStat.st_mode); + return S_ISDIR(mRawStat.st_mode); #elif defined(_MSC_VER) - return mRawStat.st_mode & S_IFDIR; + return mRawStat.st_mode & S_IFDIR; #endif - } +} - bool PathStat::IsRegFile() const { +bool PathStat::IsRegFile() const { #if defined(__linux__) - return S_ISREG(mRawStat.st_mode); + return S_ISREG(mRawStat.st_mode); #elif defined(_MSC_VER) - return mRawStat.st_mode & S_IFREG; + return mRawStat.st_mode & S_IFREG; #endif - } +} - bool PathStat::lstat(const std::string& path, PathStat& ps) { - ps.mPath = path; +bool PathStat::lstat(const std::string& path, PathStat& ps) { + ps.mPath = path; #if defined(__linux__) - return (0 == ::lstat(path.c_str(), &(ps.mRawStat))); + return (0 == ::lstat(path.c_str(), &(ps.mRawStat))); #elif defined(_MSC_VER) - return (0 == ::_stat64(path.c_str(), &(ps.mRawStat))); + return (0 == ::_stat64(path.c_str(), &(ps.mRawStat))); #endif - } +} - bool PathStat::IsLink() const { +bool PathStat::IsLink() const { #if defined(__linux__) - return S_ISLNK(mRawStat.st_mode); + return S_ISLNK(mRawStat.st_mode); #elif defined(_MSC_VER) - return false; // Windows shortcut is not symbolic link. + return false; // Windows shortcut is not symbolic link. #endif - } +} - bool PathStat::fstat(FILE* file, PathStat& ps, bool resolvePath) { +bool PathStat::fstat(FILE* file, PathStat& ps, bool resolvePath) { #if defined(__linux__) - return (0 == ::fstat(fileno(file), &(ps.mRawStat))); + return (0 == ::fstat(fileno(file), &(ps.mRawStat))); #elif defined(_MSC_VER) - return fstat(_fileno(file), ps, resolvePath); + return fstat(_fileno(file), ps, resolvePath); #endif - } +} - bool PathStat::fstat(int fd, PathStat& ps, bool resolvePath) { +bool PathStat::fstat(int fd, PathStat& ps, bool resolvePath) { #if defined(__linux__) - return (0 == ::fstat(fd, &(ps.mRawStat))); + return (0 == ::fstat(fd, &(ps.mRawStat))); #elif defined(_MSC_VER) - auto fstatRet = ::_fstat64(fd, &(ps.mRawStat)); - if (fstatRet != 0) - return false; - if (!resolvePath) - return true; + auto fstatRet = ::_fstat64(fd, &(ps.mRawStat)); + if (fstatRet != 0) + return false; + if (!resolvePath) + return true; - ps.mPath = GetFdPath(fd); - return !ps.mPath.empty(); + ps.mPath = GetFdPath(fd); + return !ps.mPath.empty(); #endif - } +} #if defined(_MSC_VER) - // FILETIME2Time converts ft to time_t. - // @return second part. - static int64_t FILETIME2Time(const FILETIME ft, int64_t* nsec = nullptr) { - ULARGE_INTEGER ui; - ui.LowPart = ft.dwLowDateTime; - ui.HighPart = ft.dwHighDateTime; - int64_t sec = static_cast(ui.QuadPart / 10000000 - 11644473600); - if (nsec != nullptr) { - *nsec = static_cast(ui.QuadPart % 10000000) * 100; - } - return sec; - } - - bool PathStat::fstat(HANDLE hFile, PathStat& ps, bool resolvePath) { - // st_mtime. - FILETIME mtim; - if (FALSE == GetFileTime(hFile, NULL, NULL, &mtim)) { - return false; - } - ps.mRawStat.st_mtime = static_cast(FILETIME2Time(mtim)); - // st_size. - LARGE_INTEGER liSize; - if (FALSE == GetFileSizeEx(hFile, &liSize)) { - return false; - } - ps.mRawStat.st_size = liSize.QuadPart; - if (!resolvePath) { - return true; - } +// FILETIME2Time converts ft to time_t. +// @return second part. +static int64_t FILETIME2Time(const FILETIME ft, int64_t* nsec = nullptr) { + ULARGE_INTEGER ui; + ui.LowPart = ft.dwLowDateTime; + ui.HighPart = ft.dwHighDateTime; + int64_t sec = static_cast(ui.QuadPart / 10000000 - 11644473600); + if (nsec != nullptr) { + *nsec = static_cast(ui.QuadPart % 10000000) * 100; + } + return sec; +} - // ps.mPath. - char filePath[MAX_PATH + 1]; - auto ret = GetFinalPathNameByHandle(hFile, filePath, MAX_PATH + 1, VOLUME_NAME_DOS); - if (ret > MAX_PATH || ret <= 0) { - return false; - } - if (0 == memcmp(filePath, "\\\\?\\", 4)) { - ps.mPath.assign(filePath + 4); - } else { - ps.mPath = filePath; - } +bool PathStat::fstat(HANDLE hFile, PathStat& ps, bool resolvePath) { + // st_mtime. + FILETIME mtim; + if (FALSE == GetFileTime(hFile, NULL, NULL, &mtim)) { + return false; + } + ps.mRawStat.st_mtime = static_cast(FILETIME2Time(mtim)); + // st_size. + LARGE_INTEGER liSize; + if (FALSE == GetFileSizeEx(hFile, &liSize)) { + return false; + } + ps.mRawStat.st_size = liSize.QuadPart; + if (!resolvePath) { return true; } -#endif - time_t PathStat::GetMtime() const { - return mRawStat.st_mtime; + // ps.mPath. + char filePath[MAX_PATH + 1]; + auto ret = GetFinalPathNameByHandle(hFile, filePath, MAX_PATH + 1, VOLUME_NAME_DOS); + if (ret > MAX_PATH || ret <= 0) { + return false; } + if (0 == memcmp(filePath, "\\\\?\\", 4)) { + ps.mPath.assign(filePath + 4); + } else { + ps.mPath = filePath; + } + return true; +} +#endif + +time_t PathStat::GetMtime() const { + return mRawStat.st_mtime; +} - void PathStat::GetLastWriteTime(int64_t& sec, int64_t& nsec) const { +void PathStat::GetLastWriteTime(int64_t& sec, int64_t& nsec) const { #if defined(__linux__) - sec = mRawStat.st_mtim.tv_sec; - nsec = mRawStat.st_mtim.tv_nsec; + sec = mRawStat.st_mtim.tv_sec; + nsec = mRawStat.st_mtim.tv_nsec; #elif defined(_MSC_VER) - HANDLE hFile = CreateFile(mPath.c_str(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - NULL, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, - NULL); - if (INVALID_HANDLE_VALUE == hFile) - return; - FILETIME mtim; - auto ret = GetFileTime(hFile, NULL, NULL, &mtim); - CloseHandle(hFile); - if (!ret) - return; - sec = FILETIME2Time(mtim, &nsec); + HANDLE hFile = CreateFile(mPath.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + NULL); + if (INVALID_HANDLE_VALUE == hFile) + return; + FILETIME mtim; + auto ret = GetFileTime(hFile, NULL, NULL, &mtim); + CloseHandle(hFile); + if (!ret) + return; + sec = FILETIME2Time(mtim, &nsec); #endif - } +} - DevInode PathStat::GetDevInode() const { +DevInode PathStat::GetDevInode() const { #if defined(__linux__) - return DevInode(mRawStat.st_dev, mRawStat.st_ino); + return DevInode(mRawStat.st_dev, mRawStat.st_ino); #elif defined(_MSC_VER) - return GetFileDevInode(mPath); + return GetFileDevInode(mPath); #endif - } +} - int64_t PathStat::GetFileSize() const { - return mRawStat.st_size; - } +int64_t PathStat::GetFileSize() const { + return mRawStat.st_size; +} } // namespace fsutil diff --git a/core/common/FileSystemUtil.h b/core/common/FileSystemUtil.h index 3ad1b046ed..d6d9088292 100644 --- a/core/common/FileSystemUtil.h +++ b/core/common/FileSystemUtil.h @@ -17,7 +17,6 @@ #pragma once #include -#include #include #include #include @@ -122,137 +121,138 @@ void Chmod(const char* filePath, mode_t mode); namespace fsutil { - class Entry { - public: - enum class Type { UNKNOWN, DIR, REG_FILE }; +class Entry { +public: + enum class Type { UNKNOWN, DIR, REG_FILE }; - Entry() {} - Entry(const std::string& name, Type type, bool isSymbolic) - : mName(name), mType(type), mIsSymbolic(isSymbolic) {} + Entry() {} + Entry(const std::string& name, Type type, bool isSymbolic) : mName(name), mType(type), mIsSymbolic(isSymbolic) {} - std::string Name() const { return mName; } - bool IsDir() const { return Type::DIR == mType; } - bool IsRegFile() const { return Type::REG_FILE == mType; } - bool IsSymbolic() const { return mIsSymbolic; } + std::string Name() const { return mName; } + bool IsDir() const { return Type::DIR == mType; } + bool IsRegFile() const { return Type::REG_FILE == mType; } + bool IsSymbolic() const { return mIsSymbolic; } - // It's caller's job to deal unknown entry type. - operator bool() const { return !mName.empty(); } + // It's caller's job to deal unknown entry type. + operator bool() const { return !mName.empty(); } - private: - std::string mName; - Type mType = Type::UNKNOWN; - bool mIsSymbolic = false; - }; +private: + std::string mName; + Type mType = Type::UNKNOWN; + bool mIsSymbolic = false; +}; - class Dir { - Dir(const Dir&) = delete; - Dir& operator=(const Dir&) = delete; +class Dir { + Dir(const Dir&) = delete; + Dir& operator=(const Dir&) = delete; - public: - Dir(const std::string& dirPath); - ~Dir(); +public: + Dir(const std::string& dirPath); + ~Dir(); - bool Open(); + bool Open(); - // Utility function to decide why Open failed. - inline static bool IsENOENT(int e) { + // Utility function to decide why Open failed. + inline static bool IsENOENT(int e) { #if defined(__linux__) - return (ENOENT == e); + return (ENOENT == e); #elif defined(_MSC_VER) - return (ERROR_PATH_NOT_FOUND == e || ERROR_FILE_NOT_FOUND == e); + return (ERROR_PATH_NOT_FOUND == e || ERROR_FILE_NOT_FOUND == e); #endif - } - inline static bool IsENOTDIR(int e) { + } + inline static bool IsENOTDIR(int e) { #if defined(__linux__) - return (ENOTDIR == e); + return (ENOTDIR == e); #elif defined(_MSC_VER) - return (ERROR_DIRECTORY == e); + return (ERROR_DIRECTORY == e); #endif - } - inline static bool IsEACCES(int e) { + } + inline static bool IsEACCES(int e) { #if defined(__linux__) - return (EACCES == e); + return (EACCES == e); #elif defined(_MSC_VER) - return (ERROR_ACCESS_DENIED == e); + return (ERROR_ACCESS_DENIED == e); #endif - } - - // ReadNext reads next entry (FILE or DIR). - // @param resolveWithStat: if true, use stat to check entry type when the entry - // type from filesystem directory iteration API is not FILE or DIR, - // eg. LNK type under Linux. - // @return: the entry of the file. If it is sure that we reached the end, an - // 'false' entry will be returned. Otherwise, we will use UNKNOWN type entry - // to indicate some errors, it is caller's job to handle this (eg. skip it). - // TODO: Most error happened when the entry is symbolic, maybe we can ignore - // this and let caller to call something else to deal it. - Entry ReadNext(bool resolveWithStat = true); - - void Close(); - - private: - std::string mDirPath; + } + + // ReadNext reads next entry (FILE or DIR). + // @param resolveWithStat: if true, use stat to check entry type when the entry + // type from filesystem directory iteration API is not FILE or DIR, + // eg. LNK type under Linux. + // @return: the entry of the file. If it is sure that we reached the end, an + // 'false' entry will be returned. Otherwise, we will use UNKNOWN type entry + // to indicate some errors, it is caller's job to handle this (eg. skip it). + // TODO: Most error happened when the entry is symbolic, maybe we can ignore + // this and let caller to call something else to deal it. + Entry ReadNext(bool resolveWithStat = true); + + void Close(); + +private: + std::string mDirPath; #if defined(__linux__) - DIR* mDir; + DIR* mDir; #elif defined(_MSC_VER) - HANDLE mFind; - // Because FindFirstFile will return first entry, we have to cache it. - Entry mCachedEntry; + HANDLE mFind; + // Because FindFirstFile will return first entry, we have to cache it. + Entry mCachedEntry; #endif - bool IsOpened() const; - }; + bool IsOpened() const; +}; #if defined(__linux__) - typedef struct stat RawStatType; +typedef struct stat RawStatType; #elif defined(_MSC_VER) - typedef struct _stat64 RawStatType; +typedef struct _stat64 RawStatType; #endif - class PathStat { - std::string mPath; - RawStatType mRawStat; - - public: - PathStat(); - ~PathStat(); - - RawStatType* GetRawStat() { return &mRawStat; } - - // stat wrappers ::stat. - static bool stat(const std::string& path, PathStat& ps); - bool IsDir() const; - bool IsRegFile() const; - - // lstat wrappers Linux ::lstat and implements it with ::stat on Windows. - static bool lstat(const std::string& path, PathStat& ps); - bool IsLink() const; - - // fstat wrappers ::fstat. - // @resolvePath only works on Windows. - // If it is true, fstat will get corresponding file path according to @file and - // set it to @ps.mPath, however, if this procedure failed, false will be returned - // although the call to ::fstat succeeded. - // **Use it when you want to call GetDevInode() or GetLastWriteTime().** - // **Disable it if you only want to call GetMtime() or GetFileSize().** - static bool fstat(FILE* file, PathStat& ps, bool resolvePath = true); - static bool fstat(int fd, PathStat& ps, bool resolvePath = true); +class PathStat { + std::string mPath; + RawStatType mRawStat; + +public: + PathStat(); + ~PathStat(); + + RawStatType* GetRawStat() { return &mRawStat; } + + // stat wrappers ::stat. + static bool stat(const std::string& path, PathStat& ps); + bool IsDir() const; + bool IsRegFile() const; + + // lstat wrappers Linux ::lstat and implements it with ::stat on Windows. + static bool lstat(const std::string& path, PathStat& ps); + bool IsLink() const; + + // fstat wrappers ::fstat. + // @resolvePath only works on Windows. + // If it is true, fstat will get corresponding file path according to @file and + // set it to @ps.mPath, however, if this procedure failed, false will be returned + // although the call to ::fstat succeeded. + // **Use it when you want to call GetDevInode() or GetLastWriteTime().** + // **Disable it if you only want to call GetMtime() or GetFileSize().** + static bool fstat(FILE* file, PathStat& ps, bool resolvePath = true); + static bool fstat(int fd, PathStat& ps, bool resolvePath = true); #if defined(_MSC_VER) - static bool fstat(HANDLE hFile, PathStat& ps, bool resolvePath = true); + static bool fstat(HANDLE hFile, PathStat& ps, bool resolvePath = true); #endif - // For Windows, GetDevInode and GetLastWriteTime have to call other system APIs to - // get information (by mPath), this will spend extra costs. - DevInode GetDevInode() const; - void GetLastWriteTime(int64_t& sec, int64_t& nsec) const; + // For Windows, GetDevInode and GetLastWriteTime have to call other system APIs to + // get information (by mPath), this will spend extra costs. + DevInode GetDevInode() const; + void GetLastWriteTime(int64_t& sec, int64_t& nsec) const; - // GetMtime and GetFileSize return st_mtime and st_size in struct stat. They needn't - // to call another system APIs. - time_t GetMtime() const; - int64_t GetFileSize() const; + // GetMtime and GetFileSize return st_mtime and st_size in struct stat. They needn't + // to call another system APIs. + time_t GetMtime() const; + int64_t GetFileSize() const; - // GetMode returns st_mode. - int GetMode() const { return static_cast(mRawStat.st_mode); } - }; + // GetMode returns st_mode. + int GetMode() const { + return static_cast(mRawStat.st_mode); + } +}; } // namespace fsutil diff --git a/core/common/Flags.h b/core/common/Flags.h index 8bd7638c66..5a8aaeb937 100644 --- a/core/common/Flags.h +++ b/core/common/Flags.h @@ -15,7 +15,7 @@ */ #pragma once -#include +#include "gflags/gflags.h" /** Macro to define INT32 flag. Must be used in global scope. */ #define DEFINE_FLAG_INT32(name, desc, value) DEFINE_int32(name, value, desc) diff --git a/core/common/HashUtil.cpp b/core/common/HashUtil.cpp index 206c872f25..e5016ab1a2 100644 --- a/core/common/HashUtil.cpp +++ b/core/common/HashUtil.cpp @@ -16,7 +16,7 @@ #include -#include +#include "boost/functional/hash.hpp" #include "FileSystemUtil.h" #include "LogFileOperator.h" diff --git a/core/common/HashUtil.h b/core/common/HashUtil.h index 5bd7204c74..55c0ba18de 100644 --- a/core/common/HashUtil.h +++ b/core/common/HashUtil.h @@ -16,6 +16,7 @@ #pragma once #include + #include // Hash and file signature utility. diff --git a/core/common/JsonUtil.cpp b/core/common/JsonUtil.cpp index 3d78c955ff..288f67283f 100644 --- a/core/common/JsonUtil.cpp +++ b/core/common/JsonUtil.cpp @@ -14,8 +14,8 @@ #include "common/JsonUtil.h" -#include #include +#include #include "common/ExceptionBase.h" #include "common/StringTools.h" @@ -169,22 +169,22 @@ int64_t GetInt64Value(const Json::Value& value, const string& name, const int64_ namespace { - // @return true if succeed to load value from env. - template - bool LoadEnvValueIfExisting(const char* envKey, T& cfgValue) { - try { - const char* value = getenv(envKey); - if (value != NULL) { - T val = StringTo(value); - cfgValue = val; - APSARA_LOG_INFO(sLogger, ("load config from env", envKey)("value", val)); - return true; - } - } catch (const exception& e) { - APSARA_LOG_WARNING(sLogger, ("load config from env error", envKey)("error", e.what())); +// @return true if succeed to load value from env. +template +bool LoadEnvValueIfExisting(const char* envKey, T& cfgValue) { + try { + const char* value = getenv(envKey); + if (value != NULL) { + T val = StringTo(value); + cfgValue = val; + APSARA_LOG_INFO(sLogger, ("load config from env", envKey)("value", val)); + return true; } - return false; + } catch (const exception& e) { + APSARA_LOG_WARNING(sLogger, ("load config from env error", envKey)("error", e.what())); } + return false; +} } // namespace diff --git a/core/common/JsonUtil.h b/core/common/JsonUtil.h index 8ebce65187..9721ec4b3c 100644 --- a/core/common/JsonUtil.h +++ b/core/common/JsonUtil.h @@ -17,9 +17,10 @@ #pragma once #include + #include -#include +#include "json/json.h" namespace logtail { diff --git a/core/common/LRUCache.h b/core/common/LRUCache.h index 7ae51d24a9..9daf20926e 100644 --- a/core/common/LRUCache.h +++ b/core/common/LRUCache.h @@ -30,63 +30,65 @@ */ #pragma once -#include +#include #include + +#include +#include #include #include #include #include #include -#include -#include namespace logtail { - namespace lru11 { - /* - * a noop lockable concept that can be used in place of std::mutex - */ - class NullLock { - public: +namespace lru11 { +/* + * a noop lockable concept that can be used in place of std::mutex + */ +class NullLock { +public: void lock() {} void unlock() {} bool try_lock() { return true; } - }; +}; - /** - * error raised when a key not in cache is passed to get() - */ - class KeyNotFound : public std::invalid_argument { - public: +/** + * error raised when a key not in cache is passed to get() + */ +class KeyNotFound : public std::invalid_argument { +public: KeyNotFound() : std::invalid_argument("key_not_found") {} - }; +}; - template - struct KeyValuePair { - public: +template +struct KeyValuePair { +public: K key; V value; std::chrono::steady_clock::time_point timestamp; KeyValuePair(K k, V v) : key(std::move(k)), value(std::move(v)), timestamp(std::chrono::steady_clock::now()) {} - }; +}; - /** - * The LRU Cache class templated by - * Key - key type - * Value - value type - * MapType - an associative container like std::unordered_map - * LockType - a lock type derived from the Lock class (default: - *NullLock = no synchronization) - * - * The default NullLock based template is not thread-safe, however passing - *Lock=std::mutex will make it - * thread-safe - */ - template >::iterator>> - class Cache { - public: +/** + * The LRU Cache class templated by + * Key - key type + * Value - value type + * MapType - an associative container like std::unordered_map + * LockType - a lock type derived from the Lock class (default: + *NullLock = no synchronization) + * + * The default NullLock based template is not thread-safe, however passing + *Lock=std::mutex will make it + * thread-safe + */ +template >::iterator>> +class Cache { +public: typedef KeyValuePair node_type; typedef std::list> list_type; typedef Map map_type; @@ -112,87 +114,87 @@ namespace logtail { } size_t size() const { - Guard g(lock_); - return cache_.size(); + Guard g(lock_); + return cache_.size(); } bool empty() const { - Guard g(lock_); - return cache_.empty(); + Guard g(lock_); + return cache_.empty(); } void clear() { - Guard g(lock_); - cache_.clear(); - keys_.clear(); + Guard g(lock_); + cache_.clear(); + keys_.clear(); } void insert(const Key& k, Value v) { - Guard g(lock_); - const auto iter = cache_.find(k); - if (iter != cache_.end()) { - iter->second->value = v; - iter->second->timestamp = std::chrono::steady_clock::now(); - keys_.splice(keys_.begin(), keys_, iter->second); - return; - } + Guard g(lock_); + const auto iter = cache_.find(k); + if (iter != cache_.end()) { + iter->second->value = v; + iter->second->timestamp = std::chrono::steady_clock::now(); + keys_.splice(keys_.begin(), keys_, iter->second); + return; + } - keys_.emplace_front(k, std::move(v)); - cache_[k] = keys_.begin(); - prune(); + keys_.emplace_front(k, std::move(v)); + cache_[k] = keys_.begin(); + prune(); } /** for backward compatibity. redirects to tryGetCopy() */ - bool tryGet(const Key& kIn, Value& vOut) { - return tryGetCopy(kIn, vOut); - } + bool tryGet(const Key& kIn, Value& vOut) { return tryGetCopy(kIn, vOut); } bool tryGetCopy(const Key& kIn, Value& vOut) { - Guard g(lock_); - Value tmp; - if (!tryGetRef_nolock(kIn, tmp)) { return false; } - vOut = tmp; - return true; + Guard g(lock_); + Value tmp; + if (!tryGetRef_nolock(kIn, tmp)) { + return false; + } + vOut = tmp; + return true; } - + bool tryGetRef(const Key& kIn, Value& vOut) { - Guard g(lock_); - return tryGetRef_nolock(kIn, vOut); + Guard g(lock_); + return tryGetRef_nolock(kIn, vOut); } /** * The const reference returned here is only - * guaranteed to be valid till the next insert/delete - * in multi-threaded apps use getCopy() to be threadsafe - */ + * guaranteed to be valid till the next insert/delete + * in multi-threaded apps use getCopy() to be threadsafe + */ const Value& getRef(const Key& k) { - Guard g(lock_); - return get_nolock(k); + Guard g(lock_); + return get_nolock(k); } /** added for backward compatibility */ - Value get(const Key& k) { return getCopy(k); } + Value get(const Key& k) { return getCopy(k); } /** * returns a copy of the stored object (if found) * safe to use/recommended in multi-threaded apps */ Value getCopy(const Key& k) { - Guard g(lock_); - return get_nolock(k); + Guard g(lock_); + return get_nolock(k); } bool remove(const Key& k) { - Guard g(lock_); - auto iter = cache_.find(k); - if (iter == cache_.end()) { - return false; - } - keys_.erase(iter->second); - cache_.erase(iter); - return true; + Guard g(lock_); + auto iter = cache_.find(k); + if (iter == cache_.end()) { + return false; + } + keys_.erase(iter->second); + cache_.erase(iter); + return true; } bool contains(const Key& k) const { - Guard g(lock_); - return cache_.find(k) != cache_.end(); + Guard g(lock_); + return cache_.find(k) != cache_.end(); } size_t getMaxSize() const { return maxSize_; } @@ -200,64 +202,64 @@ namespace logtail { size_t getMaxAllowedSize() const { return maxSize_ + elasticity_; } template void cwalk(F& f) const { - Guard g(lock_); - std::for_each(keys_.begin(), keys_.end(), f); + Guard g(lock_); + std::for_each(keys_.begin(), keys_.end(), f); } - protected: +protected: Value get_nolock(const Key& k) { - const auto iter = cache_.find(k); - if (iter == cache_.end()) { - return nullptr; - } - keys_.splice(keys_.begin(), keys_, iter->second); - return iter->second->value; + const auto iter = cache_.find(k); + if (iter == cache_.end()) { + return nullptr; + } + keys_.splice(keys_.begin(), keys_, iter->second); + return iter->second->value; } bool tryGetRef_nolock(const Key& kIn, Value& vOut) { - const auto iter = cache_.find(kIn); - if (iter == cache_.end()) { - return false; - } - keys_.splice(keys_.begin(), keys_, iter->second); - vOut = iter->second->value; - return true; + const auto iter = cache_.find(kIn); + if (iter == cache_.end()) { + return false; + } + keys_.splice(keys_.begin(), keys_, iter->second); + vOut = iter->second->value; + return true; } size_t prune() { - size_t maxAllowed = maxSize_ + elasticity_; - if (maxSize_ == 0 || cache_.size() < maxAllowed) { - return 0; - } - size_t count = 0; - while (cache_.size() > maxSize_) { - cache_.erase(keys_.back().key); - keys_.pop_back(); - ++count; - } - return count; + size_t maxAllowed = maxSize_ + elasticity_; + if (maxSize_ == 0 || cache_.size() < maxAllowed) { + return 0; + } + size_t count = 0; + while (cache_.size() > maxSize_) { + cache_.erase(keys_.back().key); + keys_.pop_back(); + ++count; + } + return count; } size_t pruneExpired() { - Guard g(lock_); - size_t count = 0; - auto now = std::chrono::steady_clock::now(); - auto cutoff = now - std::chrono::minutes(5); + Guard g(lock_); + size_t count = 0; + auto now = std::chrono::steady_clock::now(); + auto cutoff = now - std::chrono::minutes(5); - while (!keys_.empty() && keys_.back().timestamp < cutoff) { - cache_.erase(keys_.back().key); - keys_.pop_back(); - ++count; - } - return count; + while (!keys_.empty() && keys_.back().timestamp < cutoff) { + cache_.erase(keys_.back().key); + keys_.pop_back(); + ++count; + } + return count; } void pruneThreadFunc() { - while (!stop_pruning) { - pruneExpired(); - std::this_thread::sleep_for(std::chrono::seconds(60)); // 每60秒检查一次 - } + while (!stop_pruning) { + pruneExpired(); + std::this_thread::sleep_for(std::chrono::seconds(60)); // 每60秒检查一次 + } } - private: +private: // Disallow copying. Cache(const Cache&) = delete; Cache& operator=(const Cache&) = delete; @@ -270,10 +272,10 @@ namespace logtail { std::atomic stop_pruning{false}; std::thread prune_thread; - #ifdef APSARA_UNIT_TEST_MAIN +#ifdef APSARA_UNIT_TEST_MAIN friend class LRUCacheUnittest; - #endif - }; +#endif +}; - } // namespace LRUCache11 -} //namespace logtail \ No newline at end of file +} // namespace lru11 +} // namespace logtail \ No newline at end of file diff --git a/core/common/Lock.h b/core/common/Lock.h index 3f90e4ed82..561aedf175 100644 --- a/core/common/Lock.h +++ b/core/common/Lock.h @@ -18,12 +18,12 @@ #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-local-typedefs" -#include -#include -#include +#include "boost/concept_check.hpp" +#include "boost/range/algorithm/equal.hpp" +#include "boost/range/size.hpp" #pragma GCC diagnostic pop -#include -#include +#include "boost/smart_ptr/detail/spinlock.hpp" +#include "boost/thread.hpp" #ifdef _MSC_VER #define NOGDI #include diff --git a/core/common/LogFileOperator.h b/core/common/LogFileOperator.h index 8ff678ef57..1766b63797 100644 --- a/core/common/LogFileOperator.h +++ b/core/common/LogFileOperator.h @@ -15,22 +15,23 @@ */ #pragma once +#include #include + #include -#include #if defined(_MSC_VER) #include #elif defined(__linux__) +#include #include #include -#include #include #endif namespace logtail { namespace fsutil { - class PathStat; +class PathStat; } class LogFileOperator { diff --git a/core/common/MachineInfoUtil.cpp b/core/common/MachineInfoUtil.cpp index 47e0650fa7..60cec82ac8 100644 --- a/core/common/MachineInfoUtil.cpp +++ b/core/common/MachineInfoUtil.cpp @@ -32,12 +32,11 @@ #include #include #endif -#include -#include -#include - #include +#include "curl/curl.h" +#include "rapidjson/document.h" + #include "FileSystemUtil.h" #include "StringTools.h" #include "logger/Logger.h" diff --git a/core/common/MachineInfoUtil.h b/core/common/MachineInfoUtil.h index 0983cbf352..dc2ba5d97c 100644 --- a/core/common/MachineInfoUtil.h +++ b/core/common/MachineInfoUtil.h @@ -15,10 +15,11 @@ */ #pragma once + +#include + #include #include -#include -#include namespace logtail { @@ -37,7 +38,7 @@ std::string GetHostIp(const std::string& intf = ""); void GetAllPids(std::unordered_set& pids); bool GetKernelInfo(std::string& kernelRelease, int64_t& kernelVersion); bool GetRedHatReleaseInfo(std::string& os, int64_t& osVersion, std::string bashPath = ""); -bool IsDigitsDotsHostname(const char *hostname); +bool IsDigitsDotsHostname(const char* hostname); ECSMeta FetchECSMeta(); // GetAnyAvailableIP walks through all interfaces (AF_INET) to find an available IP. diff --git a/core/common/ParamExtractor.h b/core/common/ParamExtractor.h index d63e8a1895..caf0aac8ba 100644 --- a/core/common/ParamExtractor.h +++ b/core/common/ParamExtractor.h @@ -16,14 +16,15 @@ #pragma once -#include - #include + #include #include #include #include +#include "json/json.h" + #include "common/StringTools.h" #include "logger/Logger.h" #include "monitor/AlarmManager.h" diff --git a/core/common/RandomUtil.cpp b/core/common/RandomUtil.cpp index 3f2d3d55e0..842a5f11d8 100644 --- a/core/common/RandomUtil.cpp +++ b/core/common/RandomUtil.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "RandomUtil.h" + #include #if defined(__linux__) #include @@ -22,26 +23,26 @@ namespace logtail { namespace detail { - size_t countBitSize(size_t maxVal) { - if (0 == maxVal) { - return 1; - } - - size_t count = 0; - while (maxVal != 0) { - count++; - maxVal = maxVal >> 1; - } - return count; +size_t countBitSize(size_t maxVal) { + if (0 == maxVal) { + return 1; } - inline char valToHex(size_t val) { - return val < 10 ? ('0' + val) : ('A' + val - 10); + size_t count = 0; + while (maxVal != 0) { + count++; + maxVal = maxVal >> 1; } + return count; +} - inline size_t hexToVal(char hex) { - return hex >= 'A' ? 10 + hex - 'A' : hex - '0'; - } +inline char valToHex(size_t val) { + return val < 10 ? ('0' + val) : ('A' + val - 10); +} + +inline size_t hexToVal(char hex) { + return hex >= 'A' ? 10 + hex - 'A' : hex - '0'; +} } // namespace detail diff --git a/core/common/RuntimeUtil.cpp b/core/common/RuntimeUtil.cpp index 8cf2b37cc9..066e274a03 100644 --- a/core/common/RuntimeUtil.cpp +++ b/core/common/RuntimeUtil.cpp @@ -12,24 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "RuntimeUtil.h" + #include "Flags.h" #include "app_config/AppConfig.h" -#include "RuntimeUtil.h" #if defined(__linux__) -#include +#include #include #include -#include +#include #elif defined(_MSC_VER) -#include #include +#include #endif -#include #include +#include + #include -#include "logger/Logger.h" -#include "LogtailCommonFlags.h" + #include "FileSystemUtil.h" +#include "LogtailCommonFlags.h" +#include "logger/Logger.h" DECLARE_FLAG_STRING(logtail_sys_conf_dir); DECLARE_FLAG_BOOL(logtail_mode); diff --git a/core/common/SafeQueue.h b/core/common/SafeQueue.h index 9b0d11dde1..8b1c0aecdc 100644 --- a/core/common/SafeQueue.h +++ b/core/common/SafeQueue.h @@ -75,7 +75,9 @@ class SafeQueue { } #ifdef APSARA_UNIT_TEST_MAIN - void Clear() { std::queue().swap(mQueue); } + void Clear() { + std::queue().swap(mQueue); + } #endif private: diff --git a/core/common/Semaphore.h b/core/common/Semaphore.h index 669d6b5369..361bd537bf 100644 --- a/core/common/Semaphore.h +++ b/core/common/Semaphore.h @@ -15,8 +15,8 @@ */ #pragma once -#include #include +#include namespace logtail { diff --git a/core/common/SlidingWindowCounter.h b/core/common/SlidingWindowCounter.h index 99de89f307..51cfc42841 100644 --- a/core/common/SlidingWindowCounter.h +++ b/core/common/SlidingWindowCounter.h @@ -15,10 +15,10 @@ */ #pragma once -#include -#include #include +#include #include +#include namespace logtail { diff --git a/core/common/SplitedFilePath.h b/core/common/SplitedFilePath.h index bd7bcb65c8..62a1dd86b5 100644 --- a/core/common/SplitedFilePath.h +++ b/core/common/SplitedFilePath.h @@ -18,6 +18,7 @@ #define _LOG_SPLITED_FILE_PATH_H__ #include + #include "FileSystemUtil.h" namespace logtail { diff --git a/core/common/StringTools.cpp b/core/common/StringTools.cpp index 0e800cb754..ef4f46fca8 100644 --- a/core/common/StringTools.cpp +++ b/core/common/StringTools.cpp @@ -16,9 +16,9 @@ #include -#include -#include -#include +#include "boost/algorithm/string.hpp" +#include "boost/exception/all.hpp" +#include "boost/filesystem.hpp" #include "logger/Logger.h" #if defined(_MSC_VER) diff --git a/core/common/StringTools.h b/core/common/StringTools.h index 4d519dc8fa..a284e48bfa 100644 --- a/core/common/StringTools.h +++ b/core/common/StringTools.h @@ -16,11 +16,12 @@ #pragma once #include -#include -#include #include #include +#include "boost/lexical_cast.hpp" +#include "boost/regex.hpp" + #include "models/StringView.h" namespace logtail { diff --git a/core/common/Strptime.cpp b/core/common/Strptime.cpp index 33f4928cc9..5955244b92 100644 --- a/core/common/Strptime.cpp +++ b/core/common/Strptime.cpp @@ -28,9 +28,11 @@ */ #include "Strptime.h" + #include -#include #include +#include + #include "common/StringTools.h" namespace logtail { @@ -95,13 +97,13 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n if (n == 0 || (long long)(t = n) != n) { return NULL; } - #ifdef _MSC_VER +#ifdef _MSC_VER if (localtime_s(tm, &t) != 0) return NULL; - #else +#else if (NULL == localtime_r(&t, tm)) return NULL; - #endif +#endif *nanosecond = 0; *nanosecondLength = 0; @@ -143,9 +145,9 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n continue; /* - * "Alternative" modifiers. Just set the appropriate flag - * and start over again. - */ + * "Alternative" modifiers. Just set the appropriate flag + * and start over again. + */ case 'E': /* "%E?" alternative conversion modifier. */ LEGAL_ALT(0); alt_format |= ALT_E; @@ -157,8 +159,8 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n goto again; /* - * "Complex" conversion rules, implemented through recursion. - */ + * "Complex" conversion rules, implemented through recursion. + */ case 'c': /* Date and time, using the locale's format. */ // Tue Nov 20 14:12:58 2020 new_fmt = "%a %b %d %H:%M:%S %Y"; @@ -201,8 +203,8 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n continue; /* - * "Elementary" conversion rules. - */ + * "Elementary" conversion rules. + */ case 'A': /* The day of week, using the locale's form. */ case 'a': bp = find_string(bp, &tm->tm_wday, day, abday, 7); @@ -292,11 +294,11 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n case 'U': /* The week of year, beginning on sunday. */ case 'W': /* The week of year, beginning on monday. */ /* - * XXX This is bogus, as we can not assume any valid - * information present in the tm structure at this - * point to calculate a real value, so just check the - * range for now. - */ + * XXX This is bogus, as we can not assume any valid + * information present in the tm structure at this + * point to calculate a real value, so just check the + * range for now. + */ bp = conv_num(bp, &i, 0, 53); LEGAL_ALT(ALT_O); continue; @@ -314,15 +316,15 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n case 'g': /* The year corresponding to the ISO week - * number but without the century. - */ + * number but without the century. + */ bp = conv_num(bp, &i, 0, 99); continue; case 'G': /* The year corresponding to the ISO week - * number with century. - */ + * number with century. + */ do bp++; while (isdigit(*bp)); @@ -357,7 +359,8 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n continue; case 'Z': - if (CStringNCaseInsensitiveCmp((const char*)bp, gmt, 3) == 0 || CStringNCaseInsensitiveCmp((const char *)bp, utc, 3) == 0) { + if (CStringNCaseInsensitiveCmp((const char*)bp, gmt, 3) == 0 + || CStringNCaseInsensitiveCmp((const char*)bp, utc, 3) == 0) { tm->tm_isdst = 0; #ifdef TM_GMTOFF tm->TM_GMTOFF = 0; @@ -386,22 +389,22 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n case 'z': /* - * We recognize all ISO 8601 formats: - * Z = Zulu time/UTC - * [+-]hhmm - * [+-]hh:mm - * [+-]hh - * We recognize all RFC-822/RFC-2822 formats: - * UT|GMT - * North American : UTC offsets - * E[DS]T = Eastern : -4 | -5 - * C[DS]T = Central : -5 | -6 - * M[DS]T = Mountain: -6 | -7 - * P[DS]T = Pacific : -7 | -8 - * Military - * [A-IL-M] = -1 ... -9 (J not used) - * [N-Y] = +1 ... +12 - */ + * We recognize all ISO 8601 formats: + * Z = Zulu time/UTC + * [+-]hhmm + * [+-]hh:mm + * [+-]hh + * We recognize all RFC-822/RFC-2822 formats: + * UT|GMT + * North American : UTC offsets + * E[DS]T = Eastern : -4 | -5 + * C[DS]T = Central : -5 | -6 + * M[DS]T = Mountain: -6 | -7 + * P[DS]T = Pacific : -7 | -8 + * Military + * [A-IL-M] = -1 ... -9 (J not used) + * [N-Y] = +1 ... +12 + */ while (isspace(*bp)) bp++; @@ -437,7 +440,7 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n tm->TM_GMTOFF = -5 - i; #endif #ifdef TM_ZONE - tm->TM_ZONE = (char *)(nast[i]); + tm->TM_ZONE = (char*)(nast[i]); #endif bp = ep; continue; @@ -449,7 +452,7 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n tm->TM_GMTOFF = -4 - i; #endif #ifdef TM_ZONE - tm->TM_ZONE = (char *)(nadt[i]); + tm->TM_ZONE = (char*)(nadt[i]); #endif bp = ep; continue; @@ -512,8 +515,8 @@ const char* strptime_ns(const char* buf, const char* fmt, struct tm* tm, long* n continue; /* - * Miscellaneous conversions. - */ + * Miscellaneous conversions. + */ case 'n': /* Any kind of white-space. */ case 't': while (isspace(*bp)) diff --git a/core/common/Thread.h b/core/common/Thread.h index aa5ced58ad..e10381adb9 100644 --- a/core/common/Thread.h +++ b/core/common/Thread.h @@ -16,13 +16,13 @@ #pragma once -#include - #include #include #include #include +#include "boost/thread.hpp" + namespace logtail { class Thread { diff --git a/core/common/ThreadPool.h b/core/common/ThreadPool.h index 09ccd733ee..eaacae9d54 100644 --- a/core/common/ThreadPool.h +++ b/core/common/ThreadPool.h @@ -15,13 +15,13 @@ */ #pragma once -#include -#include #include #include #include -#include +#include #include +#include +#include namespace logtail { diff --git a/core/common/TimeUtil.cpp b/core/common/TimeUtil.cpp index 7521486523..7dfed25428 100644 --- a/core/common/TimeUtil.cpp +++ b/core/common/TimeUtil.cpp @@ -14,11 +14,11 @@ #include "TimeUtil.h" +#include +#include #include #include -#include -#include #include #if defined(__linux__) #include diff --git a/core/common/TimeUtil.h b/core/common/TimeUtil.h index 57370b333e..5ca04da061 100644 --- a/core/common/TimeUtil.h +++ b/core/common/TimeUtil.h @@ -16,13 +16,14 @@ #pragma once #include + #include #include #include #include "common/Strptime.h" -#include "protobuf/sls/sls_logs.pb.h" #include "pipeline/PipelineContext.h" +#include "protobuf/sls/sls_logs.pb.h" // Time and timestamp utility. namespace logtail { diff --git a/core/common/WaitObject.h b/core/common/WaitObject.h index 0421979453..15dc1ff94f 100644 --- a/core/common/WaitObject.h +++ b/core/common/WaitObject.h @@ -15,10 +15,11 @@ */ #pragma once -#include -#include -#include #include +#include + +#include +#include namespace logtail { diff --git a/core/common/YamlUtil.cpp b/core/common/YamlUtil.cpp index 1cabe106dc..c7279e57cf 100644 --- a/core/common/YamlUtil.cpp +++ b/core/common/YamlUtil.cpp @@ -13,10 +13,12 @@ // limitations under the License. #include "common/YamlUtil.h" -#include "common/ExceptionBase.h" -#include + #include #include +#include + +#include "common/ExceptionBase.h" using namespace std; @@ -42,30 +44,30 @@ bool ParseYamlTable(const string& config, YAML::Node& yamlRoot, string& errorMsg return true; } -bool VisitNode(const YAML::Node &node, std::vector& visited) { +bool VisitNode(const YAML::Node& node, std::vector& visited) { visited.push_back(node); if (node.IsMap()) { - for (const auto &child : node) { + for (const auto& child : node) { if (std::find(visited.begin(), visited.end(), child.second) != visited.end()) { - return true; // Cycle detected + return true; // Cycle detected } if (VisitNode(child.second, visited)) { - return true; // Propagate the failure up the call stack + return true; // Propagate the failure up the call stack } } } else if (node.IsSequence()) { - for (const auto &child : node) { + for (const auto& child : node) { if (std::find(visited.begin(), visited.end(), child) != visited.end()) { - return true; // Cycle detected + return true; // Cycle detected } if (VisitNode(child, visited)) { - return true; // Propagate the failure up the call stack + return true; // Propagate the failure up the call stack } } } // If the node is a scalar, we don't need to do anything special. visited.pop_back(); - return false; // No cycle detected, continue recursion + return false; // No cycle detected, continue recursion } bool CheckYamlCycle(const YAML::Node& root) { diff --git a/core/common/YamlUtil.h b/core/common/YamlUtil.h index b75e12b480..9b12097559 100644 --- a/core/common/YamlUtil.h +++ b/core/common/YamlUtil.h @@ -16,7 +16,7 @@ #include -#include +#include "json/json.h" #include "yaml-cpp/yaml.h" namespace logtail { diff --git a/core/common/compression/Compressor.h b/core/common/compression/Compressor.h index 991893e69f..42591eb49b 100644 --- a/core/common/compression/Compressor.h +++ b/core/common/compression/Compressor.h @@ -18,8 +18,8 @@ #include -#include "monitor/MetricManager.h" #include "common/compression/CompressType.h" +#include "monitor/MetricManager.h" namespace logtail { @@ -35,7 +35,9 @@ class Compressor { virtual bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) = 0; #endif - CompressType GetCompressType() const { return mType; } + CompressType GetCompressType() const { + return mType; + } void SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); protected: diff --git a/core/common/compression/CompressorFactory.h b/core/common/compression/CompressorFactory.h index 8311332b7a..eb4ebb4c30 100644 --- a/core/common/compression/CompressorFactory.h +++ b/core/common/compression/CompressorFactory.h @@ -16,14 +16,14 @@ #pragma once -#include - #include #include -#include "pipeline/PipelineContext.h" +#include "json/json.h" + #include "common/compression/CompressType.h" #include "common/compression/Compressor.h" +#include "pipeline/PipelineContext.h" namespace logtail { diff --git a/core/common/compression/LZ4Compressor.cpp b/core/common/compression/LZ4Compressor.cpp index ec54c1c4e4..5ebe0651e7 100644 --- a/core/common/compression/LZ4Compressor.cpp +++ b/core/common/compression/LZ4Compressor.cpp @@ -14,7 +14,7 @@ #include "common/compression/LZ4Compressor.h" -#include +#include "lz4/lz4.h" #include "common/StringTools.h" diff --git a/core/common/compression/LZ4Compressor.h b/core/common/compression/LZ4Compressor.h index 9fd453c163..c903e870d5 100644 --- a/core/common/compression/LZ4Compressor.h +++ b/core/common/compression/LZ4Compressor.h @@ -22,7 +22,7 @@ namespace logtail { class LZ4Compressor : public Compressor { public: - LZ4Compressor(CompressType type) : Compressor(type) {}; + LZ4Compressor(CompressType type) : Compressor(type){}; #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; diff --git a/core/common/compression/ZstdCompressor.cpp b/core/common/compression/ZstdCompressor.cpp index c482c569a1..f033ea5e66 100644 --- a/core/common/compression/ZstdCompressor.cpp +++ b/core/common/compression/ZstdCompressor.cpp @@ -14,7 +14,7 @@ #include "common/compression/ZstdCompressor.h" -#include +#include "zstd/zstd.h" using namespace std; @@ -40,8 +40,7 @@ bool ZstdCompressor::Compress(const string& input, string& output, string& error #ifdef APSARA_UNIT_TEST_MAIN bool ZstdCompressor::UnCompress(const string& input, string& output, string& errorMsg) { try { - size_t length - = ZSTD_decompress(const_cast(output.c_str()), output.size(), input.c_str(), input.size()); + size_t length = ZSTD_decompress(const_cast(output.c_str()), output.size(), input.c_str(), input.size()); if (ZSTD_isError(length)) { errorMsg = ZSTD_getErrorName(length); return false; diff --git a/core/common/compression/ZstdCompressor.h b/core/common/compression/ZstdCompressor.h index bf53c7d96d..a9bd0f2a59 100644 --- a/core/common/compression/ZstdCompressor.h +++ b/core/common/compression/ZstdCompressor.h @@ -22,7 +22,7 @@ namespace logtail { class ZstdCompressor : public Compressor { public: - ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level) {}; + ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level){}; #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; diff --git a/core/common/http/AsynCurlRunner.cpp b/core/common/http/AsynCurlRunner.cpp index 6ab352d997..82f696a80c 100644 --- a/core/common/http/AsynCurlRunner.cpp +++ b/core/common/http/AsynCurlRunner.cpp @@ -99,7 +99,9 @@ void AsynCurlRunner::DoRun() { } } - struct timeval timeout{1, 0}; + struct timeval timeout { + 1, 0 + }; long curlTimeout = -1; if ((mc = curl_multi_timeout(mClient, &curlTimeout)) != CURLM_OK) { LOG_WARNING( diff --git a/core/common/http/AsynCurlRunner.h b/core/common/http/AsynCurlRunner.h index 16a3dc3f93..67601f53a7 100644 --- a/core/common/http/AsynCurlRunner.h +++ b/core/common/http/AsynCurlRunner.h @@ -16,14 +16,14 @@ #pragma once -#include - #include #include #include #include #include +#include "curl/multi.h" + #include "common/SafeQueue.h" #include "common/http/HttpRequest.h" diff --git a/core/common/http/Constant.cpp b/core/common/http/Constant.cpp index 0a1421182c..e94de363af 100644 --- a/core/common/http/Constant.cpp +++ b/core/common/http/Constant.cpp @@ -17,7 +17,7 @@ using namespace std; namespace logtail { - + const string HTTP_POST = "POST"; const string HTTP_GET = "GET"; diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index 075ae6629d..2a2c3f9ba8 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -15,6 +15,7 @@ #include "common/http/Curl.h" #include + #include #include diff --git a/core/common/http/Curl.h b/core/common/http/Curl.h index ad21f876e8..97dc37d444 100644 --- a/core/common/http/Curl.h +++ b/core/common/http/Curl.h @@ -16,13 +16,14 @@ #pragma once -#include - #include + #include #include #include +#include "curl/curl.h" + #include "common/http/HttpRequest.h" #include "common/http/HttpResponse.h" diff --git a/core/common/http/HttpRequest.h b/core/common/http/HttpRequest.h index 2218568344..16aa3db7fa 100644 --- a/core/common/http/HttpRequest.h +++ b/core/common/http/HttpRequest.h @@ -18,6 +18,7 @@ #include #include + #include #include #include diff --git a/core/common/http/HttpResponse.h b/core/common/http/HttpResponse.h index 88bdaab1ab..2802b4a7a5 100644 --- a/core/common/http/HttpResponse.h +++ b/core/common/http/HttpResponse.h @@ -18,6 +18,7 @@ #include #include + #include #include #include @@ -110,7 +111,9 @@ class HttpResponse { *mBody = body; } - void AddHeader(const std::string& key, const std::string& value) { mHeader[key] = value; } + void AddHeader(const std::string& key, const std::string& value) { + mHeader[key] = value; + } #endif private: diff --git a/core/config/ConfigUtil.h b/core/config/ConfigUtil.h index c5fd7d0ae2..763d2cffa5 100644 --- a/core/config/ConfigUtil.h +++ b/core/config/ConfigUtil.h @@ -16,11 +16,11 @@ #pragma once -#include - #include #include +#include "json/json.h" + namespace logtail { enum class ConfigType { Pipeline, Task }; diff --git a/core/config/InstanceConfig.h b/core/config/InstanceConfig.h index e0ee08ca13..812c77e22b 100644 --- a/core/config/InstanceConfig.h +++ b/core/config/InstanceConfig.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + namespace logtail { struct InstanceConfig { diff --git a/core/config/InstanceConfigManager.cpp b/core/config/InstanceConfigManager.cpp index 24673cea85..e1b1b03aca 100644 --- a/core/config/InstanceConfigManager.cpp +++ b/core/config/InstanceConfigManager.cpp @@ -27,14 +27,16 @@ InstanceConfigManager::InstanceConfigManager() = default; void InstanceConfigManager::UpdateInstanceConfigs(InstanceConfigDiff& diff) { for (auto& config : diff.mAdded) { - mInstanceConfigMap[config.mConfigName] = - std::make_shared(config.mConfigName, std::move(config.mDetail), config.mDirName); - ConfigFeedbackReceiver::GetInstance().FeedbackInstanceConfigStatus(config.mConfigName, ConfigFeedbackStatus::APPLIED); + mInstanceConfigMap[config.mConfigName] + = std::make_shared(config.mConfigName, std::move(config.mDetail), config.mDirName); + ConfigFeedbackReceiver::GetInstance().FeedbackInstanceConfigStatus(config.mConfigName, + ConfigFeedbackStatus::APPLIED); } for (auto& config : diff.mModified) { - mInstanceConfigMap[config.mConfigName] = - std::make_shared(config.mConfigName, std::move(config.mDetail), config.mDirName); - ConfigFeedbackReceiver::GetInstance().FeedbackInstanceConfigStatus(config.mConfigName, ConfigFeedbackStatus::APPLIED); + mInstanceConfigMap[config.mConfigName] + = std::make_shared(config.mConfigName, std::move(config.mDetail), config.mDirName); + ConfigFeedbackReceiver::GetInstance().FeedbackInstanceConfigStatus(config.mConfigName, + ConfigFeedbackStatus::APPLIED); } for (const auto& configName : diff.mRemoved) { mInstanceConfigMap.erase(configName); diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index 6cafcb6ed4..aadf9795b0 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -16,6 +16,8 @@ #include +#include "boost/regex.hpp" + #include "app_config/AppConfig.h" #include "common/Flags.h" #include "common/ParamExtractor.h" diff --git a/core/config/PipelineConfig.h b/core/config/PipelineConfig.h index 6a9219cdcd..de5ea1ac65 100644 --- a/core/config/PipelineConfig.h +++ b/core/config/PipelineConfig.h @@ -16,14 +16,15 @@ #pragma once -#include - #include + #include #include #include #include +#include "json/json.h" + namespace logtail { struct PipelineConfig { diff --git a/core/config/TaskConfig.h b/core/config/TaskConfig.h index 5fafc176ef..14de23c3d9 100644 --- a/core/config/TaskConfig.h +++ b/core/config/TaskConfig.h @@ -16,12 +16,13 @@ #pragma once -#include - #include + #include #include +#include "json/json.h" + namespace logtail { struct TaskConfig { diff --git a/core/config/common_provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp index 2e278c6ecf..c254ca38db 100644 --- a/core/config/common_provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -14,12 +14,12 @@ #include "CommonConfigProvider.h" -#include - #include #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "application/Application.h" #include "common/LogtailCommonFlags.h" diff --git a/core/config/common_provider/CommonConfigProvider.h b/core/config/common_provider/CommonConfigProvider.h index f7a4e918a6..45699929f6 100644 --- a/core/config/common_provider/CommonConfigProvider.h +++ b/core/config/common_provider/CommonConfigProvider.h @@ -16,8 +16,9 @@ #pragma once -#include #include + +#include #include #include #include diff --git a/core/config/common_provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp index 9231cee6db..143f0b6e0b 100644 --- a/core/config/common_provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -14,12 +14,12 @@ #include "LegacyCommonConfigProvider.h" -#include - #include #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "application/Application.h" #include "common/EncodingUtil.h" diff --git a/core/config/common_provider/LegacyCommonConfigProvider.h b/core/config/common_provider/LegacyCommonConfigProvider.h index 7e0498c9a5..44809daf8a 100644 --- a/core/config/common_provider/LegacyCommonConfigProvider.h +++ b/core/config/common_provider/LegacyCommonConfigProvider.h @@ -16,8 +16,9 @@ #pragma once -#include #include + +#include #include #include #include diff --git a/core/config/feedbacker/ConfigFeedbackable.h b/core/config/feedbacker/ConfigFeedbackable.h index 29126c9d33..58bdfea659 100644 --- a/core/config/feedbacker/ConfigFeedbackable.h +++ b/core/config/feedbacker/ConfigFeedbackable.h @@ -17,6 +17,7 @@ #pragma once #include + #include namespace logtail { diff --git a/core/config/watcher/InstanceConfigWatcher.h b/core/config/watcher/InstanceConfigWatcher.h index c38abec9fb..6fc6820e48 100644 --- a/core/config/watcher/InstanceConfigWatcher.h +++ b/core/config/watcher/InstanceConfigWatcher.h @@ -36,7 +36,9 @@ class InstanceConfigWatcher : public ConfigWatcher { InstanceConfigDiff CheckConfigDiff(); #ifdef APSARA_UNIT_TEST_MAIN - void SetInstanceConfigManager(const InstanceConfigManager* m) { mInstanceConfigManager = m; } + void SetInstanceConfigManager(const InstanceConfigManager* m) { + mInstanceConfigManager = m; + } #endif private: diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 0cd8c00570..35c2ad7199 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -19,13 +19,13 @@ #include "common/FileSystemUtil.h" #include "config/ConfigUtil.h" #include "config/common_provider/CommonConfigProvider.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "logger/Logger.h" #include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" #include "task_pipeline/TaskPipelineManager.h" +#ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" +#endif using namespace std; diff --git a/core/config/watcher/PipelineConfigWatcher.h b/core/config/watcher/PipelineConfigWatcher.h index 766b55c512..cc7e1cf39d 100644 --- a/core/config/watcher/PipelineConfigWatcher.h +++ b/core/config/watcher/PipelineConfigWatcher.h @@ -50,7 +50,9 @@ class PipelineConfigWatcher : public ConfigWatcher { std::pair CheckConfigDiff(); #ifdef APSARA_UNIT_TEST_MAIN - void SetPipelineManager(const PipelineManager* pm) { mPipelineManager = pm; } + void SetPipelineManager(const PipelineManager* pm) { + mPipelineManager = pm; + } #endif private: diff --git a/core/constants/SpanConstants.cpp b/core/constants/SpanConstants.cpp index 262c4f15b2..3ebf741c31 100644 --- a/core/constants/SpanConstants.cpp +++ b/core/constants/SpanConstants.cpp @@ -16,28 +16,28 @@ namespace logtail { - const std::string DEFAULT_TRACE_TAG_TRACE_ID = "traceId"; - const std::string DEFAULT_TRACE_TAG_SPAN_ID = "spanId"; - const std::string DEFAULT_TRACE_TAG_PARENT_ID = "parentSpanId"; - const std::string DEFAULT_TRACE_TAG_SPAN_NAME = "spanName"; - const std::string DEFAULT_TRACE_TAG_SERVICE_NAME = "serviceName"; - const std::string DEFAULT_TRACE_TAG_START_TIME_NANO = "startTime"; - const std::string DEFAULT_TRACE_TAG_END_TIME_NANO = "endTime"; - const std::string DEFAULT_TRACE_TAG_DURATION = "duration"; - const std::string DEFAULT_TRACE_TAG_ATTRIBUTES = "attributes"; - const std::string DEFAULT_TRACE_TAG_RESOURCE = "resources"; - const std::string DEFAULT_TRACE_TAG_LINKS = "links"; - const std::string DEFAULT_TRACE_TAG_EVENTS = "events"; - const std::string DEFAULT_TRACE_TAG_TIMESTAMP = "timestamp"; - const std::string DEFAULT_TRACE_TAG_STATUS_CODE = "statusCode"; - const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE = "statusMessage"; - const std::string DEFAULT_TRACE_TAG_SPAN_KIND = "kind"; - const std::string DEFAULT_TRACE_TAG_TRACE_STATE = "traceState"; - const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME = "name"; +const std::string DEFAULT_TRACE_TAG_TRACE_ID = "traceId"; +const std::string DEFAULT_TRACE_TAG_SPAN_ID = "spanId"; +const std::string DEFAULT_TRACE_TAG_PARENT_ID = "parentSpanId"; +const std::string DEFAULT_TRACE_TAG_SPAN_NAME = "spanName"; +const std::string DEFAULT_TRACE_TAG_SERVICE_NAME = "serviceName"; +const std::string DEFAULT_TRACE_TAG_START_TIME_NANO = "startTime"; +const std::string DEFAULT_TRACE_TAG_END_TIME_NANO = "endTime"; +const std::string DEFAULT_TRACE_TAG_DURATION = "duration"; +const std::string DEFAULT_TRACE_TAG_ATTRIBUTES = "attributes"; +const std::string DEFAULT_TRACE_TAG_RESOURCE = "resources"; +const std::string DEFAULT_TRACE_TAG_LINKS = "links"; +const std::string DEFAULT_TRACE_TAG_EVENTS = "events"; +const std::string DEFAULT_TRACE_TAG_TIMESTAMP = "timestamp"; +const std::string DEFAULT_TRACE_TAG_STATUS_CODE = "statusCode"; +const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE = "statusMessage"; +const std::string DEFAULT_TRACE_TAG_SPAN_KIND = "kind"; +const std::string DEFAULT_TRACE_TAG_TRACE_STATE = "traceState"; +const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME = "name"; #ifdef __ENTERPRISE__ - // for arms - const std::string DEFAULT_TRACE_TAG_APP_ID = "pid"; - const std::string DEFAULT_TRACE_TAG_IP = "ip"; +// for arms +const std::string DEFAULT_TRACE_TAG_APP_ID = "pid"; +const std::string DEFAULT_TRACE_TAG_IP = "ip"; #endif } // namespace logtail \ No newline at end of file diff --git a/core/constants/SpanConstants.h b/core/constants/SpanConstants.h index 36a9d2f030..501de68b0a 100644 --- a/core/constants/SpanConstants.h +++ b/core/constants/SpanConstants.h @@ -18,28 +18,28 @@ #include namespace logtail { - extern const std::string DEFAULT_TRACE_TAG_TRACE_ID; - extern const std::string DEFAULT_TRACE_TAG_SPAN_ID; - extern const std::string DEFAULT_TRACE_TAG_PARENT_ID; - extern const std::string DEFAULT_TRACE_TAG_SPAN_NAME; - extern const std::string DEFAULT_TRACE_TAG_SERVICE_NAME; - extern const std::string DEFAULT_TRACE_TAG_START_TIME_NANO; - extern const std::string DEFAULT_TRACE_TAG_END_TIME_NANO; - extern const std::string DEFAULT_TRACE_TAG_DURATION; - extern const std::string DEFAULT_TRACE_TAG_ATTRIBUTES; - extern const std::string DEFAULT_TRACE_TAG_RESOURCE; - extern const std::string DEFAULT_TRACE_TAG_LINKS; - extern const std::string DEFAULT_TRACE_TAG_EVENTS; - extern const std::string DEFAULT_TRACE_TAG_TIMESTAMP; - extern const std::string DEFAULT_TRACE_TAG_STATUS_CODE; - extern const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE; - extern const std::string DEFAULT_TRACE_TAG_SPAN_KIND; - extern const std::string DEFAULT_TRACE_TAG_TRACE_STATE; - extern const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME ; +extern const std::string DEFAULT_TRACE_TAG_TRACE_ID; +extern const std::string DEFAULT_TRACE_TAG_SPAN_ID; +extern const std::string DEFAULT_TRACE_TAG_PARENT_ID; +extern const std::string DEFAULT_TRACE_TAG_SPAN_NAME; +extern const std::string DEFAULT_TRACE_TAG_SERVICE_NAME; +extern const std::string DEFAULT_TRACE_TAG_START_TIME_NANO; +extern const std::string DEFAULT_TRACE_TAG_END_TIME_NANO; +extern const std::string DEFAULT_TRACE_TAG_DURATION; +extern const std::string DEFAULT_TRACE_TAG_ATTRIBUTES; +extern const std::string DEFAULT_TRACE_TAG_RESOURCE; +extern const std::string DEFAULT_TRACE_TAG_LINKS; +extern const std::string DEFAULT_TRACE_TAG_EVENTS; +extern const std::string DEFAULT_TRACE_TAG_TIMESTAMP; +extern const std::string DEFAULT_TRACE_TAG_STATUS_CODE; +extern const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE; +extern const std::string DEFAULT_TRACE_TAG_SPAN_KIND; +extern const std::string DEFAULT_TRACE_TAG_TRACE_STATE; +extern const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME; #ifdef __ENTERPRISE__ - // for arms - extern const std::string DEFAULT_TRACE_TAG_APP_ID; - extern const std::string DEFAULT_TRACE_TAG_IP; +// for arms +extern const std::string DEFAULT_TRACE_TAG_APP_ID; +extern const std::string DEFAULT_TRACE_TAG_IP; #endif } // namespace logtail \ No newline at end of file diff --git a/core/constants/TagConstants.cpp b/core/constants/TagConstants.cpp index 4d4a865192..6dc02c71e6 100644 --- a/core/constants/TagConstants.cpp +++ b/core/constants/TagConstants.cpp @@ -17,51 +17,51 @@ namespace logtail { ////////////////////////// COMMON //////////////////////// - const std::string DEFAULT_TAG_NAMESPACE = "namespace"; - const std::string DEFAULT_TAG_HOST_NAME = "host_name"; - const std::string DEFAULT_TAG_HOST_IP = "host_ip"; - const std::string DEFAULT_TAG_POD_NAME = "pod_name"; - const std::string DEFAULT_TAG_POD_UID = "pod_uid"; - const std::string DEFAULT_TAG_CONTAINER_NAME = "container_name"; - const std::string DEFAULT_TAG_CONTAINER_IP = "container_ip"; - const std::string DEFAULT_TAG_IMAGE_NAME = "image_name"; +const std::string DEFAULT_TAG_NAMESPACE = "namespace"; +const std::string DEFAULT_TAG_HOST_NAME = "host_name"; +const std::string DEFAULT_TAG_HOST_IP = "host_ip"; +const std::string DEFAULT_TAG_POD_NAME = "pod_name"; +const std::string DEFAULT_TAG_POD_UID = "pod_uid"; +const std::string DEFAULT_TAG_CONTAINER_NAME = "container_name"; +const std::string DEFAULT_TAG_CONTAINER_IP = "container_ip"; +const std::string DEFAULT_TAG_IMAGE_NAME = "image_name"; ////////////////////////// LOG //////////////////////// #ifndef __ENTERPRISE__ // 开源版 - const std::string DEFAULT_LOG_TAG_HOST_NAME = DEFAULT_TAG_HOST_NAME; - const std::string DEFAULT_LOG_TAG_NAMESPACE = DEFAULT_TAG_NAMESPACE; - const std::string DEFAULT_LOG_TAG_POD_NAME = DEFAULT_TAG_POD_NAME; - const std::string DEFAULT_LOG_TAG_POD_UID = DEFAULT_TAG_POD_UID; - const std::string DEFAULT_LOG_TAG_CONTAINER_NAME = DEFAULT_TAG_CONTAINER_NAME; - const std::string DEFAULT_LOG_TAG_CONTAINER_IP = DEFAULT_TAG_CONTAINER_IP; - const std::string DEFAULT_LOG_TAG_IMAGE_NAME = DEFAULT_TAG_IMAGE_NAME; - const std::string DEFAULT_LOG_TAG_FILE_OFFSET = "file_offset"; - const std::string DEFAULT_LOG_TAG_FILE_INODE = "file_inode"; - const std::string DEFAULT_LOG_TAG_FILE_PATH = "file_path"; +const std::string DEFAULT_LOG_TAG_HOST_NAME = DEFAULT_TAG_HOST_NAME; +const std::string DEFAULT_LOG_TAG_NAMESPACE = DEFAULT_TAG_NAMESPACE; +const std::string DEFAULT_LOG_TAG_POD_NAME = DEFAULT_TAG_POD_NAME; +const std::string DEFAULT_LOG_TAG_POD_UID = DEFAULT_TAG_POD_UID; +const std::string DEFAULT_LOG_TAG_CONTAINER_NAME = DEFAULT_TAG_CONTAINER_NAME; +const std::string DEFAULT_LOG_TAG_CONTAINER_IP = DEFAULT_TAG_CONTAINER_IP; +const std::string DEFAULT_LOG_TAG_IMAGE_NAME = DEFAULT_TAG_IMAGE_NAME; +const std::string DEFAULT_LOG_TAG_FILE_OFFSET = "file_offset"; +const std::string DEFAULT_LOG_TAG_FILE_INODE = "file_inode"; +const std::string DEFAULT_LOG_TAG_FILE_PATH = "file_path"; - const std::string DEFAULT_LOG_TAG_HOST_IP = DEFAULT_TAG_HOST_IP; +const std::string DEFAULT_LOG_TAG_HOST_IP = DEFAULT_TAG_HOST_IP; #else - const std::string DEFAULT_LOG_TAG_HOST_NAME = "__hostname__"; - const std::string DEFAULT_LOG_TAG_NAMESPACE = "_namespace_"; - const std::string DEFAULT_LOG_TAG_POD_NAME = "_pod_name_"; - const std::string DEFAULT_LOG_TAG_POD_UID = "_pod_uid_"; - const std::string DEFAULT_LOG_TAG_CONTAINER_NAME = "_container_name_"; - const std::string DEFAULT_LOG_TAG_CONTAINER_IP = "_container_ip_"; - const std::string DEFAULT_LOG_TAG_IMAGE_NAME = "_image_name_"; - const std::string DEFAULT_LOG_TAG_FILE_OFFSET = "__file_offset__"; - const std::string DEFAULT_LOG_TAG_FILE_INODE = "__inode__"; - const std::string DEFAULT_LOG_TAG_FILE_PATH = "__path__"; - - const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID = "__user_defined_id__"; +const std::string DEFAULT_LOG_TAG_HOST_NAME = "__hostname__"; +const std::string DEFAULT_LOG_TAG_NAMESPACE = "_namespace_"; +const std::string DEFAULT_LOG_TAG_POD_NAME = "_pod_name_"; +const std::string DEFAULT_LOG_TAG_POD_UID = "_pod_uid_"; +const std::string DEFAULT_LOG_TAG_CONTAINER_NAME = "_container_name_"; +const std::string DEFAULT_LOG_TAG_CONTAINER_IP = "_container_ip_"; +const std::string DEFAULT_LOG_TAG_IMAGE_NAME = "_image_name_"; +const std::string DEFAULT_LOG_TAG_FILE_OFFSET = "__file_offset__"; +const std::string DEFAULT_LOG_TAG_FILE_INODE = "__inode__"; +const std::string DEFAULT_LOG_TAG_FILE_PATH = "__path__"; + +const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID = "__user_defined_id__"; #endif ////////////////////////// METRIC //////////////////////// - const std::string DEFAULT_METRIC_TAG_NAMESPACE = DEFAULT_TAG_NAMESPACE; - const std::string DEFAULT_METRIC_TAG_POD_NAME = DEFAULT_TAG_POD_NAME; - const std::string DEFAULT_METRIC_TAG_POD_UID = DEFAULT_TAG_POD_UID; - const std::string DEFAULT_METRIC_TAG_CONTAINER_NAME = DEFAULT_TAG_CONTAINER_NAME; - const std::string DEFAULT_METRIC_TAG_CONTAINER_IP = DEFAULT_TAG_CONTAINER_IP; - const std::string DEFAULT_METRIC_TAG_IMAGE_NAME = DEFAULT_TAG_IMAGE_NAME; +const std::string DEFAULT_METRIC_TAG_NAMESPACE = DEFAULT_TAG_NAMESPACE; +const std::string DEFAULT_METRIC_TAG_POD_NAME = DEFAULT_TAG_POD_NAME; +const std::string DEFAULT_METRIC_TAG_POD_UID = DEFAULT_TAG_POD_UID; +const std::string DEFAULT_METRIC_TAG_CONTAINER_NAME = DEFAULT_TAG_CONTAINER_NAME; +const std::string DEFAULT_METRIC_TAG_CONTAINER_IP = DEFAULT_TAG_CONTAINER_IP; +const std::string DEFAULT_METRIC_TAG_IMAGE_NAME = DEFAULT_TAG_IMAGE_NAME; ////////////////////////// TRACE //////////////////////// diff --git a/core/constants/TagConstants.h b/core/constants/TagConstants.h index 0b83d4092d..aae11b481d 100644 --- a/core/constants/TagConstants.h +++ b/core/constants/TagConstants.h @@ -20,32 +20,31 @@ namespace logtail { ////////////////////////// LOG //////////////////////// - extern const std::string DEFAULT_LOG_TAG_HOST_NAME; - extern const std::string DEFAULT_LOG_TAG_NAMESPACE; - extern const std::string DEFAULT_LOG_TAG_POD_NAME; - extern const std::string DEFAULT_LOG_TAG_POD_UID; - extern const std::string DEFAULT_LOG_TAG_CONTAINER_NAME; - extern const std::string DEFAULT_LOG_TAG_CONTAINER_IP; - extern const std::string DEFAULT_LOG_TAG_IMAGE_NAME; - extern const std::string DEFAULT_LOG_TAG_FILE_OFFSET; - extern const std::string DEFAULT_LOG_TAG_FILE_INODE; - extern const std::string DEFAULT_LOG_TAG_FILE_PATH; +extern const std::string DEFAULT_LOG_TAG_HOST_NAME; +extern const std::string DEFAULT_LOG_TAG_NAMESPACE; +extern const std::string DEFAULT_LOG_TAG_POD_NAME; +extern const std::string DEFAULT_LOG_TAG_POD_UID; +extern const std::string DEFAULT_LOG_TAG_CONTAINER_NAME; +extern const std::string DEFAULT_LOG_TAG_CONTAINER_IP; +extern const std::string DEFAULT_LOG_TAG_IMAGE_NAME; +extern const std::string DEFAULT_LOG_TAG_FILE_OFFSET; +extern const std::string DEFAULT_LOG_TAG_FILE_INODE; +extern const std::string DEFAULT_LOG_TAG_FILE_PATH; #ifndef __ENTERPRISE__ - extern const std::string DEFAULT_LOG_TAG_HOST_IP; +extern const std::string DEFAULT_LOG_TAG_HOST_IP; #else - extern const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID; +extern const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID; #endif ////////////////////////// METRIC //////////////////////// - extern const std::string DEFAULT_METRIC_TAG_NAMESPACE; - extern const std::string DEFAULT_METRIC_TAG_POD_NAME; - extern const std::string DEFAULT_METRIC_TAG_POD_UID; - extern const std::string DEFAULT_METRIC_TAG_CONTAINER_NAME; - extern const std::string DEFAULT_METRIC_TAG_CONTAINER_IP; - extern const std::string DEFAULT_METRIC_TAG_IMAGE_NAME; +extern const std::string DEFAULT_METRIC_TAG_NAMESPACE; +extern const std::string DEFAULT_METRIC_TAG_POD_NAME; +extern const std::string DEFAULT_METRIC_TAG_POD_UID; +extern const std::string DEFAULT_METRIC_TAG_CONTAINER_NAME; +extern const std::string DEFAULT_METRIC_TAG_CONTAINER_IP; +extern const std::string DEFAULT_METRIC_TAG_IMAGE_NAME; ////////////////////////// TRACE //////////////////////// - } // namespace logtail \ No newline at end of file diff --git a/core/container_manager/ConfigContainerInfoUpdateCmd.h b/core/container_manager/ConfigContainerInfoUpdateCmd.h index 3136bfe7e5..9aaff43674 100644 --- a/core/container_manager/ConfigContainerInfoUpdateCmd.h +++ b/core/container_manager/ConfigContainerInfoUpdateCmd.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + namespace logtail { struct ConfigContainerInfoUpdateCmd { diff --git a/core/container_manager/ContainerDiscoveryOptions.cpp b/core/container_manager/ContainerDiscoveryOptions.cpp index ce59ef41f5..8350df13eb 100644 --- a/core/container_manager/ContainerDiscoveryOptions.cpp +++ b/core/container_manager/ContainerDiscoveryOptions.cpp @@ -198,9 +198,8 @@ bool ContainerDiscoveryOptions::Init(const Json::Value& config, const PipelineCo return true; } -void ContainerDiscoveryOptions::GenerateContainerMetaFetchingGoPipeline(Json::Value& res, - const FileDiscoveryOptions* fileDiscovery, - const PluginInstance::PluginMeta& pluginMeta) const { +void ContainerDiscoveryOptions::GenerateContainerMetaFetchingGoPipeline( + Json::Value& res, const FileDiscoveryOptions* fileDiscovery, const PluginInstance::PluginMeta& pluginMeta) const { Json::Value plugin(Json::objectValue); Json::Value detail(Json::objectValue); Json::Value object(Json::objectValue); diff --git a/core/container_manager/ContainerDiscoveryOptions.h b/core/container_manager/ContainerDiscoveryOptions.h index 2a106c819f..bc7c48008d 100644 --- a/core/container_manager/ContainerDiscoveryOptions.h +++ b/core/container_manager/ContainerDiscoveryOptions.h @@ -16,12 +16,12 @@ #pragma once -#include - #include #include #include +#include "json/json.h" + #include "file_server/FileDiscoveryOptions.h" #include "pipeline/PipelineContext.h" #include "pipeline/plugin/instance/PluginInstance.h" diff --git a/core/ebpf/Config.h b/core/ebpf/Config.h index d760b96637..64031d9801 100644 --- a/core/ebpf/Config.h +++ b/core/ebpf/Config.h @@ -14,12 +14,12 @@ #pragma once -#include - #include #include #include +#include "json/json.h" + #include "ebpf/include/export.h" #include "pipeline/PipelineContext.h" diff --git a/core/ebpf/SourceManager.cpp b/core/ebpf/SourceManager.cpp index bec73cffeb..bde6536770 100644 --- a/core/ebpf/SourceManager.cpp +++ b/core/ebpf/SourceManager.cpp @@ -14,13 +14,12 @@ #include "ebpf/SourceManager.h" -#include - #include #include #include #include "app_config/AppConfig.h" +#include "common/Flags.h" #include "common/LogtailCommonFlags.h" #include "common/MachineInfoUtil.h" #include "common/RuntimeUtil.h" diff --git a/core/ebpf/SourceManager.h b/core/ebpf/SourceManager.h index d659f165f9..d57f7a5d16 100644 --- a/core/ebpf/SourceManager.h +++ b/core/ebpf/SourceManager.h @@ -14,12 +14,12 @@ #pragma once +#include +#include #include #include #include -#include -#include #include #include #include diff --git a/core/ebpf/eBPFServer.cpp b/core/ebpf/eBPFServer.cpp index 4106c80271..fa5a5440f4 100644 --- a/core/ebpf/eBPFServer.cpp +++ b/core/ebpf/eBPFServer.cpp @@ -14,14 +14,13 @@ #include "ebpf/eBPFServer.h" -#include - #include #include #include #include #include "app_config/AppConfig.h" +#include "common/Flags.h" #include "common/Lock.h" #include "common/LogtailCommonFlags.h" #include "common/MachineInfoUtil.h" diff --git a/core/ebpf/include/export.h b/core/ebpf/include/export.h index 96b46ed4fd..1d52cc790e 100644 --- a/core/ebpf/include/export.h +++ b/core/ebpf/include/export.h @@ -136,8 +136,8 @@ struct ApplicationBatchSpan { class SingleEvent { public: explicit __attribute__((visibility("default"))) SingleEvent() {} - explicit __attribute__((visibility("default"))) SingleEvent(std::vector>&& tags, - uint64_t ts) + explicit __attribute__((visibility("default"))) + SingleEvent(std::vector>&& tags, uint64_t ts) : tags_(tags), timestamp_(ts) {} std::vector> GetAllTags() { return tags_; } uint64_t GetTimestamp() { return timestamp_; } diff --git a/core/file_server/AdhocFileManager.cpp b/core/file_server/AdhocFileManager.cpp index 5328c89de2..46ee4783d3 100644 --- a/core/file_server/AdhocFileManager.cpp +++ b/core/file_server/AdhocFileManager.cpp @@ -15,12 +15,13 @@ */ #include "AdhocFileManager.h" + #include "logger/Logger.h" namespace logtail { -AdhocFileManager::AdhocFileManager() { - // mRunFlag = false; +AdhocFileManager::AdhocFileManager(){ + // mRunFlag = false; }; void AdhocFileManager::Run() { @@ -125,8 +126,8 @@ void AdhocFileManager::AddJob(std::string jobName, std::vector fileL // // push first file to queue // if (fileList.size() > 0) { - // Event* ev = new Event(fileList[0].filePath, fileList[0].fileName, EVENT_STATIC_FILE, -1, 0, fileList[0].Dev, fileList[0].Inode); - // PushEventQueue(ev); + // Event* ev = new Event(fileList[0].filePath, fileList[0].fileName, EVENT_STATIC_FILE, -1, 0, fileList[0].Dev, + // fileList[0].Inode); PushEventQueue(ev); // } // Run(); @@ -139,4 +140,4 @@ void AdhocFileManager::DeleteJob(std::string jobName) { // PushEventQueue(stopEvent); } -} +} // namespace logtail diff --git a/core/file_server/AdhocFileManager.h b/core/file_server/AdhocFileManager.h index 5e4ec943df..4cbfeffb6a 100644 --- a/core/file_server/AdhocFileManager.h +++ b/core/file_server/AdhocFileManager.h @@ -17,6 +17,7 @@ #pragma once #include #include + #include "checkpoint/AdhocCheckpointManager.h" #include "file_server/event/Event.h" @@ -45,6 +46,7 @@ class AdhocFileManager { std::queue mEventQueue; std::unordered_set mDeletedJobSet; std::unordered_map > mJobFileLists; + public: static AdhocFileManager* GetInstance() { static AdhocFileManager* ptr = new AdhocFileManager(); @@ -56,4 +58,4 @@ class AdhocFileManager { void DeleteJob(std::string jobName); }; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/file_server/ConfigManager.cpp b/core/file_server/ConfigManager.cpp index 1c702aac7b..90a964c343 100644 --- a/core/file_server/ConfigManager.cpp +++ b/core/file_server/ConfigManager.cpp @@ -14,18 +14,15 @@ #include "ConfigManager.h" -#include +#include #include #include #include - -#include #if defined(__linux__) #include #include #endif #include -#include #include #include diff --git a/core/file_server/ConfigManager.h b/core/file_server/ConfigManager.h index c9f718e6c0..cea0ded71e 100644 --- a/core/file_server/ConfigManager.h +++ b/core/file_server/ConfigManager.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include @@ -24,8 +25,8 @@ #include "common/Lock.h" #include "container_manager/ConfigContainerInfoUpdateCmd.h" -#include "file_server/event/Event.h" #include "file_server/FileDiscoveryOptions.h" +#include "file_server/event/Event.h" namespace logtail { diff --git a/core/file_server/ContainerInfo.h b/core/file_server/ContainerInfo.h index 0f08db7999..7465f888ca 100644 --- a/core/file_server/ContainerInfo.h +++ b/core/file_server/ContainerInfo.h @@ -16,13 +16,14 @@ #pragma once -#include - #include + #include #include #include +#include "json/json.h" + #include "container_manager/ConfigContainerInfoUpdateCmd.h" #include "protobuf/sls/sls_logs.pb.h" diff --git a/core/file_server/FileDiscoveryOptions.h b/core/file_server/FileDiscoveryOptions.h index ffe17b6e19..06747e6ec3 100644 --- a/core/file_server/FileDiscoveryOptions.h +++ b/core/file_server/FileDiscoveryOptions.h @@ -16,14 +16,15 @@ #pragma once -#include - #include + #include #include #include #include +#include "json/json.h" + #include "file_server/ContainerInfo.h" #include "pipeline/PipelineContext.h" diff --git a/core/file_server/MultilineOptions.h b/core/file_server/MultilineOptions.h index 1c5a39cd4c..274bc16372 100644 --- a/core/file_server/MultilineOptions.h +++ b/core/file_server/MultilineOptions.h @@ -16,12 +16,12 @@ #pragma once -#include - #include #include #include "boost/regex.hpp" +#include "json/json.h" + #include "pipeline/PipelineContext.h" namespace logtail { diff --git a/core/file_server/event/BlockEventManager.h b/core/file_server/event/BlockEventManager.h index f3f8546dc1..658926a9a3 100644 --- a/core/file_server/event/BlockEventManager.h +++ b/core/file_server/event/BlockEventManager.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include diff --git a/core/file_server/event/Event.h b/core/file_server/event/Event.h index cea84b5a4a..80413e1d30 100644 --- a/core/file_server/event/Event.h +++ b/core/file_server/event/Event.h @@ -16,7 +16,9 @@ #pragma once #include + #include + #include "common/DevInode.h" namespace logtail { @@ -97,7 +99,7 @@ class Event { uint64_t GetDev() const { return mDev; } uint64_t GetInode() const { return mInode; } - + int GetWd() const { return mWd; } const uint32_t GetCookie() const { return mCookie; } @@ -110,7 +112,7 @@ class Event { int64_t GetLastFilePos() const { return mLastFilePos; } - void SetSource(const std::string& source) { mSource = source; } + void SetSource(const std::string& source) { mSource = source; } void SetDev(uint64_t dev) { mDev = dev; } @@ -123,7 +125,7 @@ class Event { void SetLastReadPos(int64_t lastReadPos) { mLastReadPos = lastReadPos; } void SetLastFilePos(int64_t lastFilePos) { mLastFilePos = lastFilePos; } - + bool IsCreate() const { return mType & EVENT_CREATE; } bool IsModify() const { return mType & EVENT_MODIFY; } diff --git a/core/file_server/event/EventQueue.cpp b/core/file_server/event/EventQueue.cpp index 4e46e7b3ef..9fc7bce1b7 100644 --- a/core/file_server/event/EventQueue.cpp +++ b/core/file_server/event/EventQueue.cpp @@ -13,10 +13,11 @@ // limitations under the License. #include "EventQueue.h" -#include "file_server/event/Event.h" -#include "common/StringTools.h" + #include "common/Flags.h" +#include "common/StringTools.h" #include "common/TimeUtil.h" +#include "file_server/event/Event.h" #include "logger/Logger.h" DECLARE_FLAG_INT32(max_polling_event_queue_size); diff --git a/core/file_server/event/EventQueue.h b/core/file_server/event/EventQueue.h index aeda9d4287..d2e64ed57f 100644 --- a/core/file_server/event/EventQueue.h +++ b/core/file_server/event/EventQueue.h @@ -15,9 +15,10 @@ */ #pragma once -#include #include #include +#include + #include "common/Lock.h" namespace logtail { diff --git a/core/file_server/event_handler/HistoryFileImporter.h b/core/file_server/event_handler/HistoryFileImporter.h index e74df48b6b..c1666d21a3 100644 --- a/core/file_server/event_handler/HistoryFileImporter.h +++ b/core/file_server/event_handler/HistoryFileImporter.h @@ -17,8 +17,9 @@ #pragma once #include #include -#include "common/StringTools.h" + #include "common/CircularBuffer.h" +#include "common/StringTools.h" #include "common/Thread.h" #include "plugin/input/InputFile.h" diff --git a/core/file_server/event_handler/LogInput.cpp b/core/file_server/event_handler/LogInput.cpp index 6f0dbacebf..95ed8de7dc 100644 --- a/core/file_server/event_handler/LogInput.cpp +++ b/core/file_server/event_handler/LogInput.cpp @@ -27,6 +27,7 @@ #include "common/TimeUtil.h" #include "file_server/ConfigManager.h" #include "file_server/EventDispatcher.h" +#include "file_server/FileServer.h" #include "file_server/event/BlockEventManager.h" #include "file_server/event_handler/EventHandler.h" #include "file_server/event_handler/HistoryFileImporter.h" @@ -39,7 +40,6 @@ #include "logger/Logger.h" #include "monitor/AlarmManager.h" #include "monitor/Monitor.h" -#include "file_server/FileServer.h" using namespace std; diff --git a/core/file_server/event_listener/EventListener_Linux.cpp b/core/file_server/event_listener/EventListener_Linux.cpp index ae07e7223e..d3dc0ae4a5 100644 --- a/core/file_server/event_listener/EventListener_Linux.cpp +++ b/core/file_server/event_listener/EventListener_Linux.cpp @@ -13,15 +13,17 @@ // limitations under the License. #include "EventListener_Linux.h" + #include -#include #include -#include "logger/Logger.h" -#include "monitor/AlarmManager.h" +#include + #include "common/ErrorUtil.h" #include "common/Flags.h" #include "file_server/EventDispatcher.h" #include "file_server/event_handler/LogInput.h" +#include "logger/Logger.h" +#include "monitor/AlarmManager.h" DEFINE_FLAG_BOOL(fs_events_inotify_enable, "", true); diff --git a/core/file_server/event_listener/EventListener_Linux.h b/core/file_server/event_listener/EventListener_Linux.h index c659b9bfb7..88f7b25992 100644 --- a/core/file_server/event_listener/EventListener_Linux.h +++ b/core/file_server/event_listener/EventListener_Linux.h @@ -19,6 +19,7 @@ #include #include + #include "file_server/event/Event.h" namespace logtail { diff --git a/core/file_server/polling/PollingCache.cpp b/core/file_server/polling/PollingCache.cpp index e0f7c3806b..f4c4d525c1 100644 --- a/core/file_server/polling/PollingCache.cpp +++ b/core/file_server/polling/PollingCache.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "file_server/polling/PollingCache.h" + #include "common/Flags.h" DEFINE_FLAG_INT32(max_file_not_exist_times, "treate as deleted when file stat failed XX times, default", 10); diff --git a/core/file_server/polling/PollingCache.h b/core/file_server/polling/PollingCache.h index ea3ed8c38a..5fb7fb3303 100644 --- a/core/file_server/polling/PollingCache.h +++ b/core/file_server/polling/PollingCache.h @@ -15,12 +15,14 @@ */ #pragma once -#include -#include -#include +#include + #include +#include +#include #include -#include +#include + #include "common/SplitedFilePath.h" namespace logtail { diff --git a/core/file_server/polling/PollingDirFile.h b/core/file_server/polling/PollingDirFile.h index 8406bafdb7..64ee05a6a6 100644 --- a/core/file_server/polling/PollingDirFile.h +++ b/core/file_server/polling/PollingDirFile.h @@ -17,17 +17,17 @@ #pragma once #include -#include "file_server/polling/PollingCache.h" #include "common/Lock.h" #include "common/LogRunnable.h" #include "common/Thread.h" #include "file_server/FileDiscoveryOptions.h" +#include "file_server/polling/PollingCache.h" #include "monitor/Monitor.h" namespace logtail { namespace fsutil { - class PathStat; +class PathStat; } class PollingDirFile : public LogRunnable { diff --git a/core/file_server/polling/PollingEventQueue.cpp b/core/file_server/polling/PollingEventQueue.cpp index ebab605cf1..db66265296 100644 --- a/core/file_server/polling/PollingEventQueue.cpp +++ b/core/file_server/polling/PollingEventQueue.cpp @@ -13,8 +13,9 @@ // limitations under the License. #include "file_server/polling/PollingEventQueue.h" -#include "common/StringTools.h" + #include "common/Flags.h" +#include "common/StringTools.h" #include "common/TimeUtil.h" #include "file_server/event/Event.h" #include "logger/Logger.h" diff --git a/core/file_server/polling/PollingEventQueue.h b/core/file_server/polling/PollingEventQueue.h index 513c13cad9..3ba7c4054a 100644 --- a/core/file_server/polling/PollingEventQueue.h +++ b/core/file_server/polling/PollingEventQueue.h @@ -15,9 +15,9 @@ */ #pragma once -#include #include #include +#include namespace logtail { diff --git a/core/file_server/polling/PollingModify.h b/core/file_server/polling/PollingModify.h index 938a0596de..73a0b9c532 100644 --- a/core/file_server/polling/PollingModify.h +++ b/core/file_server/polling/PollingModify.h @@ -19,10 +19,10 @@ #include #include -#include "file_server/polling/PollingCache.h" #include "common/Lock.h" #include "common/LogRunnable.h" #include "common/Thread.h" +#include "file_server/polling/PollingCache.h" #ifdef APSARA_UINT_TEST_MAIN #include "common/SplitedFilePath.h" #endif diff --git a/core/file_server/reader/FileReaderOptions.h b/core/file_server/reader/FileReaderOptions.h index fa5f5dcd23..88bac2edd5 100644 --- a/core/file_server/reader/FileReaderOptions.h +++ b/core/file_server/reader/FileReaderOptions.h @@ -16,12 +16,13 @@ #pragma once -#include - #include + #include #include +#include "json/json.h" + #include "pipeline/PipelineContext.h" namespace logtail { diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index d5024ca48c..9e6a53308a 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -18,16 +18,17 @@ #include #include #endif -#include #include #include -#include -#include #include #include #include +#include "boost/filesystem.hpp" +#include "boost/regex.hpp" +#include "rapidjson/document.h" + #include "app_config/AppConfig.h" #include "checkpoint/CheckPointManager.h" #include "checkpoint/CheckpointManagerV2.h" @@ -52,7 +53,6 @@ #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" #include "plugin/processor/inner/ProcessorParseContainerLogNative.h" -#include "rapidjson/document.h" using namespace sls_logs; using namespace std; @@ -350,26 +350,25 @@ void LogFileReader::InitReader(bool tailExisted, FileReadPolicy policy, uint32_t namespace detail { - void updatePrimaryCheckpoint(const std::string& key, PrimaryCheckpointPB& cpt, const std::string& field) { - cpt.set_update_time(time(NULL)); - if (CheckpointManagerV2::GetInstance()->SetPB(key, cpt)) { - LOG_INFO(sLogger, ("update primary checkpoint", key)("field", field)("checkpoint", cpt.DebugString())); - } else { - LOG_WARNING(sLogger, - ("update primary checkpoint error", key)("field", field)("checkpoint", cpt.DebugString())); - } +void updatePrimaryCheckpoint(const std::string& key, PrimaryCheckpointPB& cpt, const std::string& field) { + cpt.set_update_time(time(NULL)); + if (CheckpointManagerV2::GetInstance()->SetPB(key, cpt)) { + LOG_INFO(sLogger, ("update primary checkpoint", key)("field", field)("checkpoint", cpt.DebugString())); + } else { + LOG_WARNING(sLogger, ("update primary checkpoint error", key)("field", field)("checkpoint", cpt.DebugString())); } +} - std::pair getPartitionRange(size_t idx, size_t concurrency, size_t totalPartitionCount) { - auto base = totalPartitionCount / concurrency; - auto extra = totalPartitionCount % concurrency; - if (extra == 0) { - return std::make_pair(idx * base, (idx + 1) * base - 1); - } - size_t min = idx <= extra ? idx * (base + 1) : extra * (base + 1) + (idx - extra) * base; - size_t max = idx < extra ? min + base : min + base - 1; - return std::make_pair(min, max); +std::pair getPartitionRange(size_t idx, size_t concurrency, size_t totalPartitionCount) { + auto base = totalPartitionCount / concurrency; + auto extra = totalPartitionCount % concurrency; + if (extra == 0) { + return std::make_pair(idx * base, (idx + 1) * base - 1); } + size_t min = idx <= extra ? idx * (base + 1) : extra * (base + 1) + (idx - extra) * base; + size_t max = idx < extra ? min + base : min + base - 1; + return std::make_pair(min, max); +} } // namespace detail diff --git a/core/file_server/reader/LogFileReader.h b/core/file_server/reader/LogFileReader.h index 7c62347882..91796df458 100644 --- a/core/file_server/reader/LogFileReader.h +++ b/core/file_server/reader/LogFileReader.h @@ -31,16 +31,15 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "common/memory/SourceBuffer.h" -#include "file_server/event/Event.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/FileServer.h" #include "file_server/MultilineOptions.h" -#include "protobuf/sls/sls_logs.pb.h" +#include "file_server/event/Event.h" +#include "file_server/reader/FileReaderOptions.h" #include "logger/Logger.h" #include "models/StringView.h" #include "pipeline/queue/QueueKey.h" -#include "rapidjson/allocators.h" -#include "file_server/reader/FileReaderOptions.h" +#include "protobuf/sls/sls_logs.pb.h" namespace logtail { @@ -234,7 +233,7 @@ class LogFileReader { /// @return e.g. `/home/admin/access.log` const std::string& GetConvertedPath() const; - + const std::string& GetHostLogPathFile() const { return mHostLogPathFile; } int64_t GetFileSize() const { return mLastFileSize; } diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index c1899b13df..c658cbf400 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -14,7 +14,7 @@ #include "go_pipeline/LogtailPlugin.h" -#include +#include "json/json.h" #include "app_config/AppConfig.h" #include "common/DynamicLibHelper.h" diff --git a/core/go_pipeline/LogtailPlugin.h b/core/go_pipeline/LogtailPlugin.h index 120573bff6..c4df5b947b 100644 --- a/core/go_pipeline/LogtailPlugin.h +++ b/core/go_pipeline/LogtailPlugin.h @@ -20,14 +20,15 @@ #include #endif -#include - #include + #include #include #include #include +#include "json/json.h" + #include "plugin/flusher/sls/FlusherSLS.h" #include "protobuf/sls/sls_logs.pb.h" diff --git a/core/go_pipeline/LogtailPluginAdapter.cpp b/core/go_pipeline/LogtailPluginAdapter.cpp index a5a5f9e48a..33158b1c51 100644 --- a/core/go_pipeline/LogtailPluginAdapter.cpp +++ b/core/go_pipeline/LogtailPluginAdapter.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "LogtailPluginAdapter.h" + #include IsValidToSendFun gAdapterIsValidToSendFun = NULL; diff --git a/core/logger/Logger.cpp b/core/logger/Logger.cpp index 237a33511b..3635a4aa64 100644 --- a/core/logger/Logger.cpp +++ b/core/logger/Logger.cpp @@ -14,14 +14,14 @@ #include "Logger.h" -#include -#include -#include -#include - -#include #include +#include "boost/filesystem.hpp" +#include "json/json.h" +#include "spdlog/async.h" +#include "spdlog/sinks/rotating_file_sink.h" +#include "spdlog/sinks/stdout_sinks.h" + #include "app_config/AppConfig.h" #include "common/ErrorUtil.h" #include "common/ExceptionBase.h" diff --git a/core/logger/Logger.h b/core/logger/Logger.h index b317936e67..0bcd436bf9 100644 --- a/core/logger/Logger.h +++ b/core/logger/Logger.h @@ -15,10 +15,11 @@ */ #pragma once -#include #include #include -#include +#include + +#include "spdlog/spdlog.h" namespace logtail { diff --git a/core/logtail.cpp b/core/logtail.cpp index 90a2d187e9..04463cbda6 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -75,7 +75,7 @@ void enable_core(void) { static void overwrite_community_edition_flags() { // support run in installation dir on default - if(BOOL_FLAG(logtail_mode)) { + if (BOOL_FLAG(logtail_mode)) { STRING_FLAG(logtail_sys_conf_dir) = "."; STRING_FLAG(check_point_filename) = "checkpoint/logtail_check_point"; STRING_FLAG(default_buffer_file_path) = "checkpoint"; diff --git a/core/metadata/K8sMetadata.cpp b/core/metadata/K8sMetadata.cpp index 3829e3fb91..7821790792 100644 --- a/core/metadata/K8sMetadata.cpp +++ b/core/metadata/K8sMetadata.cpp @@ -15,6 +15,7 @@ #include #include + #include #include "common/MachineInfoUtil.h" diff --git a/core/metadata/K8sMetadata.h b/core/metadata/K8sMetadata.h index 79cd9425d3..a21fad09e4 100644 --- a/core/metadata/K8sMetadata.h +++ b/core/metadata/K8sMetadata.h @@ -11,86 +11,84 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific l #pragma once + #include #include -#include -#include "common/LRUCache.h" + +#include "json/json.h" + #include "app_config/AppConfig.h" -#include #include "common/Flags.h" +#include "common/LRUCache.h" DECLARE_FLAG_STRING(loong_collector_operator_service); DECLARE_FLAG_INT32(loong_collector_k8s_meta_service_port); namespace logtail { - const static std::string appIdKey = "armsAppId"; - const static std::string imageKey = "images"; - const static std::string labelsKey = "labels"; - const static std::string namespaceKey = "namespace"; - const static std::string workloadKindKey = "workloadKind"; - const static std::string workloadNameKey = "workloadName"; - const static std::string serviceNameKey = "serviceName"; +const static std::string appIdKey = "armsAppId"; +const static std::string imageKey = "images"; +const static std::string labelsKey = "labels"; +const static std::string namespaceKey = "namespace"; +const static std::string workloadKindKey = "workloadKind"; +const static std::string workloadNameKey = "workloadName"; +const static std::string serviceNameKey = "serviceName"; + +struct k8sContainerInfo { + std::unordered_map images; + std::unordered_map labels; + std::string k8sNamespace; + std::string serviceName; + std::string workloadKind; + std::string workloadName; + std::time_t timestamp; + std::string appId; +}; + +// 定义顶层的结构体 +struct ContainerData { + std::unordered_map containers; +}; + +enum class containerInfoType { ContainerIdInfo, IpInfo }; - struct k8sContainerInfo { - std::unordered_map images; - std::unordered_map labels; - std::string k8sNamespace; - std::string serviceName; - std::string workloadKind; - std::string workloadName; - std::time_t timestamp; - std::string appId; - }; +class K8sMetadata { +private: + lru11::Cache> containerCache; + lru11::Cache> ipCache; + std::string mServiceHost; + int32_t mServicePort; + K8sMetadata(size_t cacheSize) : containerCache(cacheSize, 0), ipCache(cacheSize, 0) { + mServiceHost = STRING_FLAG(loong_collector_operator_service); + mServicePort = INT32_FLAG(loong_collector_k8s_meta_service_port); + } + K8sMetadata(const K8sMetadata&) = delete; + K8sMetadata& operator=(const K8sMetadata&) = delete; - // 定义顶层的结构体 - struct ContainerData { - std::unordered_map containers; - }; - - enum class containerInfoType { - ContainerIdInfo, - IpInfo - }; + void SetIpCache(const Json::Value& root); + void SetContainerCache(const Json::Value& root); + bool FromInfoJson(const Json::Value& json, k8sContainerInfo& info); + bool FromContainerJson(const Json::Value& json, std::shared_ptr data); - class K8sMetadata { - private: - lru11::Cache> containerCache; - lru11::Cache> ipCache; - std::string mServiceHost; - int32_t mServicePort; - K8sMetadata(size_t cacheSize) - : containerCache(cacheSize, 0), ipCache(cacheSize, 0){ - mServiceHost = STRING_FLAG(loong_collector_operator_service); - mServicePort = INT32_FLAG(loong_collector_k8s_meta_service_port); - } - K8sMetadata(const K8sMetadata&) = delete; - K8sMetadata& operator=(const K8sMetadata&) = delete; +public: + static K8sMetadata& GetInstance() { + static K8sMetadata instance(500); + return instance; + } + // 公共方法 + // if cache not have,get from server + bool GetByContainerIdsFromServer(std::vector containerIds); + void GetByLocalHostFromServer(); + bool GetByIpsFromServer(std::vector ips); + // get info by container id from cache + std::shared_ptr GetInfoByContainerIdFromCache(const std::string& containerId); + // get info by ip from cache + std::shared_ptr GetInfoByIpFromCache(const std::string& ip); + bool SendRequestToOperator(const std::string& urlHost, const std::string& output, containerInfoType infoType); - void SetIpCache(const Json::Value& root); - void SetContainerCache(const Json::Value& root); - bool FromInfoJson(const Json::Value& json, k8sContainerInfo& info); - bool FromContainerJson(const Json::Value& json, std::shared_ptr data); +#ifdef APSARA_UNIT_TEST_MAIN + friend class k8sMetadataUnittest; +#endif +}; - public: - static K8sMetadata& GetInstance() { - static K8sMetadata instance(500); - return instance; - } - // 公共方法 - //if cache not have,get from server - bool GetByContainerIdsFromServer(std::vector containerIds); - void GetByLocalHostFromServer(); - bool GetByIpsFromServer(std::vector ips); - // get info by container id from cache - std::shared_ptr GetInfoByContainerIdFromCache(const std::string& containerId); - // get info by ip from cache - std::shared_ptr GetInfoByIpFromCache(const std::string& ip); - bool SendRequestToOperator(const std::string& urlHost, const std::string& output, containerInfoType infoType); - - #ifdef APSARA_UNIT_TEST_MAIN - friend class k8sMetadataUnittest; - #endif - }; - } // namespace logtail diff --git a/core/metadata/LabelingK8sMetadata.cpp b/core/metadata/LabelingK8sMetadata.cpp index 57eee58771..7afc2b3151 100644 --- a/core/metadata/LabelingK8sMetadata.cpp +++ b/core/metadata/LabelingK8sMetadata.cpp @@ -16,17 +16,17 @@ #include "LabelingK8sMetadata.h" -#include #include +#include + +#include "K8sMetadata.h" #include "common/ParamExtractor.h" #include "logger/Logger.h" #include "models/MetricEvent.h" #include "models/SpanEvent.h" #include "monitor/metric_constants/MetricConstants.h" -#include -#include "K8sMetadata.h" -using logtail::StringView; +using logtail::StringView; namespace logtail { @@ -37,7 +37,7 @@ void LabelingK8sMetadata::AddLabelToLogGroup(PipelineEventGroup& logGroup) { EventsContainer& events = logGroup.MutableEvents(); std::vector containerVec; std::vector remoteIpVec; - std::vector cotainerNotTag; + std::vector cotainerNotTag; for (size_t rIdx = 0; rIdx < events.size(); ++rIdx) { if (!ProcessEvent(events[rIdx], containerVec, remoteIpVec)) { cotainerNotTag.push_back(rIdx); @@ -56,7 +56,9 @@ void LabelingK8sMetadata::AddLabelToLogGroup(PipelineEventGroup& logGroup) { return; } -bool LabelingK8sMetadata::ProcessEvent(PipelineEventPtr& e, std::vector& containerVec, std::vector& remoteIpVec) { +bool LabelingK8sMetadata::ProcessEvent(PipelineEventPtr& e, + std::vector& containerVec, + std::vector& remoteIpVec) { if (!IsSupportedEvent(e)) { return true; } @@ -71,9 +73,11 @@ bool LabelingK8sMetadata::ProcessEvent(PipelineEventPtr& e, std::vector -bool LabelingK8sMetadata::AddLabels(Event& e, std::vector& containerVec, std::vector& remoteIpVec) { +bool LabelingK8sMetadata::AddLabels(Event& e, + std::vector& containerVec, + std::vector& remoteIpVec) { bool res = true; - + auto& k8sMetadata = K8sMetadata::GetInstance(); StringView containerIdViewKey(containerIdKey); StringView containerIdView = e.HasTag(containerIdViewKey) ? e.GetTag(containerIdViewKey) : StringView{}; diff --git a/core/metadata/LabelingK8sMetadata.h b/core/metadata/LabelingK8sMetadata.h index c5009280e9..60322522e6 100644 --- a/core/metadata/LabelingK8sMetadata.h +++ b/core/metadata/LabelingK8sMetadata.h @@ -20,22 +20,24 @@ namespace logtail { - const static std::string containerIdKey = "container.id"; - const static std::string pidKey = "pid"; - const static std::string remoteIpKey = "remote_ip"; - const static std::string peerWorkloadNameKey = "peerWorkloadName"; - const static std::string peerWorkloadKindKey = "peerWorkloadKind"; - const static std::string peerNamespaceKey = "peerNamespace"; +const static std::string containerIdKey = "container.id"; +const static std::string pidKey = "pid"; +const static std::string remoteIpKey = "remote_ip"; +const static std::string peerWorkloadNameKey = "peerWorkloadName"; +const static std::string peerWorkloadKindKey = "peerWorkloadKind"; +const static std::string peerNamespaceKey = "peerNamespace"; - class LabelingK8sMetadata { - public: - void AddLabelToLogGroup(PipelineEventGroup& logGroup); - bool ProcessEvent(PipelineEventPtr& e, std::vector& container_vec, std::vector& remote_ip_vec); - // 声明模板函数 - template - bool AddLabels(Event& e, std::vector& containerVec, std::vector& remoteIpVec); - protected: - bool IsSupportedEvent(const PipelineEventPtr& e) const; - }; +class LabelingK8sMetadata { +public: + void AddLabelToLogGroup(PipelineEventGroup& logGroup); + bool + ProcessEvent(PipelineEventPtr& e, std::vector& container_vec, std::vector& remote_ip_vec); + // 声明模板函数 + template + bool AddLabels(Event& e, std::vector& containerVec, std::vector& remoteIpVec); + +protected: + bool IsSupportedEvent(const PipelineEventPtr& e) const; +}; } // namespace logtail diff --git a/core/models/EventPool.h b/core/models/EventPool.h index 33808e783c..f09c067a67 100644 --- a/core/models/EventPool.h +++ b/core/models/EventPool.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include @@ -32,7 +33,7 @@ class PipelineEventGroup; class EventPool { public: - EventPool(bool enableLock = true) : mEnableLock(enableLock) {}; + EventPool(bool enableLock = true) : mEnableLock(enableLock){}; ~EventPool(); EventPool(const EventPool&) = delete; EventPool& operator=(const EventPool&) = delete; diff --git a/core/models/LogEvent.cpp b/core/models/LogEvent.cpp index 3cacb6b2ea..ce27d6f098 100644 --- a/core/models/LogEvent.cpp +++ b/core/models/LogEvent.cpp @@ -88,7 +88,7 @@ void LogEvent::DelContent(StringView key) { } void LogEvent::SetLevel(const std::string& level) { - const StringBuffer& b = GetSourceBuffer()->CopyString(level); + const StringBuffer& b = GetSourceBuffer()->CopyString(level); mLevel = StringView(b.data, b.size); } diff --git a/core/models/MetricEvent.h b/core/models/MetricEvent.h index 63ed412e14..da9c47934d 100644 --- a/core/models/MetricEvent.h +++ b/core/models/MetricEvent.h @@ -33,7 +33,7 @@ class MetricEvent : public PipelineEvent { public: std::unique_ptr Copy() const override; void Reset() override; - + StringView GetName() const { return mName; } void SetName(const std::string& name); void SetNameNoCopy(StringView name); diff --git a/core/models/MetricValue.h b/core/models/MetricValue.h index a18059cec0..afbed433e7 100644 --- a/core/models/MetricValue.h +++ b/core/models/MetricValue.h @@ -20,9 +20,9 @@ #include #ifdef APSARA_UNIT_TEST_MAIN -#include - #include + +#include "json/json.h" #endif #include "common/memory/SourceBuffer.h" diff --git a/core/models/PipelineEvent.h b/core/models/PipelineEvent.h index e766a23689..56a76b779f 100644 --- a/core/models/PipelineEvent.h +++ b/core/models/PipelineEvent.h @@ -18,6 +18,7 @@ #include #include + #include #include #include @@ -27,7 +28,7 @@ #include "models/StringView.h" #ifdef APSARA_UNIT_TEST_MAIN -#include +#include "json/json.h" #endif namespace logtail { @@ -65,7 +66,9 @@ class PipelineEvent { virtual bool FromJson(const Json::Value&) = 0; std::string ToJsonString(bool enableEventMeta = false) const; bool FromJsonString(const std::string&); - PipelineEventGroup* GetPipelineEventGroupPtr() { return mPipelineEventGroupPtr; } + PipelineEventGroup* GetPipelineEventGroupPtr() { + return mPipelineEventGroupPtr; + } #endif protected: diff --git a/core/models/PipelineEventGroup.h b/core/models/PipelineEventGroup.h index 9e15733202..1f0c503016 100644 --- a/core/models/PipelineEventGroup.h +++ b/core/models/PipelineEventGroup.h @@ -20,8 +20,8 @@ #include #include "checkpoint/RangeCheckpoint.h" -#include "constants/Constants.h" #include "common/memory/SourceBuffer.h" +#include "constants/Constants.h" #include "models/PipelineEventPtr.h" namespace logtail { diff --git a/core/models/PipelineEventPtr.h b/core/models/PipelineEventPtr.h index 99d64a6238..3ab614792e 100644 --- a/core/models/PipelineEventPtr.h +++ b/core/models/PipelineEventPtr.h @@ -22,8 +22,8 @@ #include "models/LogEvent.h" #include "models/MetricEvent.h" #include "models/PipelineEvent.h" -#include "models/SpanEvent.h" #include "models/RawEvent.h" +#include "models/SpanEvent.h" namespace logtail { class EventPool; diff --git a/core/models/SpanEvent.cpp b/core/models/SpanEvent.cpp index 4396a92860..7506337e4e 100644 --- a/core/models/SpanEvent.cpp +++ b/core/models/SpanEvent.cpp @@ -15,6 +15,7 @@ */ #include "models/SpanEvent.h" + #include "constants/SpanConstants.h" using namespace std; @@ -445,10 +446,14 @@ const static std::string sSpanStatusCodeError = "ERROR"; const std::string& GetStatusString(SpanEvent::StatusCode status) { switch (status) { - case SpanEvent::StatusCode::Unset: return sSpanStatusCodeUnSet; - case SpanEvent::StatusCode::Ok: return sSpanStatusCodeOk; - case SpanEvent::StatusCode::Error: return sSpanStatusCodeError; - default: return sSpanStatusCodeUnSet; + case SpanEvent::StatusCode::Unset: + return sSpanStatusCodeUnSet; + case SpanEvent::StatusCode::Ok: + return sSpanStatusCodeOk; + case SpanEvent::StatusCode::Error: + return sSpanStatusCodeError; + default: + return sSpanStatusCodeUnSet; } } @@ -462,13 +467,20 @@ const static std::string sSpanKindUnknown = "unknown"; const std::string& GetKindString(SpanEvent::Kind kind) { switch (kind) { - case SpanEvent::Kind::Unspecified: return sSpanKindUnspecified; - case SpanEvent::Kind::Internal: return sSpanKindInternal; - case SpanEvent::Kind::Server: return sSpanKindServer; - case SpanEvent::Kind::Client: return sSpanKindClient; - case SpanEvent::Kind::Producer: return sSpanKindProducer; - case SpanEvent::Kind::Consumer: return sSpanKindConsumer; - default: return sSpanKindUnknown; + case SpanEvent::Kind::Unspecified: + return sSpanKindUnspecified; + case SpanEvent::Kind::Internal: + return sSpanKindInternal; + case SpanEvent::Kind::Server: + return sSpanKindServer; + case SpanEvent::Kind::Client: + return sSpanKindClient; + case SpanEvent::Kind::Producer: + return sSpanKindProducer; + case SpanEvent::Kind::Consumer: + return sSpanKindConsumer; + default: + return sSpanKindUnknown; } } diff --git a/core/models/SpanEvent.h b/core/models/SpanEvent.h index 780664eb1a..cac4c83ccc 100644 --- a/core/models/SpanEvent.h +++ b/core/models/SpanEvent.h @@ -19,7 +19,8 @@ #include #include #include -#include + +#include "json/json.h" #include "common/memory/SourceBuffer.h" #include "models/PipelineEvent.h" @@ -74,7 +75,8 @@ class SpanEvent : public PipelineEvent { #endif private: - SpanLink(SpanEvent* parent) : mParent(parent) {} + SpanLink(SpanEvent* parent) : mParent(parent) { + } StringView mTraceId; StringView mSpanId; @@ -114,7 +116,8 @@ class SpanEvent : public PipelineEvent { #endif private: - InnerEvent(SpanEvent* parent) : mParent(parent) {} + InnerEvent(SpanEvent* parent) : mParent(parent) { + } uint64_t mTimestampNs = 0; StringView mName; @@ -131,29 +134,51 @@ class SpanEvent : public PipelineEvent { std::unique_ptr Copy() const override; void Reset() override; - StringView GetTraceId() const { return mTraceId; } + StringView GetTraceId() const { + return mTraceId; + } void SetTraceId(const std::string& traceId); - StringView GetSpanId() const { return mSpanId; } + StringView GetSpanId() const { + return mSpanId; + } void SetSpanId(const std::string& spanId); - StringView GetTraceState() const { return mTraceState; } + StringView GetTraceState() const { + return mTraceState; + } void SetTraceState(const std::string& traceState); - StringView GetParentSpanId() const { return mParentSpanId; } + StringView GetParentSpanId() const { + return mParentSpanId; + } void SetParentSpanId(const std::string& parentSpanId); - StringView GetName() const { return mName; } + StringView GetName() const { + return mName; + } void SetName(const std::string& name); - Kind GetKind() const { return mKind; } - void SetKind(Kind kind) { mKind = kind; } - - uint64_t GetStartTimeNs() const { return mStartTimeNs; } - void SetStartTimeNs(uint64_t startTimeNs) { mStartTimeNs = startTimeNs; } - - uint64_t GetEndTimeNs() const { return mEndTimeNs; } - void SetEndTimeNs(uint64_t endTimeNs) { mEndTimeNs = endTimeNs; } + Kind GetKind() const { + return mKind; + } + void SetKind(Kind kind) { + mKind = kind; + } + + uint64_t GetStartTimeNs() const { + return mStartTimeNs; + } + void SetStartTimeNs(uint64_t startTimeNs) { + mStartTimeNs = startTimeNs; + } + + uint64_t GetEndTimeNs() const { + return mEndTimeNs; + } + void SetEndTimeNs(uint64_t endTimeNs) { + mEndTimeNs = endTimeNs; + } StringView GetTag(StringView key) const; bool HasTag(StringView key) const; @@ -162,18 +187,32 @@ class SpanEvent : public PipelineEvent { void SetTagNoCopy(const StringBuffer& key, const StringBuffer& val); void SetTagNoCopy(StringView key, StringView val); void DelTag(StringView key); - std::map::const_iterator TagsBegin() const { return mTags.mInner.begin(); } - std::map::const_iterator TagsEnd() const { return mTags.mInner.end(); } - size_t TagsSize() const { return mTags.mInner.size(); } - - const std::vector& GetEvents() const { return mEvents; } + std::map::const_iterator TagsBegin() const { + return mTags.mInner.begin(); + } + std::map::const_iterator TagsEnd() const { + return mTags.mInner.end(); + } + size_t TagsSize() const { + return mTags.mInner.size(); + } + + const std::vector& GetEvents() const { + return mEvents; + } InnerEvent* AddEvent(); - const std::vector& GetLinks() const { return mLinks; } + const std::vector& GetLinks() const { + return mLinks; + } SpanLink* AddLink(); - StatusCode GetStatus() const { return mStatus; } - void SetStatus(StatusCode status) { mStatus = status; } + StatusCode GetStatus() const { + return mStatus; + } + void SetStatus(StatusCode status) { + mStatus = status; + } StringView GetScopeTag(StringView key) const; bool HasScopeTag(StringView key) const; @@ -182,9 +221,15 @@ class SpanEvent : public PipelineEvent { void SetScopeTagNoCopy(const StringBuffer& key, const StringBuffer& val); void SetScopeTagNoCopy(StringView key, StringView val); void DelScopeTag(StringView key); - std::map::const_iterator ScopeTagsBegin() const { return mScopeTags.mInner.begin(); } - std::map::const_iterator ScopeTagsEnd() const { return mScopeTags.mInner.end(); } - size_t ScopeTagsSize() const { return mScopeTags.mInner.size(); } + std::map::const_iterator ScopeTagsBegin() const { + return mScopeTags.mInner.begin(); + } + std::map::const_iterator ScopeTagsEnd() const { + return mScopeTags.mInner.end(); + } + size_t ScopeTagsSize() const { + return mScopeTags.mInner.size(); + } size_t DataSize() const override; diff --git a/core/models/StringView.h b/core/models/StringView.h index b5cbaf4e7f..b035bae1f7 100644 --- a/core/models/StringView.h +++ b/core/models/StringView.h @@ -15,7 +15,7 @@ */ #pragma once -#include +#include "boost/utility/string_view.hpp" namespace logtail { diff --git a/core/monitor/AlarmManager.cpp b/core/monitor/AlarmManager.cpp index e5911d5c87..c72a803c7b 100644 --- a/core/monitor/AlarmManager.cpp +++ b/core/monitor/AlarmManager.cpp @@ -223,8 +223,7 @@ void AlarmManager::SendAllRegionAlarm() { logGroup.set_source(LoongCollectorMonitor::mIpAddr); logGroup.set_category(ALARM_SLS_LOGSTORE_NAME); auto now = GetCurrentLogtailTime(); - for (map>::iterator mapIter = alarmMap.begin(); - mapIter != alarmMap.end(); + for (map>::iterator mapIter = alarmMap.begin(); mapIter != alarmMap.end(); ++mapIter) { auto& messagePtr = mapIter->second; diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index b00bbb9594..f815e1ec33 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -38,14 +38,14 @@ #include "logger/Logger.h" #include "monitor/AlarmManager.h" #include "monitor/SelfMonitorServer.h" +#include "pipeline/PipelineManager.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "protobuf/sls/sls_logs.pb.h" +#include "provider/Provider.h" #include "runner/FlusherRunner.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif -#include "pipeline/PipelineManager.h" -#include "provider/Provider.h" using namespace std; using namespace sls_logs; diff --git a/core/monitor/Monitor.h b/core/monitor/Monitor.h index 3582d6cf39..6340a8e2dc 100644 --- a/core/monitor/Monitor.h +++ b/core/monitor/Monitor.h @@ -50,7 +50,9 @@ struct CpuStat { int32_t mViolateNum; float mCpuUsage; - CpuStat() { Reset(); } + CpuStat() { + Reset(); + } void Reset(); }; diff --git a/core/monitor/metric_constants/MetricCommonConstants.cpp b/core/monitor/metric_constants/MetricCommonConstants.cpp index a2a1e3542d..80b953ed33 100644 --- a/core/monitor/metric_constants/MetricCommonConstants.cpp +++ b/core/monitor/metric_constants/MetricCommonConstants.cpp @@ -32,4 +32,4 @@ const string METRIC_OUT_SIZE_BYTES = "out_size_bytes"; const string METRIC_TOTAL_DELAY_MS = "total_delay_ms"; const string METRIC_TOTAL_PROCESS_TIME_MS = "total_process_time_ms"; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/metric_constants/MetricCommonConstants.h b/core/monitor/metric_constants/MetricCommonConstants.h index 4a78b4e9d1..22d255a637 100644 --- a/core/monitor/metric_constants/MetricCommonConstants.h +++ b/core/monitor/metric_constants/MetricCommonConstants.h @@ -33,4 +33,4 @@ extern const std::string METRIC_OUT_SIZE_BYTES; extern const std::string METRIC_TOTAL_DELAY_MS; extern const std::string METRIC_TOTAL_PROCESS_TIME_MS; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/metric_models/MetricRecord.cpp b/core/monitor/metric_models/MetricRecord.cpp index cf8ca66d24..2a8146a39b 100644 --- a/core/monitor/metric_models/MetricRecord.cpp +++ b/core/monitor/metric_models/MetricRecord.cpp @@ -176,4 +176,4 @@ bool MetricsRecordRef::HasLabel(const std::string& key, const std::string& value } #endif -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/metric_models/MetricRecord.h b/core/monitor/metric_models/MetricRecord.h index 4a6e435eea..1a84eccecc 100644 --- a/core/monitor/metric_models/MetricRecord.h +++ b/core/monitor/metric_models/MetricRecord.h @@ -111,4 +111,4 @@ inline bool operator!=(std::nullptr_t lhs, const MetricsRecordRef& rhs) { return !(lhs == rhs); } -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/metric_models/MetricTypes.h b/core/monitor/metric_models/MetricTypes.h index 309c4dcaf3..493950de14 100644 --- a/core/monitor/metric_models/MetricTypes.h +++ b/core/monitor/metric_models/MetricTypes.h @@ -16,9 +16,10 @@ #pragma once -#include #include #include + +#include #include #include #include @@ -52,7 +53,7 @@ class Counter { class TimeCounter : public Counter { public: TimeCounter(const std::string& name, uint64_t val = 0) : Counter(name, val) {} - uint64_t GetValue() const { return mVal.load()/1000000; } + uint64_t GetValue() const { return mVal.load() / 1000000; } void Add(std::chrono::nanoseconds val) { mVal.fetch_add(val.count()); } TimeCounter* Collect() { return new TimeCounter(mName, mVal.exchange(0)); } }; diff --git a/core/monitor/metric_models/ReentrantMetricsRecord.cpp b/core/monitor/metric_models/ReentrantMetricsRecord.cpp index 33b9a7a5f4..95a1e6ef46 100644 --- a/core/monitor/metric_models/ReentrantMetricsRecord.cpp +++ b/core/monitor/metric_models/ReentrantMetricsRecord.cpp @@ -84,7 +84,8 @@ DoubleGaugePtr ReentrantMetricsRecord::GetDoubleGauge(const std::string& name) { return nullptr; } -ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecordRef(MetricLabels labels, DynamicMetricLabels dynamicLabels) { +ReentrantMetricsRecordRef PluginMetricManager::GetOrCreateReentrantMetricsRecordRef(MetricLabels labels, + DynamicMetricLabels dynamicLabels) { std::lock_guard lock(mutex); std::string key = GenerateKey(labels); diff --git a/core/monitor/profile_sender/ProfileSender.cpp b/core/monitor/profile_sender/ProfileSender.cpp index ec2d50ea30..25c0c4d062 100644 --- a/core/monitor/profile_sender/ProfileSender.cpp +++ b/core/monitor/profile_sender/ProfileSender.cpp @@ -14,19 +14,19 @@ #include "ProfileSender.h" -#include - #include +#include "json/json.h" + +#include "app_config/AppConfig.h" #include "common/CompressTools.h" #include "common/Flags.h" #include "common/LogtailCommonFlags.h" #include "logger/Logger.h" +#include "plugin/flusher/sls/SLSClientManager.h" #ifdef __ENTERPRISE__ #include "EnterpriseProfileSender.h" #endif -#include "app_config/AppConfig.h" -#include "plugin/flusher/sls/SLSClientManager.h" // TODO: temporarily used #include "common/compression/CompressorFactory.h" diff --git a/core/monitor/profile_sender/ProfileSender.h b/core/monitor/profile_sender/ProfileSender.h index fc91d22233..bcbbf4e97e 100644 --- a/core/monitor/profile_sender/ProfileSender.h +++ b/core/monitor/profile_sender/ProfileSender.h @@ -21,8 +21,8 @@ #include #include "common/Lock.h" -#include "protobuf/sls/sls_logs.pb.h" #include "plugin/flusher/sls/FlusherSLS.h" +#include "protobuf/sls/sls_logs.pb.h" namespace logtail { diff --git a/core/pipeline/GlobalConfig.cpp b/core/pipeline/GlobalConfig.cpp index 91a3ed43bd..ef42df30ad 100644 --- a/core/pipeline/GlobalConfig.cpp +++ b/core/pipeline/GlobalConfig.cpp @@ -14,7 +14,7 @@ #include "pipeline/GlobalConfig.h" -#include +#include "json/json.h" #include "common/ParamExtractor.h" #include "pipeline/PipelineContext.h" diff --git a/core/pipeline/GlobalConfig.h b/core/pipeline/GlobalConfig.h index 4969bf7c1b..ba73714c89 100644 --- a/core/pipeline/GlobalConfig.h +++ b/core/pipeline/GlobalConfig.h @@ -16,12 +16,13 @@ #pragma once -#include - #include + #include #include +#include "json/json.h" + namespace logtail { class PipelineContext; diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 8d4c2a75d6..408515c853 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -18,6 +18,7 @@ #include #include + #include #include "app_config/AppConfig.h" diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index fe2c79bb2e..f165f56cad 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -16,13 +16,13 @@ #pragma once -#include - #include #include #include #include +#include "json/json.h" + #include "config/PipelineConfig.h" #include "models/PipelineEventGroup.h" #include "monitor/MetricManager.h" diff --git a/core/pipeline/PipelineContext.cpp b/core/pipeline/PipelineContext.cpp index 17d63a9efa..0622131dbb 100644 --- a/core/pipeline/PipelineContext.cpp +++ b/core/pipeline/PipelineContext.cpp @@ -14,8 +14,8 @@ #include "pipeline/PipelineContext.h" -#include "plugin/flusher/sls/FlusherSLS.h" #include "pipeline/queue/QueueKeyManager.h" +#include "plugin/flusher/sls/FlusherSLS.h" using namespace std; diff --git a/core/pipeline/PipelineContext.h b/core/pipeline/PipelineContext.h index 132d0ecf14..36d0322daf 100644 --- a/core/pipeline/PipelineContext.h +++ b/core/pipeline/PipelineContext.h @@ -16,11 +16,12 @@ #pragma once -#include - #include + #include +#include "json/json.h" + #include "logger/Logger.h" #include "models/PipelineEventGroup.h" #include "monitor/AlarmManager.h" @@ -60,7 +61,7 @@ class PipelineContext { QueueKey GetLogstoreKey() const; const FlusherSLS* GetSLSInfo() const { return mSLSInfo; } void SetSLSInfo(const FlusherSLS* flusherSLS) { mSLSInfo = flusherSLS; } - + bool RequiringJsonReader() const { return mRequiringJsonReader; } void SetRequiringJsonReaderFlag(bool flag) { mRequiringJsonReader = flag; } bool IsFirstProcessorApsara() const { return mIsFirstProcessorApsara; } diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index ba5379eb4e..6567d025f0 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -23,14 +23,14 @@ #if defined(__linux__) && !defined(__ANDROID__) #include "ebpf/eBPFServer.h" #endif +#include "config/feedbacker/ConfigFeedbackReceiver.h" +#include "pipeline/queue/ProcessQueueManager.h" +#include "pipeline/queue/QueueKeyManager.h" #include "runner/ProcessorRunner.h" #if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) #include "app_config/AppConfig.h" #include "shennong/ShennongManager.h" #endif -#include "config/feedbacker/ConfigFeedbackReceiver.h" -#include "pipeline/queue/ProcessQueueManager.h" -#include "pipeline/queue/QueueKeyManager.h" using namespace std; @@ -40,7 +40,7 @@ PipelineManager::PipelineManager() : mInputRunners({ PrometheusInputRunner::GetInstance(), #if defined(__linux__) && !defined(__ANDROID__) - ebpf::eBPFServer::GetInstance(), + ebpf::eBPFServer::GetInstance(), #endif }) { } diff --git a/core/pipeline/batch/BatchItem.h b/core/pipeline/batch/BatchItem.h index d76dda00c1..2f8730773a 100644 --- a/core/pipeline/batch/BatchItem.h +++ b/core/pipeline/batch/BatchItem.h @@ -94,7 +94,8 @@ class EventBatchItem { void Add(PipelineEventPtr&& e) { mBatch.mEvents.emplace_back(std::move(e)); mStatus.Update(mBatch.mEvents.back()); - // mTotalEnqueTimeMs += std::chrono::time_point_cast(std::chrono::system_clock::now()) + // mTotalEnqueTimeMs += + // std::chrono::time_point_cast(std::chrono::system_clock::now()) // .time_since_epoch() // .count(); } diff --git a/core/pipeline/batch/BatchStatus.h b/core/pipeline/batch/BatchStatus.h index 224a83dc78..d52ea8d0b8 100644 --- a/core/pipeline/batch/BatchStatus.h +++ b/core/pipeline/batch/BatchStatus.h @@ -19,15 +19,15 @@ #include #include -#include "pipeline/batch/BatchedEvents.h" #include "models/PipelineEventPtr.h" +#include "pipeline/batch/BatchedEvents.h" namespace logtail { class EventBatchStatus { public: virtual ~EventBatchStatus() = default; - + virtual void Reset() { mCnt = 0; mSizeBytes = 0; diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index a263d82228..056ee350e4 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -16,14 +16,15 @@ #pragma once -#include - #include + #include #include #include #include +#include "json/json.h" + #include "common/Flags.h" #include "common/ParamExtractor.h" #include "models/PipelineEventGroup.h" @@ -288,8 +289,12 @@ class Batcher { } #ifdef APSARA_UNIT_TEST_MAIN - EventFlushStrategy& GetEventFlushStrategy() { return mEventFlushStrategy; } - std::optional& GetGroupFlushStrategy() { return mGroupFlushStrategy; } + EventFlushStrategy& GetEventFlushStrategy() { + return mEventFlushStrategy; + } + std::optional& GetGroupFlushStrategy() { + return mGroupFlushStrategy; + } #endif private: diff --git a/core/pipeline/batch/FlushStrategy.cpp b/core/pipeline/batch/FlushStrategy.cpp index 58a3899e69..b5704e3638 100644 --- a/core/pipeline/batch/FlushStrategy.cpp +++ b/core/pipeline/batch/FlushStrategy.cpp @@ -20,7 +20,7 @@ namespace logtail { template <> bool EventFlushStrategy::NeedFlushByTime(const SLSEventBatchStatus& status, - const PipelineEventPtr& e) { + const PipelineEventPtr& e) { return time(nullptr) - status.GetCreateTime() > mTimeoutSecs || status.GetCreateTimeMinute() != e->GetTimestamp() / 60; } diff --git a/core/pipeline/batch/FlushStrategy.h b/core/pipeline/batch/FlushStrategy.h index e2a718244e..6e69587bd8 100644 --- a/core/pipeline/batch/FlushStrategy.h +++ b/core/pipeline/batch/FlushStrategy.h @@ -16,12 +16,13 @@ #pragma once -#include - #include #include + #include +#include "json/json.h" + #include "models/PipelineEventPtr.h" #include "pipeline/batch/BatchStatus.h" diff --git a/core/pipeline/batch/TimeoutFlushManager.h b/core/pipeline/batch/TimeoutFlushManager.h index e72d90011e..de26ee24c7 100644 --- a/core/pipeline/batch/TimeoutFlushManager.h +++ b/core/pipeline/batch/TimeoutFlushManager.h @@ -18,6 +18,7 @@ #include #include + #include #include #include diff --git a/core/pipeline/limiter/ConcurrencyLimiter.cpp b/core/pipeline/limiter/ConcurrencyLimiter.cpp index 937004769b..b997c0a1cb 100644 --- a/core/pipeline/limiter/ConcurrencyLimiter.cpp +++ b/core/pipeline/limiter/ConcurrencyLimiter.cpp @@ -73,8 +73,9 @@ void ConcurrencyLimiter::Increase() { if (mCurrenctConcurrency != mMaxConcurrency) { ++mCurrenctConcurrency; if (mCurrenctConcurrency == mMaxConcurrency) { - LOG_DEBUG(sLogger, - ("increase send concurrency to maximum, type", mDescription)("concurrency", mCurrenctConcurrency)); + LOG_DEBUG( + sLogger, + ("increase send concurrency to maximum, type", mDescription)("concurrency", mCurrenctConcurrency)); } else { LOG_DEBUG(sLogger, ("increase send concurrency, type", @@ -101,23 +102,27 @@ void ConcurrencyLimiter::Decrease(double fallBackRatio) { void ConcurrencyLimiter::AdjustConcurrency(bool success, std::chrono::system_clock::time_point currentTime) { uint32_t failPercentage = 0; bool finishStatistics = false; - { + { lock_guard lock(mStatisticsMux); - mStatisticsTotal ++; + mStatisticsTotal++; if (!success) { - mStatisticsFailTotal ++; + mStatisticsFailTotal++; } if (mLastStatisticsTime == std::chrono::system_clock::time_point()) { mLastStatisticsTime = currentTime; } - if (mStatisticsTotal == CONCURRENCY_STATISTIC_THRESHOLD || chrono::duration_cast(currentTime - mLastStatisticsTime).count() > CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS) { - failPercentage = mStatisticsFailTotal*100/mStatisticsTotal; - LOG_DEBUG(sLogger,("AdjustConcurrency", mDescription)("mStatisticsFailTotal", mStatisticsFailTotal)("mStatisticsTotal", mStatisticsTotal)); + if (mStatisticsTotal == CONCURRENCY_STATISTIC_THRESHOLD + || chrono::duration_cast(currentTime - mLastStatisticsTime).count() + > CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS) { + failPercentage = mStatisticsFailTotal * 100 / mStatisticsTotal; + LOG_DEBUG(sLogger, + ("AdjustConcurrency", mDescription)("mStatisticsFailTotal", + mStatisticsFailTotal)("mStatisticsTotal", mStatisticsTotal)); mStatisticsTotal = 0; mStatisticsFailTotal = 0; mLastStatisticsTime = currentTime; finishStatistics = true; - } + } } if (finishStatistics) { if (failPercentage == 0) { @@ -128,10 +133,10 @@ void ConcurrencyLimiter::AdjustConcurrency(bool success, std::chrono::system_clo } else if (failPercentage <= SLOW_FALL_BACK_FAIL_PERCENTAGE) { // 慢回退 Decrease(mConcurrencySlowFallBackRatio); - } else { + } else { // 快速回退 Decrease(mConcurrencyFastFallBackRatio); - } + } } } diff --git a/core/pipeline/limiter/ConcurrencyLimiter.h b/core/pipeline/limiter/ConcurrencyLimiter.h index 513cea0988..481238b438 100644 --- a/core/pipeline/limiter/ConcurrencyLimiter.h +++ b/core/pipeline/limiter/ConcurrencyLimiter.h @@ -16,9 +16,10 @@ #pragma once -#include #include #include + +#include #include #include @@ -93,7 +94,6 @@ class ConcurrencyLimiter { void Increase(); void Decrease(double fallBackRatio); void AdjustConcurrency(bool success, std::chrono::system_clock::time_point currentTime); - }; } // namespace logtail diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index 6a0061c68a..e45dd6425d 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -14,13 +14,13 @@ #include "pipeline/plugin/PluginRegistry.h" +#include +#include +#include #include #include #include -#include -#include -#include #include #include "app_config/AppConfig.h" @@ -28,9 +28,6 @@ #include "plugin/flusher/blackhole/FlusherBlackHole.h" #include "plugin/flusher/file/FlusherFile.h" #include "plugin/flusher/sls/FlusherSLS.h" -#ifdef __ENTERPRISE__ -#include "plugin/flusher/sls/EnterpriseFlusherSLSMonitor.h" -#endif #include "plugin/input/InputContainerStdio.h" #include "plugin/input/InputFile.h" #include "plugin/input/InputPrometheus.h" @@ -64,7 +61,9 @@ #if defined(__linux__) && !defined(__ANDROID__) && !defined(__EXCLUDE_SPL__) #include "plugin/processor/ProcessorSPL.h" #endif - +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseFlusherSLSMonitor.h" +#endif DEFINE_FLAG_BOOL(enable_processor_spl, "", true); diff --git a/core/pipeline/plugin/creator/PluginCreator.h b/core/pipeline/plugin/creator/PluginCreator.h index 6888927186..a61d492cdb 100644 --- a/core/pipeline/plugin/creator/PluginCreator.h +++ b/core/pipeline/plugin/creator/PluginCreator.h @@ -16,8 +16,8 @@ #pragma once -#include #include +#include #include "pipeline/plugin/instance/PluginInstance.h" diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index 181e5c21e8..0c05615e3b 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -20,7 +20,10 @@ using namespace std; namespace logtail { -bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, size_t flusherIdx, Json::Value& optionalGoPipeline) { +bool FlusherInstance::Init(const Json::Value& config, + PipelineContext& context, + size_t flusherIdx, + Json::Value& optionalGoPipeline) { mPlugin->SetContext(context); mPlugin->SetPluginID(PluginID()); mPlugin->SetFlusherIndex(flusherIdx); diff --git a/core/pipeline/plugin/instance/FlusherInstance.h b/core/pipeline/plugin/instance/FlusherInstance.h index 4102616541..04b04b3517 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.h +++ b/core/pipeline/plugin/instance/FlusherInstance.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + #include "models/PipelineEventGroup.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" #include "pipeline/PipelineContext.h" diff --git a/core/pipeline/plugin/instance/InputInstance.h b/core/pipeline/plugin/instance/InputInstance.h index 139f5b554b..87c5c3455e 100644 --- a/core/pipeline/plugin/instance/InputInstance.h +++ b/core/pipeline/plugin/instance/InputInstance.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + #include "pipeline/PipelineContext.h" #include "pipeline/plugin/instance/PluginInstance.h" #include "pipeline/plugin/interface/Input.h" @@ -28,14 +28,12 @@ namespace logtail { class InputInstance : public PluginInstance { public: - InputInstance(Input* plugin, const PluginInstance::PluginMeta& pluginMeta) : PluginInstance(pluginMeta), mPlugin(plugin) {} + InputInstance(Input* plugin, const PluginInstance::PluginMeta& pluginMeta) + : PluginInstance(pluginMeta), mPlugin(plugin) {} const std::string& Name() const override { return mPlugin->Name(); } - bool Init(const Json::Value& config, - PipelineContext& context, - size_t inputIdx, - Json::Value& optionalGoPipeline); + bool Init(const Json::Value& config, PipelineContext& context, size_t inputIdx, Json::Value& optionalGoPipeline); bool Start() { return mPlugin->Start(); } bool Stop(bool isPipelineRemoving) { return mPlugin->Stop(isPipelineRemoving); } bool SupportAck() const { return mPlugin->SupportAck(); } diff --git a/core/pipeline/plugin/instance/PluginInstance.h b/core/pipeline/plugin/instance/PluginInstance.h index 34bf33eb0d..e3ffca1839 100644 --- a/core/pipeline/plugin/instance/PluginInstance.h +++ b/core/pipeline/plugin/instance/PluginInstance.h @@ -23,8 +23,7 @@ namespace logtail { class PluginInstance { public: struct PluginMeta { - PluginMeta(std::string pluginID) - : mPluginID(pluginID) {} + PluginMeta(std::string pluginID) : mPluginID(pluginID) {} std::string mPluginID; }; PluginInstance(const PluginMeta& pluginMeta) : mMeta(pluginMeta) {} diff --git a/core/pipeline/plugin/instance/ProcessorInstance.cpp b/core/pipeline/plugin/instance/ProcessorInstance.cpp index 38b7567198..fa70bd2bb9 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.cpp +++ b/core/pipeline/plugin/instance/ProcessorInstance.cpp @@ -46,7 +46,7 @@ bool ProcessorInstance::Init(const Json::Value& config, PipelineContext& context void ProcessorInstance::Process(vector& eventGroupList) { if (eventGroupList.empty()) { return; - } + } for (const auto& eventGroup : eventGroupList) { mInEventsTotal->Add(eventGroup.GetEvents().size()); mInSizeBytes->Add(eventGroup.DataSize()); @@ -59,7 +59,7 @@ void ProcessorInstance::Process(vector& eventGroupList) { for (const auto& eventGroup : eventGroupList) { mOutEventsTotal->Add(eventGroup.GetEvents().size()); mOutSizeBytes->Add(eventGroup.DataSize()); - } + } } } // namespace logtail diff --git a/core/pipeline/plugin/instance/ProcessorInstance.h b/core/pipeline/plugin/instance/ProcessorInstance.h index 24373685c6..05a6c6dc2d 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.h +++ b/core/pipeline/plugin/instance/ProcessorInstance.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + #include "models/PipelineEventGroup.h" #include "monitor/MetricManager.h" #include "pipeline/PipelineContext.h" diff --git a/core/pipeline/plugin/interface/Flusher.h b/core/pipeline/plugin/interface/Flusher.h index a59e73eb6d..73b2248786 100644 --- a/core/pipeline/plugin/interface/Flusher.h +++ b/core/pipeline/plugin/interface/Flusher.h @@ -16,11 +16,12 @@ #pragma once -#include - #include + #include +#include "json/json.h" + #include "models/PipelineEventGroup.h" #include "pipeline/plugin/interface/Plugin.h" #include "pipeline/queue/QueueKey.h" diff --git a/core/pipeline/plugin/interface/HttpFlusher.h b/core/pipeline/plugin/interface/HttpFlusher.h index f68eab2bd2..6260aa4c9a 100644 --- a/core/pipeline/plugin/interface/HttpFlusher.h +++ b/core/pipeline/plugin/interface/HttpFlusher.h @@ -29,7 +29,9 @@ class HttpFlusher : public Flusher { public: virtual ~HttpFlusher() = default; - virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) = 0; + virtual bool + BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) + = 0; virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; virtual SinkType GetSinkType() override { return SinkType::HTTP; } diff --git a/core/pipeline/plugin/interface/Input.h b/core/pipeline/plugin/interface/Input.h index 98393aeb33..3701a4f88b 100644 --- a/core/pipeline/plugin/interface/Input.h +++ b/core/pipeline/plugin/interface/Input.h @@ -16,11 +16,11 @@ #pragma once -#include - #include #include +#include "json/json.h" + #include "pipeline/plugin/instance/ProcessorInstance.h" #include "pipeline/plugin/interface/Plugin.h" diff --git a/core/pipeline/plugin/interface/Processor.h b/core/pipeline/plugin/interface/Processor.h index f41e88c471..8cfe79969e 100644 --- a/core/pipeline/plugin/interface/Processor.h +++ b/core/pipeline/plugin/interface/Processor.h @@ -16,7 +16,7 @@ #pragma once -#include +#include "json/json.h" #include "models/PipelineEventGroup.h" #include "models/PipelineEventPtr.h" diff --git a/core/pipeline/queue/BoundedProcessQueue.h b/core/pipeline/queue/BoundedProcessQueue.h index 77687ce0b9..ea3e4cfc17 100644 --- a/core/pipeline/queue/BoundedProcessQueue.h +++ b/core/pipeline/queue/BoundedProcessQueue.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include diff --git a/core/pipeline/queue/BoundedSenderQueueInterface.cpp b/core/pipeline/queue/BoundedSenderQueueInterface.cpp index 4d9f0821a1..1f55ddfeca 100644 --- a/core/pipeline/queue/BoundedSenderQueueInterface.cpp +++ b/core/pipeline/queue/BoundedSenderQueueInterface.cpp @@ -27,7 +27,8 @@ BoundedSenderQueueInterface::BoundedSenderQueueInterface( mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE}}); mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, flusherId}}); mExtraBufferSize = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE); - mFetchRejectedByRateLimiterTimesCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL); + mFetchRejectedByRateLimiterTimesCnt + = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_QUEUE_FETCH_REJECTED_BY_RATE_LIMITER_TIMES_TOTAL); mExtraBufferDataSizeBytes = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES); } @@ -45,14 +46,16 @@ void BoundedSenderQueueInterface::SetRateLimiter(uint32_t maxRate) { } } -void BoundedSenderQueueInterface::SetConcurrencyLimiters(std::unordered_map>&& concurrencyLimitersMap) { +void BoundedSenderQueueInterface::SetConcurrencyLimiters( + std::unordered_map>&& concurrencyLimitersMap) { mConcurrencyLimiters.clear(); for (const auto& item : concurrencyLimitersMap) { if (item.second == nullptr) { // should not happen continue; } - mConcurrencyLimiters.emplace_back(item.second, mMetricsRecordRef.CreateCounter(ConcurrencyLimiter::GetLimiterMetricName(item.first))); + mConcurrencyLimiters.emplace_back( + item.second, mMetricsRecordRef.CreateCounter(ConcurrencyLimiter::GetLimiterMetricName(item.first))); } } diff --git a/core/pipeline/queue/BoundedSenderQueueInterface.h b/core/pipeline/queue/BoundedSenderQueueInterface.h index 826d574a5b..e4534792f4 100644 --- a/core/pipeline/queue/BoundedSenderQueueInterface.h +++ b/core/pipeline/queue/BoundedSenderQueueInterface.h @@ -19,8 +19,8 @@ #include #include #include -#include #include +#include #include "common/FeedbackInterface.h" #include "pipeline/limiter/ConcurrencyLimiter.h" @@ -44,17 +44,22 @@ class BoundedSenderQueueInterface : public BoundedQueueInterface& item) override { return false; } virtual bool Remove(SenderQueueItem* item) = 0; - + virtual void GetAvailableItems(std::vector& items, int32_t limit) = 0; void DecreaseSendingCnt(); void SetRateLimiter(uint32_t maxRate); - void SetConcurrencyLimiters(std::unordered_map>&& concurrencyLimitersMap); + void SetConcurrencyLimiters( + std::unordered_map>&& concurrencyLimitersMap); virtual void SetPipelineForItems(const std::shared_ptr& p) const = 0; #ifdef APSARA_UNIT_TEST_MAIN - std::optional& GetRateLimiter() { return mRateLimiter; } - std::vector, CounterPtr>>& GetConcurrencyLimiters() { return mConcurrencyLimiters; } + std::optional& GetRateLimiter() { + return mRateLimiter; + } + std::vector, CounterPtr>>& GetConcurrencyLimiters() { + return mConcurrencyLimiters; + } #endif protected: diff --git a/core/pipeline/queue/CircularProcessQueue.h b/core/pipeline/queue/CircularProcessQueue.h index db05d038ea..ea5b1e6a99 100644 --- a/core/pipeline/queue/CircularProcessQueue.h +++ b/core/pipeline/queue/CircularProcessQueue.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include diff --git a/core/pipeline/queue/ExactlyOnceQueueManager.h b/core/pipeline/queue/ExactlyOnceQueueManager.h index 6fb51ab0d3..4266b0ee95 100644 --- a/core/pipeline/queue/ExactlyOnceQueueManager.h +++ b/core/pipeline/queue/ExactlyOnceQueueManager.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include diff --git a/core/pipeline/queue/ProcessQueueInterface.h b/core/pipeline/queue/ProcessQueueInterface.h index bcf5bba028..2a0a3d5a87 100644 --- a/core/pipeline/queue/ProcessQueueInterface.h +++ b/core/pipeline/queue/ProcessQueueInterface.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include diff --git a/core/pipeline/queue/ProcessQueueItem.h b/core/pipeline/queue/ProcessQueueItem.h index add8a38514..0906a104e3 100644 --- a/core/pipeline/queue/ProcessQueueItem.h +++ b/core/pipeline/queue/ProcessQueueItem.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include "models/PipelineEventGroup.h" diff --git a/core/pipeline/queue/ProcessQueueManager.h b/core/pipeline/queue/ProcessQueueManager.h index 289bafba5b..abad13836e 100644 --- a/core/pipeline/queue/ProcessQueueManager.h +++ b/core/pipeline/queue/ProcessQueueManager.h @@ -16,8 +16,9 @@ #pragma once -#include #include + +#include #include #include #include diff --git a/core/pipeline/queue/SenderQueueItem.h b/core/pipeline/queue/SenderQueueItem.h index e791479f19..d39791aa6f 100644 --- a/core/pipeline/queue/SenderQueueItem.h +++ b/core/pipeline/queue/SenderQueueItem.h @@ -16,9 +16,10 @@ #pragma once -#include #include #include + +#include #include #include diff --git a/core/pipeline/route/Condition.h b/core/pipeline/route/Condition.h index 174a78d2ce..925b28bfb1 100644 --- a/core/pipeline/route/Condition.h +++ b/core/pipeline/route/Condition.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + #include "models/PipelineEventGroup.h" #include "pipeline/PipelineContext.h" diff --git a/core/pipeline/route/Router.h b/core/pipeline/route/Router.h index b4356c05d1..da5324c2e5 100644 --- a/core/pipeline/route/Router.h +++ b/core/pipeline/route/Router.h @@ -16,11 +16,11 @@ #pragma once -#include - #include #include +#include "json/json.h" + #include "models/PipelineEventGroup.h" #include "monitor/MetricManager.h" #include "pipeline/route/Condition.h" diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index 177ddf4fa7..504a1a2b77 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -14,10 +14,10 @@ #include "pipeline/serializer/SLSSerializer.h" -#include - #include +#include "json/json.h" + #include "common/Flags.h" #include "common/compression/CompressType.h" #include "constants/SpanConstants.h" diff --git a/core/pipeline/serializer/SLSSerializer.h b/core/pipeline/serializer/SLSSerializer.h index c9bdb56798..0b92c7134b 100644 --- a/core/pipeline/serializer/SLSSerializer.h +++ b/core/pipeline/serializer/SLSSerializer.h @@ -38,8 +38,10 @@ struct CompressedLogGroup { CompressedLogGroup(std::string&& data, size_t rawSize) : mData(std::move(data)), mRawSize(rawSize) {} }; -template<> -bool Serializer>::DoSerialize(std::vector&& p, std::string& output, std::string& errorMsg); +template <> +bool Serializer>::DoSerialize(std::vector&& p, + std::string& output, + std::string& errorMsg); class SLSEventGroupListSerializer : public Serializer> { public: diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index 27cc2e0847..1bec317946 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include "models/PipelineEventPtr.h" diff --git a/core/plugin/flusher/blackhole/FlusherBlackHole.h b/core/plugin/flusher/blackhole/FlusherBlackHole.h index d0a705cf71..874c16b157 100644 --- a/core/plugin/flusher/blackhole/FlusherBlackHole.h +++ b/core/plugin/flusher/blackhole/FlusherBlackHole.h @@ -23,7 +23,7 @@ namespace logtail { class FlusherBlackHole : public Flusher { public: static const std::string sName; - + const std::string& Name() const override { return sName; } bool Init(const Json::Value& config, Json::Value& optionalGoPipeline) override; bool Send(PipelineEventGroup&& g) override; diff --git a/core/plugin/flusher/file/FlusherFile.cpp b/core/plugin/flusher/file/FlusherFile.cpp index 73eb646547..bdc12c765f 100644 --- a/core/plugin/flusher/file/FlusherFile.cpp +++ b/core/plugin/flusher/file/FlusherFile.cpp @@ -14,9 +14,9 @@ #include "plugin/flusher/file/FlusherFile.h" -#include -#include -#include +#include "spdlog/async.h" +#include "spdlog/sinks/rotating_file_sink.h" +#include "spdlog/sinks/stdout_color_sinks.h" #include "pipeline/queue/SenderQueueManager.h" diff --git a/core/plugin/flusher/file/FlusherFile.h b/core/plugin/flusher/file/FlusherFile.h index 0f8e706ac5..a9f3313b6d 100644 --- a/core/plugin/flusher/file/FlusherFile.h +++ b/core/plugin/flusher/file/FlusherFile.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "spdlog/spdlog.h" + #include "pipeline/batch/Batcher.h" #include "pipeline/plugin/interface/Flusher.h" #include "pipeline/serializer/JsonSerializer.h" diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index f613da5f30..fd4c8117d9 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -28,14 +28,14 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "plugin/flusher/sls/FlusherSLS.h" -#ifdef __ENTERPRISE__ -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" -#endif #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSConstant.h" #include "plugin/flusher/sls/SendResult.h" #include "protobuf/sls/sls_logs.pb.h" #include "provider/Provider.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif DEFINE_FLAG_INT32(write_secondary_wait_timeout, "interval of dump seconary buffer from memory to file, seconds", 2); DEFINE_FLAG_INT32(buffer_file_alive_interval, "the max alive time of a bufferfile, 5 minutes", 300); @@ -586,8 +586,7 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t } #ifdef __ENTERPRISE__ if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR) { - bool hasAuthError - = sendRes == SEND_UNAUTHORIZED && response.mErrorMsg != kAKErrorMsg; + bool hasAuthError = sendRes == SEND_UNAUTHORIZED && response.mErrorMsg != kAKErrorMsg; EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus( diff --git a/core/plugin/flusher/sls/DiskBufferWriter.h b/core/plugin/flusher/sls/DiskBufferWriter.h index 6075ffa763..725b91deaf 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.h +++ b/core/plugin/flusher/sls/DiskBufferWriter.h @@ -16,27 +16,27 @@ #pragma once -#include -#include #include #include + +#include +#include #include #include #include #include -#ifdef __ENTERPRIRSE__ -#include -#endif #include #include "common/SafeQueue.h" #include "pipeline/queue/SenderQueueItem.h" -#ifdef __ENTERPRISE__ -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" -#endif #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSResponse.h" #include "protobuf/sls/logtail_buffer_meta.pb.h" +#ifdef __ENTERPRISE__ +#include + +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif namespace logtail { diff --git a/core/plugin/flusher/sls/Exception.h b/core/plugin/flusher/sls/Exception.h index eb77722196..8ddbd91086 100644 --- a/core/plugin/flusher/sls/Exception.h +++ b/core/plugin/flusher/sls/Exception.h @@ -15,87 +15,87 @@ */ #pragma once -#include #include +#include namespace logtail { namespace sdk { - /** - *LOG Exception includes an error code and a detail message. +/** + *LOG Exception includes an error code and a detail message. + */ +class LOGException : public std::exception { + /** Constructor with error code and message. + * @param errorCode LOG error code. + * @param message Detailed information for the exception. + * @return The objcect pointer. + */ +public: + LOGException() { mHttpCode = 0; } + LOGException(const std::string& errorCode, + const std::string& message, + const std::string& requestId = "", + const int32_t httpCode = 0) + : mErrorCode(errorCode), mMessage(message), mRequestId(requestId), mHttpCode(httpCode) {} + ~LOGException() throw() {} + /** Function that return error code. + * @param void None. + * @return Error code string. + */ + std::string GetErrorCode(void) const { return mErrorCode; } + /** Function that return error message. + * @param void None. + * @return Error message string. */ - class LOGException : public std::exception { - /** Constructor with error code and message. - * @param errorCode LOG error code. - * @param message Detailed information for the exception. - * @return The objcect pointer. - */ - public: - LOGException() { mHttpCode = 0; } - LOGException(const std::string& errorCode, - const std::string& message, - const std::string& requestId = "", - const int32_t httpCode = 0) - : mErrorCode(errorCode), mMessage(message), mRequestId(requestId), mHttpCode(httpCode) {} - ~LOGException() throw() {} - /** Function that return error code. - * @param void None. - * @return Error code string. - */ - std::string GetErrorCode(void) const { return mErrorCode; } - /** Function that return error message. - * @param void None. - * @return Error message string. - */ - std::string GetMessage(void) const { return mMessage; } - // for windows compatability, to avoid conflict with the same function defined in windows.h - std::string GetMessage_() const { return mMessage; } - /** Function that return request id. - * @param void None. - * @return request id string. - */ - std::string GetRequestId(void) const { return mRequestId; } - /** Function that return http response code. - * @param void None. - * @return http response code int32_t, if client error, return 0. - */ - int32_t GetHttpCode(void) const { return mHttpCode; } + std::string GetMessage(void) const { return mMessage; } + // for windows compatability, to avoid conflict with the same function defined in windows.h + std::string GetMessage_() const { return mMessage; } + /** Function that return request id. + * @param void None. + * @return request id string. + */ + std::string GetRequestId(void) const { return mRequestId; } + /** Function that return http response code. + * @param void None. + * @return http response code int32_t, if client error, return 0. + */ + int32_t GetHttpCode(void) const { return mHttpCode; } - private: - std::string mErrorCode; - std::string mMessage; - std::string mRequestId; - int32_t mHttpCode; - }; +private: + std::string mErrorCode; + std::string mMessage; + std::string mRequestId; + int32_t mHttpCode; +}; - /** - *LOG Json Exception includes an error code and a detail message. +/** + *LOG Json Exception includes an error code and a detail message. + */ +class JsonException : public std::exception { + /** Constructor with error code and message. + * @param errorCode LOG error code. + * @param message Detailed information for the exception. + * @return The objcect pointer. + */ +public: + JsonException(const std::string& errorCode, const std::string& message) + : mErrorCode(errorCode), mMessage(message) {} + ~JsonException() throw() {} + /** Function that return error code. + * @param void None. + * @return Error code string. + */ + std::string GetErrorCode(void) const { return mErrorCode; } + /** Function that return error message. + * @param void None. + * @return Error message string. */ - class JsonException : public std::exception { - /** Constructor with error code and message. - * @param errorCode LOG error code. - * @param message Detailed information for the exception. - * @return The objcect pointer. - */ - public: - JsonException(const std::string& errorCode, const std::string& message) - : mErrorCode(errorCode), mMessage(message) {} - ~JsonException() throw() {} - /** Function that return error code. - * @param void None. - * @return Error code string. - */ - std::string GetErrorCode(void) const { return mErrorCode; } - /** Function that return error message. - * @param void None. - * @return Error message string. - */ - std::string GetMessage(void) const { return mMessage; } + std::string GetMessage(void) const { return mMessage; } - private: - std::string mErrorCode; - std::string mMessage; - }; +private: + std::string mErrorCode; + std::string mMessage; +}; } // namespace sdk } // namespace logtail diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 614f121700..d8b6b71e1f 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -22,18 +22,12 @@ #include "common/TimeUtil.h" #include "common/compression/CompressorFactory.h" #include "common/http/Constant.h" -#include "sls_logs.pb.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "pipeline/Pipeline.h" #include "pipeline/batch/FlushStrategy.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" -#ifdef __ENTERPRISE__ -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" -#endif +#include "plugin/flusher/sls/DiskBufferWriter.h" #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSConstant.h" @@ -42,11 +36,9 @@ #include "plugin/flusher/sls/SendResult.h" #include "provider/Provider.h" #include "runner/FlusherRunner.h" -// TODO: temporarily used here -#include "pipeline/PipelineManager.h" -#include "plugin/flusher/sls/DiskBufferWriter.h" +#include "sls_logs.pb.h" #ifdef __ENTERPRISE__ -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#include "config/provider/EnterpriseConfigProvider.h" #endif using namespace std; @@ -810,8 +802,8 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) // the possibility of hash key conflict is very low, so data is // dropped here. cpt->Commit(); - failDetail << ", drop exactly once log group and commit checkpoint" << " checkpointKey:" << cpt->key - << " checkpoint:" << cpt->data.DebugString(); + failDetail << ", drop exactly once log group and commit checkpoint" + << " checkpointKey:" << cpt->key << " checkpoint:" << cpt->data.DebugString(); suggestion << "no suggestion"; AlarmManager::GetInstance()->SendAlarm( EXACTLY_ONCE_ALARM, diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 24c69629b9..a228e2567d 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -16,14 +16,15 @@ #pragma once -#include - #include + #include #include #include #include +#include "json/json.h" + #include "common/compression/Compressor.h" #include "models/PipelineEventGroup.h" #include "pipeline/batch/BatchStatus.h" @@ -32,11 +33,11 @@ #include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/serializer/SLSSerializer.h" +#include "plugin/flusher/sls/SLSClientManager.h" +#include "protobuf/sls/sls_logs.pb.h" #ifdef __ENTERPRISE__ #include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #endif -#include "plugin/flusher/sls/SLSClientManager.h" -#include "protobuf/sls/sls_logs.pb.h" namespace logtail { @@ -67,7 +68,10 @@ class FlusherSLS : public HttpFlusher { bool Send(PipelineEventGroup&& g) override; bool Flush(size_t key) override; bool FlushAll() override; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override; + bool BuildRequest(SenderQueueItem* item, + std::unique_ptr& req, + bool* keepItem, + std::string* errMsg) override; void OnSendDone(const HttpResponse& response, SenderQueueItem* item) override; CompressType GetCompressType() const { return mCompressor ? mCompressor->GetCompressType() : CompressType::NONE; } diff --git a/core/plugin/flusher/sls/PackIdManager.h b/core/plugin/flusher/sls/PackIdManager.h index 18a2b022e2..3fbd51badb 100644 --- a/core/plugin/flusher/sls/PackIdManager.h +++ b/core/plugin/flusher/sls/PackIdManager.h @@ -18,6 +18,7 @@ #include #include + #include #include diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index be9e57291e..42a3b7f2f4 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -27,11 +27,11 @@ #include "common/version.h" #include "logger/Logger.h" #include "monitor/Monitor.h" +#include "plugin/flusher/sls/SLSConstant.h" +#include "plugin/flusher/sls/SLSUtil.h" #ifdef __ENTERPRISE__ #include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #endif -#include "plugin/flusher/sls/SLSConstant.h" -#include "plugin/flusher/sls/SLSUtil.h" DEFINE_FLAG_STRING(custom_user_agent, "custom user agent appended at the end of the exsiting ones", ""); DEFINE_FLAG_STRING(default_access_key_id, "", ""); diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index a49f3123a9..c5abb9e39a 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include @@ -37,7 +38,7 @@ class SLSClientManager { static SLSClientManager* GetInstance(); virtual void Init(); - virtual void Stop() {}; + virtual void Stop(){}; const std::string& GetUserAgent() const { return mUserAgent; } diff --git a/core/plugin/flusher/sls/SLSResponse.cpp b/core/plugin/flusher/sls/SLSResponse.cpp index e13847618a..a7a4b7e15d 100644 --- a/core/plugin/flusher/sls/SLSResponse.cpp +++ b/core/plugin/flusher/sls/SLSResponse.cpp @@ -14,15 +14,15 @@ #include "plugin/flusher/sls/SLSResponse.h" -#include +#include "rapidjson/document.h" +#include "Exception.h" #include "app_config/AppConfig.h" #include "common/ErrorUtil.h" #include "common/StringTools.h" #include "common/TimeUtil.h" #include "logger/Logger.h" #include "plugin/flusher/sls/SLSConstant.h" -#include "Exception.h" using namespace std; using namespace logtail::sdk; diff --git a/core/plugin/flusher/sls/SLSResponse.h b/core/plugin/flusher/sls/SLSResponse.h index 887c6f07f1..dba205acb3 100644 --- a/core/plugin/flusher/sls/SLSResponse.h +++ b/core/plugin/flusher/sls/SLSResponse.h @@ -18,6 +18,7 @@ #include #include + #include #include "common/http/HttpResponse.h" diff --git a/core/plugin/flusher/sls/SLSUtil.h b/core/plugin/flusher/sls/SLSUtil.h index a30288c90a..4a95863724 100644 --- a/core/plugin/flusher/sls/SLSUtil.h +++ b/core/plugin/flusher/sls/SLSUtil.h @@ -18,6 +18,7 @@ #include #include + #include #include diff --git a/core/plugin/input/InputContainerStdio.h b/core/plugin/input/InputContainerStdio.h index f5955b4fa6..a91cb42342 100644 --- a/core/plugin/input/InputContainerStdio.h +++ b/core/plugin/input/InputContainerStdio.h @@ -21,9 +21,9 @@ #include "container_manager/ContainerDiscoveryOptions.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/reader/FileReaderOptions.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" #include "pipeline/plugin/interface/Input.h" -#include "file_server/reader/FileReaderOptions.h" namespace logtail { diff --git a/core/plugin/input/InputFile.h b/core/plugin/input/InputFile.h index 4d966f1e67..6922ae31e0 100644 --- a/core/plugin/input/InputFile.h +++ b/core/plugin/input/InputFile.h @@ -21,9 +21,9 @@ #include "container_manager/ContainerDiscoveryOptions.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/reader/FileReaderOptions.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" #include "pipeline/plugin/interface/Input.h" -#include "file_server/reader/FileReaderOptions.h" namespace logtail { diff --git a/core/plugin/input/InputFileSecurity.cpp b/core/plugin/input/InputFileSecurity.cpp index 5c15536721..134ca2dcc9 100644 --- a/core/plugin/input/InputFileSecurity.cpp +++ b/core/plugin/input/InputFileSecurity.cpp @@ -15,8 +15,8 @@ #include "plugin/input/InputFileSecurity.h" // #include "ebpf/security/SecurityServer.h" -#include "ebpf/include/export.h" #include "ebpf/eBPFServer.h" +#include "ebpf/include/export.h" using namespace std; @@ -30,10 +30,13 @@ bool InputFileSecurity::Init(const Json::Value& config, Json::Value& optionalGoP if (!ebpf::eBPFServer::GetInstance()->IsSupportedEnv(nami::PluginType::FILE_SECURITY)) { return false; } - std::string prev_pipeline_name = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::FILE_SECURITY); + std::string prev_pipeline_name + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::FILE_SECURITY); std::string pipeline_name = mContext->GetConfigName(); if (prev_pipeline_name.size() && prev_pipeline_name != pipeline_name) { - LOG_WARNING(sLogger, ("pipeline already loaded", "FILE_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); + LOG_WARNING(sLogger, + ("pipeline already loaded", "FILE_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", + pipeline_name)); return false; } static const std::unordered_map metricKeys = { @@ -43,12 +46,14 @@ bool InputFileSecurity::Init(const Json::Value& config, Json::Value& optionalGoP {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); + mPluginMgr = std::make_shared( + GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::FILE, config, mContext, sName); } bool InputFileSecurity::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::FILE_SECURITY, mContext, &mSecurityOptions, mPluginMgr); + return ebpf::eBPFServer::GetInstance()->EnablePlugin( + mContext->GetConfigName(), mIndex, nami::PluginType::FILE_SECURITY, mContext, &mSecurityOptions, mPluginMgr); } bool InputFileSecurity::Stop(bool isPipelineRemoving) { diff --git a/core/plugin/input/InputInternalMetrics.h b/core/plugin/input/InputInternalMetrics.h index 6a11d5788e..0ffb038080 100644 --- a/core/plugin/input/InputInternalMetrics.h +++ b/core/plugin/input/InputInternalMetrics.h @@ -30,6 +30,7 @@ class InputInternalMetrics : public Input { bool Start() override; bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return true; } + private: SelfMonitorMetricRules mSelfMonitorMetricRules; #ifdef APSARA_UNIT_TEST_MAIN diff --git a/core/plugin/input/InputNetworkObserver.cpp b/core/plugin/input/InputNetworkObserver.cpp index 22b1358302..6e63dfc1ea 100644 --- a/core/plugin/input/InputNetworkObserver.cpp +++ b/core/plugin/input/InputNetworkObserver.cpp @@ -31,10 +31,13 @@ bool InputNetworkObserver::Init(const Json::Value& config, Json::Value& optional if (!ebpf::eBPFServer::GetInstance()->IsSupportedEnv(nami::PluginType::NETWORK_OBSERVE)) { return false; } - std::string prev_pipeline_name = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_OBSERVE); + std::string prev_pipeline_name + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_OBSERVE); std::string pipeline_name = mContext->GetConfigName(); if (prev_pipeline_name.size() && prev_pipeline_name != pipeline_name) { - LOG_WARNING(sLogger, ("pipeline already loaded", "NETWORK_OBSERVE")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); + LOG_WARNING(sLogger, + ("pipeline already loaded", "NETWORK_OBSERVE")("prev pipeline", prev_pipeline_name)("curr pipeline", + pipeline_name)); return false; } @@ -49,12 +52,14 @@ bool InputNetworkObserver::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_NETWORK_OBSERVER_PROTOCOL_PARSE_RECORDS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); + mPluginMgr = std::make_shared( + GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return ebpf::InitObserverNetworkOption(config, mNetworkOption, mContext, sName); } bool InputNetworkObserver::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_OBSERVE, mContext, &mNetworkOption, mPluginMgr); + return ebpf::eBPFServer::GetInstance()->EnablePlugin( + mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_OBSERVE, mContext, &mNetworkOption, mPluginMgr); } bool InputNetworkObserver::Stop(bool isPipelineRemoving) { diff --git a/core/plugin/input/InputNetworkSecurity.cpp b/core/plugin/input/InputNetworkSecurity.cpp index ccadcc26ca..db932713b3 100644 --- a/core/plugin/input/InputNetworkSecurity.cpp +++ b/core/plugin/input/InputNetworkSecurity.cpp @@ -14,8 +14,8 @@ #include "plugin/input/InputNetworkSecurity.h" -#include "ebpf/include/export.h" #include "ebpf/eBPFServer.h" +#include "ebpf/include/export.h" #include "logger/Logger.h" using namespace std; @@ -32,10 +32,13 @@ bool InputNetworkSecurity::Init(const Json::Value& config, Json::Value& optional if (!ebpf::eBPFServer::GetInstance()->IsSupportedEnv(nami::PluginType::NETWORK_SECURITY)) { return false; } - std::string prev_pipeline_name = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); + std::string prev_pipeline_name + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); std::string pipeline_name = mContext->GetConfigName(); if (prev_pipeline_name.size() && prev_pipeline_name != pipeline_name) { - LOG_WARNING(sLogger, ("pipeline already loaded", "NETWORK_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); + LOG_WARNING(sLogger, + ("pipeline already loaded", + "NETWORK_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); return false; } static const std::unordered_map metricKeys = { @@ -45,13 +48,15 @@ bool InputNetworkSecurity::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); + mPluginMgr = std::make_shared( + GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::NETWORK, config, mContext, sName); } bool InputNetworkSecurity::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_SECURITY, mContext, &mSecurityOptions, mPluginMgr); + return ebpf::eBPFServer::GetInstance()->EnablePlugin( + mContext->GetConfigName(), mIndex, nami::PluginType::NETWORK_SECURITY, mContext, &mSecurityOptions, mPluginMgr); } bool InputNetworkSecurity::Stop(bool isPipelineRemoving) { @@ -59,7 +64,8 @@ bool InputNetworkSecurity::Stop(bool isPipelineRemoving) { ebpf::eBPFServer::GetInstance()->SuspendPlugin(mContext->GetConfigName(), nami::PluginType::NETWORK_SECURITY); return true; } - return ebpf::eBPFServer::GetInstance()->DisablePlugin(mContext->GetConfigName(), nami::PluginType::NETWORK_SECURITY); + return ebpf::eBPFServer::GetInstance()->DisablePlugin(mContext->GetConfigName(), + nami::PluginType::NETWORK_SECURITY); } } // namespace logtail diff --git a/core/plugin/input/InputProcessSecurity.cpp b/core/plugin/input/InputProcessSecurity.cpp index 3baa6cbed8..867a5154f8 100644 --- a/core/plugin/input/InputProcessSecurity.cpp +++ b/core/plugin/input/InputProcessSecurity.cpp @@ -14,8 +14,8 @@ #include "plugin/input/InputProcessSecurity.h" -#include "ebpf/include/export.h" #include "ebpf/eBPFServer.h" +#include "ebpf/include/export.h" using namespace std; @@ -28,10 +28,13 @@ bool InputProcessSecurity::Init(const Json::Value& config, Json::Value& optional if (!ebpf::eBPFServer::GetInstance()->IsSupportedEnv(nami::PluginType::PROCESS_SECURITY)) { return false; } - std::string prev_pipeline_name = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY); + std::string prev_pipeline_name + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY); std::string pipeline_name = mContext->GetConfigName(); if (prev_pipeline_name.size() && prev_pipeline_name != pipeline_name) { - LOG_WARNING(sLogger, ("pipeline already loaded", "PROCESS_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); + LOG_WARNING(sLogger, + ("pipeline already loaded", + "PROCESS_SECURITY")("prev pipeline", prev_pipeline_name)("curr pipeline", pipeline_name)); return false; } @@ -42,12 +45,14 @@ bool InputProcessSecurity::Init(const Json::Value& config, Json::Value& optional {METRIC_PLUGIN_EBPF_PROCESS_CACHE_MISS_TOTAL, MetricType::METRIC_TYPE_COUNTER}, }; - mPluginMgr = std::make_shared(GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); + mPluginMgr = std::make_shared( + GetMetricsRecordRef().GetLabels(), metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); return mSecurityOptions.Init(ebpf::SecurityProbeType::PROCESS, config, mContext, sName); } bool InputProcessSecurity::Start() { - return ebpf::eBPFServer::GetInstance()->EnablePlugin(mContext->GetConfigName(), mIndex, nami::PluginType::PROCESS_SECURITY,mContext, &mSecurityOptions, mPluginMgr); + return ebpf::eBPFServer::GetInstance()->EnablePlugin( + mContext->GetConfigName(), mIndex, nami::PluginType::PROCESS_SECURITY, mContext, &mSecurityOptions, mPluginMgr); } bool InputProcessSecurity::Stop(bool isPipelineRemoving) { @@ -55,7 +60,8 @@ bool InputProcessSecurity::Stop(bool isPipelineRemoving) { ebpf::eBPFServer::GetInstance()->SuspendPlugin(mContext->GetConfigName(), nami::PluginType::PROCESS_SECURITY); return true; } - return ebpf::eBPFServer::GetInstance()->DisablePlugin(mContext->GetConfigName(), nami::PluginType::PROCESS_SECURITY); + return ebpf::eBPFServer::GetInstance()->DisablePlugin(mContext->GetConfigName(), + nami::PluginType::PROCESS_SECURITY); } } // namespace logtail diff --git a/core/plugin/input/InputPrometheus.cpp b/core/plugin/input/InputPrometheus.cpp index a1d2af138c..940e6fec6e 100644 --- a/core/plugin/input/InputPrometheus.cpp +++ b/core/plugin/input/InputPrometheus.cpp @@ -16,12 +16,12 @@ #include "plugin/input/InputPrometheus.h" -#include -#include - #include #include +#include "json/json.h" +#include "json/value.h" + #include "PluginRegistry.h" #include "logger/Logger.h" #include "pipeline/Pipeline.h" @@ -70,7 +70,8 @@ bool InputPrometheus::Start() { mTargetSubscirber->mQueueKey = mContext->GetProcessQueueKey(); auto defaultLabels = GetMetricsRecordRef()->GetLabels(); - PrometheusInputRunner::GetInstance()->UpdateScrapeInput(std::move(mTargetSubscirber), *defaultLabels, mContext->GetProjectName()); + PrometheusInputRunner::GetInstance()->UpdateScrapeInput( + std::move(mTargetSubscirber), *defaultLabels, mContext->GetProjectName()); return true; } diff --git a/core/plugin/input/InputPrometheus.h b/core/plugin/input/InputPrometheus.h index 9ae51128b6..0ad89678e5 100644 --- a/core/plugin/input/InputPrometheus.h +++ b/core/plugin/input/InputPrometheus.h @@ -1,6 +1,7 @@ #pragma once #include + #include #include diff --git a/core/plugin/input/InputStaticFile.cpp b/core/plugin/input/InputStaticFile.cpp index 9d943fff76..06e1bfb18f 100644 --- a/core/plugin/input/InputStaticFile.cpp +++ b/core/plugin/input/InputStaticFile.cpp @@ -24,7 +24,7 @@ InputStaticFile::InputStaticFile(/* args */) { InputStaticFile::~InputStaticFile() { } -void InputStaticFile::Init(PipelineConfig &&config) { +void InputStaticFile::Init(PipelineConfig&& config) { // mAdhocFileManager = AdhocFileManager::GetInstance(); // GetStaticFileList(); } @@ -46,4 +46,4 @@ void InputStaticFile::GetStaticFileList() { void InputStaticFile::SortFileList() { } -} +} // namespace logtail diff --git a/core/plugin/input/InputStaticFile.h b/core/plugin/input/InputStaticFile.h index eec8caef4c..21d8c4c4ab 100644 --- a/core/plugin/input/InputStaticFile.h +++ b/core/plugin/input/InputStaticFile.h @@ -28,14 +28,14 @@ class InputStaticFile { std::string mJobName; AdhocFileManager* mAdhocFileManager; std::vector mFileList; - + public: InputStaticFile(/* args */); ~InputStaticFile(); - void Init(PipelineConfig &&config); + void Init(PipelineConfig&& config); void Start(); void Stop(bool isRemoving); }; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/plugin/processor/CommonParserOptions.cpp b/core/plugin/processor/CommonParserOptions.cpp index d56761d69c..d728b0789a 100644 --- a/core/plugin/processor/CommonParserOptions.cpp +++ b/core/plugin/processor/CommonParserOptions.cpp @@ -16,8 +16,8 @@ #include "plugin/processor/CommonParserOptions.h" -#include "constants/Constants.h" #include "common/ParamExtractor.h" +#include "constants/Constants.h" #include "plugin/processor/inner/ProcessorParseContainerLogNative.h" using namespace std; diff --git a/core/plugin/processor/CommonParserOptions.h b/core/plugin/processor/CommonParserOptions.h index b76345b37a..f294bc0086 100644 --- a/core/plugin/processor/CommonParserOptions.h +++ b/core/plugin/processor/CommonParserOptions.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + #include "models/LogEvent.h" #include "pipeline/PipelineContext.h" diff --git a/core/plugin/processor/ProcessorDesensitizeNative.cpp b/core/plugin/processor/ProcessorDesensitizeNative.cpp index 47c94bfcf8..378b4f1829 100644 --- a/core/plugin/processor/ProcessorDesensitizeNative.cpp +++ b/core/plugin/processor/ProcessorDesensitizeNative.cpp @@ -15,12 +15,12 @@ */ #include "plugin/processor/ProcessorDesensitizeNative.h" -#include "constants/Constants.h" +#include "common/HashUtil.h" #include "common/ParamExtractor.h" +#include "constants/Constants.h" #include "models/LogEvent.h" #include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" -#include "common/HashUtil.h" namespace logtail { diff --git a/core/plugin/processor/ProcessorDesensitizeNative.h b/core/plugin/processor/ProcessorDesensitizeNative.h index b720fd7569..3deea68df1 100644 --- a/core/plugin/processor/ProcessorDesensitizeNative.h +++ b/core/plugin/processor/ProcessorDesensitizeNative.h @@ -16,7 +16,7 @@ #pragma once -#include +#include "re2/re2.h" #include "pipeline/plugin/interface/Processor.h" diff --git a/core/plugin/processor/ProcessorFilterNative.cpp b/core/plugin/processor/ProcessorFilterNative.cpp index 0b14998fe5..321038da55 100644 --- a/core/plugin/processor/ProcessorFilterNative.cpp +++ b/core/plugin/processor/ProcessorFilterNative.cpp @@ -218,7 +218,7 @@ bool ProcessorFilterNative::ProcessEvent(PipelineEventPtr& e) { } if (res && mDiscardingNonUTF8) { std::vector > newContents; - for (auto &content: sourceEvent) { + for (auto& content : sourceEvent) { if (CheckNoneUtf8(content.second)) { auto value = content.second.to_string(); FilterNoneUtf8(value); @@ -233,7 +233,7 @@ bool ProcessorFilterNative::ProcessEvent(PipelineEventPtr& e) { newContents.emplace_back(StringView(keyBuffer.data, keyBuffer.size), content.second); sourceEvent.DelContent(content.first); - } + } } for (auto& newContent : newContents) { sourceEvent.SetContentNoCopy(newContent.first, newContent.second); diff --git a/core/plugin/processor/ProcessorFilterNative.h b/core/plugin/processor/ProcessorFilterNative.h index 216bf902af..5eb49c50c1 100644 --- a/core/plugin/processor/ProcessorFilterNative.h +++ b/core/plugin/processor/ProcessorFilterNative.h @@ -16,12 +16,12 @@ #pragma once +#include "boost/regex.hpp" + #include "app_config/AppConfig.h" #include "models/LogEvent.h" #include "pipeline/plugin/interface/Processor.h" -#include - namespace logtail { // BaseFilterNode @@ -83,8 +83,7 @@ class RegexFilterValueNode : public BaseFilterNode { // UnaryFilterOperatorNode class UnaryFilterOperatorNode : public BaseFilterNode { public: - UnaryFilterOperatorNode(BaseFilterNodePtr child) - : BaseFilterNode(OPERATOR_NODE), child(child) {} + UnaryFilterOperatorNode(BaseFilterNodePtr child) : BaseFilterNode(OPERATOR_NODE), child(child) {} virtual ~UnaryFilterOperatorNode(){}; diff --git a/core/plugin/processor/ProcessorParseDelimiterNative.cpp b/core/plugin/processor/ProcessorParseDelimiterNative.cpp index 61abbfd572..8a3329c787 100644 --- a/core/plugin/processor/ProcessorParseDelimiterNative.cpp +++ b/core/plugin/processor/ProcessorParseDelimiterNative.cpp @@ -238,7 +238,7 @@ bool ProcessorParseDelimiterNative::ProcessEvent(const StringView& logPath, Pipe mOutFailedEventsTotal->Add(1); return true; } - + size_t reserveSize = mOverflowedFieldsTreatment == OverflowedFieldsTreatment::EXTEND ? (mKeys.size() + 10) : (mKeys.size() + 1); std::vector columnValues; diff --git a/core/plugin/processor/ProcessorParseJsonNative.cpp b/core/plugin/processor/ProcessorParseJsonNative.cpp index 924ff51bff..21abdab153 100644 --- a/core/plugin/processor/ProcessorParseJsonNative.cpp +++ b/core/plugin/processor/ProcessorParseJsonNative.cpp @@ -16,8 +16,9 @@ #include "plugin/processor/ProcessorParseJsonNative.h" -#include -#include +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" #include "common/ParamExtractor.h" #include "models/LogEvent.h" @@ -26,6 +27,32 @@ namespace logtail { +static std::string RapidjsonValueToString(const rapidjson::Value& value) { + if (value.IsString()) + return std::string(value.GetString(), value.GetStringLength()); + else if (value.IsBool()) + return ToString(value.GetBool()); + else if (value.IsInt()) + return ToString(value.GetInt()); + else if (value.IsUint()) + return ToString(value.GetUint()); + else if (value.IsInt64()) + return ToString(value.GetInt64()); + else if (value.IsUint64()) + return ToString(value.GetUint64()); + else if (value.IsDouble()) + return ToString(value.GetDouble()); + else if (value.IsNull()) + return ""; + else // if (value.IsObject() || value.IsArray()) + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return std::string(buffer.GetString(), buffer.GetLength()); + } +} + const std::string ProcessorParseJsonNative::sName = "processor_parse_json_native"; bool ProcessorParseJsonNative::Init(const Json::Value& config) { @@ -170,32 +197,6 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent, return true; } -std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Value& value) { - if (value.IsString()) - return std::string(value.GetString(), value.GetStringLength()); - else if (value.IsBool()) - return ToString(value.GetBool()); - else if (value.IsInt()) - return ToString(value.GetInt()); - else if (value.IsUint()) - return ToString(value.GetUint()); - else if (value.IsInt64()) - return ToString(value.GetInt64()); - else if (value.IsUint64()) - return ToString(value.GetUint64()); - else if (value.IsDouble()) - return ToString(value.GetDouble()); - else if (value.IsNull()) - return ""; - else // if (value.IsObject() || value.IsArray()) - { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - value.Accept(writer); - return std::string(buffer.GetString(), buffer.GetLength()); - } -} - void ProcessorParseJsonNative::AddLog(const StringView& key, const StringView& value, LogEvent& targetEvent, diff --git a/core/plugin/processor/ProcessorParseJsonNative.h b/core/plugin/processor/ProcessorParseJsonNative.h index 7382079585..214d4de23d 100644 --- a/core/plugin/processor/ProcessorParseJsonNative.h +++ b/core/plugin/processor/ProcessorParseJsonNative.h @@ -15,8 +15,6 @@ */ #pragma once -#include - #include "models/LogEvent.h" #include "pipeline/plugin/interface/Processor.h" #include "plugin/processor/CommonParserOptions.h" @@ -39,10 +37,12 @@ class ProcessorParseJsonNative : public Processor { bool IsSupportedEvent(const PipelineEventPtr& e) const override; private: - bool JsonLogLineParser(LogEvent& sourceEvent, const StringView& logPath, PipelineEventPtr& e, bool& sourceKeyOverwritten); + bool JsonLogLineParser(LogEvent& sourceEvent, + const StringView& logPath, + PipelineEventPtr& e, + bool& sourceKeyOverwritten); void AddLog(const StringView& key, const StringView& value, LogEvent& targetEvent, bool overwritten = true); bool ProcessEvent(const StringView& logPath, PipelineEventPtr& e); - static std::string RapidjsonValueToString(const rapidjson::Value& value); CounterPtr mDiscardedEventsTotal; CounterPtr mOutFailedEventsTotal; diff --git a/core/plugin/processor/ProcessorParseRegexNative.h b/core/plugin/processor/ProcessorParseRegexNative.h index 087a088f6c..c31efb2cd4 100644 --- a/core/plugin/processor/ProcessorParseRegexNative.h +++ b/core/plugin/processor/ProcessorParseRegexNative.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "boost/regex.hpp" + #include "models/LogEvent.h" #include "pipeline/plugin/interface/Processor.h" #include "plugin/processor/CommonParserOptions.h" diff --git a/core/plugin/processor/ProcessorParseTimestampNative.h b/core/plugin/processor/ProcessorParseTimestampNative.h index ea1a508b15..100dba1577 100644 --- a/core/plugin/processor/ProcessorParseTimestampNative.h +++ b/core/plugin/processor/ProcessorParseTimestampNative.h @@ -58,7 +58,7 @@ class ProcessorParseTimestampNative : public Processor { CounterPtr mOutKeyNotFoundEventsTotal; CounterPtr mOutSuccessfulEventsTotal; CounterPtr mHistoryFailureTotal; - + #ifdef APSARA_UNIT_TEST_MAIN friend class ProcessorParseTimestampNativeUnittest; friend class ProcessorParseLogTimeUnittest; diff --git a/core/plugin/processor/ProcessorSPL.cpp b/core/plugin/processor/ProcessorSPL.cpp index f0a2e6e177..6ebf98fba1 100644 --- a/core/plugin/processor/ProcessorSPL.cpp +++ b/core/plugin/processor/ProcessorSPL.cpp @@ -16,10 +16,6 @@ #include "plugin/processor/ProcessorSPL.h" -#include - -#include - #include "common/Flags.h" #include "common/ParamExtractor.h" #include "logger/Logger.h" diff --git a/core/plugin/processor/ProcessorSPL.h b/core/plugin/processor/ProcessorSPL.h index c5d46e05a0..e87fb7ca35 100644 --- a/core/plugin/processor/ProcessorSPL.h +++ b/core/plugin/processor/ProcessorSPL.h @@ -17,6 +17,7 @@ #include #include "spl/LoongCollectorSplPipeline.h" + #include "monitor/MetricManager.h" #include "pipeline/plugin/interface/Processor.h" diff --git a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp index e041c48971..6fe0947158 100644 --- a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp +++ b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp @@ -16,9 +16,10 @@ #include "plugin/processor/inner/ProcessorMergeMultilineLogNative.h" -#include #include +#include "boost/regex.hpp" + #include "app_config/AppConfig.h" #include "common/ParamExtractor.h" #include "logger/Logger.h" @@ -74,8 +75,7 @@ bool ProcessorMergeMultilineLogNative::Init(const Json::Value& config) { } mMergedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MERGED_EVENTS_TOTAL); - mUnmatchedEventsTotal - = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL); + mUnmatchedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL); return true; } diff --git a/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp b/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp index 4f649565ec..3172b44041 100644 --- a/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp +++ b/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp @@ -16,12 +16,7 @@ #include "plugin/processor/inner/ProcessorParseContainerLogNative.h" -#include -#include -#include - #include -#include #include "common/JsonUtil.h" #include "common/ParamExtractor.h" diff --git a/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp b/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp index 95b81a569a..679b435d0a 100644 --- a/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp +++ b/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp @@ -1,6 +1,6 @@ #include "plugin/processor/inner/ProcessorPromParseMetricNative.h" -#include +#include "json/json.h" #include "common/StringTools.h" #include "models/MetricEvent.h" diff --git a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp index 4c52663ba5..84d6aa8be1 100644 --- a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp +++ b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp @@ -15,10 +15,10 @@ */ #include "plugin/processor/inner/ProcessorPromRelabelMetricNative.h" -#include - #include +#include "json/json.h" + #include "common/Flags.h" #include "common/StringTools.h" #include "models/MetricEvent.h" diff --git a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.h b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.h index 33a00d8437..1624532e8c 100644 --- a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.h +++ b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.h @@ -26,16 +26,16 @@ namespace logtail { namespace prom { - struct AutoMetric { - double mScrapeDurationSeconds; - uint64_t mScrapeResponseSizeBytes; - uint64_t mScrapeSamplesLimit; - // uint64_t mPostRelabel; - uint64_t mScrapeSamplesScraped; - uint64_t mScrapeTimeoutSeconds; - std::string mScrapeState; - bool mUp; - }; +struct AutoMetric { + double mScrapeDurationSeconds; + uint64_t mScrapeResponseSizeBytes; + uint64_t mScrapeSamplesLimit; + // uint64_t mPostRelabel; + uint64_t mScrapeSamplesScraped; + uint64_t mScrapeTimeoutSeconds; + std::string mScrapeState; + bool mUp; +}; } // namespace prom class ProcessorPromRelabelMetricNative : public Processor { diff --git a/core/plugin/processor/inner/ProcessorSplitLogStringNative.h b/core/plugin/processor/inner/ProcessorSplitLogStringNative.h index 58057b3f8c..e08a4d2ff8 100644 --- a/core/plugin/processor/inner/ProcessorSplitLogStringNative.h +++ b/core/plugin/processor/inner/ProcessorSplitLogStringNative.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include "constants/Constants.h" diff --git a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp index 3ced98912a..8f9f38217a 100644 --- a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp +++ b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp @@ -16,9 +16,10 @@ #include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" -#include #include +#include "boost/regex.hpp" + #include "app_config/AppConfig.h" #include "common/ParamExtractor.h" #include "constants/Constants.h" diff --git a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h index 4a9d25cb57..0c74b34d1a 100644 --- a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h +++ b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include "constants/Constants.h" diff --git a/core/prometheus/Constants.h b/core/prometheus/Constants.h index d80a43c70b..f0be5177ac 100644 --- a/core/prometheus/Constants.h +++ b/core/prometheus/Constants.h @@ -1,6 +1,7 @@ #pragma once #include + #include namespace logtail::prometheus { diff --git a/core/prometheus/PromSelfMonitor.cpp b/core/prometheus/PromSelfMonitor.cpp index 410195acaf..5a20c02f5f 100644 --- a/core/prometheus/PromSelfMonitor.cpp +++ b/core/prometheus/PromSelfMonitor.cpp @@ -4,16 +4,17 @@ #include #include -#include "monitor/metric_models/MetricTypes.h" #include "monitor/metric_constants/MetricConstants.h" +#include "monitor/metric_models/MetricTypes.h" using namespace std; namespace logtail { void PromSelfMonitorUnsafe::InitMetricManager(const std::unordered_map& metricKeys, - const MetricLabels& labels) { + const MetricLabels& labels) { auto metricLabels = std::make_shared(labels); - mPluginMetricManagerPtr = std::make_shared(metricLabels, metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); + mPluginMetricManagerPtr = std::make_shared( + metricLabels, metricKeys, MetricCategory::METRIC_CATEGORY_PLUGIN_SOURCE); } void PromSelfMonitorUnsafe::AddCounter(const std::string& metricName, uint64_t statusCode, uint64_t val) { diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index cdb4ba46bf..f3686ab3c7 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -17,6 +17,7 @@ #include "PrometheusInputRunner.h" #include + #include #include diff --git a/core/prometheus/PrometheusInputRunner.h b/core/prometheus/PrometheusInputRunner.h index b8e7719171..8983bdd4d8 100644 --- a/core/prometheus/PrometheusInputRunner.h +++ b/core/prometheus/PrometheusInputRunner.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include diff --git a/core/prometheus/Utils.cpp b/core/prometheus/Utils.cpp index 8d9b743b17..57bdeb7b1e 100644 --- a/core/prometheus/Utils.cpp +++ b/core/prometheus/Utils.cpp @@ -152,34 +152,34 @@ uint64_t GetRandSleepMilliSec(const std::string& key, uint64_t intervalSeconds, namespace prom { - std::string NetworkCodeToState(NetworkCode code) { - static map sNetworkCodeMap = {{NetworkCode::Ok, "OK"}, - {NetworkCode::ConnectionFailed, "ERR_CONN_FAILED"}, - {NetworkCode::RemoteAccessDenied, "ERR_ACCESS_DENIED"}, - {NetworkCode::Timeout, "ERR_TIMEOUT"}, - {NetworkCode::SSLConnectError, "ERR_SSL_CONN_ERR"}, - {NetworkCode::SSLCertError, "ERR_SSL_CERT_ERR"}, - {NetworkCode::SSLOtherProblem, "ERR_SSL_OTHER_PROBLEM"}, - {NetworkCode::SendDataFailed, "ERR_SEND_DATA_FAILED"}, - {NetworkCode::RecvDataFailed, "ERR_RECV_DATA_FAILED"}, - {NetworkCode::Other, "ERR_UNKNOWN"}}; - static string sCurlOther = "ERR_UNKNOWN"; - if (sNetworkCodeMap.count(code)) { - return sNetworkCodeMap[code]; - } - return sCurlOther; +std::string NetworkCodeToState(NetworkCode code) { + static map sNetworkCodeMap = {{NetworkCode::Ok, "OK"}, + {NetworkCode::ConnectionFailed, "ERR_CONN_FAILED"}, + {NetworkCode::RemoteAccessDenied, "ERR_ACCESS_DENIED"}, + {NetworkCode::Timeout, "ERR_TIMEOUT"}, + {NetworkCode::SSLConnectError, "ERR_SSL_CONN_ERR"}, + {NetworkCode::SSLCertError, "ERR_SSL_CERT_ERR"}, + {NetworkCode::SSLOtherProblem, "ERR_SSL_OTHER_PROBLEM"}, + {NetworkCode::SendDataFailed, "ERR_SEND_DATA_FAILED"}, + {NetworkCode::RecvDataFailed, "ERR_RECV_DATA_FAILED"}, + {NetworkCode::Other, "ERR_UNKNOWN"}}; + static string sCurlOther = "ERR_UNKNOWN"; + if (sNetworkCodeMap.count(code)) { + return sNetworkCodeMap[code]; } + return sCurlOther; +} - std::string HttpCodeToState(uint64_t code) { - if (code > 1000) { - return "ERR_HTTP_UNKNOWN"; - } - if (code == 200) { - return "OK"; - } - string statePrefix = "ERR_HTTP_"; - return statePrefix + ToString(code); +std::string HttpCodeToState(uint64_t code) { + if (code > 1000) { + return "ERR_HTTP_UNKNOWN"; + } + if (code == 200) { + return "OK"; } + string statePrefix = "ERR_HTTP_"; + return statePrefix + ToString(code); +} } // namespace prom } // namespace logtail diff --git a/core/prometheus/Utils.h b/core/prometheus/Utils.h index c749b7f3ec..31c1d1fe7f 100644 --- a/core/prometheus/Utils.h +++ b/core/prometheus/Utils.h @@ -22,8 +22,8 @@ bool IsNumber(const std::string& str); uint64_t GetRandSleepMilliSec(const std::string& key, uint64_t intervalSeconds, uint64_t currentMilliSeconds); namespace prom { - std::string NetworkCodeToState(NetworkCode code); - std::string HttpCodeToState(uint64_t code); -} +std::string NetworkCodeToState(NetworkCode code); +std::string HttpCodeToState(uint64_t code); +} // namespace prom } // namespace logtail diff --git a/core/prometheus/async/PromHttpRequest.cpp b/core/prometheus/async/PromHttpRequest.cpp index 1f7b6a4eaa..b5ce844114 100644 --- a/core/prometheus/async/PromHttpRequest.cpp +++ b/core/prometheus/async/PromHttpRequest.cpp @@ -2,6 +2,7 @@ #include #include + #include #include diff --git a/core/prometheus/async/PromHttpRequest.h b/core/prometheus/async/PromHttpRequest.h index e9382b5470..41190b6db3 100644 --- a/core/prometheus/async/PromHttpRequest.h +++ b/core/prometheus/async/PromHttpRequest.h @@ -1,6 +1,7 @@ #pragma once #include + #include #include "common/http/HttpRequest.h" diff --git a/core/prometheus/component/StreamScraper.cpp b/core/prometheus/component/StreamScraper.cpp index da70609af9..5a573f3dbf 100644 --- a/core/prometheus/component/StreamScraper.cpp +++ b/core/prometheus/component/StreamScraper.cpp @@ -1,6 +1,7 @@ #include "prometheus/component/StreamScraper.h" #include + #include #include #include diff --git a/core/prometheus/labels/Labels.cpp b/core/prometheus/labels/Labels.cpp index e65e2adc74..f701cabd82 100644 --- a/core/prometheus/labels/Labels.cpp +++ b/core/prometheus/labels/Labels.cpp @@ -85,12 +85,12 @@ void Labels::Set(const string& k, const string& v) { void Labels::Del(const string& k) { if (mMetricEventPtr) { - if(mMetricEventPtr->HasTag(k)){ + if (mMetricEventPtr->HasTag(k)) { mMetricEventPtr->DelTag(k); } return; } - if(mLabels.count(k)){ + if (mLabels.count(k)) { mLabels.erase(k); } } diff --git a/core/prometheus/labels/Labels.h b/core/prometheus/labels/Labels.h index 27a49e9530..00cd37ef0c 100644 --- a/core/prometheus/labels/Labels.h +++ b/core/prometheus/labels/Labels.h @@ -17,6 +17,7 @@ #pragma once #include + #include #include #include diff --git a/core/prometheus/labels/Relabel.cpp b/core/prometheus/labels/Relabel.cpp index 5996208162..ff4f4a11d5 100644 --- a/core/prometheus/labels/Relabel.cpp +++ b/core/prometheus/labels/Relabel.cpp @@ -18,12 +18,13 @@ #include -#include -#include -#include #include #include +#include "boost/algorithm/string.hpp" +#include "boost/algorithm/string/join.hpp" +#include "boost/regex.hpp" + #include "common/ParamExtractor.h" #include "common/StringTools.h" #include "logger/Logger.h" @@ -31,9 +32,11 @@ using namespace std; -#define ENUM_TO_STRING_CASE(EnumValue) {Action::EnumValue, ToLowerCaseString(#EnumValue)} +#define ENUM_TO_STRING_CASE(EnumValue) \ + { Action::EnumValue, ToLowerCaseString(#EnumValue) } -#define STRING_TO_ENUM_CASE(EnumValue) {ToLowerCaseString(#EnumValue), Action::EnumValue} +#define STRING_TO_ENUM_CASE(EnumValue) \ + { ToLowerCaseString(#EnumValue), Action::EnumValue } namespace logtail { diff --git a/core/prometheus/labels/Relabel.h b/core/prometheus/labels/Relabel.h index 738e61fc0a..62b222621f 100644 --- a/core/prometheus/labels/Relabel.h +++ b/core/prometheus/labels/Relabel.h @@ -15,11 +15,11 @@ */ #pragma once -#include - -#include #include +#include "boost/regex.hpp" +#include "json/json.h" + #include "prometheus/labels/Labels.h" namespace logtail { diff --git a/core/prometheus/labels/TextParser.cpp b/core/prometheus/labels/TextParser.cpp index 100d7ddd2e..63176b6635 100644 --- a/core/prometheus/labels/TextParser.cpp +++ b/core/prometheus/labels/TextParser.cpp @@ -16,10 +16,12 @@ #include "prometheus/labels/TextParser.h" -#include #include + #include +#include "boost/algorithm/string.hpp" + #include "logger/Logger.h" #include "models/MetricEvent.h" #include "models/PipelineEventGroup.h" diff --git a/core/prometheus/schedulers/ScrapeConfig.cpp b/core/prometheus/schedulers/ScrapeConfig.cpp index 3bc05e9f65..1a09acad63 100644 --- a/core/prometheus/schedulers/ScrapeConfig.cpp +++ b/core/prometheus/schedulers/ScrapeConfig.cpp @@ -1,16 +1,16 @@ #include "prometheus/schedulers/ScrapeConfig.h" -#include - #include +#include "json/value.h" + +#include "common/EncodingUtil.h" #include "common/FileSystemUtil.h" #include "common/StringTools.h" #include "logger/Logger.h" #include "prometheus/Constants.h" #include "prometheus/Utils.h" -#include "common/EncodingUtil.h" using namespace std; diff --git a/core/prometheus/schedulers/ScrapeConfig.h b/core/prometheus/schedulers/ScrapeConfig.h index 38e5087ccc..5d2d1ea0b4 100644 --- a/core/prometheus/schedulers/ScrapeConfig.h +++ b/core/prometheus/schedulers/ScrapeConfig.h @@ -1,12 +1,13 @@ #pragma once -#include - #include + #include #include #include +#include "json/value.h" + #include "common/http/HttpRequest.h" #include "prometheus/labels/Relabel.h" diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index c37ebaf23f..88f7c7e216 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -17,6 +17,7 @@ #include "prometheus/schedulers/ScrapeScheduler.h" #include + #include #include #include diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index a5540c8958..f3f7647adc 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -17,6 +17,7 @@ #include "prometheus/schedulers/TargetSubscriberScheduler.h" #include + #include #include diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.h b/core/prometheus/schedulers/TargetSubscriberScheduler.h index af5e1493fd..1ea5db03de 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.h +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.h @@ -16,12 +16,13 @@ #pragma once -#include - #include + #include #include +#include "json/json.h" + #include "common/http/HttpResponse.h" #include "common/timer/Timer.h" #include "pipeline/queue/QueueKey.h" diff --git a/core/protobuf/models/ProtocolConversion.cpp b/core/protobuf/models/ProtocolConversion.cpp index 65b786d3fd..44cb5f86f1 100644 --- a/core/protobuf/models/ProtocolConversion.cpp +++ b/core/protobuf/models/ProtocolConversion.cpp @@ -4,52 +4,53 @@ using namespace std; namespace logtail { -bool TransferPBToPipelineEventGroup(const logtail::models::PipelineEventGroup& src, logtail::PipelineEventGroup& dst, std::string& errMsg) { +bool TransferPBToPipelineEventGroup(const logtail::models::PipelineEventGroup& src, + logtail::PipelineEventGroup& dst, + std::string& errMsg) { // events - switch (src.PipelineEvents_case()) - { - case logtail::models::PipelineEventGroup::PipelineEventsCase::kLogs: - if (src.logs().events_size() == 0) { - errMsg = "error transfer PB to PipelineEventGroup: no log events"; - return false; - } - dst.MutableEvents().reserve(src.logs().events_size()); - for (auto& logSrc : src.logs().events()) { - auto logDst = dst.AddLogEvent(); - if (!TransferPBToLogEvent(logSrc, *logDst, errMsg)) { + switch (src.PipelineEvents_case()) { + case logtail::models::PipelineEventGroup::PipelineEventsCase::kLogs: + if (src.logs().events_size() == 0) { + errMsg = "error transfer PB to PipelineEventGroup: no log events"; return false; } - } - break; - case logtail::models::PipelineEventGroup::PipelineEventsCase::kMetrics: - if (src.metrics().events_size() == 0) { - errMsg = "error transfer PB to PipelineEventGroup: no metric events"; - return false; - } - dst.MutableEvents().reserve(src.metrics().events_size()); - for (auto& metricSrc : src.metrics().events()) { - auto metricDst = dst.AddMetricEvent(); - if (!TransferPBToMetricEvent(metricSrc, *metricDst, errMsg)) { + dst.MutableEvents().reserve(src.logs().events_size()); + for (auto& logSrc : src.logs().events()) { + auto logDst = dst.AddLogEvent(); + if (!TransferPBToLogEvent(logSrc, *logDst, errMsg)) { + return false; + } + } + break; + case logtail::models::PipelineEventGroup::PipelineEventsCase::kMetrics: + if (src.metrics().events_size() == 0) { + errMsg = "error transfer PB to PipelineEventGroup: no metric events"; return false; } - } - break; - case logtail::models::PipelineEventGroup::PipelineEventsCase::kSpans: - if (src.spans().events_size() == 0) { - errMsg = "error transfer PB to PipelineEventGroup: no span events"; - return false; - } - dst.MutableEvents().reserve(src.spans().events_size()); - for (auto& spanSrc : src.spans().events()) { - auto spanDst = dst.AddSpanEvent(); - if (!TransferPBToSpanEvent(spanSrc, *spanDst, errMsg)) { + dst.MutableEvents().reserve(src.metrics().events_size()); + for (auto& metricSrc : src.metrics().events()) { + auto metricDst = dst.AddMetricEvent(); + if (!TransferPBToMetricEvent(metricSrc, *metricDst, errMsg)) { + return false; + } + } + break; + case logtail::models::PipelineEventGroup::PipelineEventsCase::kSpans: + if (src.spans().events_size() == 0) { + errMsg = "error transfer PB to PipelineEventGroup: no span events"; return false; } - } - break; - default: - errMsg = "error transfer PB to PipelineEventGroup: unsupported event type"; - return false; + dst.MutableEvents().reserve(src.spans().events_size()); + for (auto& spanSrc : src.spans().events()) { + auto spanDst = dst.AddSpanEvent(); + if (!TransferPBToSpanEvent(spanSrc, *spanDst, errMsg)) { + return false; + } + } + break; + default: + errMsg = "error transfer PB to PipelineEventGroup: unsupported event type"; + return false; } // tags @@ -69,8 +70,8 @@ bool TransferPBToPipelineEventGroup(const logtail::models::PipelineEventGroup& s bool TransferPBToLogEvent(const logtail::models::LogEvent& src, logtail::LogEvent& dst, std::string& errMsg) { // timestamp - std::chrono::nanoseconds tns(src.timestamp()); - std::chrono::seconds ts = std::chrono::duration_cast(tns); + std::chrono::nanoseconds tns(src.timestamp()); + std::chrono::seconds ts = std::chrono::duration_cast(tns); dst.SetTimestamp(ts.count(), tns.count() - ts.count() * 1000000000); // contents for (auto& contentPair : src.contents()) { @@ -85,19 +86,19 @@ bool TransferPBToLogEvent(const logtail::models::LogEvent& src, logtail::LogEven bool TransferPBToMetricEvent(const logtail::models::MetricEvent& src, logtail::MetricEvent& dst, std::string& errMsg) { // timestamp - std::chrono::nanoseconds tns(src.timestamp()); - std::chrono::seconds ts = std::chrono::duration_cast(tns); + std::chrono::nanoseconds tns(src.timestamp()); + std::chrono::seconds ts = std::chrono::duration_cast(tns); dst.SetTimestamp(ts.count(), tns.count() - ts.count() * 1000000000); // name dst.SetName(src.name()); // value switch (src.Value_case()) { - case logtail::models::MetricEvent::ValueCase::kUntypedSingleValue: - dst.SetValue(logtail::UntypedSingleValue{src.untypedsinglevalue().value()}); - break; - default: - errMsg = "error transfer PB to MetricEvent: unsupported value type"; - return false; + case logtail::models::MetricEvent::ValueCase::kUntypedSingleValue: + dst.SetValue(logtail::UntypedSingleValue{src.untypedsinglevalue().value()}); + break; + default: + errMsg = "error transfer PB to MetricEvent: unsupported value type"; + return false; } // tags for (auto& tagPair : src.tags()) { @@ -108,8 +109,8 @@ bool TransferPBToMetricEvent(const logtail::models::MetricEvent& src, logtail::M bool TransferPBToSpanEvent(const logtail::models::SpanEvent& src, logtail::SpanEvent& dst, std::string& errMsg) { // timestamp - std::chrono::nanoseconds tns(src.timestamp()); - std::chrono::seconds ts = std::chrono::duration_cast(tns); + std::chrono::nanoseconds tns(src.timestamp()); + std::chrono::seconds ts = std::chrono::duration_cast(tns); dst.SetTimestamp(ts.count(), tns.count() - ts.count() * 1000000000); dst.SetTraceId(src.traceid()); @@ -155,7 +156,9 @@ bool TransferPBToSpanEvent(const logtail::models::SpanEvent& src, logtail::SpanE return true; } -bool TransferPipelineEventGroupToPB(const logtail::PipelineEventGroup& src, logtail::models::PipelineEventGroup& dst, std::string& errMsg) { +bool TransferPipelineEventGroupToPB(const logtail::PipelineEventGroup& src, + logtail::models::PipelineEventGroup& dst, + std::string& errMsg) { // events if (src.GetEvents().empty()) { errMsg = "error transfer PipelineEventGroup to PB: events empty"; @@ -163,51 +166,54 @@ bool TransferPipelineEventGroupToPB(const logtail::PipelineEventGroup& src, logt } switch (src.GetEvents()[0]->GetType()) { - case logtail::PipelineEvent::Type::LOG: - dst.mutable_logs()->mutable_events()->Reserve(src.GetEvents().size()); - for (const auto& event : src.GetEvents()) { - if (!event.Is()) { - errMsg = "error transfer PipelineEventGroup to PB: unsupport pipelineEventGroup with multi types of events"; - return false; - } - const auto& logSrc = event.Cast(); - auto logDst = dst.mutable_logs()->add_events(); - if (!TransferLogEventToPB(logSrc, *logDst, errMsg)) { - return false; + case logtail::PipelineEvent::Type::LOG: + dst.mutable_logs()->mutable_events()->Reserve(src.GetEvents().size()); + for (const auto& event : src.GetEvents()) { + if (!event.Is()) { + errMsg = "error transfer PipelineEventGroup to PB: unsupport pipelineEventGroup with multi types " + "of events"; + return false; + } + const auto& logSrc = event.Cast(); + auto logDst = dst.mutable_logs()->add_events(); + if (!TransferLogEventToPB(logSrc, *logDst, errMsg)) { + return false; + } } - } - break; - case logtail::PipelineEvent::Type::METRIC: - dst.mutable_metrics()->mutable_events()->Reserve(src.GetEvents().size()); - for (const auto& event : src.GetEvents()) { - if (!event.Is()) { - errMsg = "error transfer PipelineEventGroup to PB: unsupport pipelineEventGroup with multi types of events"; - return false; + break; + case logtail::PipelineEvent::Type::METRIC: + dst.mutable_metrics()->mutable_events()->Reserve(src.GetEvents().size()); + for (const auto& event : src.GetEvents()) { + if (!event.Is()) { + errMsg = "error transfer PipelineEventGroup to PB: unsupport pipelineEventGroup with multi types " + "of events"; + return false; + } + const auto& metricSrc = event.Cast(); + auto metricDst = dst.mutable_metrics()->add_events(); + if (!TransferMetricEventToPB(metricSrc, *metricDst, errMsg)) { + return false; + } } - const auto& metricSrc = event.Cast(); - auto metricDst = dst.mutable_metrics()->add_events(); - if (!TransferMetricEventToPB(metricSrc, *metricDst, errMsg)) { - return false; + break; + case logtail::PipelineEvent::Type::SPAN: + dst.mutable_spans()->mutable_events()->Reserve(src.GetEvents().size()); + for (const auto& event : src.GetEvents()) { + if (!event.Is()) { + errMsg = "error transfer PipelineEventGroup to PB: unsupport pipelineEventGroup with multi types " + "of events"; + return false; + } + const auto& spanSrc = event.Cast(); + auto spanDst = dst.mutable_spans()->add_events(); + if (!TransferSpanEventToPB(spanSrc, *spanDst, errMsg)) { + return false; + } } - } - break; - case logtail::PipelineEvent::Type::SPAN: - dst.mutable_spans()->mutable_events()->Reserve(src.GetEvents().size()); - for (const auto& event : src.GetEvents()) { - if (!event.Is()) { - errMsg = "error transfer PipelineEventGroup to PB: unsupport pipelineEventGroup with multi types of events"; - return false; - } - const auto& spanSrc = event.Cast(); - auto spanDst = dst.mutable_spans()->add_events(); - if (!TransferSpanEventToPB(spanSrc, *spanDst, errMsg)) { - return false; - } - } - break; - default: - errMsg = "error transfer PipelineEventGroup to PB: unsupported event type"; - return false; + break; + default: + errMsg = "error transfer PipelineEventGroup to PB: unsupported event type"; + return false; } // tags @@ -225,7 +231,7 @@ bool TransferPipelineEventGroupToPB(const logtail::PipelineEventGroup& src, logt bool TransferLogEventToPB(const logtail::LogEvent& src, logtail::models::LogEvent& dst, std::string& errMsg) { // timestamp - std::chrono::seconds ts(src.GetTimestamp()); + std::chrono::seconds ts(src.GetTimestamp()); dst.set_timestamp(ts.count() * 1000000000 + src.GetTimestampNanosecond().value_or(0)); // contents @@ -248,7 +254,7 @@ bool TransferLogEventToPB(const logtail::LogEvent& src, logtail::models::LogEven bool TransferMetricEventToPB(const logtail::MetricEvent& src, logtail::models::MetricEvent& dst, std::string& errMsg) { // timestamp - std::chrono::seconds ts(src.GetTimestamp()); + std::chrono::seconds ts(src.GetTimestamp()); dst.set_timestamp(ts.count() * 1000000000 + src.GetTimestampNanosecond().value_or(0)); // name @@ -273,7 +279,7 @@ bool TransferMetricEventToPB(const logtail::MetricEvent& src, logtail::models::M bool TransferSpanEventToPB(const logtail::SpanEvent& src, logtail::models::SpanEvent& dst, std::string& errMsg) { // timestamp - std::chrono::seconds ts(src.GetTimestamp()); + std::chrono::seconds ts(src.GetTimestamp()); dst.set_timestamp(ts.count() * 1000000000 + src.GetTimestampNanosecond().value_or(0)); dst.set_traceid(src.GetTraceId().to_string()); diff --git a/core/protobuf/models/ProtocolConversion.h b/core/protobuf/models/ProtocolConversion.h index aac227cfc2..7a3c52b34e 100644 --- a/core/protobuf/models/ProtocolConversion.h +++ b/core/protobuf/models/ProtocolConversion.h @@ -16,23 +16,27 @@ #pragma once -#include "models/PipelineEventGroup.h" #include "models/LogEvent.h" #include "models/MetricEvent.h" +#include "models/PipelineEventGroup.h" #include "models/SpanEvent.h" -#include "protobuf/models/pipeline_event_group.pb.h" #include "protobuf/models/log_event.pb.h" #include "protobuf/models/metric_event.pb.h" +#include "protobuf/models/pipeline_event_group.pb.h" #include "protobuf/models/span_event.pb.h" namespace logtail { -bool TransferPBToPipelineEventGroup(const models::PipelineEventGroup& src, PipelineEventGroup& dst, std::string& errMsg); +bool TransferPBToPipelineEventGroup(const models::PipelineEventGroup& src, + PipelineEventGroup& dst, + std::string& errMsg); bool TransferPBToLogEvent(const models::LogEvent& src, LogEvent& dst, std::string& errMsg); bool TransferPBToMetricEvent(const models::MetricEvent& src, MetricEvent& dst, std::string& errMsg); bool TransferPBToSpanEvent(const models::SpanEvent& src, SpanEvent& dst, std::string& errMsg); -bool TransferPipelineEventGroupToPB(const PipelineEventGroup& src, models::PipelineEventGroup& dst, std::string& errMsg); +bool TransferPipelineEventGroupToPB(const PipelineEventGroup& src, + models::PipelineEventGroup& dst, + std::string& errMsg); bool TransferLogEventToPB(const LogEvent& src, models::LogEvent& dst, std::string& errMsg); bool TransferMetricEventToPB(const MetricEvent& src, models::MetricEvent& dst, std::string& errMsg); bool TransferSpanEventToPB(const SpanEvent& src, models::SpanEvent& dst, std::string& errMsg); diff --git a/core/protobuf/sls/LogGroupSerializer.h b/core/protobuf/sls/LogGroupSerializer.h index 394bbb9a6d..cd533138d6 100644 --- a/core/protobuf/sls/LogGroupSerializer.h +++ b/core/protobuf/sls/LogGroupSerializer.h @@ -17,10 +17,11 @@ #pragma once #include + #include -#include "models/StringView.h" #include "models/MetricEvent.h" +#include "models/StringView.h" namespace logtail { diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 1e985cecb2..1dbfde0132 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -154,8 +154,9 @@ void FlusherRunner::Run() { mLastRunTime->Set(chrono::duration_cast(curTime.time_since_epoch()).count()); vector items; - int32_t limit - = Application::GetInstance()->IsExiting() ? -1 : AppConfig::GetInstance()->GetSendRequestGlobalConcurrency(); + int32_t limit = Application::GetInstance()->IsExiting() + ? -1 + : AppConfig::GetInstance()->GetSendRequestGlobalConcurrency(); SenderQueueManager::GetInstance()->GetAvailableItems(items, limit); if (items.empty()) { SenderQueueManager::GetInstance()->Wait(1000); diff --git a/core/runner/FlusherRunner.h b/core/runner/FlusherRunner.h index d433d6138b..03944ac3e6 100644 --- a/core/runner/FlusherRunner.h +++ b/core/runner/FlusherRunner.h @@ -16,8 +16,9 @@ #pragma once -#include #include + +#include #include #include "monitor/MetricManager.h" diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index aae72adad0..d9e937fe57 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -58,8 +58,7 @@ void ProcessorRunner::Stop() { if (!mThreadRes[threadNo].valid()) { continue; } - future_status s - = mThreadRes[threadNo].wait_for(chrono::seconds(INT32_FLAG(processor_runner_exit_timeout_sec))); + future_status s = mThreadRes[threadNo].wait_for(chrono::seconds(INT32_FLAG(processor_runner_exit_timeout_sec))); if (s == future_status::ready) { LOG_INFO(sLogger, ("processor runner", "stopped successfully")("threadNo", threadNo)); } else { diff --git a/core/runner/ProcessorRunner.h b/core/runner/ProcessorRunner.h index 6841569edc..10612456dd 100644 --- a/core/runner/ProcessorRunner.h +++ b/core/runner/ProcessorRunner.h @@ -16,8 +16,9 @@ #pragma once -#include #include + +#include #include #include #include diff --git a/core/runner/sink/http/HttpSink.h b/core/runner/sink/http/HttpSink.h index a6ce67d882..e0c903f761 100644 --- a/core/runner/sink/http/HttpSink.h +++ b/core/runner/sink/http/HttpSink.h @@ -16,8 +16,6 @@ #pragma once -#include - #include #include #include @@ -26,6 +24,7 @@ #include "monitor/MetricManager.h" #include "runner/sink/Sink.h" #include "runner/sink/http/HttpSinkRequest.h" + namespace logtail { class HttpSink : public Sink { diff --git a/core/runner/sink/http/HttpSinkRequest.h b/core/runner/sink/http/HttpSinkRequest.h index d9ed07ce41..35e856274e 100644 --- a/core/runner/sink/http/HttpSinkRequest.h +++ b/core/runner/sink/http/HttpSinkRequest.h @@ -34,9 +34,9 @@ struct HttpSinkRequest : public AsynHttpRequest { const std::string& body, SenderQueueItem* item, uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)), - uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)) - ) - : AsynHttpRequest(method, httpsFlag, host, port, url, query, header, body, HttpResponse(), timeout, maxTryCnt), mItem(item) {} + uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt))) + : AsynHttpRequest(method, httpsFlag, host, port, url, query, header, body, HttpResponse(), timeout, maxTryCnt), + mItem(item) {} bool IsContextValid() const override { return true; } void OnSendDone(HttpResponse& response) override {} diff --git a/core/task_pipeline/Task.h b/core/task_pipeline/Task.h index c3d8eedb5b..0310617530 100644 --- a/core/task_pipeline/Task.h +++ b/core/task_pipeline/Task.h @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include "json/json.h" + #include "logger/Logger.h" #include "monitor/AlarmManager.h" diff --git a/core/task_pipeline/TaskPipeline.h b/core/task_pipeline/TaskPipeline.h index fc5b95a1f3..3ade8110f5 100644 --- a/core/task_pipeline/TaskPipeline.h +++ b/core/task_pipeline/TaskPipeline.h @@ -16,11 +16,11 @@ #pragma once -#include - #include #include +#include "json/json.h" + #include "config/TaskConfig.h" #include "task_pipeline/Task.h" @@ -35,7 +35,9 @@ class TaskPipeline { const Json::Value& GetConfig() const { return *mConfig; } #ifdef APSARA_UNIT_TEST_MAIN - Task* GetPlugin() const { return mPlugin.get(); } + Task* GetPlugin() const { + return mPlugin.get(); + } #endif private: diff --git a/core/task_pipeline/TaskPipelineManager.h b/core/task_pipeline/TaskPipelineManager.h index 60776f6fd4..01fb1f8449 100644 --- a/core/task_pipeline/TaskPipelineManager.h +++ b/core/task_pipeline/TaskPipelineManager.h @@ -43,7 +43,9 @@ class TaskPipelineManager { std::vector GetAllPipelineNames() const; #ifdef APSARA_UNIT_TEST_MAIN - void ClearEnvironment() { mPipelineNameEntityMap.clear(); } + void ClearEnvironment() { + mPipelineNameEntityMap.clear(); + } #endif private: diff --git a/core/unittest/Unittest.h b/core/unittest/Unittest.h index 05e7c1a4c9..2ea5c6ec2f 100644 --- a/core/unittest/Unittest.h +++ b/core/unittest/Unittest.h @@ -19,13 +19,15 @@ #include #endif #include -#include -#include -#include "logger/Logger.h" + +#include "boost/filesystem.hpp" +#include "gtest/gtest.h" + +#include "common/HashUtil.h" #include "common/LogtailCommonFlags.h" #include "common/RuntimeUtil.h" -#include "common/HashUtil.h" #include "common/TimeUtil.h" +#include "logger/Logger.h" namespace bfs = boost::filesystem; diff --git a/core/unittest/UnittestHelper.h b/core/unittest/UnittestHelper.h index c6b5945371..d239c9086f 100644 --- a/core/unittest/UnittestHelper.h +++ b/core/unittest/UnittestHelper.h @@ -14,10 +14,12 @@ * limitations under the License. */ -#include -#include #include +#include +#include + #include + #include "logger/Logger.h" #include "protobuf/sls/sls_logs.pb.h" diff --git a/core/unittest/app_config/AppConfigUnittest.cpp b/core/unittest/app_config/AppConfigUnittest.cpp index 01b35f6bbb..e1224d201a 100644 --- a/core/unittest/app_config/AppConfigUnittest.cpp +++ b/core/unittest/app_config/AppConfigUnittest.cpp @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "app_config/AppConfig.h" +#include "common/FileSystemUtil.h" #include "common/Flags.h" #include "common/JsonUtil.h" #include "unittest/Unittest.h" -#include "common/FileSystemUtil.h" -#include "app_config/AppConfig.h" DECLARE_FLAG_INT32(checkpoint_find_max_file_count); DECLARE_FLAG_INT32(ebpf_receive_event_chan_cap); @@ -42,7 +42,7 @@ class AppConfigUnittest : public ::testing::Test { OverwriteFile(STRING_FLAG(ilogtail_config), v.toStyledString()); } else { CreateAgentDir(); - std::string conf = GetAgentConfDir() + "/instance_config/local/loongcollector_config.json"; + std::string conf = GetAgentConfDir() + "/instance_config/local/loongcollector_config.json"; AppConfig::GetInstance()->LoadAppConfig(conf); OverwriteFile(conf, v.toStyledString()); } @@ -84,7 +84,7 @@ void AppConfigUnittest::TestRecurseParseJsonToFlags() { AppConfig* app_config = AppConfig::GetInstance(); app_config->LoadAppConfig(STRING_FLAG(ilogtail_config)); APSARA_TEST_EQUAL(INT32_FLAG(checkpoint_find_max_file_count), 600); - + // test multi-layer json, include bool, string, int, double configStr = R"( { diff --git a/core/unittest/app_config/AppConfigUnittestLegal.cpp b/core/unittest/app_config/AppConfigUnittestLegal.cpp index 2dd1b3890f..388d2c8132 100644 --- a/core/unittest/app_config/AppConfigUnittestLegal.cpp +++ b/core/unittest/app_config/AppConfigUnittestLegal.cpp @@ -13,12 +13,13 @@ // limitations under the License. #include -#include "unittest/Unittest.h" -#include "common/StringTools.h" + +#include "app_config/AppConfig.h" #include "common/FileSystemUtil.h" #include "common/RuntimeUtil.h" -#include "app_config/AppConfig.h" +#include "common/StringTools.h" #include "file_server/reader/LogFileReader.h" +#include "unittest/Unittest.h" DECLARE_FLAG_STRING(ilogtail_config); DECLARE_FLAG_INT32(logreader_filedeleted_remove_interval); diff --git a/core/unittest/checkpoint/AdhocCheckpointManagerUnittest.cpp b/core/unittest/checkpoint/AdhocCheckpointManagerUnittest.cpp index d23d4a5bbf..39a79842ab 100644 --- a/core/unittest/checkpoint/AdhocCheckpointManagerUnittest.cpp +++ b/core/unittest/checkpoint/AdhocCheckpointManagerUnittest.cpp @@ -13,12 +13,14 @@ // limitations under the License. #include -#include + #include -#include "unittest/Unittest.h" +#include + +#include "AppConfig.h" #include "checkpoint/AdhocCheckpointManager.h" #include "common/FileSystemUtil.h" -#include "AppConfig.h" +#include "unittest/Unittest.h" namespace logtail { diff --git a/core/unittest/checkpoint/CheckpointManagerUnittest.cpp b/core/unittest/checkpoint/CheckpointManagerUnittest.cpp index 864b9de15a..35839ba234 100644 --- a/core/unittest/checkpoint/CheckpointManagerUnittest.cpp +++ b/core/unittest/checkpoint/CheckpointManagerUnittest.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include "checkpoint/CheckPointManager.h" #include "common/FileSystemUtil.h" #include "common/Flags.h" +#include "unittest/Unittest.h" DECLARE_FLAG_INT32(checkpoint_find_max_file_count); diff --git a/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp b/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp index 3c9f6c3fe0..a49c0f650e 100644 --- a/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp +++ b/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" -#include "common/Flags.h" #include "app_config/AppConfig.h" -#include "protobuf/sls/sls_logs.pb.h" #include "checkpoint/CheckpointManagerV2.h" +#include "common/Flags.h" +#include "protobuf/sls/sls_logs.pb.h" +#include "unittest/Unittest.h" DECLARE_FLAG_INT32(logtail_checkpoint_check_gc_interval_sec); DECLARE_FLAG_INT32(logtail_checkpoint_expired_threshold_sec); @@ -274,7 +274,7 @@ void CheckpointManagerV2Unittest::TestScanCheckpoints() { namespace detail { - std::string extractPrimaryKeyFromRangeKey(const char* data, size_t size); +std::string extractPrimaryKeyFromRangeKey(const char* data, size_t size); } diff --git a/core/unittest/common/DevInodeUnittest.h b/core/unittest/common/DevInodeUnittest.h index 3cd9105926..3d8400bbfb 100644 --- a/core/unittest/common/DevInodeUnittest.h +++ b/core/unittest/common/DevInodeUnittest.h @@ -14,10 +14,11 @@ * limitations under the License. */ -#include "unittest/Unittest.h" #include -#include "common/RuntimeUtil.h" + #include "common/DevInode.h" +#include "common/RuntimeUtil.h" +#include "unittest/Unittest.h" namespace logtail { diff --git a/core/unittest/common/EncodingConverterUnittest.cpp b/core/unittest/common/EncodingConverterUnittest.cpp index edf7905125..de2a48446f 100644 --- a/core/unittest/common/EncodingConverterUnittest.cpp +++ b/core/unittest/common/EncodingConverterUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include "common/EncodingConverter.h" +#include "unittest/Unittest.h" #if defined(__linux__) #include "unittest/UnittestHelper.h" #endif diff --git a/core/unittest/common/FileSystemUtilUnittest.h b/core/unittest/common/FileSystemUtilUnittest.h index 30c782f38d..c77162a9a6 100644 --- a/core/unittest/common/FileSystemUtilUnittest.h +++ b/core/unittest/common/FileSystemUtilUnittest.h @@ -14,10 +14,11 @@ * limitations under the License. */ -#include #include #include +#include "boost/format.hpp" + #include "common/FileSystemUtil.h" #include "common/LogtailCommonFlags.h" #include "common/RuntimeUtil.h" @@ -130,9 +131,7 @@ TEST_F(FileSystemUtilUnittest, TestDirNormal) { #ifndef _MSC_VER TEST_F(FileSystemUtilUnittest, TestDirSymbolic) { - { - std::ofstream((mTestRoot / "f1").string()); - } + { std::ofstream((mTestRoot / "f1").string()); } bfs::create_directory(mTestRoot / "d1"); std::map symbolics = {{"s1", "f1"}, {"s2", "d1"}}; for (auto& s : symbolics) { @@ -226,9 +225,7 @@ TEST_F(FileSystemUtilUnittest, TestCheckExistance) { TEST_F(FileSystemUtilUnittest, TestPathStat_fstat) { auto currentTime = time(NULL); auto filePath = ((mTestRoot / "file").string()); - { - std::ofstream(filePath).write("xxx", 3); - } + { std::ofstream(filePath).write("xxx", 3); } FILE* file = fopen(filePath.c_str(), "r"); EXPECT_TRUE(file != NULL); @@ -269,9 +266,7 @@ TEST_F(FileSystemUtilUnittest, TestPathStat_fstat) { TEST_F(FileSystemUtilUnittest, TestPathStat_GetLastWriteTime) { auto filePath = ((mTestRoot / "file").string()); - { - std::ofstream(filePath).write("xxx", 3); - } + { std::ofstream(filePath).write("xxx", 3); } { int64_t sec = -1, nsec = -1; @@ -295,9 +290,7 @@ TEST_F(FileSystemUtilUnittest, TestPathStat_GetLastWriteTime) { TEST_F(FileSystemUtilUnittest, TestFileReadOnlyOpen) { auto filePath = ((mTestRoot / "file").string()); const std::string fileContent{"xxx"}; - { - std::ofstream(filePath) << fileContent; - } + { std::ofstream(filePath) << fileContent; } // Open the file and delete it before closing. // File can still be read after deleting. @@ -364,9 +357,7 @@ TEST_F(FileSystemUtilUnittest, TestFileWriteOnlyOpen) { // Case #2: File is existing, open will truncate it. { - { - std::ofstream(filePath) << fileContent; - } + { std::ofstream(filePath) << fileContent; } auto file = FileWriteOnlyOpen(filePath.c_str(), "w"); ASSERT_TRUE(file != NULL); @@ -418,9 +409,7 @@ TEST_F(FileSystemUtilUnittest, TestFileAppendOpen) { // Case #3: Open existing file, check its cursor position. { - { - std::ofstream(filePath) << fileContent; - } + { std::ofstream(filePath) << fileContent; } auto file = FileAppendOpen(filePath.c_str(), "a"); EXPECT_EQ(ftell(file), fileContentLen); diff --git a/core/unittest/common/LoadParameterUnittest.h b/core/unittest/common/LoadParameterUnittest.h index 755e2ed8bc..c03b1f8abc 100644 --- a/core/unittest/common/LoadParameterUnittest.h +++ b/core/unittest/common/LoadParameterUnittest.h @@ -14,11 +14,11 @@ * limitations under the License. */ -#include "unittest/Unittest.h" #include "common/Flags.h" #include "common/JsonUtil.h" -#include "common/StringTools.h" #include "common/LogtailCommonFlags.h" +#include "common/StringTools.h" +#include "unittest/Unittest.h" DECLARE_FLAG_INT32(batch_send_interval); DECLARE_FLAG_STRING(check_point_filename); diff --git a/core/unittest/common/LockUnittest.h b/core/unittest/common/LockUnittest.h index cd75ebb3c7..0e6227a2e0 100644 --- a/core/unittest/common/LockUnittest.h +++ b/core/unittest/common/LockUnittest.h @@ -14,11 +14,12 @@ * limitations under the License. */ -#include "unittest/Unittest.h" -#include #include +#include + #include "common/Lock.h" #include "common/TimeUtil.h" +#include "unittest/Unittest.h" namespace logtail { diff --git a/core/unittest/common/LogFileOperatorUnittest.cpp b/core/unittest/common/LogFileOperatorUnittest.cpp index b5b27ce7a1..88546f06ca 100644 --- a/core/unittest/common/LogFileOperatorUnittest.cpp +++ b/core/unittest/common/LogFileOperatorUnittest.cpp @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" -#include #include + +#include + +#include "unittest/Unittest.h" #if defined(__linux__) #include #endif -#include "common/LogFileOperator.h" #include "common/FileSystemUtil.h" +#include "common/LogFileOperator.h" #if defined(__linux__) #include "unittest/UnittestHelper.h" #endif diff --git a/core/unittest/common/MachineInfoUtilUnittest.cpp b/core/unittest/common/MachineInfoUtilUnittest.cpp index 28652ca788..c0af3078c2 100644 --- a/core/unittest/common/MachineInfoUtilUnittest.cpp +++ b/core/unittest/common/MachineInfoUtilUnittest.cpp @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include "common/MachineInfoUtil.h" +#include "unittest/Unittest.h" namespace logtail { class HostnameValidationUnittest : public ::testing::Test { public: void DecHostnameValidationTest() { std::string hostname; - hostname = "4294967296"; // 2^32 + hostname = "4294967296"; // 2^32 EXPECT_TRUE(IsDigitsDotsHostname(hostname.c_str())); - hostname = "4294967295"; // 2^32-1 + hostname = "4294967295"; // 2^32-1 EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); hostname = "0"; EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); @@ -35,7 +35,7 @@ class HostnameValidationUnittest : public ::testing::Test { EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); hostname = "0.123"; EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); - hostname = "255.16777216"; // 2^24 + hostname = "255.16777216"; // 2^24 EXPECT_TRUE(IsDigitsDotsHostname(hostname.c_str())); hostname = "255.16777215"; // 2^24-1 EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); @@ -102,16 +102,16 @@ class HostnameValidationUnittest : public ::testing::Test { void OctHostnameValidationTest() { std::string hostname; - hostname = "040000000000"; // 2^32 + hostname = "040000000000"; // 2^32 EXPECT_TRUE(IsDigitsDotsHostname(hostname.c_str())); - hostname = "037777777777"; // 2^32-1 + hostname = "037777777777"; // 2^32-1 EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); hostname = "0400.0123"; EXPECT_TRUE(IsDigitsDotsHostname(hostname.c_str())); hostname = "0377.0123"; EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); - hostname = "0377.0100000000"; // 2^24 + hostname = "0377.0100000000"; // 2^24 EXPECT_TRUE(IsDigitsDotsHostname(hostname.c_str())); hostname = "0377.077777777"; // 2^24-1 EXPECT_FALSE(IsDigitsDotsHostname(hostname.c_str())); diff --git a/core/unittest/common/RandomUtilUnittest.h b/core/unittest/common/RandomUtilUnittest.h index 8b64de08e1..6e279c9ab6 100644 --- a/core/unittest/common/RandomUtilUnittest.h +++ b/core/unittest/common/RandomUtilUnittest.h @@ -14,10 +14,11 @@ * limitations under the License. */ -#include "unittest/Unittest.h" -#include "common/RandomUtil.h" #include +#include "common/RandomUtil.h" +#include "unittest/Unittest.h" + namespace logtail { class RandomUtilUnittest : public ::testing::Test { @@ -45,7 +46,7 @@ void RandomUtilUnittest::TestGenerateRandomHashKey() { } namespace detail { - size_t countBitSize(size_t maxVal); +size_t countBitSize(size_t maxVal); } void RandomUtilUnittest::TestCountBitSize() { diff --git a/core/unittest/common/SimpleUtilsUnittest.cpp b/core/unittest/common/SimpleUtilsUnittest.cpp index b088a4f469..a549d0c78d 100644 --- a/core/unittest/common/SimpleUtilsUnittest.cpp +++ b/core/unittest/common/SimpleUtilsUnittest.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "DevInodeUnittest.h" +#include "FileSystemUtilUnittest.h" #include "LoadParameterUnittest.h" #include "RandomUtilUnittest.h" -#include "FileSystemUtilUnittest.h" -#include "DevInodeUnittest.h" #include "TimeUtilUnittest.h" int main(int argc, char** argv) { diff --git a/core/unittest/common/SlidingWindowCounterUnittest.cpp b/core/unittest/common/SlidingWindowCounterUnittest.cpp index cecc36ede7..922090ca97 100644 --- a/core/unittest/common/SlidingWindowCounterUnittest.cpp +++ b/core/unittest/common/SlidingWindowCounterUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include "common/SlidingWindowCounter.h" +#include "unittest/Unittest.h" namespace logtail { diff --git a/core/unittest/common/StringPieceUnittest.cpp b/core/unittest/common/StringPieceUnittest.cpp index 1b3a7a36ad..0e22d4d104 100644 --- a/core/unittest/common/StringPieceUnittest.cpp +++ b/core/unittest/common/StringPieceUnittest.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" -#include "common/FileSystemUtil.h" -#include "app_config/AppConfig.h" #include "StringPiece.h" +#include "app_config/AppConfig.h" +#include "common/FileSystemUtil.h" +#include "unittest/Unittest.h" namespace logtail { diff --git a/core/unittest/common/StringToolsUnittest.cpp b/core/unittest/common/StringToolsUnittest.cpp index c96a5b0a78..6f8df61cde 100644 --- a/core/unittest/common/StringToolsUnittest.cpp +++ b/core/unittest/common/StringToolsUnittest.cpp @@ -118,10 +118,10 @@ TEST_F(StringToolsUnittest, TestGetTopicNames) { TEST_F(StringToolsUnittest, TestRemoveFilePathTrailingSlash) { std::string filePath = "/aaa/aa/"; RemoveFilePathTrailingSlash(filePath); - APSARA_TEST_EQUAL("/aaa/aa",filePath); + APSARA_TEST_EQUAL("/aaa/aa", filePath); filePath = "/"; RemoveFilePathTrailingSlash(filePath); - APSARA_TEST_EQUAL("/",filePath); + APSARA_TEST_EQUAL("/", filePath); } TEST_F(StringToolsUnittest, TestBoostRegexSearch) { diff --git a/core/unittest/common/TimeUtilUnittest.h b/core/unittest/common/TimeUtilUnittest.h index 37392d639f..42cd72964e 100644 --- a/core/unittest/common/TimeUtilUnittest.h +++ b/core/unittest/common/TimeUtilUnittest.h @@ -14,12 +14,13 @@ * limitations under the License. */ -#include "unittest/Unittest.h" #include #include -#include "common/TimeUtil.h" + #include "common/StringTools.h" #include "common/Strptime.h" +#include "common/TimeUtil.h" +#include "unittest/Unittest.h" namespace logtail { @@ -132,12 +133,36 @@ void TimeUtilUnittest::TestStrptimeNanosecond() { std::vector cases{ {"2012-01-01 15:05:07.123456", "%Y-%m-%d %H:%M:%S.%f", "2012-01-01 15:05:07", "%Y-%m-%d %H:%M:%S", 123456000}, - {"[2012-01-01 15:05:07.123456]", "[%Y-%m-%d %H:%M:%S.%f]", "[2012-01-01 15:05:07]", "[%Y-%m-%d %H:%M:%S]", 123456000}, - {"01 Jan 12 15:05:07.123456 MST", "%d %b %y %H:%M:%S.%f", "01 Jan 12 15:05:07 MST", "%d %b %y %H:%M:%S", 123456000}, - {"01 Jan 12 15:05:07.123456 -0700", "%d %b %y %H:%M:%S.%f", "01 Jan 12 15:05:07 -0700", "%d %b %y %H:%M:%S", 123456000}, - {"Sunday, 01-Jan-12 15:05:07.123456 MST", "%A, %d-%b-%y %H:%M:%S.%f", "Sunday, 01-Jan-12 15:05:07 MST", "%A, %d-%b-%y %H:%M:%S", 123456000}, - {"Sun, 01 Jan 12 15:05:07.123456 MST", "%A, %d %b %Y %H:%M:%S.%f", "Sun, 01 Jan 12 15:05:07 MST", "%A, %d %b %Y %H:%M:%S", 123456000}, - {"2012-01-01T15:05:07.123456Z07:00", "%Y-%m-%dT%H:%M:%S.%f", "2012-01-01T15:05:07Z07:00", "%Y-%m-%dT%H:%M:%S", 123456000}, + {"[2012-01-01 15:05:07.123456]", + "[%Y-%m-%d %H:%M:%S.%f]", + "[2012-01-01 15:05:07]", + "[%Y-%m-%d %H:%M:%S]", + 123456000}, + {"01 Jan 12 15:05:07.123456 MST", + "%d %b %y %H:%M:%S.%f", + "01 Jan 12 15:05:07 MST", + "%d %b %y %H:%M:%S", + 123456000}, + {"01 Jan 12 15:05:07.123456 -0700", + "%d %b %y %H:%M:%S.%f", + "01 Jan 12 15:05:07 -0700", + "%d %b %y %H:%M:%S", + 123456000}, + {"Sunday, 01-Jan-12 15:05:07.123456 MST", + "%A, %d-%b-%y %H:%M:%S.%f", + "Sunday, 01-Jan-12 15:05:07 MST", + "%A, %d-%b-%y %H:%M:%S", + 123456000}, + {"Sun, 01 Jan 12 15:05:07.123456 MST", + "%A, %d %b %Y %H:%M:%S.%f", + "Sun, 01 Jan 12 15:05:07 MST", + "%A, %d %b %Y %H:%M:%S", + 123456000}, + {"2012-01-01T15:05:07.123456Z07:00", + "%Y-%m-%dT%H:%M:%S.%f", + "2012-01-01T15:05:07Z07:00", + "%Y-%m-%dT%H:%M:%S", + 123456000}, {"1325430307", "%s", "1325430307", "%s", 0}, {"325430307", "%s", "325430307", "%s", 0}, // the behavior of `strptime_ns` is different with C++ `strptime` in this case diff --git a/core/unittest/common/YamlUtilUnittest.cpp b/core/unittest/common/YamlUtilUnittest.cpp index 581b324e79..71dcf00330 100644 --- a/core/unittest/common/YamlUtilUnittest.cpp +++ b/core/unittest/common/YamlUtilUnittest.cpp @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" - #include #include "common/YamlUtil.h" +#include "unittest/Unittest.h" namespace logtail { diff --git a/core/unittest/compression/CompressorFactoryUnittest.cpp b/core/unittest/compression/CompressorFactoryUnittest.cpp index e9244c3904..1128d77680 100644 --- a/core/unittest/compression/CompressorFactoryUnittest.cpp +++ b/core/unittest/compression/CompressorFactoryUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/metric_constants/MetricConstants.h" #include "common/compression/CompressorFactory.h" +#include "monitor/metric_constants/MetricConstants.h" #include "unittest/Unittest.h" using namespace std; @@ -98,7 +98,8 @@ void CompressorFactoryUnittest::TestMetric() { APSARA_TEST_EQUAL(4U, compressor->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); - APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR)); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, + METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR)); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, mFlusherId)); } diff --git a/core/unittest/compression/CompressorUnittest.cpp b/core/unittest/compression/CompressorUnittest.cpp index a678d692e7..e3579da06d 100644 --- a/core/unittest/compression/CompressorUnittest.cpp +++ b/core/unittest/compression/CompressorUnittest.cpp @@ -22,7 +22,7 @@ namespace logtail { class CompressorMock : public Compressor { public: - CompressorMock(CompressType type) : Compressor(type) {}; + CompressorMock(CompressType type) : Compressor(type){}; bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override { return true; } diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index 4543eff7c6..d0796a6955 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -13,7 +13,7 @@ // limitations under the License. -#include +#include "json/json.h" #include "AppConfig.h" #include "common/FileSystemUtil.h" @@ -21,9 +21,6 @@ #include "config/ConfigDiff.h" #include "config/InstanceConfigManager.h" #include "config/common_provider/CommonConfigProvider.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "config/watcher/InstanceConfigWatcher.h" #include "config/watcher/PipelineConfigWatcher.h" #include "file_server/FileServer.h" @@ -31,6 +28,9 @@ #include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" #include "unittest/Unittest.h" +#ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" +#endif DECLARE_FLAG_BOOL(logtail_mode); diff --git a/core/unittest/config/ConfigContainerUnittest.cpp b/core/unittest/config/ConfigContainerUnittest.cpp index b85f6c49f9..06cb643f17 100644 --- a/core/unittest/config/ConfigContainerUnittest.cpp +++ b/core/unittest/config/ConfigContainerUnittest.cpp @@ -22,8 +22,8 @@ #include "app_config/AppConfig.h" #include "common/Flags.h" #include "file_server/ConfigManager.h" -#include "file_server/event/Event.h" #include "file_server/FileServer.h" +#include "file_server/event/Event.h" #include "unittest/Unittest.h" using namespace std; diff --git a/core/unittest/config/ConfigMatchUnittest.cpp b/core/unittest/config/ConfigMatchUnittest.cpp index ba5f72587c..2f4f9b1473 100644 --- a/core/unittest/config/ConfigMatchUnittest.cpp +++ b/core/unittest/config/ConfigMatchUnittest.cpp @@ -12,39 +12,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include + +#include "unittest/Unittest.h" #if defined(__linux__) -#include +#include #include #include -#include +#include #endif -#include -#include -#include +#include #include -#include #include -#include -#include -#include -#include +#include +#include +#include +#include + #include +#include #include -#include #include -#include -#include "common/Flags.h" +#include +#include +#include + +#include "json/json.h" + +#include "app_config/AppConfig.h" #include "common/FileSystemUtil.h" +#include "common/Flags.h" +#include "file_server/ConfigManager.h" +#include "file_server/EventDispatcher.h" +#include "file_server/event/Event.h" #include "file_server/event_handler/EventHandler.h" +#include "file_server/event_handler/LogInput.h" #include "file_server/polling/PollingEventQueue.h" -#include "file_server/EventDispatcher.h" -#include "app_config/AppConfig.h" -#include "file_server/ConfigManager.h" #include "file_server/reader/LogFileReader.h" -#include "file_server/event_handler/LogInput.h" -#include "file_server/event/Event.h" #include "logger/Logger.h" using namespace std; @@ -1043,7 +1047,8 @@ void ConfigMatchUnittest::TestChinesePathAndFilePattern() { filePattern = EncodingConverter::GetInstance()->FromACPToUTF8(filePattern); #endif { - PipelineConfig cfg(basePath, filePattern, LogType::REGEX_LOG, "log", ".*", "", "", "project", true, 3, 3, "logstore"); + PipelineConfig cfg( + basePath, filePattern, LogType::REGEX_LOG, "log", ".*", "", "", "project", true, 3, 3, "logstore"); fsutil::Dir dir(pathRoot); APSARA_TEST_TRUE(dir.Open()); while (fsutil::Entry ent = dir.ReadNext()) { diff --git a/core/unittest/config/ConfigUpdateUnittest.cpp b/core/unittest/config/ConfigUpdateUnittest.cpp index 17f5f3589e..77c2333b9e 100644 --- a/core/unittest/config/ConfigUpdateUnittest.cpp +++ b/core/unittest/config/ConfigUpdateUnittest.cpp @@ -20,9 +20,6 @@ #include "config/PipelineConfig.h" #include "config/common_provider/CommonConfigProvider.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "config/watcher/PipelineConfigWatcher.h" #include "file_server/FileServer.h" #include "pipeline/Pipeline.h" @@ -32,6 +29,9 @@ #include "unittest/Unittest.h" #include "unittest/config/PipelineManagerMock.h" #include "unittest/plugin/PluginMock.h" +#ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" +#endif using namespace std; diff --git a/core/unittest/config/ConfigUpdatorUnittest.cpp b/core/unittest/config/ConfigUpdatorUnittest.cpp index 1ef9a6825f..e5965c8953 100644 --- a/core/unittest/config/ConfigUpdatorUnittest.cpp +++ b/core/unittest/config/ConfigUpdatorUnittest.cpp @@ -16,15 +16,16 @@ #if defined(__linux__) #include #endif +#include #include -#include #include -#include #include #include #include +#include "json/json.h" + #include "AlarmManager.h" #include "AppConfig.h" #include "CheckPointManager.h" diff --git a/core/unittest/config/ConfigWatcherUnittest.cpp b/core/unittest/config/ConfigWatcherUnittest.cpp index 2da9c9dc9e..86e68ea72a 100644 --- a/core/unittest/config/ConfigWatcherUnittest.cpp +++ b/core/unittest/config/ConfigWatcherUnittest.cpp @@ -17,13 +17,13 @@ #include "config/ConfigDiff.h" #include "config/common_provider/CommonConfigProvider.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "config/watcher/InstanceConfigWatcher.h" #include "config/watcher/PipelineConfigWatcher.h" #include "pipeline/plugin/PluginRegistry.h" #include "unittest/Unittest.h" +#ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" +#endif using namespace std; diff --git a/core/unittest/config/PipelineConfigUnittest.cpp b/core/unittest/config/PipelineConfigUnittest.cpp index e3dcfc5d4f..c40f13380b 100644 --- a/core/unittest/config/PipelineConfigUnittest.cpp +++ b/core/unittest/config/PipelineConfigUnittest.cpp @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include +#include "json/json.h" + #include "common/JsonUtil.h" #include "config/PipelineConfig.h" #include "pipeline/plugin/PluginRegistry.h" diff --git a/core/unittest/config/PipelineConfigWatcherUnittest.cpp b/core/unittest/config/PipelineConfigWatcherUnittest.cpp index 920aec0882..5fb2ec779a 100644 --- a/core/unittest/config/PipelineConfigWatcherUnittest.cpp +++ b/core/unittest/config/PipelineConfigWatcherUnittest.cpp @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include +#include "json/json.h" + #include "common/JsonUtil.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "config/watcher/PipelineConfigWatcher.h" #include "plugin/PluginRegistry.h" #include "unittest/Unittest.h" #include "unittest/config/PipelineManagerMock.h" +#ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" +#endif using namespace std; @@ -225,13 +225,13 @@ void PipelineConfigWatcherUnittest::TestLoadAddedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: added -> added, first > second -> first < second - // should not happen - } - { // case: added -> added, first < second -> first > second - // should not happen - } - { // case: added -> modified, first > second + { + // case: added -> added, first > second -> first < second + // should not happen + } { + // case: added -> added, first < second -> first > second + // should not happen + } { // case: added -> modified, first > second PrepareConfig(); ofstream fout(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; @@ -303,13 +303,13 @@ void PipelineConfigWatcherUnittest::TestLoadAddedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: added -> modified, first > second -> first < second - // should not happen - } - { // case: added -> modified, first < second -> first > second - // should not happen - } - { // case: added -> removed, first > second + { + // case: added -> modified, first > second -> first < second + // should not happen + } { + // case: added -> modified, first < second -> first > second + // should not happen + } { // case: added -> removed, first > second PrepareConfig(); ofstream fout(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; @@ -367,13 +367,13 @@ void PipelineConfigWatcherUnittest::TestLoadAddedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } - { // case: added -> removed, first > second -> first < second - // should not happen - } - { // case: added -> removed, first < second -> first > second - // should not happen - } - { // case: added -> unchanged, first > second + { + // case: added -> removed, first > second -> first < second + // should not happen + } { + // case: added -> removed, first < second -> first > second + // should not happen + } { // case: added -> unchanged, first > second PrepareConfig(); ofstream fout(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; @@ -431,11 +431,11 @@ void PipelineConfigWatcherUnittest::TestLoadAddedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: added -> unchanged, first > second -> first < second - // should not happen - } - { // case: added -> unchanged, first < second -> first > second - // should not happen + { + // case: added -> unchanged, first > second -> first < second + // should not happen + } { // case: added -> unchanged, first < second -> first > second + // should not happen } } @@ -506,13 +506,13 @@ void PipelineConfigWatcherUnittest::TestLoadModifiedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: modified -> added, first > second -> first < second - // should not happen - } - { // case: modified -> added, first < second -> first > second - // should not happen - } - { // case: modified -> modified, first > second + { + // case: modified -> added, first > second -> first < second + // should not happen + } { + // case: modified -> added, first < second -> first > second + // should not happen + } { // case: modified -> modified, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -734,13 +734,13 @@ void PipelineConfigWatcherUnittest::TestLoadModifiedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } - { // case: modified -> removed, first > second -> first < second - // should not happen - } - { // case: modified -> removed, first < second -> first > second - // should not happen - } - { // case: modified -> unchanged, first > second + { + // case: modified -> removed, first > second -> first < second + // should not happen + } { + // case: modified -> removed, first < second -> first > second + // should not happen + } { // case: modified -> unchanged, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -940,13 +940,13 @@ void PipelineConfigWatcherUnittest::TestLoadRemovedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: removed -> added, first > second -> first < second - // should not happen - } - { // case: removed -> added, first < second -> first > second - // should not happen - } - { // case: removed -> modified, first > second + { + // case: removed -> added, first > second -> first < second + // should not happen + } { + // case: removed -> added, first < second -> first > second + // should not happen + } { // case: removed -> modified, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -1020,13 +1020,13 @@ void PipelineConfigWatcherUnittest::TestLoadRemovedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: removed -> modified, first > second -> first < second - // should not happen - } - { // case: removed -> modified, first < second -> first > second - // should not happen - } - { // case: removed -> removed, first > second + { + // case: removed -> modified, first > second -> first < second + // should not happen + } { + // case: removed -> modified, first < second -> first > second + // should not happen + } { // case: removed -> removed, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -1084,13 +1084,13 @@ void PipelineConfigWatcherUnittest::TestLoadRemovedSingletonConfig() { PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); ClearConfig(); } - { // case: removed -> removed, first > second -> first < second - // should not happen - } - { // case: removed -> removed, first < second -> first > second - // should not happen - } - { // case: removed -> unchanged, first > second + { + // case: removed -> removed, first > second -> first < second + // should not happen + } { + // case: removed -> removed, first < second -> first > second + // should not happen + } { // case: removed -> unchanged, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -1150,11 +1150,11 @@ void PipelineConfigWatcherUnittest::TestLoadRemovedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: removed -> unchanged, first > second -> first < second - // should not happen - } - { // case: removed -> unchanged, first < second -> first > second - // should not happen + { + // case: removed -> unchanged, first > second -> first < second + // should not happen + } { // case: removed -> unchanged, first < second -> first > second + // should not happen } } @@ -1221,13 +1221,13 @@ void PipelineConfigWatcherUnittest::TestLoadUnchangedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: unchanged -> added, first > second -> first < second - // should not happen - } - { // case: unchanged -> added, first < second -> first > second - // should not happen - } - { // case: unchanged -> modified, first > second + { + // case: unchanged -> added, first > second -> first < second + // should not happen + } { + // case: unchanged -> added, first < second -> first > second + // should not happen + } { // case: unchanged -> modified, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -1425,13 +1425,13 @@ void PipelineConfigWatcherUnittest::TestLoadUnchangedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } - { // case: unchanged -> removed, first > second -> first < second - // should not happen - } - { // case: unchanged -> removed, first < second -> first > second - // should not happen - } - { // case: unchanged -> unchanged, first > second + { + // case: unchanged -> removed, first > second -> first < second + // should not happen + } { + // case: unchanged -> removed, first < second -> first > second + // should not happen + } { // case: unchanged -> unchanged, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -1489,11 +1489,11 @@ void PipelineConfigWatcherUnittest::TestLoadUnchangedSingletonConfig() { APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[builtinPipelineCnt + 1]); ClearConfig(); } - { // case: unchanged -> unchanged, first > second -> first < second - // should not happen - } - { // case: unchanged -> unchanged, first < second -> first > second - // should not happen + { + // case: unchanged -> unchanged, first > second -> first < second + // should not happen + } { // case: unchanged -> unchanged, first < second -> first > second + // should not happen } } diff --git a/core/unittest/config/TaskConfigUnittest.cpp b/core/unittest/config/TaskConfigUnittest.cpp index 658b459914..ab822dc53c 100644 --- a/core/unittest/config/TaskConfigUnittest.cpp +++ b/core/unittest/config/TaskConfigUnittest.cpp @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include +#include "json/json.h" + #include "common/JsonUtil.h" #include "config/TaskConfig.h" #include "task_pipeline/TaskRegistry.h" diff --git a/core/unittest/container_manager/ContainerDiscoveryOptionsUnittest.cpp b/core/unittest/container_manager/ContainerDiscoveryOptionsUnittest.cpp index 2f40343a24..2485e2b3cd 100644 --- a/core/unittest/container_manager/ContainerDiscoveryOptionsUnittest.cpp +++ b/core/unittest/container_manager/ContainerDiscoveryOptionsUnittest.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include "json/json.h" #include "common/JsonUtil.h" #include "container_manager/ContainerDiscoveryOptions.h" diff --git a/core/unittest/controller/EventDispatcherDirUnittest.cpp b/core/unittest/controller/EventDispatcherDirUnittest.cpp index f08938ac52..9671fe614c 100644 --- a/core/unittest/controller/EventDispatcherDirUnittest.cpp +++ b/core/unittest/controller/EventDispatcherDirUnittest.cpp @@ -12,16 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include #include #include -#include -#include + #include +#include + #include "common/Flags.h" #include "file_server/EventDispatcher.h" #include "file_server/event/Event.h" #include "file_server/event_handler/EventHandler.h" +#include "unittest/Unittest.h" using namespace std; DECLARE_FLAG_STRING(ilogtail_config); diff --git a/core/unittest/ebpf/eBPFServerUnittest.cpp b/core/unittest/ebpf/eBPFServerUnittest.cpp index 5fb3a98bcd..25fae38fb8 100644 --- a/core/unittest/ebpf/eBPFServerUnittest.cpp +++ b/core/unittest/ebpf/eBPFServerUnittest.cpp @@ -1,9 +1,9 @@ -#include - #include #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "common/FileSystemUtil.h" #include "common/JsonUtil.h" diff --git a/core/unittest/event/EventUnittest.cpp b/core/unittest/event/EventUnittest.cpp index 2644b4539b..4cfecd0605 100644 --- a/core/unittest/event/EventUnittest.cpp +++ b/core/unittest/event/EventUnittest.cpp @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include #include #include -#include -#include + #include +#include + #include "common/Flags.h" #include "file_server/event/Event.h" +#include "unittest/Unittest.h" using namespace std; DECLARE_FLAG_STRING(ilogtail_config); diff --git a/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp b/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp index 906b432bd4..fbbb9d1762 100644 --- a/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp +++ b/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp @@ -12,16 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include #include #include -#include -#include + #include +#include + #include "common/Flags.h" #include "file_server/ConfigManager.h" #include "file_server/event/Event.h" #include "file_server/event_handler/EventHandler.h" +#include "unittest/Unittest.h" using namespace std; DECLARE_FLAG_STRING(ilogtail_config); diff --git a/core/unittest/event_handler/LogInputUnittest.cpp b/core/unittest/event_handler/LogInputUnittest.cpp index 17c79fb578..14ad196fad 100644 --- a/core/unittest/event_handler/LogInputUnittest.cpp +++ b/core/unittest/event_handler/LogInputUnittest.cpp @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include #include #include -#include -#include + #include -#include "common/Flags.h" +#include + #include "common/FileSystemUtil.h" -#include "file_server/polling/PollingEventQueue.h" +#include "common/Flags.h" #include "file_server/event/Event.h" #include "file_server/event_handler/LogInput.h" +#include "file_server/polling/PollingEventQueue.h" +#include "unittest/Unittest.h" using namespace std; DECLARE_FLAG_STRING(ilogtail_config); diff --git a/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp b/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp index bed23e1854..12e8dc9b0b 100644 --- a/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp +++ b/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp @@ -16,7 +16,7 @@ #include #include -#include +#include "json/json.h" #include "common/JsonUtil.h" #include "file_server/FileDiscoveryOptions.h" diff --git a/core/unittest/file_source/MultilineOptionsUnittest.cpp b/core/unittest/file_source/MultilineOptionsUnittest.cpp index 481064b659..2197231b99 100644 --- a/core/unittest/file_source/MultilineOptionsUnittest.cpp +++ b/core/unittest/file_source/MultilineOptionsUnittest.cpp @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include +#include "json/json.h" + #include "common/JsonUtil.h" #include "file_server/MultilineOptions.h" #include "pipeline/PipelineContext.h" diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 3639b7b44c..d4bd8ff314 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -12,19 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" -#include "common/http/Constant.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "common/compression/CompressorFactory.h" +#include "common/http/Constant.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineContext.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" @@ -32,15 +29,13 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" -#ifdef __ENTERPRISE__ -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" -#endif #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSConstant.h" #include "unittest/Unittest.h" #ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" #include "unittest/flusher/SLSNetworkRequestMock.h" #endif @@ -718,9 +713,9 @@ void FlusherSLSUnittest::TestBuildRequest() { FlusherSLS::GetRegionConcurrencyLimiter(flusher.mRegion)->GetCurrentLimit()); } EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", - EndpointMode::DEFAULT, - "test_project.test_region-b.log.aliyuncs.com", - chrono::milliseconds(100)); + EndpointMode::DEFAULT, + "test_project.test_region-b.log.aliyuncs.com", + chrono::milliseconds(100)); flusher.mCandidateHostsInfo->SelectBestHost(); #endif // log telemetry type @@ -1045,9 +1040,9 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", - EndpointMode::ACCELERATE, - "test_project." + kAccelerationDataEndpoint, - chrono::milliseconds(10)); + EndpointMode::ACCELERATE, + "test_project." + kAccelerationDataEndpoint, + chrono::milliseconds(10)); flusher.mCandidateHostsInfo->SelectBestHost(); APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); diff --git a/core/unittest/input/InputContainerStdioUnittest.cpp b/core/unittest/input/InputContainerStdioUnittest.cpp index 60d353b86c..0fee30f961 100644 --- a/core/unittest/input/InputContainerStdioUnittest.cpp +++ b/core/unittest/input/InputContainerStdioUnittest.cpp @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "file_server/FileServer.h" -#include "plugin/input/InputContainerStdio.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineContext.h" #include "pipeline/plugin/PluginRegistry.h" +#include "plugin/input/InputContainerStdio.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(default_plugin_log_queue_size); diff --git a/core/unittest/input/InputFileSecurityUnittest.cpp b/core/unittest/input/InputFileSecurityUnittest.cpp index a9acb471df..aec3981df4 100644 --- a/core/unittest/input/InputFileSecurityUnittest.cpp +++ b/core/unittest/input/InputFileSecurityUnittest.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "ebpf/Config.h" diff --git a/core/unittest/input/InputFileUnittest.cpp b/core/unittest/input/InputFileUnittest.cpp index 057dff17e2..65287e0e83 100644 --- a/core/unittest/input/InputFileUnittest.cpp +++ b/core/unittest/input/InputFileUnittest.cpp @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "file_server/FileServer.h" diff --git a/core/unittest/input/InputInternalMetricsUnittest.cpp b/core/unittest/input/InputInternalMetricsUnittest.cpp index e9e01455b5..9e4b8d418b 100644 --- a/core/unittest/input/InputInternalMetricsUnittest.cpp +++ b/core/unittest/input/InputInternalMetricsUnittest.cpp @@ -44,7 +44,7 @@ class InputInternalMetricsUnittest : public testing::Test { PluginRegistry::GetInstance()->LoadPlugins(); } - static void TearDownTestCase() { + static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); LoongCollectorMonitor::GetInstance()->Stop(); } @@ -166,8 +166,10 @@ void InputInternalMetricsUnittest::OnPipelineUpdate() { APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mPipelineMetricsRule.mInterval, 9); APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mPluginMetricsRule.mEnable, false); APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mPluginMetricsRule.mInterval, 11); - APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mPluginSourceMetricsRule.mEnable, false); - APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mPluginSourceMetricsRule.mInterval, 10); + APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mPluginSourceMetricsRule.mEnable, + false); + APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mPluginSourceMetricsRule.mInterval, + 10); APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mRunnerMetricsRule.mEnable, true); APSARA_TEST_EQUAL(SelfMonitorServer::GetInstance()->mSelfMonitorMetricRules->mRunnerMetricsRule.mInterval, 8); diff --git a/core/unittest/input/InputNetworkObserverUnittest.cpp b/core/unittest/input/InputNetworkObserverUnittest.cpp index 1ec6688984..a7bf27c752 100644 --- a/core/unittest/input/InputNetworkObserverUnittest.cpp +++ b/core/unittest/input/InputNetworkObserverUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include "json/json.h" #include "app_config/AppConfig.h" #include "common/JsonUtil.h" diff --git a/core/unittest/input/InputNetworkSecurityUnittest.cpp b/core/unittest/input/InputNetworkSecurityUnittest.cpp index 23e264ba55..8b63d47de0 100644 --- a/core/unittest/input/InputNetworkSecurityUnittest.cpp +++ b/core/unittest/input/InputNetworkSecurityUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include "json/json.h" #include "app_config/AppConfig.h" #include "common/JsonUtil.h" @@ -207,7 +207,8 @@ void InputNetworkSecurityUnittest::OnSuccessfulStart() { input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); - string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); + string serverPipelineName + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); string pipelineName = input->GetContext().GetConfigName(); APSARA_TEST_TRUE(serverPipelineName.size() && serverPipelineName == pipelineName); } @@ -237,7 +238,8 @@ void InputNetworkSecurityUnittest::OnSuccessfulStop() { input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); - string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); + string serverPipelineName + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::NETWORK_SECURITY); string pipelineName = input->GetContext().GetConfigName(); APSARA_TEST_TRUE(serverPipelineName.size() && serverPipelineName == pipelineName); APSARA_TEST_TRUE(input->Stop(false)); diff --git a/core/unittest/input/InputProcessSecurityUnittest.cpp b/core/unittest/input/InputProcessSecurityUnittest.cpp index 4fd25d65dc..d94958d4fd 100644 --- a/core/unittest/input/InputProcessSecurityUnittest.cpp +++ b/core/unittest/input/InputProcessSecurityUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include "json/json.h" #include "app_config/AppConfig.h" #include "common/JsonUtil.h" @@ -99,7 +99,8 @@ void InputProcessSecurityUnittest::OnSuccessfulStart() { input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); - string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY); + string serverPipelineName + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY); string pipelineName = input->GetContext().GetConfigName(); APSARA_TEST_TRUE(serverPipelineName.size() && serverPipelineName == pipelineName); } @@ -120,7 +121,8 @@ void InputProcessSecurityUnittest::OnSuccessfulStop() { input->SetMetricsRecordRef("test", "1"); APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(input->Start()); - string serverPipelineName = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY); + string serverPipelineName + = ebpf::eBPFServer::GetInstance()->CheckLoadedPipelineName(nami::PluginType::PROCESS_SECURITY); string pipelineName = input->GetContext().GetConfigName(); APSARA_TEST_TRUE(serverPipelineName.size() && serverPipelineName == pipelineName); APSARA_TEST_TRUE(input->Stop(false)); diff --git a/core/unittest/input/InputPrometheusUnittest.cpp b/core/unittest/input/InputPrometheusUnittest.cpp index 2b11218651..06e9e32c07 100644 --- a/core/unittest/input/InputPrometheusUnittest.cpp +++ b/core/unittest/input/InputPrometheusUnittest.cpp @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include +#include "json/json.h" + #include "PluginRegistry.h" #include "app_config/AppConfig.h" #include "common/JsonUtil.h" @@ -50,9 +50,7 @@ class InputPrometheusUnittest : public testing::Test { ctx.SetConfigName("test_config"); ctx.SetPipeline(p); } - static void TearDownTestCase() { - PluginRegistry::GetInstance()->UnloadPlugins(); - } + static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } private: Pipeline p; diff --git a/core/unittest/logger/logger_unittest.cpp b/core/unittest/logger/logger_unittest.cpp index e34b58606e..4ebb2bb4dd 100644 --- a/core/unittest/logger/logger_unittest.cpp +++ b/core/unittest/logger/logger_unittest.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "logger/Logger.h" + #include int main() { diff --git a/core/unittest/metadata/K8sMetadataUnittest.cpp b/core/unittest/metadata/K8sMetadataUnittest.cpp index 46b077e186..3b794ca557 100644 --- a/core/unittest/metadata/K8sMetadataUnittest.cpp +++ b/core/unittest/metadata/K8sMetadataUnittest.cpp @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" -#include #include +#include #include -#include "metadata/LabelingK8sMetadata.h" + #include "metadata/K8sMetadata.h" +#include "metadata/LabelingK8sMetadata.h" #include "models/PipelineEventGroup.h" +#include "unittest/Unittest.h" using namespace std; @@ -32,11 +33,12 @@ class k8sMetadataUnittest : public ::testing::Test { void TearDown() override { // Clean up after each test case if needed } - + public: void TestGetByContainerIds() { LOG_INFO(sLogger, ("TestGetByContainerIds() begin", time(NULL))); - const std::string jsonData = R"({"containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144":{"namespace":"default","workloadName":"oneagent-demo-658648895b","workloadKind":"replicaset","serviceName":"","labels":{"app":"oneagent-demo","pod-template-hash":"658648895b"},"envs":{},"images":{"oneagent-demo":"sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/centos7-cve-fix:1.0.0"}}})"; + const std::string jsonData + = R"({"containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144":{"namespace":"default","workloadName":"oneagent-demo-658648895b","workloadKind":"replicaset","serviceName":"","labels":{"app":"oneagent-demo","pod-template-hash":"658648895b"},"envs":{},"images":{"oneagent-demo":"sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/centos7-cve-fix:1.0.0"}}})"; Json::Value root; Json::CharReaderBuilder readerBuilder; @@ -51,11 +53,12 @@ class k8sMetadataUnittest : public ::testing::Test { auto& k8sMetadata = K8sMetadata::GetInstance(); k8sMetadata.SetContainerCache(root); k8sMetadata.GetByLocalHostFromServer(); - + // Assume GetInfoByContainerIdFromCache returns non-null shared_ptr for valid IDs, // and check for some expectations. - APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByContainerIdFromCache("containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144") != nullptr); - + APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByContainerIdFromCache( + "containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144") + != nullptr); } void TestGetByLocalHost() { @@ -139,7 +142,7 @@ class k8sMetadataUnittest : public ::testing::Test { auto& k8sMetadata = K8sMetadata::GetInstance(); k8sMetadata.SetIpCache(root); - + auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::string eventStr = R"({ @@ -178,7 +181,7 @@ class k8sMetadataUnittest : public ::testing::Test { APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByIpFromCache("10.41.0.2") != nullptr); } - void TestAddLabelToSpan() { + void TestAddLabelToSpan() { LOG_INFO(sLogger, ("TestProcessEventForSpan() begin", time(NULL))); // Sample JSON data const std::string jsonData = R"({ @@ -292,7 +295,7 @@ class k8sMetadataUnittest : public ::testing::Test { } - void TestAddLabelToMetric() { + void TestAddLabelToMetric() { LOG_INFO(sLogger, ("TestGetByLocalHost() begin", time(NULL))); // Sample JSON data const std::string jsonData = R"({ @@ -374,7 +377,7 @@ class k8sMetadataUnittest : public ::testing::Test { auto& k8sMetadata = K8sMetadata::GetInstance(); k8sMetadata.SetIpCache(root); k8sMetadata.GetByLocalHostFromServer(); - + auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::string eventStr = R"({ @@ -423,7 +426,6 @@ APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestAddLabelToMetric, 2); APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestAddLabelToSpan, 3); - } // end of namespace logtail int main(int argc, char** argv) { diff --git a/core/unittest/models/EventGroupBenchmark.cpp b/core/unittest/models/EventGroupBenchmark.cpp index ca53df4951..98d7ba43c8 100644 --- a/core/unittest/models/EventGroupBenchmark.cpp +++ b/core/unittest/models/EventGroupBenchmark.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include + #include "common/JsonUtil.h" #include "common/TimeUtil.h" #include "models/LogEvent.h" diff --git a/core/unittest/models/EventPoolUnittest.cpp b/core/unittest/models/EventPoolUnittest.cpp index 7c1e220d2e..0861e5993c 100644 --- a/core/unittest/models/EventPoolUnittest.cpp +++ b/core/unittest/models/EventPoolUnittest.cpp @@ -116,7 +116,6 @@ void EventPoolUnittest::TestLock() { APSARA_TEST_EQUAL(0U, pool.mLogEventPoolBak.size()); APSARA_TEST_EQUAL(1U, pool.mLogEventPool.size()); delete e2; - } { auto e = pool.AcquireMetricEvent(mGroup.get()); diff --git a/core/unittest/models/MetricEventUnittest.cpp b/core/unittest/models/MetricEventUnittest.cpp index b148cce39b..f14cc99684 100644 --- a/core/unittest/models/MetricEventUnittest.cpp +++ b/core/unittest/models/MetricEventUnittest.cpp @@ -161,7 +161,8 @@ void MetricEventUnittest::TestUntypedSingleValueSize() { void MetricEventUnittest::TestUntypedMultiDoubleValuesSize() { mMetricEvent->SetName("test"); mMetricEvent->SetValue(map{}); - size_t basicSize = sizeof(time_t) + sizeof(long) + sizeof(UntypedMultiDoubleValues) + sizeof(map); + size_t basicSize + = sizeof(time_t) + sizeof(long) + sizeof(UntypedMultiDoubleValues) + sizeof(map); basicSize += 4; // add tag, and key not existed @@ -357,7 +358,8 @@ void MetricEventUnittest::TestCopy() { APSARA_TEST_EQUAL(10.0, val); APSARA_TEST_EQUAL(true, newMetricEvent.GetValue()->GetValue("test-2", val)); APSARA_TEST_EQUAL(2.0, val); - APSARA_TEST_NOT_EQUAL(newMetricEvent.GetValue()->mMetricEventPtr, oldMetricEvent->GetValue()->mMetricEventPtr); + APSARA_TEST_NOT_EQUAL(newMetricEvent.GetValue()->mMetricEventPtr, + oldMetricEvent->GetValue()->mMetricEventPtr); } UNIT_TEST_CASE(MetricEventUnittest, TestName) diff --git a/core/unittest/monitor/MetricManagerUnittest.cpp b/core/unittest/monitor/MetricManagerUnittest.cpp index 343b6b9129..ba89466745 100644 --- a/core/unittest/monitor/MetricManagerUnittest.cpp +++ b/core/unittest/monitor/MetricManagerUnittest.cpp @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include #include -#include #include -#include #include -#include "MetricManager.h" + +#include "json/json.h" + #include "MetricConstants.h" +#include "MetricManager.h" +#include "unittest/Unittest.h" namespace logtail { @@ -53,7 +55,8 @@ void MetricManagerUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); APSARA_TEST_EQUAL(fileMetric->GetLabels()->size(), 3); @@ -91,7 +94,8 @@ void MetricManagerUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric2; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter2 = fileMetric2.CreateCounter("filed2"); fileCounter2->Add(222UL); } @@ -102,7 +106,8 @@ void MetricManagerUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("logstore", "logstore1")); labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric3; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter3 = fileMetric3.CreateCounter("filed3"); fileCounter3->Add(333UL); } @@ -136,7 +141,8 @@ void createMetrics(int count) { labels.emplace_back(std::make_pair("count", std::to_string(count))); labels.emplace_back(std::make_pair("region", "cn-beijing")); MetricsRecordRef fileMetric; - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + fileMetric, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric.CreateCounter("filed1"); fileCounter->Add(111UL); } @@ -204,7 +210,8 @@ void MetricManagerUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test1")); labels.emplace_back(std::make_pair("logstore", "test1")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + *fileMetric1, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric1->CreateCounter("filed1"); fileCounter->Add(111UL); @@ -213,7 +220,8 @@ void MetricManagerUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test2")); labels.emplace_back(std::make_pair("logstore", "test2")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + *fileMetric2, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric2->CreateCounter("filed1"); fileCounter->Add(111UL); } @@ -223,7 +231,8 @@ void MetricManagerUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test3")); labels.emplace_back(std::make_pair("logstore", "test3")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + *fileMetric3, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(labels)); CounterPtr fileCounter = fileMetric3->CreateCounter("filed1"); fileCounter->Add(111UL); } diff --git a/core/unittest/monitor/PluginMetricManagerUnittest.cpp b/core/unittest/monitor/PluginMetricManagerUnittest.cpp index 77246a322b..1ff567868a 100644 --- a/core/unittest/monitor/PluginMetricManagerUnittest.cpp +++ b/core/unittest/monitor/PluginMetricManagerUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/metric_models/ReentrantMetricsRecord.h" #include "monitor/metric_constants/MetricConstants.h" +#include "monitor/metric_models/ReentrantMetricsRecord.h" #include "unittest/Unittest.h" namespace logtail { @@ -28,7 +28,8 @@ class PluginMetricManagerUnittest : public ::testing::Test { defaultLabels->emplace_back(METRIC_LABEL_KEY_PIPELINE_NAME, "default_config"); defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_TYPE, "default_plugin"); defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_ID, "default_id"); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(*defaultLabels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_UNKNOWN, std::move(*defaultLabels)); std::unordered_map metricKeys; metricKeys.emplace("default_counter", MetricType::METRIC_TYPE_COUNTER); metricKeys.emplace("default_gauge", MetricType::METRIC_TYPE_INT_GAUGE); diff --git a/core/unittest/pipeline/ConcurrencyLimiterUnittest.cpp b/core/unittest/pipeline/ConcurrencyLimiterUnittest.cpp index 973b1b06b3..9eb615a85a 100644 --- a/core/unittest/pipeline/ConcurrencyLimiterUnittest.cpp +++ b/core/unittest/pipeline/ConcurrencyLimiterUnittest.cpp @@ -29,7 +29,8 @@ void ConcurrencyLimiterUnittest::TestLimiter() const { int maxConcurrency = 80; int minConcurrency = 20; - shared_ptr sConcurrencyLimiter = make_shared("", maxConcurrency, minConcurrency); + shared_ptr sConcurrencyLimiter + = make_shared("", maxConcurrency, minConcurrency); // fastFallBack APSARA_TEST_EQUAL(true, sConcurrencyLimiter->IsValidToPop()); for (uint32_t i = 0; i < sConcurrencyLimiter->GetStatisticThreshold(); i++) { @@ -42,7 +43,7 @@ void ConcurrencyLimiterUnittest::TestLimiter() const { APSARA_TEST_EQUAL(40U, sConcurrencyLimiter->GetCurrentLimit()); APSARA_TEST_EQUAL(0U, sConcurrencyLimiter->GetInSendingCount()); - // success one time + // success one time APSARA_TEST_EQUAL(true, sConcurrencyLimiter->IsValidToPop()); for (uint32_t i = 0; i < sConcurrencyLimiter->GetStatisticThreshold(); i++) { APSARA_TEST_EQUAL(true, sConcurrencyLimiter->IsValidToPop()); @@ -71,7 +72,7 @@ void ConcurrencyLimiterUnittest::TestLimiter() const { sConcurrencyLimiter->OnFail(curSystemTime); sConcurrencyLimiter->OnSendDone(); } - uint32_t expect = 41*0.8; + uint32_t expect = 41 * 0.8; APSARA_TEST_EQUAL(0U, sConcurrencyLimiter->GetInSendingCount()); APSARA_TEST_EQUAL(expect, sConcurrencyLimiter->GetCurrentLimit()); @@ -134,7 +135,7 @@ void ConcurrencyLimiterUnittest::TestLimiter() const { sConcurrencyLimiter->OnSuccess(curSystemTime); sConcurrencyLimiter->OnSendDone(); } - expect = 40*0.8; + expect = 40 * 0.8; APSARA_TEST_EQUAL(0U, sConcurrencyLimiter->GetInSendingCount()); APSARA_TEST_EQUAL(expect, sConcurrencyLimiter->GetCurrentLimit()); } diff --git a/core/unittest/pipeline/GlobalConfigUnittest.cpp b/core/unittest/pipeline/GlobalConfigUnittest.cpp index d646d17251..db79d54583 100644 --- a/core/unittest/pipeline/GlobalConfigUnittest.cpp +++ b/core/unittest/pipeline/GlobalConfigUnittest.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include "json/json.h" #include "common/JsonUtil.h" #include "pipeline/GlobalConfig.h" diff --git a/core/unittest/pipeline/HttpSinkMock.h b/core/unittest/pipeline/HttpSinkMock.h index 38a3a9882e..58c04c52ef 100644 --- a/core/unittest/pipeline/HttpSinkMock.h +++ b/core/unittest/pipeline/HttpSinkMock.h @@ -20,9 +20,9 @@ #include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "plugin/flusher/sls/FlusherSLS.h" +#include "plugin/flusher/sls/SLSConstant.h" #include "runner/FlusherRunner.h" #include "runner/sink/http/HttpSink.h" -#include "plugin/flusher/sls/SLSConstant.h" namespace logtail { class HttpSinkMock : public HttpSink { diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 89df992071..8e0421b596 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "config/PipelineConfig.h" diff --git a/core/unittest/pipeline/PipelineUpdateUnittest.cpp b/core/unittest/pipeline/PipelineUpdateUnittest.cpp index 1301e4046d..37feb58b26 100644 --- a/core/unittest/pipeline/PipelineUpdateUnittest.cpp +++ b/core/unittest/pipeline/PipelineUpdateUnittest.cpp @@ -66,7 +66,10 @@ class FlusherSLSMock : public FlusherSLS { public: static const std::string sName; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override { + bool BuildRequest(SenderQueueItem* item, + std::unique_ptr& req, + bool* keepItem, + std::string* errMsg) override { auto data = static_cast(item); std::map header; req = std::make_unique( @@ -144,7 +147,9 @@ class PipelineUpdateUnittest : public testing::Test { AppConfig::GetInstance()->mSendRequestGlobalConcurrency = 200; } - static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } + static void TearDownTestCase() { + PluginRegistry::GetInstance()->UnloadPlugins(); + } void SetUp() override { LogInput::GetInstance()->CleanEnviroments(); diff --git a/core/unittest/plugin/PluginMock.h b/core/unittest/plugin/PluginMock.h index e61410df75..12ba09a7d6 100644 --- a/core/unittest/plugin/PluginMock.h +++ b/core/unittest/plugin/PluginMock.h @@ -151,7 +151,10 @@ class FlusherHttpMock : public HttpFlusher { return true; } bool FlushAll() override { return mIsValid; } - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override { + bool BuildRequest(SenderQueueItem* item, + std::unique_ptr& req, + bool* keepItem, + std::string* errMsg) override { if (item->mData == "invalid_keep") { *keepItem = true; return false; diff --git a/core/unittest/plugin/PluginRegistryUnittest.cpp b/core/unittest/plugin/PluginRegistryUnittest.cpp index 086a6638a0..a012015489 100644 --- a/core/unittest/plugin/PluginRegistryUnittest.cpp +++ b/core/unittest/plugin/PluginRegistryUnittest.cpp @@ -46,7 +46,8 @@ void PluginRegistryUnittest::TestCreateInput() const { } void PluginRegistryUnittest::TestCreateProcessor() const { - unique_ptr processor = PluginRegistry::GetInstance()->CreateProcessor(ProcessorMock::sName, {"0"}); + unique_ptr processor + = PluginRegistry::GetInstance()->CreateProcessor(ProcessorMock::sName, {"0"}); APSARA_TEST_NOT_EQUAL_FATAL(nullptr, processor); APSARA_TEST_EQUAL_FATAL("0", processor->PluginID()); } diff --git a/core/unittest/plugin/ProcessorInstanceUnittest.cpp b/core/unittest/plugin/ProcessorInstanceUnittest.cpp index 90621bf159..6f5ab4e244 100644 --- a/core/unittest/plugin/ProcessorInstanceUnittest.cpp +++ b/core/unittest/plugin/ProcessorInstanceUnittest.cpp @@ -30,12 +30,14 @@ class ProcessorInstanceUnittest : public testing::Test { }; void ProcessorInstanceUnittest::TestName() const { - unique_ptr processor = make_unique(new ProcessorMock(), PluginInstance::PluginMeta("0")); + unique_ptr processor + = make_unique(new ProcessorMock(), PluginInstance::PluginMeta("0")); APSARA_TEST_EQUAL(ProcessorMock::sName, processor->Name()); } void ProcessorInstanceUnittest::TestInit() const { - unique_ptr processor = make_unique(new ProcessorMock(), PluginInstance::PluginMeta("0")); + unique_ptr processor + = make_unique(new ProcessorMock(), PluginInstance::PluginMeta("0")); Json::Value config; PipelineContext context; APSARA_TEST_TRUE(processor->Init(config, context)); @@ -43,7 +45,8 @@ void ProcessorInstanceUnittest::TestInit() const { } void ProcessorInstanceUnittest::TestProcess() const { - unique_ptr processor = make_unique(new ProcessorMock(), PluginInstance::PluginMeta("0")); + unique_ptr processor + = make_unique(new ProcessorMock(), PluginInstance::PluginMeta("0")); Json::Value config; PipelineContext context; processor->Init(config, context); diff --git a/core/unittest/plugin/StaticFlusherCreatorUnittest.cpp b/core/unittest/plugin/StaticFlusherCreatorUnittest.cpp index 42cb639f5e..a4a2735554 100644 --- a/core/unittest/plugin/StaticFlusherCreatorUnittest.cpp +++ b/core/unittest/plugin/StaticFlusherCreatorUnittest.cpp @@ -14,8 +14,8 @@ #include "pipeline/plugin/creator/StaticFlusherCreator.h" #include "pipeline/plugin/instance/PluginInstance.h" -#include "unittest/plugin/PluginMock.h" #include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" using namespace std; diff --git a/core/unittest/plugin/StaticInputCreatorUnittest.cpp b/core/unittest/plugin/StaticInputCreatorUnittest.cpp index e7d039eddf..083dd252fe 100644 --- a/core/unittest/plugin/StaticInputCreatorUnittest.cpp +++ b/core/unittest/plugin/StaticInputCreatorUnittest.cpp @@ -14,8 +14,8 @@ #include "pipeline/plugin/creator/StaticInputCreator.h" #include "pipeline/plugin/instance/PluginInstance.h" -#include "unittest/plugin/PluginMock.h" #include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" using namespace std; diff --git a/core/unittest/plugin/StaticProcessorCreatorUnittest.cpp b/core/unittest/plugin/StaticProcessorCreatorUnittest.cpp index 1fb2afffbb..af83d6f0d1 100644 --- a/core/unittest/plugin/StaticProcessorCreatorUnittest.cpp +++ b/core/unittest/plugin/StaticProcessorCreatorUnittest.cpp @@ -14,8 +14,8 @@ #include "pipeline/plugin/creator/StaticProcessorCreator.h" #include "pipeline/plugin/instance/PluginInstance.h" -#include "unittest/plugin/PluginMock.h" #include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" using namespace std; diff --git a/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp b/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp index 1e860271bd..8dc4103e1c 100644 --- a/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp +++ b/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp @@ -1,9 +1,10 @@ -#include - #include // Include the header for sleep_for + #include // Include the header for this_thread +#include "json/json.h" + #include "application/Application.h" #include "common/Flags.h" #include "common/JsonUtil.h" diff --git a/core/unittest/polling/PollingUnittest.cpp b/core/unittest/polling/PollingUnittest.cpp index 9dd2ea3c1d..d27983b99d 100644 --- a/core/unittest/polling/PollingUnittest.cpp +++ b/core/unittest/polling/PollingUnittest.cpp @@ -12,49 +12,53 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include "json/json.h" + +#include "app_config/AppConfig.h" #include "common/Flags.h" -#include -#include "file_server/EventDispatcher.h" +#include "common/StringTools.h" #include "file_server/ConfigManager.h" -#include "app_config/AppConfig.h" -#include "file_server/reader/LogFileReader.h" +#include "file_server/EventDispatcher.h" +#include "file_server/event/Event.h" #include "file_server/event_handler/EventHandler.h" -#include "monitor/Monitor.h" -#include "common/StringTools.h" +#include "file_server/reader/LogFileReader.h" #include "logger/Logger.h" -#include "file_server/event/Event.h" +#include "monitor/Monitor.h" +#include "unittest/Unittest.h" #if defined(__linux__) #include #include #endif #include -#include -#include -#include -#include +#include #include -#include +#include #include +#include +#include +#include +#include + +#include +#include +#include +#include +#include #include -#include #include -#include -#include -#include -#include -#include "protobuf/sls/metric.pb.h" -#include "protobuf/sls/sls_logs.pb.h" -#include "monitor/AlarmManager.h" -#include "file_server/event_handler/LogInput.h" +#include + +#include "boost/regex.hpp" + #include "common/FileEncryption.h" #include "common/FileSystemUtil.h" -#include -#include -#include +#include "file_server/event_handler/LogInput.h" #include "file_server/polling/PollingDirFile.h" -#include "file_server/polling/PollingModify.h" #include "file_server/polling/PollingEventQueue.h" +#include "file_server/polling/PollingModify.h" +#include "monitor/AlarmManager.h" +#include "protobuf/sls/metric.pb.h" +#include "protobuf/sls/sls_logs.pb.h" using namespace std; using namespace sls_logs; diff --git a/core/unittest/processor/BoostRegexBenchmark.cpp b/core/unittest/processor/BoostRegexBenchmark.cpp index 42000106b7..1b1ce56c8b 100644 --- a/core/unittest/processor/BoostRegexBenchmark.cpp +++ b/core/unittest/processor/BoostRegexBenchmark.cpp @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include + #include #include +#include "boost/regex.hpp" + #include "unittest/Unittest.h" diff --git a/core/unittest/processor/ParseContainerLogBenchmark.cpp b/core/unittest/processor/ParseContainerLogBenchmark.cpp index c48d7970b2..df7796dc17 100644 --- a/core/unittest/processor/ParseContainerLogBenchmark.cpp +++ b/core/unittest/processor/ParseContainerLogBenchmark.cpp @@ -14,6 +14,7 @@ #include #include + #include #include diff --git a/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp b/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp index 4ed95ecc73..6b0a1fd338 100644 --- a/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorDesensitizeNativeUnittest.cpp @@ -54,7 +54,7 @@ UNIT_TEST_CASE(ProcessorDesensitizeNativeUnittest, TestMultipleLines); UNIT_TEST_CASE(ProcessorDesensitizeNativeUnittest, TestMultipleLinesWithProcessorMergeMultilineLogNative); -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } diff --git a/core/unittest/processor/ProcessorFilterNativeUnittest.cpp b/core/unittest/processor/ProcessorFilterNativeUnittest.cpp index 71c416da80..30c9f6a0b4 100644 --- a/core/unittest/processor/ProcessorFilterNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorFilterNativeUnittest.cpp @@ -41,7 +41,7 @@ UNIT_TEST_CASE(ProcessorFilterNativeUnittest, TestLogFilterRule) UNIT_TEST_CASE(ProcessorFilterNativeUnittest, TestBaseFilter) UNIT_TEST_CASE(ProcessorFilterNativeUnittest, TestFilterNoneUtf8) -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -449,7 +449,8 @@ void ProcessorFilterNativeUnittest::TestBaseFilter() { })"; APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - APSARA_TEST_EQUAL_FATAL(2, processorInstance.mInEventsTotal->GetValue() - processorInstance.mOutEventsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL( + 2, processorInstance.mInEventsTotal->GetValue() - processorInstance.mOutEventsTotal->GetValue()); } { const char* jsonStr = "{\n" diff --git a/core/unittest/processor/ProcessorMergeMultilineLogNativeUnittest.cpp b/core/unittest/processor/ProcessorMergeMultilineLogNativeUnittest.cpp index 6c3e0e774b..3a787f5b94 100644 --- a/core/unittest/processor/ProcessorMergeMultilineLogNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorMergeMultilineLogNativeUnittest.cpp @@ -13,9 +13,9 @@ // limitations under the License. #include -#include "constants/Constants.h" #include "common/JsonUtil.h" #include "config/PipelineConfig.h" +#include "constants/Constants.h" #include "models/LogEvent.h" #include "plugin/processor/inner/ProcessorMergeMultilineLogNative.h" #include "plugin/processor/inner/ProcessorSplitLogStringNative.h" @@ -44,235 +44,229 @@ UNIT_TEST_CASE(ProcessorMergeMultilineLogNativeUnittest, TestProcess); void ProcessorMergeMultilineLogNativeUnittest::TestInit() { // 测试合法的正则配置 + {// start init通过，IsMultiline为true + {Json::Value config; + config["StartPattern"] = "123123123.*"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); +} +// start + continue init通过，IsMultiline为true +{ + Json::Value config; + config["StartPattern"] = "123123123.*"; + config["ContinuePattern"] = "123123.*"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); +} +// continue + end init通过，IsMultiline为true +{ + Json::Value config; + config["ContinuePattern"] = "123123.*"; + config["EndPattern"] = "123.*"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); +} +// end init通过，IsMultiline为true +{ + Json::Value config; + config["EndPattern"] = "123.*"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); +} +// start + end init通过，IsMultiline为true +{ + Json::Value config; + config["StartPattern"] = "123123123.*"; + config["EndPattern"] = "123.*"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); +} +} // namespace logtail +// 测试非法的正则配置 +{// start + continue + end init通过，IsMultiline为true + {Json::Value config; +config["StartPattern"] = "123123123.*"; +config["ContinuePattern"] = "123123.*"; +config["EndPattern"] = "123.*"; +config["MergeType"] = "regex"; +ProcessorMergeMultilineLogNative processor; +processor.SetContext(mContext); +processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); +APSARA_TEST_TRUE(processor.Init(config)); +APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); +} +// continue init通过，IsMultiline为false +{ + Json::Value config; + config["ContinuePattern"] = "123123.*"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); +} +// 正则格式非法 init通过，IsMultiline为false +{ + Json::Value config; + config["StartPattern"] = 1; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); +} +// 正则格式非法 init通过，IsMultiline为false +{ + Json::Value config; + config["StartPattern"] = "******"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); +} +// 正则格式缺失 init通过，IsMultiline为false +{ + Json::Value config; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); +} +} +// 测试mergetype +{// regex init通过 + {Json::Value config; +config["StartPattern"] = ".*"; +config["MergeType"] = "regex"; +ProcessorMergeMultilineLogNative processor; +processor.SetContext(mContext); +processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); +APSARA_TEST_TRUE(processor.Init(config)); +APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); +} +// flag init通过 +{ + Json::Value config; + config["MergeType"] = "flag"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); +} +// unknown init不通过 +{ + Json::Value config; + config["StartPattern"] = ".*"; + config["MergeType"] = "unknown"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_FALSE(processor.Init(config)); +} +// 格式错误 init不通过 +{ + Json::Value config; + config["StartPattern"] = ".*"; + config["MergeType"] = 1; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_FALSE(processor.Init(config)); +} +// 不存在 init不通过 +{ + Json::Value config; + config["StartPattern"] = ".*"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_FALSE(processor.Init(config)); +} +} +// 测试UnmatchedContentTreatment +{ + // single_line init通过 { - // start init通过，IsMultiline为true - { - Json::Value config; - config["StartPattern"] = "123123123.*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); - } - // start + continue init通过，IsMultiline为true - { - Json::Value config; - config["StartPattern"] = "123123123.*"; - config["ContinuePattern"] = "123123.*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); - } - // continue + end init通过，IsMultiline为true - { - Json::Value config; - config["ContinuePattern"] = "123123.*"; - config["EndPattern"] = "123.*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); - } - // end init通过，IsMultiline为true - { - Json::Value config; - config["EndPattern"] = "123.*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); - } - // start + end init通过，IsMultiline为true - { - Json::Value config; - config["StartPattern"] = "123123123.*"; - config["EndPattern"] = "123.*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); - } + Json::Value config; + config["StartPattern"] = ".*"; + config["MergeType"] = "regex"; + config["UnmatchedContentTreatment"] = "single_line"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); } - // 测试非法的正则配置 + // discard init通过 { - // start + continue + end init通过，IsMultiline为true - { - Json::Value config; - config["StartPattern"] = "123123123.*"; - config["ContinuePattern"] = "123123.*"; - config["EndPattern"] = "123.*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_TRUE(processor.mMultiline.IsMultiline()); - } - // continue init通过，IsMultiline为false - { - Json::Value config; - config["ContinuePattern"] = "123123.*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); - } - // 正则格式非法 init通过，IsMultiline为false - { - Json::Value config; - config["StartPattern"] = 1; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); - } - // 正则格式非法 init通过，IsMultiline为false - { - Json::Value config; - config["StartPattern"] = "******"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); - } - // 正则格式缺失 init通过，IsMultiline为false - { - Json::Value config; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); - } + Json::Value config; + config["StartPattern"] = ".*"; + config["MergeType"] = "regex"; + config["UnmatchedContentTreatment"] = "discard"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); } - // 测试mergetype + // unknown init通过 { - // regex init通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - APSARA_TEST_FALSE(processor.mMultiline.IsMultiline()); - } - // flag init通过 - { - Json::Value config; - config["MergeType"] = "flag"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - } - // unknown init不通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = "unknown"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_FALSE(processor.Init(config)); - } - // 格式错误 init不通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = 1; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_FALSE(processor.Init(config)); - } - // 不存在 init不通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_FALSE(processor.Init(config)); - } + Json::Value config; + config["StartPattern"] = ".*"; + config["MergeType"] = "regex"; + config["UnmatchedContentTreatment"] = "unknown"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); } - // 测试UnmatchedContentTreatment + // 格式错误 init通过 { - // single_line init通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = "regex"; - config["UnmatchedContentTreatment"] = "single_line"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - } - // discard init通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = "regex"; - config["UnmatchedContentTreatment"] = "discard"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - } - // unknown init通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = "regex"; - config["UnmatchedContentTreatment"] = "unknown"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - } - // 格式错误 init通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = "regex"; - config["UnmatchedContentTreatment"] = 1; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - } - // 不存在 init通过 - { - Json::Value config; - config["StartPattern"] = ".*"; - config["MergeType"] = "regex"; - ProcessorMergeMultilineLogNative processor; - processor.SetContext(mContext); - processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); - APSARA_TEST_TRUE(processor.Init(config)); - } + Json::Value config; + config["StartPattern"] = ".*"; + config["MergeType"] = "regex"; + config["UnmatchedContentTreatment"] = 1; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); + } + // 不存在 init通过 + { + Json::Value config; + config["StartPattern"] = ".*"; + config["MergeType"] = "regex"; + ProcessorMergeMultilineLogNative processor; + processor.SetContext(mContext); + processor.SetMetricsRecordRef(ProcessorMergeMultilineLogNative::sName, "1"); + APSARA_TEST_TRUE(processor.Init(config)); } } +} void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { // make config @@ -304,12 +298,10 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } // 存在不支持的event类型 - { - // 某个unmatch 后出现了一个不支持 - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::string inJson1 = R"({ + {// 某个unmatch 后出现了一个不支持 + {auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson1 = R"({ "events" : [ { @@ -323,9 +315,9 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.FromJsonString(inJson1); - eventGroup.AddMetricEvent(); - std::string inJson2 = R"({ + eventGroup.FromJsonString(inJson1); + eventGroup.AddMetricEvent(); + std::string inJson2 = R"({ "events" : [ { @@ -339,12 +331,12 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.FromJsonString(inJson2); - // run test function - processorSplitLogStringNative.Process(eventGroup); - processorMergeMultilineLogNative.Process(eventGroup); - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson2); + // run test function + processorSplitLogStringNative.Process(eventGroup); + processorMergeMultilineLogNative.Process(eventGroup); + std::stringstream expectJson; + expectJson << R"({ "events": [ { "contents": { @@ -404,15 +396,15 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} - // 正在匹配过程中 出现了一个不支持 - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::string inJson = R"({ +// 正在匹配过程中 出现了一个不支持 +{ + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ "events" : [ { @@ -426,9 +418,9 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.FromJsonString(inJson); - eventGroup.AddMetricEvent(); - inJson = R"({ + eventGroup.FromJsonString(inJson); + eventGroup.AddMetricEvent(); + inJson = R"({ "events" : [ { @@ -442,12 +434,12 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.FromJsonString(inJson); - // run test function - processorSplitLogStringNative.Process(eventGroup); - processorMergeMultilineLogNative.Process(eventGroup); - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson); + // run test function + processorSplitLogStringNative.Process(eventGroup); + processorMergeMultilineLogNative.Process(eventGroup); + std::stringstream expectJson; + expectJson << R"({ "events": [ { "contents": { @@ -491,15 +483,15 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} - // 第一个event就是不支持 - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::string inJson = R"({ +// 第一个event就是不支持 +{ + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ "events" : [ { @@ -513,13 +505,13 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.AddMetricEvent(); - eventGroup.FromJsonString(inJson); - // run test function - processorSplitLogStringNative.Process(eventGroup); - processorMergeMultilineLogNative.Process(eventGroup); - std::stringstream expectJson; - expectJson << R"({ + eventGroup.AddMetricEvent(); + eventGroup.FromJsonString(inJson); + // run test function + processorSplitLogStringNative.Process(eventGroup); + processorMergeMultilineLogNative.Process(eventGroup); + std::stringstream expectJson; + expectJson << R"({ "events": [ { "name": "", @@ -563,17 +555,17 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - } - // event group中某条event没有mSourceKey + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +} +// event group中某条event没有mSourceKey +{ + // 某个unmatch 后出现一个没有mSourceKey的event { - // 某个unmatch 后出现一个没有mSourceKey的event - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::string inJson = R"({ + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ "events" : [ { @@ -605,12 +597,12 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.FromJsonString(inJson); - // run test function - processorSplitLogStringNative.Process(eventGroup); - processorMergeMultilineLogNative.Process(eventGroup); - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson); + // run test function + processorSplitLogStringNative.Process(eventGroup); + processorMergeMultilineLogNative.Process(eventGroup); + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -674,14 +666,14 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // 正在匹配过程中出现没有mSourceKey的event - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::string inJson = R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // 正在匹配过程中出现没有mSourceKey的event + { + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ "events" : [ { @@ -713,12 +705,12 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.FromJsonString(inJson); - // run test function - processorSplitLogStringNative.Process(eventGroup); - processorMergeMultilineLogNative.Process(eventGroup); - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson); + // run test function + processorSplitLogStringNative.Process(eventGroup); + processorMergeMultilineLogNative.Process(eventGroup); + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -768,14 +760,14 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // 第一个就出现没有mSourceKey的event - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::string inJson = R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // 第一个就出现没有mSourceKey的event + { + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ "events" : [ { @@ -798,12 +790,12 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - eventGroup.FromJsonString(inJson); - // run test function - processorSplitLogStringNative.Process(eventGroup); - processorMergeMultilineLogNative.Process(eventGroup); - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson); + // run test function + processorSplitLogStringNative.Process(eventGroup); + processorMergeMultilineLogNative.Process(eventGroup); + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -853,11 +845,11 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() { } ] })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); } } +} class ProcessEventsWithPartLogUnittest : public ::testing::Test { public: diff --git a/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp b/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp index e047a874bb..cad9a5ba1d 100644 --- a/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp @@ -12,18 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include + +#include #include #include #include #include #include "boost/utility/string_view.hpp" -#include "constants/Constants.h" + #include "common/JsonUtil.h" #include "config/PipelineConfig.h" +#include "constants/Constants.h" #include "models/LogEvent.h" #include "plugin/processor/inner/ProcessorMergeMultilineLogNative.h" #include "plugin/processor/inner/ProcessorParseContainerLogNative.h" @@ -1225,14 +1227,12 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { processor.SetMetricsRecordRef(ProcessorParseContainerLogNative::sName, "1"); APSARA_TEST_TRUE_FATAL(processor.Init(config)); // log 测试 - { - // log 不存在情况下 - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + {// log 不存在情况下 + {auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1244,12 +1244,12 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson.str()); + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1266,16 +1266,16 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // log为空 - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +// log为空 +{ + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1287,12 +1287,12 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson.str()); + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1311,16 +1311,16 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // log为不是string - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +// log为不是string +{ + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1332,12 +1332,12 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson.str()); + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1354,20 +1354,18 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - } - // time - { - // time不存在 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +} // namespace logtail +// time +{// time不存在 + {// make eventGroup + auto sourceBuffer = std::make_shared(); +PipelineEventGroup eventGroup(sourceBuffer); +std::stringstream inJson; +eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); +inJson << R"({ "events": [ { "contents": { @@ -1379,13 +1377,13 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); +eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ +// run test function +processor.Process(eventGroup); +// judge result +std::stringstream expectJson; +expectJson << R"({ "events" : [ { @@ -1402,17 +1400,17 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // time 为空 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ +std::string outJson = eventGroup.ToJsonString(); +APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +// time 为空 +{ + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1424,13 +1422,13 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1447,17 +1445,17 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // time 不是string - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +// time 不是string +{ + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1469,13 +1467,13 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1492,20 +1490,20 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - } - // stream + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +} +// stream +{ + // stream不存在 { - // stream不存在 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1517,13 +1515,13 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1540,17 +1538,17 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // stream非法 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // stream非法 + { + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1562,13 +1560,13 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1585,17 +1583,17 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // stream不是string - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // stream不是string + { + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1607,13 +1605,13 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events" : [ { @@ -1630,11 +1628,11 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); } } +} void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() { // containerd_text @@ -1767,15 +1765,12 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() processor.SetMetricsRecordRef(ProcessorParseContainerLogNative::sName, "1"); APSARA_TEST_TRUE_FATAL(processor.Init(config)); // log 测试 - { - // log 不存在情况下 - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + {// log 不存在情况下 + {auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1787,27 +1782,26 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson.str()); + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // log为空 - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // log为空 + { + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1819,12 +1813,12 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson.str()); + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "events": [ { "contents" : @@ -1842,17 +1836,16 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // log为不是string - { - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // log为不是string + { + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1864,31 +1857,28 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + eventGroup.FromJsonString(inJson.str()); + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - } - // time - { - // time不存在 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } +} +// time +{// time不存在 + {// make eventGroup + auto sourceBuffer = std::make_shared(); +PipelineEventGroup eventGroup(sourceBuffer); +std::stringstream inJson; +eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); +inJson << R"({ "events": [ { "contents": { @@ -1900,29 +1890,28 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); +eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ +// run test function +processor.Process(eventGroup); +// judge result +std::stringstream expectJson; +expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // time 为空 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ +std::string outJson = eventGroup.ToJsonString(); +APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +// time 为空 +{ + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1934,29 +1923,28 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // time 不是string - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +// time 不是string +{ + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -1968,32 +1956,31 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - } - // stream - { - // stream不存在 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); +} +} +// stream +{ + // stream不存在 + { + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -2005,29 +1992,28 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // stream非法 - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // stream非法 + { + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -2039,29 +2025,28 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - // stream不是string - { - // make eventGroup - auto sourceBuffer = std::make_shared(); - PipelineEventGroup eventGroup(sourceBuffer); - std::stringstream inJson; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, - ProcessorParseContainerLogNative::DOCKER_JSON_FILE); - inJson << R"({ + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } + // stream不是string + { + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ "events": [ { "contents": { @@ -2073,23 +2058,23 @@ void ProcessorParseContainerLogNativeUnittest::TestKeepingSourceWhenParseFail() } ] })"; - eventGroup.FromJsonString(inJson.str()); + eventGroup.FromJsonString(inJson.str()); - // run test function - processor.Process(eventGroup); - // judge result - std::stringstream expectJson; - expectJson << R"({ + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ "metadata":{ "container.type":"docker_json-file" } })"; - std::string outJson = eventGroup.ToJsonString(); - APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); - } - } + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); } } +} +} void ProcessorParseContainerLogNativeUnittest::TestParseDockerLog() { { diff --git a/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp b/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp index f50d573f1c..3dd3298e35 100644 --- a/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include + #include #include @@ -52,7 +53,7 @@ UNIT_TEST_CASE(ProcessorParseTimestampNativeUnittest, TestProcessHistoryDiscard) UNIT_TEST_CASE(ProcessorParseTimestampNativeUnittest, TestProcessEventPreciseTimestampLegacy); UNIT_TEST_CASE(ProcessorParseTimestampNativeUnittest, TestCheckTime); -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -364,7 +365,7 @@ void ProcessorParseTimestampNativeUnittest::TestProcessRegularFormat() { std::vector eventGroupList; eventGroupList.emplace_back(std::move(eventGroup)); processorInstance.Process(eventGroupList); - + // judge result std::string outJson = eventGroupList[0].ToJsonString(); std::stringstream expectJsonSs; @@ -541,7 +542,7 @@ void ProcessorParseTimestampNativeUnittest::TestProcessRegularFormatFailed() { std::vector eventGroupList; eventGroupList.emplace_back(std::move(eventGroup)); processorInstance.Process(eventGroupList); - + // judge result std::string outJson = eventGroupList[0].ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(inJson).c_str(), CompactJson(outJson).c_str()); @@ -598,7 +599,7 @@ void ProcessorParseTimestampNativeUnittest::TestProcessHistoryDiscard() { std::vector eventGroupList; eventGroupList.emplace_back(std::move(eventGroup)); processorInstance.Process(eventGroupList); - + // check observablity APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mInEventsTotal->GetValue()); // discard history, so output is 0 diff --git a/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp b/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp index d48ec7b4a4..8486a41a98 100644 --- a/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorSplitLogStringNativeUnittest.cpp @@ -13,11 +13,12 @@ // limitations under the License. #include + #include -#include "constants/Constants.h" #include "common/JsonUtil.h" #include "config/PipelineConfig.h" +#include "constants/Constants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" #include "plugin/processor/inner/ProcessorSplitLogStringNative.h" #include "unittest/Unittest.h" @@ -41,7 +42,7 @@ UNIT_TEST_CASE(ProcessorSplitLogStringNativeUnittest, TestProcessJson) UNIT_TEST_CASE(ProcessorSplitLogStringNativeUnittest, TestProcessCommon) UNIT_TEST_CASE(ProcessorSplitLogStringNativeUnittest, TestEnableRawContent) -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } diff --git a/core/unittest/prometheus/LabelsUnittest.cpp b/core/unittest/prometheus/LabelsUnittest.cpp index 1b17de1148..4bd3ef9470 100644 --- a/core/unittest/prometheus/LabelsUnittest.cpp +++ b/core/unittest/prometheus/LabelsUnittest.cpp @@ -15,6 +15,7 @@ */ #include + #include #include "prometheus/Constants.h" @@ -113,8 +114,6 @@ void LabelsUnittest::TestRange() { } - - UNIT_TEST_CASE(LabelsUnittest, TestGet) UNIT_TEST_CASE(LabelsUnittest, TestSet) UNIT_TEST_CASE(LabelsUnittest, TestRange) diff --git a/core/unittest/prometheus/PrometheusInputRunnerUnittest.cpp b/core/unittest/prometheus/PrometheusInputRunnerUnittest.cpp index 8b8501a358..96981885fd 100644 --- a/core/unittest/prometheus/PrometheusInputRunnerUnittest.cpp +++ b/core/unittest/prometheus/PrometheusInputRunnerUnittest.cpp @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "json/json.h" +#include "json/value.h" #include "common/JsonUtil.h" #include "prometheus/PrometheusInputRunner.h" diff --git a/core/unittest/prometheus/RelabelUnittest.cpp b/core/unittest/prometheus/RelabelUnittest.cpp index 3b58f6c374..502237ef51 100644 --- a/core/unittest/prometheus/RelabelUnittest.cpp +++ b/core/unittest/prometheus/RelabelUnittest.cpp @@ -14,11 +14,11 @@ * limitations under the License. */ -#include - -#include #include +#include "boost/regex.hpp" +#include "json/json.h" + #include "common/JsonUtil.h" #include "prometheus/labels/Relabel.h" #include "unittest/Unittest.h" diff --git a/core/unittest/prometheus/ScrapeConfigUnittest.cpp b/core/unittest/prometheus/ScrapeConfigUnittest.cpp index cceb39089f..ab5c38d473 100644 --- a/core/unittest/prometheus/ScrapeConfigUnittest.cpp +++ b/core/unittest/prometheus/ScrapeConfigUnittest.cpp @@ -1,10 +1,12 @@ #include + #include -#include "FileSystemUtil.h" #include "JsonUtil.h" #include "json/value.h" + +#include "FileSystemUtil.h" #include "prometheus/schedulers/ScrapeConfig.h" #include "unittest/Unittest.h" diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index 1eb84623a3..3f832ab3b8 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -72,8 +72,8 @@ void ScrapeSchedulerUnittest::TestProcess() { labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); ScrapeScheduler event(mScrapeConfig, "localhost", 8080, labels, 0, 0); - HttpResponse httpResponse - = HttpResponse(&event.mPromStreamScraper, [](void*) {}, prom::StreamScraper::MetricWriteCallback); + HttpResponse httpResponse = HttpResponse( + &event.mPromStreamScraper, [](void*) {}, prom::StreamScraper::MetricWriteCallback); auto defaultLabels = MetricLabels(); event.InitSelfMonitor(defaultLabels); APSARA_TEST_EQUAL(event.GetId(), "test_jobhttp://localhost:8080/metrics" + ToString(labels.Hash())); @@ -132,8 +132,8 @@ void ScrapeSchedulerUnittest::TestStreamMetricWriteCallback() { labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); ScrapeScheduler event(mScrapeConfig, "localhost", 8080, labels, 0, 0); - HttpResponse httpResponse - = HttpResponse(&event.mPromStreamScraper, [](void*) {}, prom::StreamScraper::MetricWriteCallback); + HttpResponse httpResponse = HttpResponse( + &event.mPromStreamScraper, [](void*) {}, prom::StreamScraper::MetricWriteCallback); APSARA_TEST_EQUAL(event.GetId(), "test_jobhttp://localhost:8080/metrics" + ToString(labels.Hash())); string body1 = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" diff --git a/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp b/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp index 0fd23e6cc1..58544f1c22 100644 --- a/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp +++ b/core/unittest/prometheus/TargetSubscriberSchedulerUnittest.cpp @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include "json/json.h" + #include "common/JsonUtil.h" #include "prometheus/labels/Labels.h" #include "prometheus/schedulers/TargetSubscriberScheduler.h" @@ -188,7 +188,8 @@ void TargetSubscriberSchedulerUnittest::TestParseTargetGroups() { APSARA_TEST_TRUE(targetSubscriber->Init(mConfig["ScrapeConfig"])); std::vector newScrapeSchedulerSet; - APSARA_TEST_TRUE(targetSubscriber->ParseScrapeSchedulerGroup(*mHttpResponse.GetBody(), newScrapeSchedulerSet)); + APSARA_TEST_TRUE( + targetSubscriber->ParseScrapeSchedulerGroup(*mHttpResponse.GetBody(), newScrapeSchedulerSet)); APSARA_TEST_EQUAL(3UL, newScrapeSchedulerSet.size()); } @@ -197,7 +198,8 @@ void TargetSubscriberSchedulerUnittest::TestBuildScrapeSchedulerSet() { std::shared_ptr targetSubscriber = std::make_shared(); APSARA_TEST_TRUE(targetSubscriber->Init(mConfig["ScrapeConfig"])); std::vector newScrapeSchedulerSet; - APSARA_TEST_TRUE(targetSubscriber->ParseScrapeSchedulerGroup(*mHttpResponse.GetBody(), newScrapeSchedulerSet)); + APSARA_TEST_TRUE( + targetSubscriber->ParseScrapeSchedulerGroup(*mHttpResponse.GetBody(), newScrapeSchedulerSet)); APSARA_TEST_EQUAL(3UL, newScrapeSchedulerSet.size()); auto result = targetSubscriber->BuildScrapeSchedulerSet(newScrapeSchedulerSet); diff --git a/core/unittest/provider/ProviderUnittest.cpp b/core/unittest/provider/ProviderUnittest.cpp index 9eb039db4c..fa84b65ba4 100644 --- a/core/unittest/provider/ProviderUnittest.cpp +++ b/core/unittest/provider/ProviderUnittest.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include "provider/Provider.h" +#include "unittest/Unittest.h" namespace logtail { @@ -22,7 +22,6 @@ class ProviderUnittest : public testing::Test { public: void TestGetRemoteConfigProvider(); void TestGetProfileSender(); - }; void ProviderUnittest::TestGetRemoteConfigProvider() { auto remoteConfigProviders = GetRemoteConfigProviders(); diff --git a/core/unittest/queue/BoundedProcessQueueUnittest.cpp b/core/unittest/queue/BoundedProcessQueueUnittest.cpp index 08a178b207..4ca5101338 100644 --- a/core/unittest/queue/BoundedProcessQueueUnittest.cpp +++ b/core/unittest/queue/BoundedProcessQueueUnittest.cpp @@ -121,7 +121,8 @@ void BoundedProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, + METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "bounded")); auto item = GenerateItem(); diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index cd80c823ad..fae5662755 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -154,7 +154,8 @@ void CircularProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, + METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "circular")); auto item = GenerateItem(2); diff --git a/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp b/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp index aee40f3be6..4551d8435b 100644 --- a/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp +++ b/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "plugin/flusher/sls/FlusherSLS.h" #include "pipeline/queue/ExactlyOnceSenderQueue.h" #include "pipeline/queue/SLSSenderQueueItem.h" +#include "plugin/flusher/sls/FlusherSLS.h" #include "unittest/Unittest.h" #include "unittest/queue/FeedbackInterfaceMock.h" @@ -81,8 +81,8 @@ void ExactlyOnceSenderQueueUnittest::TestPush() { APSARA_TEST_EQUAL(100U, mQueue->mRateLimiter->mMaxSendBytesPerSecond); APSARA_TEST_EQUAL(3U, mQueue->mConcurrencyLimiters.size()); - //APSARA_TEST_EQUAL(FlusherSLS::GetRegionConcurrencyLimiter("region"), mQueue->mConcurrencyLimiters[0]); - //APSARA_TEST_EQUAL(FlusherSLS::GetProjectConcurrencyLimiter("project"), mQueue->mConcurrencyLimiters[1]); + // APSARA_TEST_EQUAL(FlusherSLS::GetRegionConcurrencyLimiter("region"), mQueue->mConcurrencyLimiters[0]); + // APSARA_TEST_EQUAL(FlusherSLS::GetProjectConcurrencyLimiter("project"), mQueue->mConcurrencyLimiters[1]); // reach high water mark APSARA_TEST_TRUE(mQueue->Push(GenerateItem())); diff --git a/core/unittest/queue/SenderQueueManagerUnittest.cpp b/core/unittest/queue/SenderQueueManagerUnittest.cpp index 6dc4c2f092..f07bbe0aa2 100644 --- a/core/unittest/queue/SenderQueueManagerUnittest.cpp +++ b/core/unittest/queue/SenderQueueManagerUnittest.cpp @@ -92,15 +92,14 @@ void SenderQueueManagerUnittest::TestCreateQueue() { { // new queue uint32_t maxRate = 100U; - APSARA_TEST_TRUE(sManager->CreateQueue( - 0, sFlusherId, sCtx, {{"region", sConcurrencyLimiter}}, maxRate)); + APSARA_TEST_TRUE(sManager->CreateQueue(0, sFlusherId, sCtx, {{"region", sConcurrencyLimiter}}, maxRate)); APSARA_TEST_EQUAL(1U, sManager->mQueues.size()); auto& queue = sManager->mQueues.at(0); APSARA_TEST_EQUAL(sManager->mDefaultQueueParam.GetCapacity(), queue.mCapacity); APSARA_TEST_EQUAL(sManager->mDefaultQueueParam.GetLowWatermark(), queue.mLowWatermark); APSARA_TEST_EQUAL(sManager->mDefaultQueueParam.GetHighWatermark(), queue.mHighWatermark); APSARA_TEST_EQUAL(1U, queue.mConcurrencyLimiters.size()); - //APSARA_TEST_EQUAL(sConcurrencyLimiter, queue.mConcurrencyLimiters[0]); + // APSARA_TEST_EQUAL(sConcurrencyLimiter, queue.mConcurrencyLimiters[0]); APSARA_TEST_TRUE(queue.mRateLimiter.has_value()); APSARA_TEST_EQUAL(maxRate, queue.mRateLimiter->mMaxSendBytesPerSecond); } @@ -108,12 +107,11 @@ void SenderQueueManagerUnittest::TestCreateQueue() { // resued queue shared_ptr newLimiter = make_shared("", 80); uint32_t maxRate = 10U; - APSARA_TEST_TRUE( - sManager->CreateQueue(0, sFlusherId, sCtx, {{"region", newLimiter}}, maxRate)); + APSARA_TEST_TRUE(sManager->CreateQueue(0, sFlusherId, sCtx, {{"region", newLimiter}}, maxRate)); APSARA_TEST_EQUAL(1U, sManager->mQueues.size()); auto& queue = sManager->mQueues.at(0); APSARA_TEST_EQUAL(1U, queue.mConcurrencyLimiters.size()); - //APSARA_TEST_EQUAL(newLimiter, queue.mConcurrencyLimiters[0]); + // APSARA_TEST_EQUAL(newLimiter, queue.mConcurrencyLimiters[0]); APSARA_TEST_TRUE(queue.mRateLimiter.has_value()); APSARA_TEST_EQUAL(maxRate, queue.mRateLimiter->mMaxSendBytesPerSecond); } @@ -127,10 +125,8 @@ void SenderQueueManagerUnittest::TestDeleteQueue() { QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("name_1"); QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("name_2"); - sManager->CreateQueue( - key1, sFlusherId, sCtx, {{"region", sConcurrencyLimiter}}, sMaxRate); - sManager->CreateQueue( - key2, sFlusherId, sCtx, {{"region", sConcurrencyLimiter}}, sMaxRate); + sManager->CreateQueue(key1, sFlusherId, sCtx, {{"region", sConcurrencyLimiter}}, sMaxRate); + sManager->CreateQueue(key2, sFlusherId, sCtx, {{"region", sConcurrencyLimiter}}, sMaxRate); sManager->PushQueue(key2, GenerateItem()); // queue exists and not marked deleted @@ -185,11 +181,7 @@ void SenderQueueManagerUnittest::TestPushQueue() { void SenderQueueManagerUnittest::TestGetAvailableItems() { // prepare nomal queue sManager->CreateQueue( - 0, - sFlusherId, - sCtx, - {{"region", FlusherSLS::GetRegionConcurrencyLimiter(mFlusher.mRegion)}}, - sMaxRate); + 0, sFlusherId, sCtx, {{"region", FlusherSLS::GetRegionConcurrencyLimiter(mFlusher.mRegion)}}, sMaxRate); for (size_t i = 0; i <= sManager->mDefaultQueueParam.GetCapacity(); ++i) { sManager->PushQueue(0, GenerateItem()); } diff --git a/core/unittest/queue/SenderQueueUnittest.cpp b/core/unittest/queue/SenderQueueUnittest.cpp index 301b3a87d2..b2b0ff9b94 100644 --- a/core/unittest/queue/SenderQueueUnittest.cpp +++ b/core/unittest/queue/SenderQueueUnittest.cpp @@ -177,7 +177,8 @@ void SenderQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE)); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, + METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE)); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "bounded")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_PLUGIN_ID, sFlusherId)); diff --git a/core/unittest/reader/DeletedFileUnittest.cpp b/core/unittest/reader/DeletedFileUnittest.cpp index 23bf0500e7..32cfbdbc42 100644 --- a/core/unittest/reader/DeletedFileUnittest.cpp +++ b/core/unittest/reader/DeletedFileUnittest.cpp @@ -33,7 +33,7 @@ class DeletedFileUnittest : public testing::Test { DevInode(), make_pair(&readerOpts, &ctx), make_pair(&multilineOpts, &ctx))); - } + } void TearDown() override { INT32_FLAG(force_release_deleted_file_fd_timeout) = -1; } diff --git a/core/unittest/reader/FileReaderOptionsUnittest.cpp b/core/unittest/reader/FileReaderOptionsUnittest.cpp index 762370eda5..9b27d21e3f 100644 --- a/core/unittest/reader/FileReaderOptionsUnittest.cpp +++ b/core/unittest/reader/FileReaderOptionsUnittest.cpp @@ -15,12 +15,12 @@ #include #include -#include +#include "json/json.h" #include "common/Flags.h" #include "common/JsonUtil.h" -#include "pipeline/PipelineContext.h" #include "file_server/reader/FileReaderOptions.h" +#include "pipeline/PipelineContext.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(default_tail_limit_kb); @@ -56,7 +56,8 @@ void FileReaderOptionsUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_reader_flush_timeout)), config->mFlushTimeoutSecs); APSARA_TEST_EQUAL(0U, config->mReadDelaySkipThresholdBytes); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(delay_bytes_upperlimit)), config->mReadDelayAlertThresholdBytes); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(reader_close_unused_file_time)), config->mCloseUnusedReaderIntervalSec); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(reader_close_unused_file_time)), + config->mCloseUnusedReaderIntervalSec); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(logreader_max_rotate_queue_size)), config->mRotatorQueueSize); APSARA_TEST_FALSE(config->mAppendingLogPositionMeta); @@ -110,7 +111,8 @@ void FileReaderOptionsUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_reader_flush_timeout)), config->mFlushTimeoutSecs); APSARA_TEST_EQUAL(0U, config->mReadDelaySkipThresholdBytes); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(delay_bytes_upperlimit)), config->mReadDelayAlertThresholdBytes); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(reader_close_unused_file_time)), config->mCloseUnusedReaderIntervalSec); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(reader_close_unused_file_time)), + config->mCloseUnusedReaderIntervalSec); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(logreader_max_rotate_queue_size)), config->mRotatorQueueSize); APSARA_TEST_FALSE(config->mAppendingLogPositionMeta); diff --git a/core/unittest/reader/ForceReadUnittest.cpp b/core/unittest/reader/ForceReadUnittest.cpp index a14e4bf6cc..fe0665d32d 100644 --- a/core/unittest/reader/ForceReadUnittest.cpp +++ b/core/unittest/reader/ForceReadUnittest.cpp @@ -286,16 +286,13 @@ void ForceReadUnittest::TestTimeoutForceRead() { { // TODO: difficult to test, the behavior should be // read -> add timeout event -> handle timeout -> write -> event valid -> read not empty -> rollback - } - { + } { // TODO: difficult to test, the behavior should be // read -> add timeout event -> handle timeout -> event valid -> write -> read not empty -> rollback - } - { + } { // TODO: difficult to test, the behavior should be // read -> add timeout event -> handle timeout -> event valid -> read empty -> write -> not rollback - } - { + } { // TODO: difficult to test, the behavior should be // read -> add timeout event -> handle timeout -> event valid -> read empty -> not rollback -> write } diff --git a/core/unittest/reader/GetLastLineDataUnittest.cpp b/core/unittest/reader/GetLastLineDataUnittest.cpp index 761c225a57..b866d5a8d3 100644 --- a/core/unittest/reader/GetLastLineDataUnittest.cpp +++ b/core/unittest/reader/GetLastLineDataUnittest.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include "common/FileSystemUtil.h" -#include "file_server/reader/LogFileReader.h" #include "common/memory/SourceBuffer.h" +#include "file_server/reader/LogFileReader.h" #include "unittest/Unittest.h" namespace logtail { @@ -88,424 +88,420 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); // 异常情况+有回车 - { - // case: PartLogFlag存在，第三个空格存在但空格后无内容 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P \n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第二个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第一个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - } - // 异常情况+无回车 - { - // case: PartLogFlag存在，第三个空格存在但空格后无内容 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P "; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第二个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(testLog, line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第一个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(testLog, line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - } - // case: F + P + P - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456"; - std::string expectedLog = LOG_FULL + "789\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: F + P + P + '\n' - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_FULL + "789\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } + {// case: PartLogFlag存在，第三个空格存在但空格后无内容 + {std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P \n"; - // case: F + P + P + F - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; - std::string expectedLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: F + P + P + F + '\n' - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; - std::string expectedLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - // F + errorLog - { - std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789"; - std::string expectedLog = LOG_FULL + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // F + errorLog + '\n' - { - std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789\n"; - std::string expectedLog = LOG_FULL + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // case: PartLogFlag存在，第三个空格不存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; - // case: P + P + F - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: P + P + F + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - // case: P + P + error - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: P + P + error + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789\n"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // case: PartLogFlag不存在，第二个空格存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } - // case: P + P - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456"; - std::string expectedLog = LOG_PART + "123\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + // case: 第二个空格不存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } - // case: P + P + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_PART + "123\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + // case: 第一个空格不存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } +} +// 异常情况+无回车 +{// case: PartLogFlag存在，第三个空格存在但空格后无内容 + {std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P "; + +int32_t size = testLog.size(); +int32_t endPs; // the position of \n or \0 +if (testLog[size - 1] == '\n') { + endPs = size - 1; +} else { + endPs = size; +} +LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + +APSARA_TEST_EQUAL("", line.data.to_string()); +APSARA_TEST_EQUAL(0, line.lineBegin); +APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); +APSARA_TEST_EQUAL(false, line.fullLine); +} // namespace logtail +// case: PartLogFlag存在，第三个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag不存在，第二个空格存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: 第二个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(testLog, line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: 第一个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(testLog, line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +} +// case: F + P + P +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456"; + std::string expectedLog = LOG_FULL + "789\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: F + P + P + '\n' +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_FULL + "789\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// case: F + P + P + F +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; + std::string expectedLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: F + P + P + F + '\n' +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; + std::string expectedLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// F + errorLog +{ + std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789"; + std::string expectedLog = LOG_FULL + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// F + errorLog + '\n' +{ + std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789\n"; + std::string expectedLog = LOG_FULL + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// case: P + P + F +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: P + P + F + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// case: P + P + error +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: P + P + error + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789\n"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: P + P +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456"; + std::string expectedLog = LOG_PART + "123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: P + P + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_PART + "123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +} } void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { @@ -517,425 +513,421 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); // 异常情况+有回车 - { - // case: PartLogFlag存在，第三个空格存在但空格后无内容 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P \n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第二个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第一个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - } - // 异常情况+无回车 - { - // case: PartLogFlag存在，第三个空格存在但空格后无内容 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P "; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第二个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL(testLog, line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: 第一个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL(testLog, line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - } - // case: F + P + P - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456"; - std::string expectedLog = LOG_FULL + "789\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: F + P + P + '\n' - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_FULL + "789\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } + {// case: PartLogFlag存在，第三个空格存在但空格后无内容 + {std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P \n"; - // case: F + P + P + F - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; - std::string expectedLog = LOG_FULL + "789\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("123456789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: F + P + P + F + '\n' - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; - std::string expectedLog = LOG_FULL + "789\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("123456789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); - // F + errorLog - { - std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789"; - std::string expectedLog = LOG_FULL + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // F + errorLog + '\n' - { - std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789\n"; - std::string expectedLog = LOG_FULL + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // case: PartLogFlag存在，第三个空格不存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; - // case: P + P + F - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("123456789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: P + P + F + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("123456789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); - // case: P + P + error - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("123456" + LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: P + P + error + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789\n"; - std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("123456" + LOG_ERROR + "789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // case: PartLogFlag不存在，第二个空格存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } - // case: P + P - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456"; - std::string expectedLog = LOG_PART + "123\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + // case: 第二个空格不存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } - // case: P + P + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_PART + "123\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); + } + // case: 第一个空格不存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); } } +// 异常情况+无回车 +{// case: PartLogFlag存在，第三个空格存在但空格后无内容 + {std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P "; + +int32_t size = testLog.size(); +int32_t endPs; // the position of \n or \0 +if (testLog[size - 1] == '\n') { + endPs = size - 1; +} else { + endPs = size; +} +LineInfo line = logFileReader.GetLastLine(testLog, endPs); + +APSARA_TEST_EQUAL("", line.data.to_string()); +APSARA_TEST_EQUAL(0, line.lineBegin); +APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); +APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag存在，第三个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag不存在，第二个空格存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: 第二个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(testLog, line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: 第一个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(testLog, line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +} +// case: F + P + P +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456"; + std::string expectedLog = LOG_FULL + "789\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: F + P + P + '\n' +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_FULL + "789\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// case: F + P + P + F +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; + std::string expectedLog = LOG_FULL + "789\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123456789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: F + P + P + F + '\n' +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; + std::string expectedLog = LOG_FULL + "789\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123456789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// F + errorLog +{ + std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789"; + std::string expectedLog = LOG_FULL + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// F + errorLog + '\n' +{ + std::string testLog = LOG_FULL + "456\n" + LOG_ERROR + "789\n"; + std::string expectedLog = LOG_FULL + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(int(expectedLog.size()), line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// case: P + P + F +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123456789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: P + P + F + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_FULL + "789\n"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123456789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} + +// case: P + P + error +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123456" + LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: P + P + error + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n" + LOG_ERROR + "789\n"; + std::string expectedLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("123456" + LOG_ERROR + "789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(3, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: P + P +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456"; + std::string expectedLog = LOG_PART + "123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: P + P + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_PART + "123\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(2, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +} +} class LastMatchedDockerJsonFileUnittest : public ::testing::Test { public: @@ -1000,194 +992,190 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { logFileReader.mLineParsers.emplace_back(baseLineParsePtr); // 不带回车 // 不带回车 - { - // 合法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", - line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // log非法 - { - std::string testLog - = R"({"log1":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // stream非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // time非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // 非法json - { - std::string testLog - = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } + {// 合法 + {std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(true, line.fullLine); + } + // log非法 + { + std::string testLog + = R"({"log1":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // stream非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // time非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // 非法json + { + std::string testLog + = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } +} +// 带回车 +{ + // 合法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(true, line.fullLine); + } + // log非法 + { + std::string testLog + = R"({"log1":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } - // 带回车 - { - // 合法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", - line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // log非法 - { - std::string testLog - = R"({"log1":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // stream非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // time非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // 非法json - { - std::string testLog - = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // stream非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); } + // time非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // 非法json + { + std::string testLog + = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } +} +} } diff --git a/core/unittest/reader/LogFileReaderUnittest.cpp b/core/unittest/reader/LogFileReaderUnittest.cpp index 165e4e258d..e4e146e08e 100644 --- a/core/unittest/reader/LogFileReaderUnittest.cpp +++ b/core/unittest/reader/LogFileReaderUnittest.cpp @@ -21,8 +21,8 @@ #include "common/RuntimeUtil.h" #include "common/memory/SourceBuffer.h" #include "file_server/FileServer.h" -#include "protobuf/sls/sls_logs.pb.h" #include "file_server/reader/LogFileReader.h" +#include "protobuf/sls/sls_logs.pb.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(force_release_deleted_file_fd_timeout); diff --git a/core/unittest/reader/SourceBufferUnittest.cpp b/core/unittest/reader/SourceBufferUnittest.cpp index ac1088defa..5c9bc01776 100644 --- a/core/unittest/reader/SourceBufferUnittest.cpp +++ b/core/unittest/reader/SourceBufferUnittest.cpp @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include -#include + +#include "json/json.h" + #include "file_server/reader/LogFileReader.h" +#include "unittest/Unittest.h" DECLARE_FLAG_INT32(force_release_deleted_file_fd_timeout); diff --git a/core/unittest/sender/FlusherRunnerUnittest.cpp b/core/unittest/sender/FlusherRunnerUnittest.cpp index eac4f5ff70..17f501b64c 100644 --- a/core/unittest/sender/FlusherRunnerUnittest.cpp +++ b/core/unittest/sender/FlusherRunnerUnittest.cpp @@ -32,10 +32,8 @@ class FlusherRunnerUnittest : public ::testing::Test { void TestPushToHttpSink(); protected: - static void SetUpTestCase() { - AppConfig::GetInstance()->mSendRequestGlobalConcurrency = 10; - } - + static void SetUpTestCase() { AppConfig::GetInstance()->mSendRequestGlobalConcurrency = 10; } + void TearDown() override { SenderQueueManager::GetInstance()->Clear(); HttpSink::GetInstance()->mQueue.Clear(); diff --git a/core/unittest/sender/SenderUnittest.cpp b/core/unittest/sender/SenderUnittest.cpp index 0f01300bb2..12875eb95c 100644 --- a/core/unittest/sender/SenderUnittest.cpp +++ b/core/unittest/sender/SenderUnittest.cpp @@ -13,7 +13,8 @@ // limitations under the License. #include -#include + +#include "json/json.h" #include "app_config/AppConfig.h" #include "file_server/ConfigManager.h" @@ -36,7 +37,6 @@ #include #include -#include #include #include #include @@ -45,6 +45,8 @@ #include #include +#include "boost/regex.hpp" + #include "checkpoint/CheckpointManagerV2.h" #include "common/FileEncryption.h" #include "common/FileSystemUtil.h" diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index d542ec95cb..018cc42b98 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -52,318 +52,314 @@ unique_ptr SLSSerializerUnittest::sFlusher; void SLSSerializerUnittest::TestSerializeEventGroup() { SLSEventGroupSerializer serializer(sFlusher.get()); - { - // log - { - // nano second disabled, and set - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); - APSARA_TEST_STREQ("key", logGroup.logs(0).contents(0).key().c_str()); - APSARA_TEST_STREQ("value", logGroup.logs(0).contents(0).value().c_str()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - APSARA_TEST_EQUAL(1, logGroup.logtags_size()); - APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); - APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); - APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); - APSARA_TEST_STREQ("source", logGroup.source().c_str()); - APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); - } - { - // nano second enabled, and set - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(true, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_EQUAL(1U, logGroup.logs(0).time_ns()); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; - } - { - // nano second enabled, not set - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; - } - { - // empty log content - string res, errorMsg; - APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedLogEvents(false, true), res, errorMsg)); - } - } - { - // metric - { - // only 1 tag - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false, true), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); - APSARA_TEST_EQUAL(1, logGroup.logtags_size()); - APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); - APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); - APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); - APSARA_TEST_STREQ("source", logGroup.source().c_str()); - APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); - } - { - // nano second disabled - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); - APSARA_TEST_EQUAL(1, logGroup.logtags_size()); - APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); - APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); - APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); - APSARA_TEST_STREQ("source", logGroup.source().c_str()); - APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); - } - { - // nano second enabled, less than 9 digits - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1, false, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890000000001"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); - APSARA_TEST_EQUAL(1, logGroup.logtags_size()); - APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); - APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); - APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); - APSARA_TEST_STREQ("source", logGroup.source().c_str()); - APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); - } - { - // nano second enabled, exactly 9 digits - string res, errorMsg; - APSARA_TEST_TRUE( - serializer.DoSerialize(CreateBatchedMetricEvents(true, 999999999, false, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890999999999"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); - APSARA_TEST_EQUAL(1, logGroup.logtags_size()); - APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); - APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); - APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); - APSARA_TEST_STREQ("source", logGroup.source().c_str()); - APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); - } - { - // empty metric value - string res, errorMsg; - APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true, false), res, errorMsg)); - } - } - { - // span - string res, errorMsg; - auto events = CreateBatchedSpanEvents(); - APSARA_TEST_EQUAL(events.mEvents.size(), 1U); - APSARA_TEST_TRUE(events.mEvents[0]->GetType() == PipelineEvent::Type::SPAN); - APSARA_TEST_TRUE(serializer.DoSerialize(std::move(events), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(13, logGroup.logs(0).contents_size()); - // traceid - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "traceId"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "trace-1-2-3-4-5"); - // span id - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "spanId"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "span-1-2-3-4-5"); - // parent span id - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "parentSpanId"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "parent-1-2-3-4-5"); - // spanName - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "spanName"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "/oneagent/qianlu/local/1"); - // kind - APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).key(), "kind"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).value(), "client"); - // code - APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).key(), "statusCode"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).value(), "OK"); - // traceState - APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).key(), "traceState"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).value(), "test-state"); - // attributes - APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).key(), "attributes"); - auto attrs = logGroup.logs(0).contents(7).value(); - Json::Value jsonVal; - Json::CharReaderBuilder readerBuilder; - std::string errs; - - std::istringstream s(attrs); - bool ret = Json::parseFromStream(readerBuilder, s, &jsonVal, &errs); - APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 10U); - APSARA_TEST_EQUAL(jsonVal["rpcType"].asString(), "25"); - APSARA_TEST_EQUAL(jsonVal["scope-tag-0"].asString(), "scope-value-0"); - // APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).value(), ""); - // links - APSARA_TEST_EQUAL(logGroup.logs(0).contents(8).key(), "links"); - - auto linksStr = logGroup.logs(0).contents(8).value(); - - std::istringstream ss(linksStr); - ret = Json::parseFromStream(readerBuilder, ss, &jsonVal, &errs); - APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 1U); - for (auto& link : jsonVal) { - APSARA_TEST_EQUAL(link["spanId"].asString(), "inner-link-spanid"); - APSARA_TEST_EQUAL(link["traceId"].asString(), "inner-link-traceid"); - APSARA_TEST_EQUAL(link["traceState"].asString(), "inner-link-trace-state"); - } - // events - APSARA_TEST_EQUAL(logGroup.logs(0).contents(9).key(), "events"); - auto eventsStr = logGroup.logs(0).contents(9).value(); - std::istringstream sss(eventsStr); - ret = Json::parseFromStream(readerBuilder, sss, &jsonVal, &errs); - APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 1U); - for (auto& event : jsonVal) { - APSARA_TEST_EQUAL(event["name"].asString(), "inner-event"); - APSARA_TEST_EQUAL(event["timestamp"].asString(), "1000"); - } - // start - APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).key(), "startTime"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).value(), "1000"); - - // end - APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).key(), "endTime"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).value(), "2000"); - - // duration - APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).key(), "duration"); - APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).value(), "1000"); - } - { - // raw - { - // nano second disabled, and set - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedRawEvents(false, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1, logGroup.logs_size()); - APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); - APSARA_TEST_STREQ("content", logGroup.logs(0).contents(0).key().c_str()); - APSARA_TEST_STREQ("value", logGroup.logs(0).contents(0).value().c_str()); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - APSARA_TEST_EQUAL(1, logGroup.logtags_size()); - APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); - APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); - APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); - APSARA_TEST_STREQ("source", logGroup.source().c_str()); - APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); - } - { - // nano second enabled, and set - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedRawEvents(true, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_EQUAL(1U, logGroup.logs(0).time_ns()); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; - } - { - // nano second enabled, not set - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; - string res, errorMsg; - APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedRawEvents(false, false), res, errorMsg)); - sls_logs::LogGroup logGroup; - APSARA_TEST_TRUE(logGroup.ParseFromString(res)); - APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); - APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); - const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; - } - { - // empty log content - string res, errorMsg; - APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedRawEvents(false, true), res, errorMsg)); - } - } - { - // log group exceed size limit - INT32_FLAG(max_send_log_group_size) = 0; - string res, errorMsg; - APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedLogEvents(true, false), res, errorMsg)); - INT32_FLAG(max_send_log_group_size) = 10 * 1024 * 1024; + {// log + {// nano second disabled, and set + string res, + errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); + APSARA_TEST_STREQ("key", logGroup.logs(0).contents(0).key().c_str()); + APSARA_TEST_STREQ("value", logGroup.logs(0).contents(0).value().c_str()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); +} +{ + // nano second enabled, and set + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(true, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_EQUAL(1U, logGroup.logs(0).time_ns()); + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; +} +{ + // nano second enabled, not set + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedLogEvents(false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; +} +{ + // empty log content + string res, errorMsg; + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedLogEvents(false, true), res, errorMsg)); +} +} // namespace logtail +{// metric + {// only 1 tag + string res, + errorMsg; +APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false, true), res, errorMsg)); +sls_logs::LogGroup logGroup; +APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + +APSARA_TEST_EQUAL(1, logGroup.logs_size()); +APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); +APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); +APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1"); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890"); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); +APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); +APSARA_TEST_EQUAL(1, logGroup.logtags_size()); +APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); +APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); +APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); +APSARA_TEST_STREQ("source", logGroup.source().c_str()); +APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); +} +{ + // nano second disabled + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); +} +{ + // nano second enabled, less than 9 digits + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1, false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890000000001"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); +} +{ + // nano second enabled, exactly 9 digits + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 999999999, false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + APSARA_TEST_EQUAL(logGroup.logs(0).contents_size(), 4); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "__labels__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "key1#$#value1|key2#$#value2"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "__time_nano__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "1234567890999999999"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "__value__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "0.100000"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "__name__"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "test_gauge"); + APSARA_TEST_EQUAL(1, logGroup.logtags_size()); + APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); + APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); + APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); + APSARA_TEST_STREQ("source", logGroup.source().c_str()); + APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); +} +{ + // empty metric value + string res, errorMsg; + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true, false), res, errorMsg)); +} +} +{ + // span + string res, errorMsg; + auto events = CreateBatchedSpanEvents(); + APSARA_TEST_EQUAL(events.mEvents.size(), 1U); + APSARA_TEST_TRUE(events.mEvents[0]->GetType() == PipelineEvent::Type::SPAN); + APSARA_TEST_TRUE(serializer.DoSerialize(std::move(events), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(13, logGroup.logs(0).contents_size()); + // traceid + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "traceId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "trace-1-2-3-4-5"); + // span id + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "spanId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "span-1-2-3-4-5"); + // parent span id + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "parentSpanId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "parent-1-2-3-4-5"); + // spanName + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "spanName"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "/oneagent/qianlu/local/1"); + // kind + APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).key(), "kind"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).value(), "client"); + // code + APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).key(), "statusCode"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).value(), "OK"); + // traceState + APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).key(), "traceState"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).value(), "test-state"); + // attributes + APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).key(), "attributes"); + auto attrs = logGroup.logs(0).contents(7).value(); + Json::Value jsonVal; + Json::CharReaderBuilder readerBuilder; + std::string errs; + + std::istringstream s(attrs); + bool ret = Json::parseFromStream(readerBuilder, s, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 10U); + APSARA_TEST_EQUAL(jsonVal["rpcType"].asString(), "25"); + APSARA_TEST_EQUAL(jsonVal["scope-tag-0"].asString(), "scope-value-0"); + // APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).value(), ""); + // links + APSARA_TEST_EQUAL(logGroup.logs(0).contents(8).key(), "links"); + + auto linksStr = logGroup.logs(0).contents(8).value(); + + std::istringstream ss(linksStr); + ret = Json::parseFromStream(readerBuilder, ss, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 1U); + for (auto& link : jsonVal) { + APSARA_TEST_EQUAL(link["spanId"].asString(), "inner-link-spanid"); + APSARA_TEST_EQUAL(link["traceId"].asString(), "inner-link-traceid"); + APSARA_TEST_EQUAL(link["traceState"].asString(), "inner-link-trace-state"); } - { - // empty log group - PipelineEventGroup group(make_shared()); - BatchedEvents batch(std::move(group.MutableEvents()), - std::move(group.GetSizedTags()), - std::move(group.GetSourceBuffer()), - group.GetMetadata(EventGroupMetaKey::SOURCE_ID), - std::move(group.GetExactlyOnceCheckpoint())); - string res, errorMsg; - APSARA_TEST_FALSE(serializer.DoSerialize(std::move(batch), res, errorMsg)); + // events + APSARA_TEST_EQUAL(logGroup.logs(0).contents(9).key(), "events"); + auto eventsStr = logGroup.logs(0).contents(9).value(); + std::istringstream sss(eventsStr); + ret = Json::parseFromStream(readerBuilder, sss, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 1U); + for (auto& event : jsonVal) { + APSARA_TEST_EQUAL(event["name"].asString(), "inner-event"); + APSARA_TEST_EQUAL(event["timestamp"].asString(), "1000"); } + // start + APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).key(), "startTime"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).value(), "1000"); + + // end + APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).key(), "endTime"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).value(), "2000"); + + // duration + APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).key(), "duration"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).value(), "1000"); +} +{// raw + {// nano second disabled, and set + string res, + errorMsg; +APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedRawEvents(false, false), res, errorMsg)); +sls_logs::LogGroup logGroup; +APSARA_TEST_TRUE(logGroup.ParseFromString(res)); +APSARA_TEST_EQUAL(1, logGroup.logs_size()); +APSARA_TEST_EQUAL(1, logGroup.logs(0).contents_size()); +APSARA_TEST_STREQ("content", logGroup.logs(0).contents(0).key().c_str()); +APSARA_TEST_STREQ("value", logGroup.logs(0).contents(0).value().c_str()); +APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); +APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); +APSARA_TEST_EQUAL(1, logGroup.logtags_size()); +APSARA_TEST_STREQ("__pack_id__", logGroup.logtags(0).key().c_str()); +APSARA_TEST_STREQ("pack_id", logGroup.logtags(0).value().c_str()); +APSARA_TEST_STREQ("machine_uuid", logGroup.machineuuid().c_str()); +APSARA_TEST_STREQ("source", logGroup.source().c_str()); +APSARA_TEST_STREQ("topic", logGroup.topic().c_str()); +} +{ + // nano second enabled, and set + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedRawEvents(true, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_EQUAL(1U, logGroup.logs(0).time_ns()); + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; +} +{ + // nano second enabled, not set + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; + string res, errorMsg; + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedRawEvents(false, false), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); + APSARA_TEST_FALSE(logGroup.logs(0).has_time_ns()); + const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = false; +} +{ + // empty log content + string res, errorMsg; + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedRawEvents(false, true), res, errorMsg)); +} +} +{ + // log group exceed size limit + INT32_FLAG(max_send_log_group_size) = 0; + string res, errorMsg; + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedLogEvents(true, false), res, errorMsg)); + INT32_FLAG(max_send_log_group_size) = 10 * 1024 * 1024; +} +{ + // empty log group + PipelineEventGroup group(make_shared()); + BatchedEvents batch(std::move(group.MutableEvents()), + std::move(group.GetSizedTags()), + std::move(group.GetSourceBuffer()), + group.GetMetadata(EventGroupMetaKey::SOURCE_ID), + std::move(group.GetExactlyOnceCheckpoint())); + string res, errorMsg; + APSARA_TEST_FALSE(serializer.DoSerialize(std::move(batch), res, errorMsg)); +} } void SLSSerializerUnittest::TestSerializeEventGroupList() { @@ -477,11 +473,11 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedSpanEvents() { group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "source"); group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "aaa"); - group.SetTag(LOG_RESERVED_KEY_PACKAGE_ID, "bbb"); + group.SetTag(LOG_RESERVED_KEY_PACKAGE_ID, "bbb"); auto now = std::chrono::system_clock::now(); auto duration = now.time_since_epoch(); auto seconds = std::chrono::duration_cast(duration).count(); - // auto nano = std::chrono::duration_cast(duration).count(); + // auto nano = std::chrono::duration_cast(duration).count(); StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); diff --git a/core/unittest/serializer/SerializerUnittest.cpp b/core/unittest/serializer/SerializerUnittest.cpp index 9d08338794..b411b50f5b 100644 --- a/core/unittest/serializer/SerializerUnittest.cpp +++ b/core/unittest/serializer/SerializerUnittest.cpp @@ -24,7 +24,7 @@ namespace logtail { class SerializerMock : public Serializer { public: - SerializerMock(Flusher* f) : Serializer(f) {}; + SerializerMock(Flusher* f) : Serializer(f){}; private: bool Serialize(BatchedEvents&& p, std::string& res, std::string& errorMsg) override { diff --git a/core/unittest/spl/SplBenchmark.cpp b/core/unittest/spl/SplBenchmark.cpp index 098a2189b2..2f0ef6fab7 100644 --- a/core/unittest/spl/SplBenchmark.cpp +++ b/core/unittest/spl/SplBenchmark.cpp @@ -12,24 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include +#include #include "common/JsonUtil.h" +#include "common/TimeUtil.h" #include "config/PipelineConfig.h" -#include "plugin/processor/ProcessorSPL.h" -#include "plugin/processor/ProcessorParseRegexNative.h" -#include "plugin/processor/ProcessorParseJsonNative.h" -#include "plugin/processor/ProcessorParseDelimiterNative.h" #include "models/LogEvent.h" #include "pipeline/plugin/instance/ProcessorInstance.h" -#include -#include -#include "common/TimeUtil.h" +#include "plugin/processor/ProcessorParseDelimiterNative.h" +#include "plugin/processor/ProcessorParseJsonNative.h" +#include "plugin/processor/ProcessorParseRegexNative.h" +#include "plugin/processor/ProcessorSPL.h" +#include "unittest/Unittest.h" using namespace logtail; -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -52,7 +52,7 @@ Json::Value GetCastConfig(std::string spl) { Json::Value config; config["Script"] = Json::Value(spl); config["TimeoutMilliSeconds"] = Json::Value(1000); - config["MaxMemoryBytes"] = Json::Value(50*1024*1024); + config["MaxMemoryBytes"] = Json::Value(50 * 1024 * 1024); return config; } @@ -64,15 +64,24 @@ static void BM_SplRegex(int size, int batchSize) { mContext.SetConfigName("project##config_0"); // make config - std::string spl = "* |parse-regexp content, '^(\\S+)\\s-\\s\\[([^]]+)]\\s-\\s(\\S+)\\s\\[(\\S+)\\s\\S+\\s\"(\\w+)\\s(\\S+)\\s([^\"]+)\"\\s(\\d+)\\s(\\d+)\\s\"([^\"]*)\"\\s\"([^\"]*)\"\\s(\\S+)\\s(\\S+)+\\s\\[([^]]*)]\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s*(\\S*).*' as client_ip,x_forward_for,remote_user,time,method,url,version,status,body_bytes_sent,http_referer,http_user_agent,request_length,request_time,proxy_upstream_name,upstream_addr,upstream_response_length,upstream_response_time,upstream_status,req_id,host"; + std::string spl = "* |parse-regexp content, " + "'^(\\S+)\\s-\\s\\[([^]]+)]\\s-\\s(\\S+)\\s\\[(\\S+)\\s\\S+\\s\"(\\w+)\\s(\\S+)\\s([^\"]+)\"\\s(" + "\\d+)\\s(\\d+)\\s\"([^\"]*)\"\\s\"([^\"]*)\"\\s(\\S+)\\s(\\S+)+\\s\\[([^]]*)]\\s(\\S+)\\s(\\S+)" + "\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s*(\\S*).*' as " + "client_ip,x_forward_for,remote_user,time,method,url,version,status,body_bytes_sent,http_referer," + "http_user_agent,request_length,request_time,proxy_upstream_name,upstream_addr,upstream_response_" + "length,upstream_response_time,upstream_status,req_id,host"; Json::Value config = GetCastConfig(spl); - - std::string data = "106.14.76.139 - [106.14.76.139] - - [08/Nov/2023:13:12:52 +0800] \"POST /api/v1/trade/queryLast HTTP/1.1\" 200 34 \"-\" \"okhttp/3.14.9\" 1313 0.003 [sas-devops-202210191027-svc-80] 10.33.95.216:7001 34 0.003 200 d82accba8c35ad7de27a3a64926a03d0 stosas-test.sto.cn"; - + + std::string data + = "106.14.76.139 - [106.14.76.139] - - [08/Nov/2023:13:12:52 +0800] \"POST /api/v1/trade/queryLast HTTP/1.1\" " + "200 34 \"-\" \"okhttp/3.14.9\" 1313 0.003 [sas-devops-202210191027-svc-80] 10.33.95.216:7001 34 0.003 200 " + "d82accba8c35ad7de27a3a64926a03d0 stosas-test.sto.cn"; + // make events Json::Value root; Json::Value events; - for (int i = 0; i < size; i ++) { + for (int i = 0; i < size; i++) { Json::Value event; event["type"] = 1; event["timestamp"] = 1234567890; @@ -93,17 +102,17 @@ static void BM_SplRegex(int size, int batchSize) { std::ostringstream oss; writer->write(root, &oss); std::string inJson = oss.str(); - //std::cout << "inJson: " << inJson << std::endl; + // std::cout << "inJson: " << inJson << std::endl; ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + bool init = processorInstance.Init(config, mContext); if (init) { - // Perform setup here + // Perform setup here int count = 0; uint64_t durationTime = 0; - for (int i = 0; i < batchSize; i ++) { + for (int i = 0; i < batchSize; i++) { count++; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); @@ -116,14 +125,16 @@ static void BM_SplRegex(int size, int batchSize) { uint64_t startTime = GetCurrentTimeInMicroSeconds(); processorInstance.Process(logGroupList); durationTime += GetCurrentTimeInMicroSeconds() - startTime; - //for (auto& logGroup : logGroupList) { - // std::string outJson = logGroup.ToJsonString(); - // std::cout << "outJson: " << outJson << std::endl; - //} + // for (auto& logGroup : logGroupList) { + // std::string outJson = logGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; + // } } std::cout << "spl regex count: " << count << std::endl; std::cout << "durationTime: " << durationTime << std::endl; - std::cout << "process: " << formatSize((data.size() + 7)*(uint64_t)count*1000000*(uint64_t)size/durationTime) << std::endl; + std::cout << "process: " + << formatSize((data.size() + 7) * (uint64_t)count * 1000000 * (uint64_t)size / durationTime) + << std::endl; } } @@ -136,7 +147,9 @@ static void BM_RawRegex(int size, int batchSize) { // make config Json::Value config; config["SourceKey"] = "content"; - config["Regex"] = "^(\\S+)\\s-\\s\\[([^]]+)]\\s-\\s(\\S+)\\s\\[(\\S+)\\s\\S+\\s\"(\\w+)\\s(\\S+)\\s([^\"]+)\"\\s(\\d+)\\s(\\d+)\\s\"([^\"]*)\"\\s\"([^\"]*)\"\\s(\\S+)\\s(\\S+)+\\s\\[([^]]*)]\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s*(\\S*).*"; + config["Regex"] = "^(\\S+)\\s-\\s\\[([^]]+)]\\s-\\s(\\S+)\\s\\[(\\S+)\\s\\S+\\s\"(\\w+)\\s(\\S+)\\s([^\"]+)\"\\s(" + "\\d+)\\s(\\d+)\\s\"([^\"]*)\"\\s\"([^\"]*)\"\\s(\\S+)\\s(\\S+)+\\s\\[([^]]*)]\\s(\\S+)\\s(\\S+)" + "\\s(\\S+)\\s(\\S+)\\s(\\S+)\\s*(\\S*).*"; config["Keys"] = Json::arrayValue; config["Keys"].append("client_ip"); config["Keys"].append("x_forward_for"); @@ -165,13 +178,15 @@ static void BM_RawRegex(int size, int batchSize) { config["RenamedSourceKey"] = "rawLog"; - - std::string data = "106.14.76.139 - [106.14.76.139] - - [08/Nov/2023:13:12:52 +0800] \"POST /api/v1/trade/queryLast HTTP/1.1\" 200 34 \"-\" \"okhttp/3.14.9\" 1313 0.003 [sas-devops-202210191027-svc-80] 10.33.95.216:7001 34 0.003 200 d82accba8c35ad7de27a3a64926a03d0 stosas-test.sto.cn"; + std::string data + = "106.14.76.139 - [106.14.76.139] - - [08/Nov/2023:13:12:52 +0800] \"POST /api/v1/trade/queryLast HTTP/1.1\" " + "200 34 \"-\" \"okhttp/3.14.9\" 1313 0.003 [sas-devops-202210191027-svc-80] 10.33.95.216:7001 34 0.003 200 " + "d82accba8c35ad7de27a3a64926a03d0 stosas-test.sto.cn"; // make events Json::Value root; Json::Value events; - for (int i = 0; i < size; i ++) { + for (int i = 0; i < size; i++) { Json::Value event; event["type"] = 1; event["timestamp"] = 1234567890; @@ -192,20 +207,20 @@ static void BM_RawRegex(int size, int batchSize) { std::ostringstream oss; writer->write(root, &oss); std::string inJson = oss.str(); - //std::cout << "inJson: " << inJson << std::endl; + // std::cout << "inJson: " << inJson << std::endl; // run function ProcessorParseRegexNative& processor = *(new ProcessorParseRegexNative); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + bool init = processorInstance.Init(config, mContext); if (init) { int count = 0; // Perform setup here uint64_t durationTime = 0; - for (int i = 0; i < batchSize; i ++) { - count ++; + for (int i = 0; i < batchSize; i++) { + count++; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); // This code gets timed @@ -215,16 +230,18 @@ static void BM_RawRegex(int size, int batchSize) { uint64_t startTime = GetCurrentTimeInMicroSeconds(); processorInstance.Process(logGroupList); durationTime += GetCurrentTimeInMicroSeconds() - startTime; - //std::string outJson = eventGroup.ToJsonString(); - //std::cout << "outJson: " << outJson << std::endl; + // std::string outJson = eventGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; } - + std::cout << "raw regex count: " << count << std::endl; std::cout << "durationTime: " << durationTime << std::endl; - std::cout << "process: " << formatSize((data.size() + 7)*(uint64_t)count*1000000*(uint64_t)size/durationTime) << std::endl; - - //std::string outJson = eventGroup.ToJsonString(); - //std::cout << "outJson: " << outJson << std::endl; + std::cout << "process: " + << formatSize((data.size() + 7) * (uint64_t)count * 1000000 * (uint64_t)size / durationTime) + << std::endl; + + // std::string outJson = eventGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; } } @@ -236,12 +253,40 @@ static void BM_SplJson(int size, int batchSize) { // make config Json::Value config = GetCastConfig(R"(* | parse-json content)"); - std::string data = "{\"_time_\":\"2023-11-15T01:04:21.80553511Z\",\"_source_\":\"stdout\",\"_pod_name_\":\"gpassport-37games-deployment-6d68b45779-rgfcz\",\"_namespace_\":\"go-app\",\"_pod_uid_\":\"22d6acfa-d55e-4be0-bb3f-ca91584a4f49\",\"_container_ip_\":\"10.101.31.136\",\"_image_name_\":\"686337631058.dkr.ecr.ap-southeast-1.amazonaws.com/gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf\",\"_container_name_\":\"gpassport-37games\",\"traceId\":\"44507629d8ebd96a6ff7810618d020ee\",\"logType\":\"http_access_log\",\"level\":\"INFO\",\"request\":\"/direct_login\",\"clientip\":\"218.225.227.156\",\"x_true_client_ip\":\"218.225.227.156\",\"real_ip_remote\":\"10.101.128.113\",\"xforward\":\"218.225.227.156, 70.132.19.70\",\"xforwardProto\":\"https\",\"method\":\"POST\",\"status\":\"200\",\"agent\":\"okhttp/3.12.13\",\"cost\":\"0.020\",\"bytes\":\"1409\",\"host\":\"http://gpassport.superfastgame.com\",\"remove_host\":\"http://gpassport.superfastgame.com\",\"referer\":\"-\",\"httpversion\":\"HTTP/1.1\",\"postData\":\"gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-a61881cfbde7&country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610.011%29&advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=0&ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&phoneBrand=TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=81444cf49a3f0f014d30b3e0571d894e&userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=android&timeZone=JST&mac=&isVpnOn=0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=5fd790e62c8e791388d913e808504c03&thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&osVersion=9&customUserId=b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&sdkVersion=326&ptCode=global&gameCode=ztmslg&att=1&battery=68\",\"cookieData\":\"-\",\"content_length\":\"986\",\"@timestamp\":\"2023-11-15T09:04:21+08:00\",\"__pack_meta__\":\"1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|426\",\"__topic__\":\"\",\"__source__\":\"10.101.29.105\",\"__tag__:__pack_id__\":\"5BCAE694BB74A062-38D81B\",\"__tag__:_node_name_\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:_node_ip_\":\"10.101.29.105\",\"__tag__:__hostname__\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:__client_ip__\":\"54.251.11.83\",\"__tag__:__receive_time__\":\"1700010262\"}"; + std::string data + = "{\"_time_\":\"2023-11-15T01:04:21.80553511Z\",\"_source_\":\"stdout\",\"_pod_name_\":\"gpassport-37games-" + "deployment-6d68b45779-rgfcz\",\"_namespace_\":\"go-app\",\"_pod_uid_\":\"22d6acfa-d55e-4be0-bb3f-" + "ca91584a4f49\",\"_container_ip_\":\"10.101.31.136\",\"_image_name_\":\"686337631058.dkr.ecr.ap-southeast-1." + "amazonaws.com/" + "gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf\",\"_container_name_\":\"gpassport-" + "37games\",\"traceId\":\"44507629d8ebd96a6ff7810618d020ee\",\"logType\":\"http_access_log\",\"level\":" + "\"INFO\",\"request\":\"/" + "direct_login\",\"clientip\":\"218.225.227.156\",\"x_true_client_ip\":\"218.225.227.156\",\"real_ip_remote\":" + "\"10.101.128.113\",\"xforward\":\"218.225.227.156, " + "70.132.19.70\",\"xforwardProto\":\"https\",\"method\":\"POST\",\"status\":\"200\",\"agent\":\"okhttp/" + "3.12.13\",\"cost\":\"0.020\",\"bytes\":\"1409\",\"host\":\"http://" + "gpassport.superfastgame.com\",\"remove_host\":\"http://" + "gpassport.superfastgame.com\",\"referer\":\"-\",\"httpversion\":\"HTTP/" + "1.1\",\"postData\":\"gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-" + "a61881cfbde7&country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610." + "011%29&advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=" + "0&ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&" + "phoneBrand=TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=" + "81444cf49a3f0f014d30b3e0571d894e&userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=" + "android&timeZone=JST&mac=&isVpnOn=0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=" + "5fd790e62c8e791388d913e808504c03&thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&" + "osVersion=9&customUserId=b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&" + "sdkVersion=326&ptCode=global&gameCode=ztmslg&att=1&battery=68\",\"cookieData\":\"-\",\"content_length\":" + "\"986\",\"@timestamp\":\"2023-11-15T09:04:21+08:00\",\"__pack_meta__\":\"1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|" + "426\",\"__topic__\":\"\",\"__source__\":\"10.101.29.105\",\"__tag__:__pack_id__\":\"5BCAE694BB74A062-" + "38D81B\",\"__tag__:_node_name_\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:_node_ip_\":" + "\"10.101.29.105\",\"__tag__:__hostname__\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:__" + "client_ip__\":\"54.251.11.83\",\"__tag__:__receive_time__\":\"1700010262\"}"; // make events Json::Value root; Json::Value events; - for (int i = 0; i < size; i ++) { + for (int i = 0; i < size; i++) { Json::Value event; event["type"] = 1; event["timestamp"] = 1234567890; @@ -262,18 +307,18 @@ static void BM_SplJson(int size, int batchSize) { std::ostringstream oss; writer->write(root, &oss); std::string inJson = oss.str(); - //std::cout << "inJson: " << inJson << std::endl; + // std::cout << "inJson: " << inJson << std::endl; ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + bool init = processorInstance.Init(config, mContext); if (init) { int count = 0; uint64_t durationTime = 0; - for (int i = 0; i < batchSize; i ++) { - count ++; + for (int i = 0; i < batchSize; i++) { + count++; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); eventGroup.FromJsonString(inJson); @@ -283,14 +328,16 @@ static void BM_SplJson(int size, int batchSize) { uint64_t startTime = GetCurrentTimeInMicroSeconds(); processorInstance.Process(logGroupList); durationTime += GetCurrentTimeInMicroSeconds() - startTime; - //for (auto& logGroup : logGroupList) { - // std::string outJson = logGroup.ToJsonString(); - // std::cout << "outJson: " << outJson << std::endl; - //} + // for (auto& logGroup : logGroupList) { + // std::string outJson = logGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; + // } } std::cout << "spl json count: " << count << std::endl; std::cout << "durationTime: " << durationTime << std::endl; - std::cout << "process: " << formatSize((data.size() + 7)*(uint64_t)count*1000000*(uint64_t)size/durationTime) << std::endl; + std::cout << "process: " + << formatSize((data.size() + 7) * (uint64_t)count * 1000000 * (uint64_t)size / durationTime) + << std::endl; } } @@ -309,11 +356,39 @@ static void BM_RawJson(int size, int batchSize) { config["RenamedSourceKey"] = "rawLog"; - std::string data = "{\"_time_\":\"2023-11-15T01:04:21.80553511Z\",\"_source_\":\"stdout\",\"_pod_name_\":\"gpassport-37games-deployment-6d68b45779-rgfcz\",\"_namespace_\":\"go-app\",\"_pod_uid_\":\"22d6acfa-d55e-4be0-bb3f-ca91584a4f49\",\"_container_ip_\":\"10.101.31.136\",\"_image_name_\":\"686337631058.dkr.ecr.ap-southeast-1.amazonaws.com/gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf\",\"_container_name_\":\"gpassport-37games\",\"traceId\":\"44507629d8ebd96a6ff7810618d020ee\",\"logType\":\"http_access_log\",\"level\":\"INFO\",\"request\":\"/direct_login\",\"clientip\":\"218.225.227.156\",\"x_true_client_ip\":\"218.225.227.156\",\"real_ip_remote\":\"10.101.128.113\",\"xforward\":\"218.225.227.156, 70.132.19.70\",\"xforwardProto\":\"https\",\"method\":\"POST\",\"status\":\"200\",\"agent\":\"okhttp/3.12.13\",\"cost\":\"0.020\",\"bytes\":\"1409\",\"host\":\"http://gpassport.superfastgame.com\",\"remove_host\":\"http://gpassport.superfastgame.com\",\"referer\":\"-\",\"httpversion\":\"HTTP/1.1\",\"postData\":\"gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-a61881cfbde7&country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610.011%29&advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=0&ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&phoneBrand=TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=81444cf49a3f0f014d30b3e0571d894e&userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=android&timeZone=JST&mac=&isVpnOn=0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=5fd790e62c8e791388d913e808504c03&thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&osVersion=9&customUserId=b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&sdkVersion=326&ptCode=global&gameCode=ztmslg&att=1&battery=68\",\"cookieData\":\"-\",\"content_length\":\"986\",\"@timestamp\":\"2023-11-15T09:04:21+08:00\",\"__pack_meta__\":\"1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|426\",\"__topic__\":\"\",\"__source__\":\"10.101.29.105\",\"__tag__:__pack_id__\":\"5BCAE694BB74A062-38D81B\",\"__tag__:_node_name_\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:_node_ip_\":\"10.101.29.105\",\"__tag__:__hostname__\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:__client_ip__\":\"54.251.11.83\",\"__tag__:__receive_time__\":\"1700010262\"}"; + std::string data + = "{\"_time_\":\"2023-11-15T01:04:21.80553511Z\",\"_source_\":\"stdout\",\"_pod_name_\":\"gpassport-37games-" + "deployment-6d68b45779-rgfcz\",\"_namespace_\":\"go-app\",\"_pod_uid_\":\"22d6acfa-d55e-4be0-bb3f-" + "ca91584a4f49\",\"_container_ip_\":\"10.101.31.136\",\"_image_name_\":\"686337631058.dkr.ecr.ap-southeast-1." + "amazonaws.com/" + "gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf\",\"_container_name_\":\"gpassport-" + "37games\",\"traceId\":\"44507629d8ebd96a6ff7810618d020ee\",\"logType\":\"http_access_log\",\"level\":" + "\"INFO\",\"request\":\"/" + "direct_login\",\"clientip\":\"218.225.227.156\",\"x_true_client_ip\":\"218.225.227.156\",\"real_ip_remote\":" + "\"10.101.128.113\",\"xforward\":\"218.225.227.156, " + "70.132.19.70\",\"xforwardProto\":\"https\",\"method\":\"POST\",\"status\":\"200\",\"agent\":\"okhttp/" + "3.12.13\",\"cost\":\"0.020\",\"bytes\":\"1409\",\"host\":\"http://" + "gpassport.superfastgame.com\",\"remove_host\":\"http://" + "gpassport.superfastgame.com\",\"referer\":\"-\",\"httpversion\":\"HTTP/" + "1.1\",\"postData\":\"gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-" + "a61881cfbde7&country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610." + "011%29&advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=" + "0&ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&" + "phoneBrand=TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=" + "81444cf49a3f0f014d30b3e0571d894e&userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=" + "android&timeZone=JST&mac=&isVpnOn=0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=" + "5fd790e62c8e791388d913e808504c03&thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&" + "osVersion=9&customUserId=b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&" + "sdkVersion=326&ptCode=global&gameCode=ztmslg&att=1&battery=68\",\"cookieData\":\"-\",\"content_length\":" + "\"986\",\"@timestamp\":\"2023-11-15T09:04:21+08:00\",\"__pack_meta__\":\"1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|" + "426\",\"__topic__\":\"\",\"__source__\":\"10.101.29.105\",\"__tag__:__pack_id__\":\"5BCAE694BB74A062-" + "38D81B\",\"__tag__:_node_name_\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:_node_ip_\":" + "\"10.101.29.105\",\"__tag__:__hostname__\":\"ip-10-101-29-105.ap-southeast-1.compute.internal\",\"__tag__:__" + "client_ip__\":\"54.251.11.83\",\"__tag__:__receive_time__\":\"1700010262\"}"; // make events Json::Value root; Json::Value events; - for (int i = 0; i < size; i ++) { + for (int i = 0; i < size; i++) { Json::Value event; event["type"] = 1; event["timestamp"] = 1234567890; @@ -334,20 +409,20 @@ static void BM_RawJson(int size, int batchSize) { std::ostringstream oss; writer->write(root, &oss); std::string inJson = oss.str(); - //std::cout << "inJson: " << inJson << std::endl; + // std::cout << "inJson: " << inJson << std::endl; // run function ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + bool init = processorInstance.Init(config, mContext); if (init) { int count = 0; // Perform setup here uint64_t durationTime = 0; - for (int i = 0; i < batchSize; i ++) { - count ++; + for (int i = 0; i < batchSize; i++) { + count++; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); // This code gets timed @@ -359,13 +434,14 @@ static void BM_RawJson(int size, int batchSize) { processorInstance.Process(logGroupList); durationTime += GetCurrentTimeInMicroSeconds() - startTime; - //std::string outJson = eventGroup.ToJsonString(); - //std::cout << "outJson: " << outJson << std::endl; + // std::string outJson = eventGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; } std::cout << "raw json count: " << count << std::endl; std::cout << "durationTime: " << durationTime << std::endl; - std::cout << "process: " << formatSize((data.size() + 7)*(uint64_t)count*1000000*(uint64_t)size/durationTime) << std::endl; - + std::cout << "process: " + << formatSize((data.size() + 7) * (uint64_t)count * 1000000 * (uint64_t)size / durationTime) + << std::endl; } } @@ -376,13 +452,38 @@ static void BM_SplSplit(int size, int batchSize) { mContext.SetConfigName("project##config_0"); // make config - std::string spl = "* | parse-csv content as _time_,_source_,_pod_name_,_namespace_,_pod_uid_,_container_ip_,_image_name_,_container_name_,traceId,logType,level,request,clientip,x_true_client_ip,real_ip_remote,xforward,xforwardProto,method,status,agent,cost,bytes,host,remove_host,referer,httpversion,postData,cookieData,content_length,\"@timestamp\",__pack_meta__,__topic__,__source__,__tag__:__pack_id__,__tag__:_node_name_,__tag__:_node_ip_,__tag__:__hostname__,__tag__:__client_ip__,__tag__:__receive_time__,__time__,other"; + std::string spl + = "* | parse-csv content as " + "_time_,_source_,_pod_name_,_namespace_,_pod_uid_,_container_ip_,_image_name_,_container_name_,traceId," + "logType,level,request,clientip,x_true_client_ip,real_ip_remote,xforward,xforwardProto,method,status,agent," + "cost,bytes,host,remove_host,referer,httpversion,postData,cookieData,content_length,\"@timestamp\",__pack_" + "meta__,__topic__,__source__,__tag__:__pack_id__,__tag__:_node_name_,__tag__:_node_ip_,__tag__:__hostname__,_" + "_tag__:__client_ip__,__tag__:__receive_time__,__time__,other"; Json::Value config = GetCastConfig(spl); - std::string data = "2023-11-15T01:04:21.80553511Z,stdout,gpassport-37games-deployment-6d68b45779-rgfcz,go-app,22d6acfa-d55e-4be0-bb3f-ca91584a4f49,10.101.31.136,686337631058.dkr.ecr.ap-southeast-1.amazonaws.com/gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf,gpassport-37games,44507629d8ebd96a6ff7810618d020ee,http_access_log,INFO,/direct_login,218.225.227.156,218.225.227.156,10.101.128.113,218.225.227.156, 70.132.19.70,https,POST,200,okhttp/3.12.13,0.020,1409,http://gpassport.superfastgame.com,http://gpassport.superfastgame.com,-,HTTP/1.1,gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-a61881cfbde7&country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610.011%29&advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=0&ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&phoneBrand=TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=81444cf49a3f0f014d30b3e0571d894e&userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=android&timeZone=JST&mac=&isVpnOn=0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=5fd790e62c8e791388d913e808504c03&thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&osVersion=9&customUserId=b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&sdkVersion=326&ptCode=global&gameCode=ztmslg&att=1&battery=68,-,986,2023-11-15T09:04:21+08:00,1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|426,,10.101.29.105,5BCAE694BB74A062-38D81B,ip-10-101-29-105.ap-southeast-1.compute.internal,10.101.29.105,ip-10-101-29-105.ap-southeast-1.compute.internal,54.251.11.83,1700010262,1700010261"; + std::string data + = "2023-11-15T01:04:21.80553511Z,stdout,gpassport-37games-deployment-6d68b45779-rgfcz,go-app,22d6acfa-d55e-" + "4be0-bb3f-ca91584a4f49,10.101.31.136,686337631058.dkr.ecr.ap-southeast-1.amazonaws.com/" + "gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf,gpassport-37games," + "44507629d8ebd96a6ff7810618d020ee,http_access_log,INFO,/" + "direct_login,218.225.227.156,218.225.227.156,10.101.128.113,218.225.227.156, " + "70.132.19.70,https,POST,200,okhttp/3.12.13,0.020,1409,http://gpassport.superfastgame.com,http://" + "gpassport.superfastgame.com,-,HTTP/" + "1.1,gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-a61881cfbde7&" + "country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610.011%29&" + "advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=0&" + "ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&phoneBrand=" + "TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=81444cf49a3f0f014d30b3e0571d894e&" + "userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=android&timeZone=JST&mac=&isVpnOn=" + "0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=5fd790e62c8e791388d913e808504c03&" + "thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&osVersion=9&customUserId=" + "b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&sdkVersion=326&ptCode=" + "global&gameCode=ztmslg&att=1&battery=68,-,986,2023-11-15T09:04:21+08:00,1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|" + "426,,10.101.29.105,5BCAE694BB74A062-38D81B,ip-10-101-29-105.ap-southeast-1.compute.internal,10.101.29.105," + "ip-10-101-29-105.ap-southeast-1.compute.internal,54.251.11.83,1700010262,1700010261"; // make events Json::Value root; Json::Value events; - for (int i = 0; i < size; i ++) { + for (int i = 0; i < size; i++) { Json::Value event; event["type"] = 1; event["timestamp"] = 1234567890; @@ -403,34 +504,36 @@ static void BM_SplSplit(int size, int batchSize) { std::ostringstream oss; writer->write(root, &oss); std::string inJson = oss.str(); - //std::cout << "inJson: " << inJson << std::endl; + // std::cout << "inJson: " << inJson << std::endl; ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + bool init = processorInstance.Init(config, mContext); if (init) { int count = 0; uint64_t durationTime = 0; - for (int i = 0; i < batchSize; i ++) { - count ++; + for (int i = 0; i < batchSize; i++) { + count++; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); eventGroup.FromJsonString(inJson); std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); - + uint64_t startTime = GetCurrentTimeInMicroSeconds(); processorInstance.Process(logGroupList); durationTime += GetCurrentTimeInMicroSeconds() - startTime; - //for (auto& logGroup : logGroupList) { - // std::string outJson = logGroup.ToJsonString(); - // std::cout << "outJson: " << outJson << std::endl; - //} + // for (auto& logGroup : logGroupList) { + // std::string outJson = logGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; + // } } std::cout << "spl split count: " << count << std::endl; std::cout << "durationTime: " << durationTime << std::endl; - std::cout << "process: " << formatSize((data.size() + 7)*(uint64_t)count*1000000*(uint64_t)size/durationTime) << std::endl; + std::cout << "process: " + << formatSize((data.size() + 7) * (uint64_t)count * 1000000 * (uint64_t)size / durationTime) + << std::endl; } } @@ -495,11 +598,30 @@ static void BM_RawSplit(int size, int batchSize) { config["RenamedSourceKey"] = "rawLog"; config["AllowingShortenedFields"] = false; - std::string data = "2023-11-15T01:04:21.80553511Z,stdout,gpassport-37games-deployment-6d68b45779-rgfcz,go-app,22d6acfa-d55e-4be0-bb3f-ca91584a4f49,10.101.31.136,686337631058.dkr.ecr.ap-southeast-1.amazonaws.com/gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf,gpassport-37games,44507629d8ebd96a6ff7810618d020ee,http_access_log,INFO,/direct_login,218.225.227.156,218.225.227.156,10.101.128.113,218.225.227.156, 70.132.19.70,https,POST,200,okhttp/3.12.13,0.020,1409,http://gpassport.superfastgame.com,http://gpassport.superfastgame.com,-,HTTP/1.1,gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-a61881cfbde7&country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610.011%29&advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=0&ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&phoneBrand=TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=81444cf49a3f0f014d30b3e0571d894e&userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=android&timeZone=JST&mac=&isVpnOn=0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=5fd790e62c8e791388d913e808504c03&thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&osVersion=9&customUserId=b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&sdkVersion=326&ptCode=global&gameCode=ztmslg&att=1&battery=68,-,986,2023-11-15T09:04:21+08:00,1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|426,,10.101.29.105,5BCAE694BB74A062-38D81B,ip-10-101-29-105.ap-southeast-1.compute.internal,10.101.29.105,ip-10-101-29-105.ap-southeast-1.compute.internal,54.251.11.83,1700010262,1700010261"; + std::string data + = "2023-11-15T01:04:21.80553511Z,stdout,gpassport-37games-deployment-6d68b45779-rgfcz,go-app,22d6acfa-d55e-" + "4be0-bb3f-ca91584a4f49,10.101.31.136,686337631058.dkr.ecr.ap-southeast-1.amazonaws.com/" + "gpassport-37games:master-ceb4bb745aa101731616baad3c2920a3a0b11dbf,gpassport-37games," + "44507629d8ebd96a6ff7810618d020ee,http_access_log,INFO,/" + "direct_login,218.225.227.156,218.225.227.156,10.101.128.113,218.225.227.156, " + "70.132.19.70,https,POST,200,okhttp/3.12.13,0.020,1409,http://gpassport.superfastgame.com,http://" + "gpassport.superfastgame.com,-,HTTP/" + "1.1,gpid=393ed90f-9de0-4343-80bc-a61881cfbde7&language=ja-JP&gaid=393ed90f-9de0-4343-80bc-a61881cfbde7&" + "country=JP&userAgent=Dalvik%2F2.1.0+%28Linux%3B+U%3B+Android+9%3B+TONE+e20+Build%2FPPR1.180610.011%29&" + "advertiser=global&channelId=googlePlay&installTime=1694994381280&jgPid=&phoneModel=TONE+e20&Isdblink=0&" + "ratio=720x1520&gameId=191&netType=MOBILE&phoneTablet=Phone&deepLinkURL=&timeStamp=1700010260521&phoneBrand=" + "TONE&apps=1694994408269-2661115393006544017&packageVersion=146&androidid=81444cf49a3f0f014d30b3e0571d894e&" + "userMode=2&sdkVersionName=3.2.6_beta_1b09b7&isTrackEnabled=1&devicePlate=android&timeZone=JST&mac=&isVpnOn=" + "0&appLanguage=ja-JP&imei=&ueAndroidId=e3010c3cc52667ae&isFirst=0&sign=5fd790e62c8e791388d913e808504c03&" + "thirdPlatForm=mac&packageName=com.global.ztmslg&publishPlatForm=googlePlay&osVersion=9&customUserId=" + "b7c47cec-2c1f-4b5f-8a86-1f27884da5f0&loginId=393ed90f-9de0-4343-80bc-a61881cfbde7&sdkVersion=326&ptCode=" + "global&gameCode=ztmslg&att=1&battery=68,-,986,2023-11-15T09:04:21+08:00,1|MTY5MzU5Njg0MTIwODU1NjgwOQ==|437|" + "426,,10.101.29.105,5BCAE694BB74A062-38D81B,ip-10-101-29-105.ap-southeast-1.compute.internal,10.101.29.105," + "ip-10-101-29-105.ap-southeast-1.compute.internal,54.251.11.83,1700010262,1700010261"; // make events Json::Value root; Json::Value events; - for (int i = 0; i < size; i ++) { + for (int i = 0; i < size; i++) { Json::Value event; event["type"] = 1; event["timestamp"] = 1234567890; @@ -520,20 +642,20 @@ static void BM_RawSplit(int size, int batchSize) { std::ostringstream oss; writer->write(root, &oss); std::string inJson = oss.str(); - //std::cout << "inJson: " << inJson << std::endl; + // std::cout << "inJson: " << inJson << std::endl; // run function ProcessorParseDelimiterNative& processor = *(new ProcessorParseDelimiterNative); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + bool init = processorInstance.Init(config, mContext); if (init) { int count = 0; // Perform setup here uint64_t durationTime = 0; - for (int i = 0; i < batchSize; i ++) { - count ++; + for (int i = 0; i < batchSize; i++) { + count++; auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); // This code gets timed @@ -544,13 +666,14 @@ static void BM_RawSplit(int size, int batchSize) { processorInstance.Process(logGroupList); durationTime += GetCurrentTimeInMicroSeconds() - startTime; - //std::string outJson = eventGroup.ToJsonString(); - //std::cout << "outJson: " << outJson << std::endl; + // std::string outJson = eventGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; } std::cout << "raw split count: " << count << std::endl; std::cout << "durationTime: " << durationTime << std::endl; - std::cout << "process: " << formatSize((data.size() + 7)*(uint64_t)count*1000000*(uint64_t)size/durationTime) << std::endl; - + std::cout << "process: " + << formatSize((data.size() + 7) * (uint64_t)count * 1000000 * (uint64_t)size / durationTime) + << std::endl; } } @@ -559,7 +682,7 @@ int main(int argc, char** argv) { logtail::Logger::Instance().InitGlobalLoggers(); #ifdef NDEBUG std::cout << "release" << std::endl; -#else +#else std::cout << "debug" << std::endl; #endif diff --git a/core/unittest/spl/SplUnittest.cpp b/core/unittest/spl/SplUnittest.cpp index 92184ef310..3a1dc68e9e 100644 --- a/core/unittest/spl/SplUnittest.cpp +++ b/core/unittest/spl/SplUnittest.cpp @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include +#include #include "common/JsonUtil.h" #include "config/PipelineConfig.h" -#include "plugin/processor/ProcessorSPL.h" #include "models/LogEvent.h" #include "pipeline/plugin/instance/ProcessorInstance.h" -#include -#include +#include "plugin/processor/ProcessorSPL.h" +#include "unittest/Unittest.h" namespace logtail { @@ -29,9 +29,7 @@ static std::atomic_bool running(true); class SplUnittest : public ::testing::Test { public: - void SetUp() override { - mContext.SetConfigName("project##config_0"); - } + void SetUp() override { mContext.SetConfigName("project##config_0"); } PipelineContext mContext; Json::Value GetCastConfig(std::string spl); void TestInit(); @@ -43,10 +41,10 @@ class SplUnittest : public ::testing::Test { void TestRegexCSV(); void TestTag(); - //void TestMultiParse(); + // void TestMultiParse(); }; -//APSARA_UNIT_TEST_CASE(SplUnittest, TestInit, 8); +// APSARA_UNIT_TEST_CASE(SplUnittest, TestInit, 8); APSARA_UNIT_TEST_CASE(SplUnittest, TestWhere, 0); APSARA_UNIT_TEST_CASE(SplUnittest, TestExtend, 1); APSARA_UNIT_TEST_CASE(SplUnittest, TestJsonParse, 2); @@ -54,9 +52,9 @@ APSARA_UNIT_TEST_CASE(SplUnittest, TestRegexParse, 3); APSARA_UNIT_TEST_CASE(SplUnittest, TestRegexCSV, 4); APSARA_UNIT_TEST_CASE(SplUnittest, TestRegexKV, 5); APSARA_UNIT_TEST_CASE(SplUnittest, TestTag, 6); -//APSARA_UNIT_TEST_CASE(SplUnittest, TestMultiParse, 7); +// APSARA_UNIT_TEST_CASE(SplUnittest, TestMultiParse, 7); -PluginInstance::PluginMeta getPluginMeta(){ +PluginInstance::PluginMeta getPluginMeta() { PluginInstance::PluginMeta pluginMeta{"1"}; return pluginMeta; } @@ -65,7 +63,7 @@ Json::Value SplUnittest::GetCastConfig(std::string spl) { Json::Value config; config["Script"] = Json::Value(spl); config["TimeoutMilliSeconds"] = Json::Value(1000); - config["MaxMemoryBytes"] = Json::Value(50*1024*1024); + config["MaxMemoryBytes"] = Json::Value(50 * 1024 * 1024); return config; } @@ -74,15 +72,15 @@ void SplUnittest::TestInit() { std::ifstream input("spl.txt"); std::string line; - while( std::getline( input, line ) ) { + while (std::getline(input, line)) { Json::Value config = GetCastConfig(line); ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + bool init = processorInstance.Init(config, mContext); if (!init) { - std::cout<<"error:" < logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); @@ -137,8 +135,8 @@ void SplUnittest::TestWhere() { StringView content = log->GetContent("content"); APSARA_TEST_EQUAL("value_3_0", content); } - //std::string outJson = logGroup.ToJsonString(); - //std::cout << "outJson: " << outJson << std::endl; + // std::string outJson = logGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; } } @@ -148,7 +146,8 @@ void SplUnittest::TestWhere() { void SplUnittest::TestExtend() { // make config - Json::Value config = GetCastConfig(R"(* | extend a=json_extract(content, '$.body.a'), b=json_extract(content, '$.body.b'))"); + Json::Value config + = GetCastConfig(R"(* | extend a=json_extract(content, '$.body.a'), b=json_extract(content, '$.body.b'))"); // make events auto sourceBuffer = std::make_shared(); @@ -168,13 +167,13 @@ void SplUnittest::TestExtend() { ] })"; eventGroup.FromJsonString(inJson); - + std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); @@ -187,7 +186,7 @@ void SplUnittest::TestExtend() { APSARA_TEST_EQUAL("1", content); content = log->GetContent("b"); APSARA_TEST_EQUAL("2", content); - } + } } return; } @@ -226,13 +225,13 @@ void SplUnittest::TestJsonParse() { } })"; eventGroup.FromJsonString(inJson); - + std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); APSARA_TEST_EQUAL(logGroupList.size(), 1); @@ -296,13 +295,13 @@ void SplUnittest::TestRegexParse() { } })"; eventGroup.FromJsonString(inJson); - + std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); @@ -367,13 +366,13 @@ void SplUnittest::TestRegexCSV() { } })"; eventGroup.FromJsonString(inJson); - + std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); @@ -409,7 +408,6 @@ void SplUnittest::TestRegexCSV() { } - void SplUnittest::TestRegexKV() { // make config Json::Value config = GetCastConfig(R"(* | parse-kv -delims='&?' content)"); @@ -444,13 +442,13 @@ void SplUnittest::TestRegexKV() { } })"; eventGroup.FromJsonString(inJson); - + std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); @@ -485,7 +483,6 @@ void SplUnittest::TestRegexKV() { } - void SplUnittest::TestTag() { // make config Json::Value config = GetCastConfig(R"(* | parse-json content | project-rename __tag__:taiye2=a1)"); @@ -520,13 +517,13 @@ void SplUnittest::TestTag() { } })"; eventGroup.FromJsonString(inJson); - + std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); @@ -539,8 +536,8 @@ void SplUnittest::TestTag() { tag = logGroup.GetTag("taiye"); APSARA_TEST_EQUAL("123", tag); - //std::string outJson = logGroup.ToJsonString(); - //std::cout << "outJson: " << outJson << std::endl; + // std::string outJson = logGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; } { auto& logGroup = logGroupList[1]; @@ -549,20 +546,19 @@ void SplUnittest::TestTag() { tag = logGroup.GetTag("taiye"); APSARA_TEST_EQUAL("123", tag); - //std::string outJson = logGroup.ToJsonString(); - //std::cout << "outJson: " << outJson << std::endl; + // std::string outJson = logGroup.ToJsonString(); + // std::cout << "outJson: " << outJson << std::endl; } } - + return; } - /* void SplUnittest::TestMultiParse() { // make config - Json::Value config = GetCastConfig(R"(.let src = * + Json::Value config = GetCastConfig(R"(.let src = * | parse-json content; .let ds1 = $src | where type = 'kv' @@ -604,13 +600,13 @@ void SplUnittest::TestMultiParse() { } })"; eventGroup.FromJsonString(inJson); - + std::vector logGroupList; logGroupList.emplace_back(std::move(eventGroup)); // run function ProcessorSPL& processor = *(new ProcessorSPL); ProcessorInstance processorInstance(&processor, getPluginMeta()); - + APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext)); processor.Process(logGroupList); @@ -650,7 +646,7 @@ void SplUnittest::TestMultiParse() { APSARA_TEST_EQUAL("c", content); } } - } + } return; } */ diff --git a/core/unittest/task_pipeline/TaskPipelineUnittest.cpp b/core/unittest/task_pipeline/TaskPipelineUnittest.cpp index 9332784277..436ef0c120 100644 --- a/core/unittest/task_pipeline/TaskPipelineUnittest.cpp +++ b/core/unittest/task_pipeline/TaskPipelineUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include "json/json.h" #include "common/JsonUtil.h" #include "task_pipeline/TaskPipeline.h" From 10afd9e36d23d42c0af7e9bb2db34477e69b64c8 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 31 Dec 2024 08:54:18 +0000 Subject: [PATCH 2/8] polish --- .../reader/GetLastLineDataUnittest.cpp | 742 +++++----- .../RemoveLastIncompleteLogUnittest.cpp | 1200 ++++++++--------- 2 files changed, 981 insertions(+), 961 deletions(-) diff --git a/core/unittest/reader/GetLastLineDataUnittest.cpp b/core/unittest/reader/GetLastLineDataUnittest.cpp index 1edbb4f4e7..b458630d20 100644 --- a/core/unittest/reader/GetLastLineDataUnittest.cpp +++ b/core/unittest/reader/GetLastLineDataUnittest.cpp @@ -100,15 +100,15 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // case: PartLogFlag存在，第三个空格不存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -119,15 +119,15 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // case: PartLogFlag不存在，第二个空格存在 + { + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -193,15 +193,15 @@ if (testLog[size - 1] == '\n') { } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; +APSARA_TEST_EQUAL("", line.data.to_string()); +APSARA_TEST_EQUAL(0, line.lineBegin); +APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); +APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); +APSARA_TEST_EQUAL(false, line.fullLine); +} // namespace logtail +// case: PartLogFlag存在，第三个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -212,15 +212,15 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag不存在，第二个空格存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -287,16 +287,16 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: F + P + P + '\n' - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_FULL + "789\n"; + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: F + P + P + '\n' +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_FULL + "789\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -307,12 +307,12 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} // case: F + P + P + F { @@ -483,16 +483,16 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: P + P + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_PART + "123\n"; + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: P + P + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_PART + "123\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -503,13 +503,13 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); } LineInfo line = logFileReader.GetLastLine(testLog, endPs, true); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - } + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +} } void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { @@ -520,49 +520,7 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); - // 异常情况+有回车 - { - // case: PartLogFlag存在，第三个空格存在但空格后无内容 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P \n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; + {{std::string testLog = "\n2024-01-05T23:28:06.818486411+08:00 stdout P 123123\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -575,12 +533,13 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { APSARA_TEST_EQUAL("", line.data.to_string()); APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, line.lineEnd); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } - // case: 第二个空格不存在 { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout\n"; + std::string testLog = "\n2024-01-05T23:28:06.818486411+08:00 stdout F 123123\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -591,29 +550,105 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { } LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL("123123", line.data.to_string()); + APSARA_TEST_EQUAL(1, line.lineBegin); + APSARA_TEST_EQUAL(endPs, line.lineEnd); APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); APSARA_TEST_EQUAL(true, line.fullLine); } - // case: 第一个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout\n"; +} +// 异常情况+有回车 +{// case: PartLogFlag存在，第三个空格存在但空格后无内容 + {std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P \n"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); +int32_t size = testLog.size(); +int32_t endPs; // the position of \n or \0 +if (testLog[size - 1] == '\n') { + endPs = size - 1; +} else { + endPs = size; +} +LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); +APSARA_TEST_EQUAL("", line.data.to_string()); +APSARA_TEST_EQUAL(0, line.lineBegin); +APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); +APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); +APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag存在，第三个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag不存在，第二个空格存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout \n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: 第二个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: 第一个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00stdout\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + + APSARA_TEST_EQUAL(testLog.substr(0, size - 1), line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} } // 异常情况+无回车 {// case: PartLogFlag存在，第三个空格存在但空格后无内容 @@ -628,15 +663,15 @@ if (testLog[size - 1] == '\n') { } LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag存在，第三个空格不存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; +APSARA_TEST_EQUAL("", line.data.to_string()); +APSARA_TEST_EQUAL(0, line.lineBegin); +APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); +APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); +APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag存在，第三个空格不存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout P"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -647,15 +682,15 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs); } LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: PartLogFlag不存在，第二个空格存在 - { - std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: PartLogFlag不存在，第二个空格存在 +{ + std::string testLog = "2024-01-05T23:28:06.818486411+08:00 stdout "; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -722,16 +757,16 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs); } LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // case: F + P + P + '\n' - { - std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_FULL + "789\n"; + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} +// case: F + P + P + '\n' +{ + std::string testLog = LOG_FULL + "789\n" + LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_FULL + "789\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -742,12 +777,12 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs); } LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL("789", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(true, line.fullLine); - } + APSARA_TEST_EQUAL("789", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(true, line.fullLine); +} // case: F + P + P + F { @@ -918,16 +953,16 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs); } LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // case: P + P + '\n' - { - std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; - std::string expectedLog = LOG_PART + "123\n"; + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +// case: P + P + '\n' +{ + std::string testLog = LOG_PART + "123\n" + LOG_PART + "456\n"; + std::string expectedLog = LOG_PART + "123\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -938,13 +973,13 @@ LineInfo line = logFileReader.GetLastLine(testLog, endPs); } LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL(false, line.fullLine); - } - } + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(2, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL(false, line.fullLine); +} +} } class LastMatchedDockerJsonFileUnittest : public ::testing::Test { @@ -1010,105 +1045,102 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { logFileReader.mLineParsers.emplace_back(baseLineParsePtr); // 不带回车 // 不带回车 - { - // 合法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", - line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(true, line.fullLine); - } - // log非法 - { - std::string testLog - = R"({"log1":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // stream非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // time非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // 非法json - { - std::string testLog - = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } + {// 合法 + {std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } - // 带回车 - { - // 合法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(1, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(R"(Exception in thread "main" java.lang.NullPoinntterException)", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(true, line.fullLine); + } + // log非法 + { + std::string testLog + = R"({"log1":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // stream非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // time非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // 非法json + { + std::string testLog + = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } +} +// 带回车 +{ + // 合法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; int32_t size = testLog.size(); int32_t endPs; // the position of \n or \0 @@ -1129,83 +1161,83 @@ void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { = R"({"log1":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; testLog += "\n"; - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // stream非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // time非法 - { - std::string testLog - = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } - // 非法json - { - std::string testLog - = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; - testLog += "\n"; - - int32_t size = testLog.size(); - int32_t endPs; // the position of \n or \0 - if (testLog[size - 1] == '\n') { - endPs = size - 1; - } else { - endPs = size; - } - LineInfo line = logFileReader.GetLastLine(testLog, endPs); - APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); - APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); - APSARA_TEST_EQUAL("", line.data.to_string()); - APSARA_TEST_EQUAL(0, line.lineBegin); - APSARA_TEST_EQUAL(false, line.fullLine); - } + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // stream非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream1":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // time非法 + { + std::string testLog + = R"({"log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time1":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; + } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); + } + // 非法json + { + std::string testLog + = R"(log":"Exception in thread \"main\" java.lang.NullPoinntterException\n","stream":"stdout","time":"2024-02-19T03:49:37.793533014Z"})"; + testLog += "\n"; + + int32_t size = testLog.size(); + int32_t endPs; // the position of \n or \0 + if (testLog[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + LineInfo line = logFileReader.GetLastLine(testLog, endPs); + APSARA_TEST_EQUAL(0, line.rollbackLineFeedCount); + APSARA_TEST_EQUAL(1, line.forceRollbackLineFeedCount); + APSARA_TEST_EQUAL("", line.data.to_string()); + APSARA_TEST_EQUAL(0, line.lineBegin); + APSARA_TEST_EQUAL(false, line.fullLine); } } +} +} class LastMatchedContainerdTextWithDockerJsonUnittest : public ::testing::Test { diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 8e514a08d2..97855d5ee5 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + #include "common/FileSystemUtil.h" #include "common/memory/SourceBuffer.h" #include "file_server/reader/LogFileReader.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/writer.h" #include "unittest/Unittest.h" namespace logtail { @@ -418,31 +419,41 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn LogFileReader logFileReader( "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; - { // case: end with end - { - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); - } - { - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL(0, matchSize); - APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), ""); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - } - { // case: end with unmatch - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_UNMATCH + "\n"; + {// case: end with end + {std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + std::string testLog = std::string(expectMatch.data()); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); +} +{ + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING; + std::string testLog = std::string(expectMatch.data()); + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), ""); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); +} +} // namespace logtail +{ // case: end with unmatch + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_UNMATCH + "\n"; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); +} +{ // case: all unmatch + { + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); @@ -450,29 +461,18 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } - { // case: all unmatch - { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_UNMATCH; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); - } - { - std::string expectMatch = "\n\n" + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } + { + std::string expectMatch = "\n\n" + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); } } +} class GetLastLineUnittest : public ::testing::Test { public: @@ -612,181 +612,178 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); - { // case: end with begin - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } - } - { // case: end with unmatch - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - } - { // case: all unmatch - { - std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } - { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } - { - std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - { - std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } + {// case: end with begin + {std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +{ + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +{ + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +{ + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +} +{// case: end with unmatch + {std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; +std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + +int32_t rollbackLineFeedCount = 0; +int32_t matchSize + = logFileReader.RemoveLastIncompleteLog(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); +const auto& matchLog = std::string(testLog.data(), matchSize); + +APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); +APSARA_TEST_EQUAL(expectMatch, matchLog); +APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +{ + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +} +{// case: all unmatch + {std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; +std::string testLog = expectMatch; + +int32_t rollbackLineFeedCount = 0; +int32_t matchSize + = logFileReader.RemoveLastIncompleteLog(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); +const auto& matchLog = std::string(testLog.data(), matchSize); + +APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); +APSARA_TEST_EQUAL(expectMatch, matchLog); +APSARA_TEST_EQUAL(0, rollbackLineFeedCount); +} +{ + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +{ + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +{ + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch + LOG_PART + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +} +{ // case: end with part log + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" + + LOG_PART + LOG_BEGIN_STRING + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } - { // case: end with part log - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" - + LOG_PART + LOG_BEGIN_STRING + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" - + LOG_PART + LOG_BEGIN_STRING; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } + { + std::string expectMatch + = LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_BEGIN_STRING + "\n" + LOG_PART + LOG_BEGIN_STRING + "\n" + + LOG_PART + LOG_BEGIN_STRING; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } } +} void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { Json::Value config; @@ -798,68 +795,109 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); - { // case: end with end - { - std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART - + LOG_END_STRING + '\n' + LOG_FULL + LOG_UNMATCH; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(0, matchSize); - APSARA_TEST_EQUAL("", matchLog); - APSARA_TEST_EQUAL(4, rollbackLineFeedCount); - } - { - std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART - + LOG_END_STRING + '\n' + LOG_FULL + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(0, matchSize); - APSARA_TEST_EQUAL("", matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - } - { // case: end with unmatch + {// case: end with end + {std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART + + LOG_END_STRING + '\n' + LOG_FULL + LOG_UNMATCH; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(4, rollbackLineFeedCount); +} +{ + std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_PART + LOG_END_STRING + + '\n' + LOG_FULL + LOG_UNMATCH + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); +} +{ + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); +} +{ + std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); +} +} +{ // case: end with unmatch + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +{// case: all unmatch + {std::string expectMatch = "\n\n"; +std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; + +int32_t rollbackLineFeedCount = 0; +int32_t matchSize + = logFileReader.RemoveLastIncompleteLog(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); +const auto& matchLog = std::string(testLog.data(), matchSize); + +APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); +APSARA_TEST_EQUAL(expectMatch, matchLog); +APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +{ + std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); +} +} +{ // case: end with part log + { std::string expectMatch = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH + "\n"; + std::string testLog + = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH; int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( @@ -868,84 +906,40 @@ void ContainerdTextRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncom APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } - { // case: all unmatch - { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + LOG_FULL + LOG_UNMATCH; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } - { - std::string expectMatch = "\n\n" + LOG_FULL + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + + LOG_UNMATCH + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } - { // case: end with part log - { - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog - = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART - + LOG_UNMATCH + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" - + "2021-08-25T07:00:00.000000000Z"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } + { + std::string expectMatch + = LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_UNMATCH + "\n" + LOG_FULL + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_PART + LOG_UNMATCH + "\n" + LOG_PART + LOG_UNMATCH + "\n" + + "2021-08-25T07:00:00.000000000Z"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } } +} class DockerJsonRemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { public: @@ -998,155 +992,152 @@ void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncomplet BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(0); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); - { // case: end with begin - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING); - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } - } - { // case: end with unmatch - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(2, rollbackLineFeedCount); - } - } - { // case: all unmatch - { - std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } - { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } + {// case: end with begin + {std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +{ + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +{ + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +{ + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +} +{// case: end with unmatch + {std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; +std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + +int32_t rollbackLineFeedCount = 0; +int32_t matchSize + = logFileReader.RemoveLastIncompleteLog(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); +const auto& matchLog = std::string(testLog.data(), matchSize); + +APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); +APSARA_TEST_EQUAL(expectMatch, matchLog); +APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +{ + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(2, rollbackLineFeedCount); +} +} +{// case: all unmatch + {std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; +std::string testLog = expectMatch; + +int32_t rollbackLineFeedCount = 0; +int32_t matchSize + = logFileReader.RemoveLastIncompleteLog(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); +const auto& matchLog = std::string(testLog.data(), matchSize); + +APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); +APSARA_TEST_EQUAL(expectMatch, matchLog); +APSARA_TEST_EQUAL(0, rollbackLineFeedCount); +} +{ + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +} +{ // case: end with part log + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) + "\n" + + BuildLog(LOG_BEGIN_STRING, false) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } - { // case: end with part log - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) - + "\n" + BuildLog(LOG_BEGIN_STRING, false) + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) - + "\n" + BuildLog(LOG_BEGIN_STRING, false); - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } + { + std::string expectMatch + = BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_BEGIN_STRING) + "\n" + BuildLog(LOG_BEGIN_STRING, false) + "\n" + + BuildLog(LOG_BEGIN_STRING, false); + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } } +} void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { Json::Value config; @@ -1158,40 +1149,81 @@ void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncomplet BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(0); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); - { // case: end with end - { - std::string expectMatch - = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING); - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(0, matchSize); - APSARA_TEST_EQUAL("", matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } - } - { // case: end with unmatch + {// case: end with end + {std::string expectMatch = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING); + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(0, matchSize); + APSARA_TEST_EQUAL("", matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); +} +{ + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); +} +} +{ // case: end with unmatch + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +{// case: all unmatch + {std::string expectMatch = "\n\n"; +std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); + +int32_t rollbackLineFeedCount = 0; +int32_t matchSize + = logFileReader.RemoveLastIncompleteLog(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); +const auto& matchLog = std::string(testLog.data(), matchSize); + +APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); +APSARA_TEST_EQUAL(expectMatch, matchLog); +APSARA_TEST_EQUAL(1, rollbackLineFeedCount); +} +{ + std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(0, rollbackLineFeedCount); +} +} +{ // case: end with part log + { std::string expectMatch = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n"; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + BuildLog(LOG_UNMATCH, false); int32_t rollbackLineFeedCount = 0; int32_t matchSize = logFileReader.RemoveLastIncompleteLog( @@ -1200,84 +1232,40 @@ void DockerJsonRemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncomplet APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } - { // case: all unmatch - { - std::string expectMatch = "\n\n"; - std::string testLog = expectMatch + BuildLog(LOG_UNMATCH); - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(1, rollbackLineFeedCount); - } - { - std::string expectMatch = "\n\n" + BuildLog(LOG_UNMATCH) + "\n"; - std::string testLog = expectMatch; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(0, rollbackLineFeedCount); - } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + BuildLog(LOG_UNMATCH, false) + "\n"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } - { // case: end with part log - { - std::string expectMatch - = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" - + BuildLog(LOG_UNMATCH, false); - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" - + BuildLog(LOG_UNMATCH, false) + "\n"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } - { - std::string expectMatch - = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; - std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" - + "2021-08-25T07:00:00.000000000Z"; - - int32_t rollbackLineFeedCount = 0; - int32_t matchSize = logFileReader.RemoveLastIncompleteLog( - const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - const auto& matchLog = std::string(testLog.data(), matchSize); - - APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL(expectMatch, matchLog); - APSARA_TEST_EQUAL(3, rollbackLineFeedCount); - } + { + std::string expectMatch + = BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_END_STRING) + '\n'; + std::string testLog = expectMatch + BuildLog(LOG_UNMATCH) + "\n" + BuildLog(LOG_UNMATCH, false) + "\n" + + "2021-08-25T07:00:00.000000000Z"; + + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + const auto& matchLog = std::string(testLog.data(), matchSize); + + APSARA_TEST_EQUAL(int32_t(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL(expectMatch, matchLog); + APSARA_TEST_EQUAL(3, rollbackLineFeedCount); } } +} } // namespace logtail UNIT_TEST_MAIN From 496cbf2721e480b6b8cecc185cb8b304b23c3aff Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 31 Dec 2024 09:07:24 +0000 Subject: [PATCH 3/8] polish --- core/runner/sink/http/HttpSink.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/runner/sink/http/HttpSink.h b/core/runner/sink/http/HttpSink.h index e0c903f761..e06a35af75 100644 --- a/core/runner/sink/http/HttpSink.h +++ b/core/runner/sink/http/HttpSink.h @@ -21,6 +21,8 @@ #include #include +#include "curl/multi.h" + #include "monitor/MetricManager.h" #include "runner/sink/Sink.h" #include "runner/sink/http/HttpSinkRequest.h" From ad5072a989185e0a604dbbd7fc7f35da78fbdb81 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 2 Jan 2025 01:53:58 +0000 Subject: [PATCH 4/8] polish --- core/common/MachineInfoUtil.cpp | 26 +++++++++---------- .../input/InputInternalMetricsUnittest.cpp | 4 +-- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/core/common/MachineInfoUtil.cpp b/core/common/MachineInfoUtil.cpp index 5675cf2ea2..0efa6dcee0 100644 --- a/core/common/MachineInfoUtil.cpp +++ b/core/common/MachineInfoUtil.cpp @@ -14,12 +14,20 @@ #include "MachineInfoUtil.h" -#include -#include -#include +#include -#include "AppConfig.h" +#include + +#include "curl/curl.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" +#include "AppConfig.h" +#include "FileSystemUtil.h" +#include "StringTools.h" +#include "common/FileSystemUtil.h" +#include "common/UUIDUtil.h" +#include "logger/Logger.h" #if defined(__linux__) #include #include @@ -37,16 +45,6 @@ #include #include #endif -#include - -#include "curl/curl.h" -#include "rapidjson/document.h" - -#include "FileSystemUtil.h" -#include "StringTools.h" -#include "common/FileSystemUtil.h" -#include "common/UUIDUtil.h" -#include "logger/Logger.h" DEFINE_FLAG_STRING(agent_host_id, "", ""); diff --git a/core/unittest/input/InputInternalMetricsUnittest.cpp b/core/unittest/input/InputInternalMetricsUnittest.cpp index 9e4b8d418b..e7aa6b6844 100644 --- a/core/unittest/input/InputInternalMetricsUnittest.cpp +++ b/core/unittest/input/InputInternalMetricsUnittest.cpp @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include #include +#include "json/json.h" + #include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "monitor/Monitor.h" From 72ff3364c0aa308369176cea54e469271e7a1f92 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 2 Jan 2025 02:01:09 +0000 Subject: [PATCH 5/8] polish --- .clang-format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index f911ab3561..f783f467eb 100644 --- a/.clang-format +++ b/.clang-format @@ -70,7 +70,7 @@ SpacesInSquareBrackets: false IncludeCategories: - Regex: '^<.*\.h>' Priority: 1 - - Regex: '^' + - Regex: '^' # exclude non-c standard library headers like or Priority: 2 - Regex: '^ Date: Thu, 2 Jan 2025 02:30:11 +0000 Subject: [PATCH 6/8] polish --- .clang-format | 1 + 1 file changed, 1 insertion(+) diff --git a/.clang-format b/.clang-format index f783f467eb..b217c85d2c 100644 --- a/.clang-format +++ b/.clang-format @@ -67,6 +67,7 @@ SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false +# InsertNewlineAtEOF: true # uncomment when clang-format is upgraded to 16.0.0 IncludeCategories: - Regex: '^<.*\.h>' Priority: 1 From fd1d1be1287687029e163c344d96c6fffe5d0e76 Mon Sep 17 00:00:00 2001 From: Abingcbc Date: Wed, 27 Nov 2024 16:20:10 +0800 Subject: [PATCH 7/8] chore: add C++ core code lint in CI --- .github/workflows/static-check.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static-check.yaml b/.github/workflows/static-check.yaml index b7684a54aa..f5870f2cb8 100644 --- a/.github/workflows/static-check.yaml +++ b/.github/workflows/static-check.yaml @@ -83,7 +83,13 @@ jobs: if: matrix.runner == 'ubuntu' run: make check-dependency-licenses - - name: Lint + - name: C++ Core Lint + uses: jidicula/clang-format-action@v4.13.0 + with: + clang-format-version: '13' + check-path: 'core' + + - name: Go Plugin Lint run: make lint - name: UnitTest From 5f0b8961fafebc946aa88a2d6d803f530c067221 Mon Sep 17 00:00:00 2001 From: Abingcbc Date: Thu, 2 Jan 2025 10:33:33 +0800 Subject: [PATCH 8/8] fix --- .github/workflows/static-check.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static-check.yaml b/.github/workflows/static-check.yaml index f5870f2cb8..f7be089dd7 100644 --- a/.github/workflows/static-check.yaml +++ b/.github/workflows/static-check.yaml @@ -86,8 +86,10 @@ jobs: - name: C++ Core Lint uses: jidicula/clang-format-action@v4.13.0 with: - clang-format-version: '13' + clang-format-version: '14' check-path: 'core' + include-regex: '.*\.(cpp|h)$' + fallback-style: 'Google' - name: Go Plugin Lint run: make lint