From 5c0d9c842eb0451ec37242dab59005fbd4264faa Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 28 Nov 2024 14:44:42 +0800 Subject: [PATCH 01/15] feat: support singleton input --- core/config/watcher/PipelineConfigWatcher.cpp | 20 +++ core/config/watcher/PipelineConfigWatcher.h | 8 +- core/pipeline/PipelineManager.cpp | 43 +++++- core/pipeline/PipelineManager.h | 2 + core/pipeline/plugin/PluginRegistry.cpp | 49 +++++-- core/pipeline/plugin/PluginRegistry.h | 25 ++-- core/pipeline/plugin/creator/PluginCreator.h | 2 +- core/pipeline/plugin/instance/InputInstance.h | 8 +- core/plugin/input/InputContainerStdio.h | 2 +- core/plugin/input/InputFile.h | 2 +- core/plugin/input/InputFileSecurity.h | 4 +- core/plugin/input/InputInternalMetrics.h | 1 + core/plugin/input/InputNetworkObserver.h | 2 +- core/plugin/input/InputNetworkSecurity.h | 2 +- core/plugin/input/InputProcessSecurity.h | 2 +- core/unittest/config/CMakeLists.txt | 4 + .../config/PipelineConfigWatcherUnittest.cpp | 136 ++++++++++++++++++ .../pipeline/PipelineManagerUnittest.cpp | 35 +++++ 18 files changed, 308 insertions(+), 39 deletions(-) create mode 100644 core/unittest/config/PipelineConfigWatcherUnittest.cpp diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 7c91068b72..dc0f698897 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -80,6 +80,7 @@ pair PipelineConfigWatcher::CheckConfigDiff( LOG_DEBUG(sLogger, ("config files scan done", "no task update")); } + SortPipelineConfigDiff(pDiff); return make_pair(std::move(pDiff), std::move(tDiff)); } @@ -405,4 +406,23 @@ bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, return true; } +void PipelineConfigWatcher::SortPipelineConfigDiff(PipelineConfigDiff& pDiff) { + // sort rule + // 1. sort by create time first, if create time is 0 (include local config), put it back + // 2. if create time is the same, sort by name + auto cmp = [](const PipelineConfig& a, const PipelineConfig& b) { + if (a.mCreateTime == b.mCreateTime) { + return a.mName < b.mName; + } else if (a.mCreateTime == 0 && b.mCreateTime != 0) { + return false; + } else if (a.mCreateTime != 0 && b.mCreateTime == 0) { + return true; + } else { + return a.mCreateTime < b.mCreateTime; + } + }; + sort(pDiff.mAdded.begin(), pDiff.mAdded.end(), cmp); + sort(pDiff.mModified.begin(), pDiff.mModified.end(), cmp); +} + } // namespace logtail diff --git a/core/config/watcher/PipelineConfigWatcher.h b/core/config/watcher/PipelineConfigWatcher.h index 28a7f00f97..138483db36 100644 --- a/core/config/watcher/PipelineConfigWatcher.h +++ b/core/config/watcher/PipelineConfigWatcher.h @@ -46,7 +46,8 @@ class PipelineConfigWatcher : public ConfigWatcher { PipelineConfigWatcher(); ~PipelineConfigWatcher() = default; - void InsertInnerPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set& configSet); + void + InsertInnerPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set& configSet); void InsertPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set& configSet); bool CheckAddedConfig(const std::string& configName, std::unique_ptr&& configDetail, @@ -56,9 +57,14 @@ class PipelineConfigWatcher : public ConfigWatcher { std::unique_ptr&& configDetail, PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff); + void SortPipelineConfigDiff(PipelineConfigDiff& pDiff); const PipelineManager* mPipelineManager = nullptr; const TaskPipelineManager* mTaskPipelineManager = nullptr; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class PipelineConfigWatcherUnittest; +#endif }; } // namespace logtail diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 57abcde874..fc6c442bff 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -29,8 +29,7 @@ #include "shennong/ShennongManager.h" #endif #include "config/feedbacker/ConfigFeedbackReceiver.h" -#include "pipeline/queue/ProcessQueueManager.h" -#include "pipeline/queue/QueueKeyManager.h" +#include "plugin/PluginRegistry.h" using namespace std; @@ -40,7 +39,7 @@ PipelineManager::PipelineManager() : mInputRunners({ PrometheusInputRunner::GetInstance(), #if defined(__linux__) && !defined(__ANDROID__) - ebpf::eBPFServer::GetInstance(), + ebpf::eBPFServer::GetInstance(), #endif }) { } @@ -82,6 +81,10 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { ConfigFeedbackStatus::DELETED); } for (auto& config : diff.mModified) { + if (!PreCheckPipelineConfig(config)) { + continue; + } + auto p = BuildPipeline(std::move(config)); // auto reuse old pipeline's process queue and sender queue if (!p) { LOG_WARNING(sLogger, @@ -97,6 +100,7 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { ConfigFeedbackStatus::FAILED); continue; } + LOG_INFO(sLogger, ("pipeline building for existing config succeeded", "stop the old pipeline and start the new one")("config", config.mName)); @@ -111,6 +115,10 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { ConfigFeedbackStatus::APPLIED); } for (auto& config : diff.mAdded) { + if (!PreCheckPipelineConfig(config)) { + continue; + } + auto p = BuildPipeline(std::move(config)); if (!p) { LOG_WARNING(sLogger, @@ -238,6 +246,35 @@ void PipelineManager::DecreasePluginUsageCnt(const unordered_mapSendAlarm( + CATEGORY_CONFIG_ALARM, + "global singleton input plugin is already loaded: skip current object, config: " + config.mName, + config.mProject, + config.mLogstore, + config.mRegion); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::FAILED); + return false; + } + return true; +} + +bool PipelineManager::CheckIfGlobalSingletonInputLoaded(std::vector& inputConfig) { + for (const auto& input : inputConfig) { + auto inputType = (*input)["Type"].asString(); + if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(inputType) + && mPluginCntMap["inputs"][inputType] > 0) { + return true; + } + } + return false; +} + bool PipelineManager::CheckIfFileServerUpdated(const Json::Value& config) { string inputType = config["Type"].asString(); return inputType == "input_file" || inputType == "input_container_stdio"; diff --git a/core/pipeline/PipelineManager.h b/core/pipeline/PipelineManager.h index 5dc1535f77..b4880b59ca 100644 --- a/core/pipeline/PipelineManager.h +++ b/core/pipeline/PipelineManager.h @@ -58,6 +58,8 @@ class PipelineManager { void DecreasePluginUsageCnt( const std::unordered_map>& statistics); void FlushAllBatch(); + bool PreCheckPipelineConfig(PipelineConfig& config); + bool CheckIfGlobalSingletonInputLoaded(std::vector& inputConfig); // TODO: 长期过渡使用 bool CheckIfFileServerUpdated(const Json::Value& config); diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index bf0d7acbe6..bbb81753f6 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -127,13 +127,13 @@ bool PluginRegistry::IsValidNativeFlusherPlugin(const string& name) const { void PluginRegistry::LoadStaticPlugins() { RegisterInputCreator(new StaticInputCreator()); RegisterInputCreator(new StaticInputCreator()); - RegisterInputCreator(new StaticInputCreator()); + RegisterInputCreator(new StaticInputCreator(), true); #if defined(__linux__) && !defined(__ANDROID__) RegisterInputCreator(new StaticInputCreator()); - RegisterInputCreator(new StaticInputCreator()); - RegisterInputCreator(new StaticInputCreator()); - RegisterInputCreator(new StaticInputCreator()); - RegisterInputCreator(new StaticInputCreator()); + RegisterInputCreator(new StaticInputCreator(), true); + RegisterInputCreator(new StaticInputCreator(), true); + RegisterInputCreator(new StaticInputCreator(), true); + RegisterInputCreator(new StaticInputCreator(), true); #endif RegisterProcessorCreator(new StaticProcessorCreator()); @@ -183,16 +183,16 @@ void PluginRegistry::LoadDynamicPlugins(const set& plugins) { } } -void PluginRegistry::RegisterInputCreator(PluginCreator* creator) { - RegisterCreator(INPUT_PLUGIN, creator); +void PluginRegistry::RegisterInputCreator(PluginCreator* creator, bool isSingleton) { + RegisterCreator(INPUT_PLUGIN, creator, isSingleton); } -void PluginRegistry::RegisterProcessorCreator(PluginCreator* creator) { - RegisterCreator(PROCESSOR_PLUGIN, creator); +void PluginRegistry::RegisterProcessorCreator(PluginCreator* creator, bool isSingleton) { + RegisterCreator(PROCESSOR_PLUGIN, creator, isSingleton); } -void PluginRegistry::RegisterFlusherCreator(PluginCreator* creator) { - RegisterCreator(FLUSHER_PLUGIN, creator); +void PluginRegistry::RegisterFlusherCreator(PluginCreator* creator, bool isSingleton) { + RegisterCreator(FLUSHER_PLUGIN, creator, isSingleton); } PluginCreator* PluginRegistry::LoadProcessorPlugin(DynamicLibLoader& loader, const string pluginType) { @@ -217,11 +217,12 @@ PluginCreator* PluginRegistry::LoadProcessorPlugin(DynamicLibLoader& loader, con return new DynamicCProcessorCreator(plugin, loader.Release()); } -void PluginRegistry::RegisterCreator(PluginCat cat, PluginCreator* creator) { +void PluginRegistry::RegisterCreator(PluginCat cat, PluginCreator* creator, bool isSingleton) { if (!creator) { return; } - mPluginDict.emplace(PluginKey(cat, creator->Name()), shared_ptr(creator)); + mPluginDict.emplace(PluginKey(cat, creator->Name()), + PluginCreatorWithInfo(shared_ptr(creator), isSingleton)); } unique_ptr @@ -229,9 +230,29 @@ PluginRegistry::Create(PluginCat cat, const string& name, const PluginInstance:: unique_ptr ins; auto creatorEntry = mPluginDict.find(PluginKey(cat, name)); if (creatorEntry != mPluginDict.end()) { - ins = creatorEntry->second->Create(pluginMeta); + ins = creatorEntry->second.first->Create(pluginMeta); } return ins; } +bool PluginRegistry::IsGlobalSingletonInputPlugin(const string& name) const { + return IsGlobalSingleton(INPUT_PLUGIN, name); +} + +bool PluginRegistry::IsGlobalSingletonProcessorPlugin(const string& name) const { + return IsGlobalSingleton(PROCESSOR_PLUGIN, name); +} + +bool PluginRegistry::IsGlobalSingletonFlusherPlugin(const string& name) const { + return IsGlobalSingleton(FLUSHER_PLUGIN, name); +} + +bool PluginRegistry::IsGlobalSingleton(PluginCat cat, const string& name) const { + auto creatorEntry = mPluginDict.find(PluginKey(cat, name)); + if (creatorEntry != mPluginDict.end()) { + return creatorEntry->second.second; + } + return false; +} + } // namespace logtail \ No newline at end of file diff --git a/core/pipeline/plugin/PluginRegistry.h b/core/pipeline/plugin/PluginRegistry.h index 22213d6c39..9a094e16fa 100644 --- a/core/pipeline/plugin/PluginRegistry.h +++ b/core/pipeline/plugin/PluginRegistry.h @@ -46,12 +46,17 @@ class PluginRegistry { void LoadPlugins(); void UnloadPlugins(); std::unique_ptr CreateInput(const std::string& name, const PluginInstance::PluginMeta& pluginMeta); - std::unique_ptr CreateProcessor(const std::string& name, const PluginInstance::PluginMeta& pluginMeta); - std::unique_ptr CreateFlusher(const std::string& name, const PluginInstance::PluginMeta& pluginMeta); + std::unique_ptr CreateProcessor(const std::string& name, + const PluginInstance::PluginMeta& pluginMeta); + std::unique_ptr CreateFlusher(const std::string& name, + const PluginInstance::PluginMeta& pluginMeta); bool IsValidGoPlugin(const std::string& name) const; bool IsValidNativeInputPlugin(const std::string& name) const; bool IsValidNativeProcessorPlugin(const std::string& name) const; bool IsValidNativeFlusherPlugin(const std::string& name) const; + bool IsGlobalSingletonInputPlugin(const std::string& name) const; + bool IsGlobalSingletonProcessorPlugin(const std::string& name) const; + bool IsGlobalSingletonFlusherPlugin(const std::string& name) const; private: enum PluginCat { INPUT_PLUGIN, PROCESSOR_PLUGIN, FLUSHER_PLUGIN }; @@ -69,19 +74,23 @@ class PluginRegistry { } }; + using PluginCreatorWithInfo = std::pair, bool>; + PluginRegistry() {} ~PluginRegistry() = default; void LoadStaticPlugins(); void LoadDynamicPlugins(const std::set& plugins); - void RegisterInputCreator(PluginCreator* creator); - void RegisterProcessorCreator(PluginCreator* creator); - void RegisterFlusherCreator(PluginCreator* creator); + void RegisterInputCreator(PluginCreator* creator, bool isSingleton = false); + void RegisterProcessorCreator(PluginCreator* creator, bool isSingleton = false); + void RegisterFlusherCreator(PluginCreator* creator, bool isSingleton = false); PluginCreator* LoadProcessorPlugin(DynamicLibLoader& loader, const std::string pluginType); - void RegisterCreator(PluginCat cat, PluginCreator* creator); - std::unique_ptr Create(PluginCat cat, const std::string& name, const PluginInstance::PluginMeta& pluginMeta); + void RegisterCreator(PluginCat cat, PluginCreator* creator, bool isSingleton); + std::unique_ptr + Create(PluginCat cat, const std::string& name, const PluginInstance::PluginMeta& pluginMeta); + bool IsGlobalSingleton(PluginCat cat, const std::string& name) const; - std::unordered_map, PluginKeyHash> mPluginDict; + std::unordered_map mPluginDict; #ifdef APSARA_UNIT_TEST_MAIN friend class PluginRegistryUnittest; diff --git a/core/pipeline/plugin/creator/PluginCreator.h b/core/pipeline/plugin/creator/PluginCreator.h index 6888927186..a61d492cdb 100644 --- a/core/pipeline/plugin/creator/PluginCreator.h +++ b/core/pipeline/plugin/creator/PluginCreator.h @@ -16,8 +16,8 @@ #pragma once -#include #include +#include #include "pipeline/plugin/instance/PluginInstance.h" diff --git a/core/pipeline/plugin/instance/InputInstance.h b/core/pipeline/plugin/instance/InputInstance.h index 139f5b554b..8505ecb130 100644 --- a/core/pipeline/plugin/instance/InputInstance.h +++ b/core/pipeline/plugin/instance/InputInstance.h @@ -28,14 +28,12 @@ namespace logtail { class InputInstance : public PluginInstance { public: - InputInstance(Input* plugin, const PluginInstance::PluginMeta& pluginMeta) : PluginInstance(pluginMeta), mPlugin(plugin) {} + InputInstance(Input* plugin, const PluginInstance::PluginMeta& pluginMeta) + : PluginInstance(pluginMeta), mPlugin(plugin) {} const std::string& Name() const override { return mPlugin->Name(); } - bool Init(const Json::Value& config, - PipelineContext& context, - size_t inputIdx, - Json::Value& optionalGoPipeline); + bool Init(const Json::Value& config, PipelineContext& context, size_t inputIdx, Json::Value& optionalGoPipeline); bool Start() { return mPlugin->Start(); } bool Stop(bool isPipelineRemoving) { return mPlugin->Stop(isPipelineRemoving); } bool SupportAck() const { return mPlugin->SupportAck(); } diff --git a/core/plugin/input/InputContainerStdio.h b/core/plugin/input/InputContainerStdio.h index a9d1e51aed..67ad10e550 100644 --- a/core/plugin/input/InputContainerStdio.h +++ b/core/plugin/input/InputContainerStdio.h @@ -21,9 +21,9 @@ #include "container_manager/ContainerDiscoveryOptions.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/reader/FileReaderOptions.h" #include "monitor/PluginMetricManager.h" #include "pipeline/plugin/interface/Input.h" -#include "file_server/reader/FileReaderOptions.h" namespace logtail { diff --git a/core/plugin/input/InputFile.h b/core/plugin/input/InputFile.h index ee8275ef7c..641f1067e5 100644 --- a/core/plugin/input/InputFile.h +++ b/core/plugin/input/InputFile.h @@ -21,9 +21,9 @@ #include "container_manager/ContainerDiscoveryOptions.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/reader/FileReaderOptions.h" #include "monitor/PluginMetricManager.h" #include "pipeline/plugin/interface/Input.h" -#include "file_server/reader/FileReaderOptions.h" namespace logtail { diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index fea0b459fc..3eac3a7a9f 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -19,9 +19,9 @@ #include #include "ebpf/config.h" -#include "pipeline/plugin/interface/Input.h" #include "ebpf/eBPFServer.h" #include "monitor/PluginMetricManager.h" +#include "pipeline/plugin/interface/Input.h" namespace logtail { @@ -34,7 +34,7 @@ class InputFileSecurity : public Input { bool Start() override; bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return false; } - + ebpf::SecurityOptions mSecurityOptions; PluginMetricManagerPtr mPluginMgr; }; diff --git a/core/plugin/input/InputInternalMetrics.h b/core/plugin/input/InputInternalMetrics.h index 694edf85af..c59cf89608 100644 --- a/core/plugin/input/InputInternalMetrics.h +++ b/core/plugin/input/InputInternalMetrics.h @@ -30,6 +30,7 @@ class InputInternalMetrics : public Input { bool Start() override; bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return true; } + private: SelfMonitorMetricRules mSelfMonitorMetricRules; }; diff --git a/core/plugin/input/InputNetworkObserver.h b/core/plugin/input/InputNetworkObserver.h index 7f204a2c90..72db90501d 100644 --- a/core/plugin/input/InputNetworkObserver.h +++ b/core/plugin/input/InputNetworkObserver.h @@ -19,9 +19,9 @@ #include #include "ebpf/config.h" -#include "pipeline/plugin/interface/Input.h" #include "ebpf/include/export.h" #include "monitor/PluginMetricManager.h" +#include "pipeline/plugin/interface/Input.h" namespace logtail { diff --git a/core/plugin/input/InputNetworkSecurity.h b/core/plugin/input/InputNetworkSecurity.h index cda3a7c170..84a3294a03 100644 --- a/core/plugin/input/InputNetworkSecurity.h +++ b/core/plugin/input/InputNetworkSecurity.h @@ -19,8 +19,8 @@ #include #include "ebpf/config.h" -#include "pipeline/plugin/interface/Input.h" #include "monitor/PluginMetricManager.h" +#include "pipeline/plugin/interface/Input.h" namespace logtail { diff --git a/core/plugin/input/InputProcessSecurity.h b/core/plugin/input/InputProcessSecurity.h index d26d7a95e3..46cd00fcbe 100644 --- a/core/plugin/input/InputProcessSecurity.h +++ b/core/plugin/input/InputProcessSecurity.h @@ -19,8 +19,8 @@ #include #include "ebpf/config.h" -#include "pipeline/plugin/interface/Input.h" #include "monitor/PluginMetricManager.h" +#include "pipeline/plugin/interface/Input.h" namespace logtail { diff --git a/core/unittest/config/CMakeLists.txt b/core/unittest/config/CMakeLists.txt index 560d47d393..7efb993180 100644 --- a/core/unittest/config/CMakeLists.txt +++ b/core/unittest/config/CMakeLists.txt @@ -44,6 +44,9 @@ target_link_libraries(config_feedbackable_unittest ${UT_BASE_TARGET}) add_executable(common_config_provider_unittest CommonConfigProviderUnittest.cpp) target_link_libraries(common_config_provider_unittest ${UT_BASE_TARGET}) +add_executable(pipeline_config_watcher_unittest PipelineConfigWatcherUnittest.cpp) +target_link_libraries(pipeline_config_watcher_unittest ${UT_BASE_TARGET}) + include(GoogleTest) gtest_discover_tests(pipeline_config_unittest) gtest_discover_tests(task_config_unittest) @@ -54,3 +57,4 @@ if (ENABLE_ENTERPRISE) endif () gtest_discover_tests(config_feedbackable_unittest) gtest_discover_tests(common_config_provider_unittest) +gtest_discover_tests(pipeline_config_watcher_unittest) diff --git a/core/unittest/config/PipelineConfigWatcherUnittest.cpp b/core/unittest/config/PipelineConfigWatcherUnittest.cpp new file mode 100644 index 0000000000..0d301f34f7 --- /dev/null +++ b/core/unittest/config/PipelineConfigWatcherUnittest.cpp @@ -0,0 +1,136 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include + +#include "config/ConfigDiff.h" +#include "config/PipelineConfig.h" +#include "config/watcher/PipelineConfigWatcher.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class PipelineConfigWatcherUnittest : public testing::Test { +public: + void TestSortPipelineConfigDiff() const; + +private: + const string configName = "test"; +}; + +void PipelineConfigWatcherUnittest::TestSortPipelineConfigDiff() const { + PipelineConfigDiff diff; + auto configName1 = "test1"; + auto configJson1 = std::make_unique(); + (*configJson1)["name"] = configName1; + auto config1 = PipelineConfig(configName1, std::move(configJson1)); + config1.Parse(); + + auto configName2 = "test2"; + auto configJson2 = std::make_unique(); + (*configJson2)["name"] = configName2; + (*configJson2)["createTime"] = 1; + auto config2 = PipelineConfig(configName2, std::move(configJson2)); + config2.Parse(); + + auto configName3 = "test3"; + auto configJson3 = std::make_unique(); + (*configJson3)["name"] = configName3; + (*configJson3)["createTime"] = 2; + auto config3 = PipelineConfig(configName3, std::move(configJson3)); + config3.Parse(); + + auto configName4 = "test4"; + auto configJson4 = std::make_unique(); + (*configJson4)["name"] = configName4; + auto config4 = PipelineConfig(configName4, std::move(configJson4)); + config4.Parse(); + + auto configName5 = "test5"; + auto configJson5 = std::make_unique(); + (*configJson5)["name"] = configName5; + (*configJson5)["createTime"] = 1; + auto config5 = PipelineConfig(configName5, std::move(configJson5)); + config5.Parse(); + + diff.mAdded.push_back(std::move(config1)); + diff.mAdded.push_back(std::move(config2)); + diff.mAdded.push_back(std::move(config3)); + diff.mAdded.push_back(std::move(config4)); + diff.mAdded.push_back(std::move(config5)); + + auto configName6 = "test6"; + auto configJson6 = std::make_unique(); + (*configJson6)["name"] = configName6; + auto config6 = PipelineConfig(configName6, std::move(configJson6)); + config6.Parse(); + + auto configName7 = "test7"; + auto configJson7 = std::make_unique(); + (*configJson7)["name"] = configName7; + (*configJson7)["createTime"] = 1; + auto config7 = PipelineConfig(configName7, std::move(configJson7)); + config7.Parse(); + + auto configName8 = "test8"; + auto configJson8 = std::make_unique(); + (*configJson8)["name"] = configName8; + (*configJson8)["createTime"] = 2; + auto config8 = PipelineConfig(configName8, std::move(configJson8)); + config8.Parse(); + + auto configName9 = "test9"; + auto configJson9 = std::make_unique(); + (*configJson9)["name"] = configName9; + auto config9 = PipelineConfig(configName9, std::move(configJson9)); + config9.Parse(); + + auto configName10 = "test10"; + auto configJson10 = std::make_unique(); + (*configJson10)["name"] = configName10; + (*configJson10)["createTime"] = 1; + auto config10 = PipelineConfig(configName10, std::move(configJson10)); + config10.Parse(); + + diff.mModified.push_back(std::move(config6)); + diff.mModified.push_back(std::move(config7)); + diff.mModified.push_back(std::move(config8)); + diff.mModified.push_back(std::move(config9)); + diff.mModified.push_back(std::move(config10)); + + PipelineConfigWatcher::GetInstance()->SortPipelineConfigDiff(diff); + + APSARA_TEST_EQUAL(diff.mAdded[0].mName, configName2); + APSARA_TEST_EQUAL(diff.mAdded[1].mName, configName5); + APSARA_TEST_EQUAL(diff.mAdded[2].mName, configName3); + APSARA_TEST_EQUAL(diff.mAdded[3].mName, configName1); + APSARA_TEST_EQUAL(diff.mAdded[4].mName, configName4); + + APSARA_TEST_EQUAL(diff.mModified[0].mName, configName10); + APSARA_TEST_EQUAL(diff.mModified[1].mName, configName7); + APSARA_TEST_EQUAL(diff.mModified[2].mName, configName8); + APSARA_TEST_EQUAL(diff.mModified[3].mName, configName6); + APSARA_TEST_EQUAL(diff.mModified[4].mName, configName9); +} + +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestSortPipelineConfigDiff) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/pipeline/PipelineManagerUnittest.cpp b/core/unittest/pipeline/PipelineManagerUnittest.cpp index 29c56da393..31e258c72f 100644 --- a/core/unittest/pipeline/PipelineManagerUnittest.cpp +++ b/core/unittest/pipeline/PipelineManagerUnittest.cpp @@ -14,6 +14,8 @@ #include "pipeline/Pipeline.h" #include "pipeline/PipelineManager.h" +#include "pipeline/plugin/PluginRegistry.h" +#include "plugin/input/InputNetworkSecurity.h" #include "unittest/Unittest.h" using namespace std; @@ -23,6 +25,11 @@ namespace logtail { class PipelineManagerUnittest : public testing::Test { public: void TestPipelineManagement() const; + void TestCheckIfGlobalSingletonInputLoaded() const; + +protected: + static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); } + static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } }; void PipelineManagerUnittest::TestPipelineManagement() const { @@ -34,7 +41,35 @@ void PipelineManagerUnittest::TestPipelineManagement() const { APSARA_TEST_EQUAL(nullptr, PipelineManager::GetInstance()->FindConfigByName("test3")); } +void PipelineManagerUnittest::TestCheckIfGlobalSingletonInputLoaded() const { + { // test not singleton input + auto inputConfig = Json::Value(); + inputConfig["Type"] = InputFile::sName; + std::vector inputConfigs = {&inputConfig}; + + PipelineManager::GetInstance()->mPluginCntMap["inputs"][InputFile::sName] = 1; + APSARA_TEST_EQUAL(false, PipelineManager::GetInstance()->CheckIfGlobalSingletonInputLoaded(inputConfigs)); + } + { // test singleton input not loaded + auto inputConfig = Json::Value(); + inputConfig["Type"] = InputNetworkSecurity::sName; + std::vector inputConfigs = {&inputConfig}; + + PipelineManager::GetInstance()->mPluginCntMap["inputs"][InputNetworkSecurity::sName] = 0; + APSARA_TEST_EQUAL(false, PipelineManager::GetInstance()->CheckIfGlobalSingletonInputLoaded(inputConfigs)); + } + { // test singleton input loaded + auto inputConfig = Json::Value(); + inputConfig["Type"] = InputNetworkSecurity::sName; + std::vector inputConfigs = {&inputConfig}; + + PipelineManager::GetInstance()->mPluginCntMap["inputs"][InputNetworkSecurity::sName] = 1; + APSARA_TEST_EQUAL(true, PipelineManager::GetInstance()->CheckIfGlobalSingletonInputLoaded(inputConfigs)); + } +} + UNIT_TEST_CASE(PipelineManagerUnittest, TestPipelineManagement) +UNIT_TEST_CASE(PipelineManagerUnittest, TestCheckIfGlobalSingletonInputLoaded) } // namespace logtail From 57cdfd11d73ed853871bfee724b7643f1a1fad89 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Mon, 2 Dec 2024 23:46:59 +0800 Subject: [PATCH 02/15] fix --- core/config/ConfigUtil.cpp | 16 + core/config/ConfigUtil.h | 1 + core/config/watcher/PipelineConfigWatcher.cpp | 179 +++++---- core/config/watcher/PipelineConfigWatcher.h | 12 +- core/pipeline/PipelineManager.cpp | 37 -- core/pipeline/PipelineManager.h | 2 - core/pipeline/plugin/PluginRegistry.cpp | 10 +- .../config/CommonConfigProviderUnittest.cpp | 15 +- .../config/PipelineConfigWatcherUnittest.cpp | 360 +++++++++++++----- .../pipeline/PipelineManagerUnittest.cpp | 29 -- 10 files changed, 405 insertions(+), 256 deletions(-) diff --git a/core/config/ConfigUtil.cpp b/core/config/ConfigUtil.cpp index 2c3975f4c7..f60c90cf3b 100644 --- a/core/config/ConfigUtil.cpp +++ b/core/config/ConfigUtil.cpp @@ -80,6 +80,22 @@ bool IsConfigEnabled(const string& name, const Json::Value& detail) { return true; } +void GetAllInputTypes(const Json::Value& detail, std::vector& inputTypes) { + const char* key = "inputs"; + const Json::Value* inputs = detail.find(key, key + strlen(key)); + if (inputs == nullptr || !inputs->isArray()) { + return; + } + for (const Json::Value& input : *inputs) { + const char* typeKey = "Type"; + const Json::Value* type = input.find(typeKey, typeKey + strlen(typeKey)); + if (type == nullptr || !type->isString()) { + continue; + } + inputTypes.push_back(type->asString()); + } +} + ConfigType GetConfigType(const Json::Value& detail) { return detail.isMember("task") ? ConfigType::Task : ConfigType::Pipeline; } diff --git a/core/config/ConfigUtil.h b/core/config/ConfigUtil.h index c5fd7d0ae2..1d1c708b42 100644 --- a/core/config/ConfigUtil.h +++ b/core/config/ConfigUtil.h @@ -31,6 +31,7 @@ bool ParseConfigDetail(const std::string& content, Json::Value& detail, std::string& errorMsg); bool IsConfigEnabled(const std::string& name, const Json::Value& detail); +void GetAllInputTypes(const Json::Value& detail, std::vector& inputTypes); ConfigType GetConfigType(const Json::Value& detail); } // namespace logtail diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index dc0f698897..1751a55195 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -17,10 +17,12 @@ #include #include "common/FileSystemUtil.h" +#include "common/ParamExtractor.h" #include "config/ConfigUtil.h" #include "logger/Logger.h" #include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" +#include "pipeline/plugin/PluginRegistry.h" #include "task_pipeline/TaskPipelineManager.h" using namespace std; @@ -79,8 +81,6 @@ pair PipelineConfigWatcher::CheckConfigDiff( } else { LOG_DEBUG(sLogger, ("config files scan done", "no task update")); } - - SortPipelineConfigDiff(pDiff); return make_pair(std::move(pDiff), std::move(tDiff)); } @@ -170,6 +170,8 @@ void PipelineConfigWatcher::InsertInnerPipelines(PipelineConfigDiff& pDiff, void PipelineConfigWatcher::InsertPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set& configSet) { + std::unordered_map>> toBeDiffedConfigs; + std::unordered_map singletonConfigs; for (const auto& dir : mSourceDir) { error_code ec; filesystem::file_status s = filesystem::status(dir, ec); @@ -212,64 +214,66 @@ void PipelineConfigWatcher::InsertPipelines(PipelineConfigDiff& pDiff, } configSet.insert(configName); - auto iter = mFileInfoMap.find(filepath); - uintmax_t size = filesystem::file_size(path, ec); - filesystem::file_time_type mTime = filesystem::last_write_time(path, ec); - if (iter == mFileInfoMap.end()) { - mFileInfoMap[filepath] = make_pair(size, mTime); - unique_ptr detail = make_unique(); - if (!LoadConfigDetailFromFile(path, *detail)) { - continue; - } - if (!IsConfigEnabled(configName, *detail)) { - LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", configName)); - continue; - } - if (!CheckAddedConfig(configName, std::move(detail), pDiff, tDiff)) { - continue; - } - } else if (iter->second.first != size || iter->second.second != mTime) { - // for config currently running, we leave it untouched if new config is invalid - mFileInfoMap[filepath] = make_pair(size, mTime); - unique_ptr detail = make_unique(); - if (!LoadConfigDetailFromFile(path, *detail)) { - continue; - } - if (!IsConfigEnabled(configName, *detail)) { - switch (GetConfigType(*detail)) { - case ConfigType::Pipeline: - if (mPipelineManager->FindConfigByName(configName)) { - pDiff.mRemoved.push_back(configName); - LOG_INFO(sLogger, - ("existing valid config modified and disabled", - "prepare to stop current running pipeline")("config", configName)); - } else { - LOG_INFO(sLogger, - ("existing invalid config modified and disabled", - "skip current object")("config", configName)); - } - break; - case ConfigType::Task: - if (mTaskPipelineManager->FindPipelineByName(configName)) { - tDiff.mRemoved.push_back(configName); - LOG_INFO(sLogger, - ("existing valid config modified and disabled", - "prepare to stop current running task")("config", configName)); - } else { - LOG_INFO(sLogger, - ("existing invalid config modified and disabled", - "skip current object")("config", configName)); - } - break; - } - continue; - } - if (!CheckModifiedConfig(configName, std::move(detail), pDiff, tDiff)) { - continue; + unique_ptr detail = make_unique(); + if (!LoadConfigDetailFromFile(path, *detail)) { + continue; + } + PreCheckConfig(configName, path, std::move(detail), toBeDiffedConfigs, singletonConfigs); + } + } + + for (auto& config : toBeDiffedConfigs) { + error_code ec; + const filesystem::path& path = config.second.first; + const string& configName = path.stem().string(); + const string& filepath = path.string(); + + auto iter = mFileInfoMap.find(filepath); + uintmax_t size = filesystem::file_size(path, ec); + filesystem::file_time_type mTime = filesystem::last_write_time(path, ec); + unique_ptr detail = std::move(config.second.second); + if (iter == mFileInfoMap.end()) { + mFileInfoMap[filepath] = make_pair(size, mTime); + if (!CheckAddedConfig(configName, std::move(detail), pDiff, tDiff)) { + continue; + } + } else if (iter->second.first != size || iter->second.second != mTime) { + // for config currently running, we leave it untouched if new config is invalid + mFileInfoMap[filepath] = make_pair(size, mTime); + if (!IsConfigEnabled(configName, *detail)) { + switch (GetConfigType(*detail)) { + case ConfigType::Pipeline: + if (mPipelineManager->FindConfigByName(configName)) { + pDiff.mRemoved.push_back(configName); + LOG_INFO(sLogger, + ("existing valid config modified and disabled", + "prepare to stop current running pipeline")("config", configName)); + } else { + LOG_INFO(sLogger, + ("existing invalid config modified and disabled", + "skip current object")("config", configName)); + } + break; + case ConfigType::Task: + if (mTaskPipelineManager->FindPipelineByName(configName)) { + tDiff.mRemoved.push_back(configName); + LOG_INFO(sLogger, + ("existing valid config modified and disabled", + "prepare to stop current running task")("config", configName)); + } else { + LOG_INFO(sLogger, + ("existing invalid config modified and disabled", + "skip current object")("config", configName)); + } + break; } - } else { - LOG_DEBUG(sLogger, ("existing config file unchanged", "skip current object")); + continue; + } + if (!CheckModifiedConfig(configName, std::move(detail), pDiff, tDiff)) { + continue; } + } else { + LOG_DEBUG(sLogger, ("existing config file unchanged", "skip current object")); } } } @@ -406,23 +410,50 @@ bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, return true; } -void PipelineConfigWatcher::SortPipelineConfigDiff(PipelineConfigDiff& pDiff) { - // sort rule - // 1. sort by create time first, if create time is 0 (include local config), put it back - // 2. if create time is the same, sort by name - auto cmp = [](const PipelineConfig& a, const PipelineConfig& b) { - if (a.mCreateTime == b.mCreateTime) { - return a.mName < b.mName; - } else if (a.mCreateTime == 0 && b.mCreateTime != 0) { - return false; - } else if (a.mCreateTime != 0 && b.mCreateTime == 0) { - return true; - } else { - return a.mCreateTime < b.mCreateTime; +bool PipelineConfigWatcher::PreCheckConfig(const string& configName, + const filesystem::path& path, + unique_ptr&& configDetail, + std::unordered_map& toBeDiffedConfigs, + std::unordered_map& singletonConfigs) { + // check enabled + if (!IsConfigEnabled(configName, *configDetail)) { + LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", configName)); + return false; + } + + // check singleton input, and only check this, others will be checked in Parse() + std::string errorMsg; + uint32_t createTime = 0; + if (!GetOptionalUIntParam(*configDetail, "createTime", createTime, errorMsg)) { + createTime = 0; + } + + std::vector inputTypes; + GetAllInputTypes(*configDetail, inputTypes); + for (const auto& inputType : inputTypes) { + if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(inputType)) { + auto it = singletonConfigs.find(inputType); + if (it != singletonConfigs.end()) { + if (it->second.first < createTime + || (it->second.first == createTime && it->second.second < configName)) { + LOG_WARNING(sLogger, + ("global singleton plugin found, but another older config or smaller name config " + "already exists", + "skip current object")("config", configName)); + return false; + } + toBeDiffedConfigs.erase(it->second.second); + } + } + } + for (const auto& inputType : inputTypes) { + if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(inputType)) { + singletonConfigs[inputType] = make_pair(createTime, configName); } - }; - sort(pDiff.mAdded.begin(), pDiff.mAdded.end(), cmp); - sort(pDiff.mModified.begin(), pDiff.mModified.end(), cmp); + } + toBeDiffedConfigs[configName] = make_pair(path, std::move(configDetail)); + return true; } + } // namespace logtail diff --git a/core/config/watcher/PipelineConfigWatcher.h b/core/config/watcher/PipelineConfigWatcher.h index 138483db36..cb13e93b5c 100644 --- a/core/config/watcher/PipelineConfigWatcher.h +++ b/core/config/watcher/PipelineConfigWatcher.h @@ -16,6 +16,9 @@ #pragma once +#include +#include +#include #include #include "config/ConfigDiff.h" @@ -23,6 +26,9 @@ namespace logtail { +using ConfigWithPath = std::pair>; +using ConfigPriority = std::pair; + class PipelineManager; class TaskPipelineManager; @@ -57,7 +63,11 @@ class PipelineConfigWatcher : public ConfigWatcher { std::unique_ptr&& configDetail, PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff); - void SortPipelineConfigDiff(PipelineConfigDiff& pDiff); + bool PreCheckConfig(const std::string& configName, + const std::filesystem::path& path, + std::unique_ptr&& configDetail, + std::unordered_map& toBeDiffedConfigs, + std::unordered_map& singletonConfigs); const PipelineManager* mPipelineManager = nullptr; const TaskPipelineManager* mTaskPipelineManager = nullptr; diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index fc6c442bff..0855869938 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -81,10 +81,6 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { ConfigFeedbackStatus::DELETED); } for (auto& config : diff.mModified) { - if (!PreCheckPipelineConfig(config)) { - continue; - } - auto p = BuildPipeline(std::move(config)); // auto reuse old pipeline's process queue and sender queue if (!p) { LOG_WARNING(sLogger, @@ -115,10 +111,6 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { ConfigFeedbackStatus::APPLIED); } for (auto& config : diff.mAdded) { - if (!PreCheckPipelineConfig(config)) { - continue; - } - auto p = BuildPipeline(std::move(config)); if (!p) { LOG_WARNING(sLogger, @@ -246,35 +238,6 @@ void PipelineManager::DecreasePluginUsageCnt(const unordered_mapSendAlarm( - CATEGORY_CONFIG_ALARM, - "global singleton input plugin is already loaded: skip current object, config: " + config.mName, - config.mProject, - config.mLogstore, - config.mRegion); - ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, - ConfigFeedbackStatus::FAILED); - return false; - } - return true; -} - -bool PipelineManager::CheckIfGlobalSingletonInputLoaded(std::vector& inputConfig) { - for (const auto& input : inputConfig) { - auto inputType = (*input)["Type"].asString(); - if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(inputType) - && mPluginCntMap["inputs"][inputType] > 0) { - return true; - } - } - return false; -} - bool PipelineManager::CheckIfFileServerUpdated(const Json::Value& config) { string inputType = config["Type"].asString(); return inputType == "input_file" || inputType == "input_container_stdio"; diff --git a/core/pipeline/PipelineManager.h b/core/pipeline/PipelineManager.h index b4880b59ca..5dc1535f77 100644 --- a/core/pipeline/PipelineManager.h +++ b/core/pipeline/PipelineManager.h @@ -58,8 +58,6 @@ class PipelineManager { void DecreasePluginUsageCnt( const std::unordered_map>& statistics); void FlushAllBatch(); - bool PreCheckPipelineConfig(PipelineConfig& config); - bool CheckIfGlobalSingletonInputLoaded(std::vector& inputConfig); // TODO: 长期过渡使用 bool CheckIfFileServerUpdated(const Json::Value& config); diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index bbb81753f6..acb0d82b50 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -239,14 +239,6 @@ bool PluginRegistry::IsGlobalSingletonInputPlugin(const string& name) const { return IsGlobalSingleton(INPUT_PLUGIN, name); } -bool PluginRegistry::IsGlobalSingletonProcessorPlugin(const string& name) const { - return IsGlobalSingleton(PROCESSOR_PLUGIN, name); -} - -bool PluginRegistry::IsGlobalSingletonFlusherPlugin(const string& name) const { - return IsGlobalSingleton(FLUSHER_PLUGIN, name); -} - bool PluginRegistry::IsGlobalSingleton(PluginCat cat, const string& name) const { auto creatorEntry = mPluginDict.find(PluginKey(cat, name)); if (creatorEntry != mPluginDict.end()) { @@ -255,4 +247,4 @@ bool PluginRegistry::IsGlobalSingleton(PluginCat cat, const string& name) const return false; } -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index c08df532a7..b2928709d1 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -436,19 +436,22 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); APSARA_TEST_TRUE(!pipelineConfigDiff.first.IsEmpty()); APSARA_TEST_EQUAL(2U, pipelineConfigDiff.first.mAdded.size()); - APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[0].mName, LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); - APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[1].mName, "config1"); + APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[0].mName, "config1"); + APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[1].mName, + LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames().size(), 2); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], "config1"); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], + LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], "config1"); // 再次处理 pipelineconfig pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); APSARA_TEST_TRUE(pipelineConfigDiff.first.IsEmpty()); APSARA_TEST_TRUE(pipelineConfigDiff.first.mAdded.empty()); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames().size(), 2); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], "config1"); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], + LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], "config1"); APSARA_TEST_EQUAL(provider.mInstanceConfigInfoMap.size(), 2); diff --git a/core/unittest/config/PipelineConfigWatcherUnittest.cpp b/core/unittest/config/PipelineConfigWatcherUnittest.cpp index 0d301f34f7..e1fcd49c62 100644 --- a/core/unittest/config/PipelineConfigWatcherUnittest.cpp +++ b/core/unittest/config/PipelineConfigWatcherUnittest.cpp @@ -17,9 +17,9 @@ #include #include -#include "config/ConfigDiff.h" -#include "config/PipelineConfig.h" +#include "common/JsonUtil.h" #include "config/watcher/PipelineConfigWatcher.h" +#include "plugin/PluginRegistry.h" #include "unittest/Unittest.h" using namespace std; @@ -28,108 +28,272 @@ namespace logtail { class PipelineConfigWatcherUnittest : public testing::Test { public: - void TestSortPipelineConfigDiff() const; + void TestPreCheckConfig() const; + void TestPreCheckConfigWithInvalidConfig() const; + +protected: + static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); } + static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } private: - const string configName = "test"; }; -void PipelineConfigWatcherUnittest::TestSortPipelineConfigDiff() const { - PipelineConfigDiff diff; - auto configName1 = "test1"; - auto configJson1 = std::make_unique(); - (*configJson1)["name"] = configName1; - auto config1 = PipelineConfig(configName1, std::move(configJson1)); - config1.Parse(); - - auto configName2 = "test2"; - auto configJson2 = std::make_unique(); - (*configJson2)["name"] = configName2; - (*configJson2)["createTime"] = 1; - auto config2 = PipelineConfig(configName2, std::move(configJson2)); - config2.Parse(); - - auto configName3 = "test3"; - auto configJson3 = std::make_unique(); - (*configJson3)["name"] = configName3; - (*configJson3)["createTime"] = 2; - auto config3 = PipelineConfig(configName3, std::move(configJson3)); - config3.Parse(); - - auto configName4 = "test4"; - auto configJson4 = std::make_unique(); - (*configJson4)["name"] = configName4; - auto config4 = PipelineConfig(configName4, std::move(configJson4)); - config4.Parse(); - - auto configName5 = "test5"; - auto configJson5 = std::make_unique(); - (*configJson5)["name"] = configName5; - (*configJson5)["createTime"] = 1; - auto config5 = PipelineConfig(configName5, std::move(configJson5)); - config5.Parse(); - - diff.mAdded.push_back(std::move(config1)); - diff.mAdded.push_back(std::move(config2)); - diff.mAdded.push_back(std::move(config3)); - diff.mAdded.push_back(std::move(config4)); - diff.mAdded.push_back(std::move(config5)); - - auto configName6 = "test6"; - auto configJson6 = std::make_unique(); - (*configJson6)["name"] = configName6; - auto config6 = PipelineConfig(configName6, std::move(configJson6)); - config6.Parse(); - - auto configName7 = "test7"; - auto configJson7 = std::make_unique(); - (*configJson7)["name"] = configName7; - (*configJson7)["createTime"] = 1; - auto config7 = PipelineConfig(configName7, std::move(configJson7)); - config7.Parse(); - - auto configName8 = "test8"; - auto configJson8 = std::make_unique(); - (*configJson8)["name"] = configName8; - (*configJson8)["createTime"] = 2; - auto config8 = PipelineConfig(configName8, std::move(configJson8)); - config8.Parse(); - - auto configName9 = "test9"; - auto configJson9 = std::make_unique(); - (*configJson9)["name"] = configName9; - auto config9 = PipelineConfig(configName9, std::move(configJson9)); - config9.Parse(); - - auto configName10 = "test10"; - auto configJson10 = std::make_unique(); - (*configJson10)["name"] = configName10; - (*configJson10)["createTime"] = 1; - auto config10 = PipelineConfig(configName10, std::move(configJson10)); - config10.Parse(); - - diff.mModified.push_back(std::move(config6)); - diff.mModified.push_back(std::move(config7)); - diff.mModified.push_back(std::move(config8)); - diff.mModified.push_back(std::move(config9)); - diff.mModified.push_back(std::move(config10)); - - PipelineConfigWatcher::GetInstance()->SortPipelineConfigDiff(diff); - - APSARA_TEST_EQUAL(diff.mAdded[0].mName, configName2); - APSARA_TEST_EQUAL(diff.mAdded[1].mName, configName5); - APSARA_TEST_EQUAL(diff.mAdded[2].mName, configName3); - APSARA_TEST_EQUAL(diff.mAdded[3].mName, configName1); - APSARA_TEST_EQUAL(diff.mAdded[4].mName, configName4); - - APSARA_TEST_EQUAL(diff.mModified[0].mName, configName10); - APSARA_TEST_EQUAL(diff.mModified[1].mName, configName7); - APSARA_TEST_EQUAL(diff.mModified[2].mName, configName8); - APSARA_TEST_EQUAL(diff.mModified[3].mName, configName6); - APSARA_TEST_EQUAL(diff.mModified[4].mName, configName9); +void PipelineConfigWatcherUnittest::TestPreCheckConfig() const { + unique_ptr configJson; + string configStr, errorMsg; + std::unordered_map toBeDiffedConfigs; + std::unordered_map singletonConfigs; + + configStr = R"( + { + "inputs": [ + { + "Type": "input_file" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName1 = "test1"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); + + configStr = R"( + { + "inputs": [ + { + "Type": "input_file" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName2 = "test2"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName2, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(2, toBeDiffedConfigs.size()); + + // case: singleton input + configStr = R"( + { + "createTime": 123456, + "inputs": [ + { + "Type": "input_network_observer" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName3 = "test3"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName3, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); + APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName3); + + // case: compare by create time + configStr = R"( + { + "inputs": [ + { + "Type": "input_network_observer" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName4 = "test4"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName4, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); + APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName4); + + // case: compare by name + configStr = R"( + { + "inputs": [ + { + "Type": "input_network_observer" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName5 = "a-test5"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName5, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); + APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName5); + + // case: load config with singleton input and not singleton input + configStr = R"( + { + "inputs": [ + { + "Type": "input_file" + }, + { + "Type": "input_network_observer" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName6 = "ab-test6"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName6, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); + APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName5); + + // case: load config with different singleton input + configStr = R"( + { + "inputs": [ + { + "Type": "input_network_security" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName7 = "test7"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName7, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(4, toBeDiffedConfigs.size()); + APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_security"].second, configName7); + + // case: load config with two singleton input, one valid and one invalid + configStr = R"( + { + "inputs": [ + { + "Type": "input_network_observer" + }, + { + "Type": "input_network_security" + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName8 = "a-test8"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName8, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(4, toBeDiffedConfigs.size()); + APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName5); + APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_security"].second, configName7); +} + +void PipelineConfigWatcherUnittest::TestPreCheckConfigWithInvalidConfig() const { + { + unique_ptr configJson; + string configStr, errorMsg; + std::unordered_map toBeDiffedConfigs; + std::unordered_map singletonConfigs; + + configStr = R"( + { + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName1 = "test1"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); + } + { + unique_ptr configJson; + string configStr, errorMsg; + std::unordered_map toBeDiffedConfigs; + std::unordered_map singletonConfigs; + + configStr = R"( + { + "inputs": 1 + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName1 = "test1"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); + } + { + unique_ptr configJson; + string configStr, errorMsg; + std::unordered_map toBeDiffedConfigs; + std::unordered_map singletonConfigs; + + configStr = R"( + { + "inputs": [], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName1 = "test1"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); + } + { + unique_ptr configJson; + string configStr, errorMsg; + std::unordered_map toBeDiffedConfigs; + std::unordered_map singletonConfigs; + + configStr = R"( + { + "inputs": [ + { + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName1 = "test1"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); + } + { + unique_ptr configJson; + string configStr, errorMsg; + std::unordered_map toBeDiffedConfigs; + std::unordered_map singletonConfigs; + + configStr = R"( + { + "inputs": [ + { + "Type": 1 + } + ], + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + std::string configName1 = "test1"; + PipelineConfigWatcher::GetInstance()->PreCheckConfig( + configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); + APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); + } } -UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestSortPipelineConfigDiff) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestPreCheckConfig) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestPreCheckConfigWithInvalidConfig) } // namespace logtail diff --git a/core/unittest/pipeline/PipelineManagerUnittest.cpp b/core/unittest/pipeline/PipelineManagerUnittest.cpp index 31e258c72f..df0c771343 100644 --- a/core/unittest/pipeline/PipelineManagerUnittest.cpp +++ b/core/unittest/pipeline/PipelineManagerUnittest.cpp @@ -25,7 +25,6 @@ namespace logtail { class PipelineManagerUnittest : public testing::Test { public: void TestPipelineManagement() const; - void TestCheckIfGlobalSingletonInputLoaded() const; protected: static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); } @@ -41,35 +40,7 @@ void PipelineManagerUnittest::TestPipelineManagement() const { APSARA_TEST_EQUAL(nullptr, PipelineManager::GetInstance()->FindConfigByName("test3")); } -void PipelineManagerUnittest::TestCheckIfGlobalSingletonInputLoaded() const { - { // test not singleton input - auto inputConfig = Json::Value(); - inputConfig["Type"] = InputFile::sName; - std::vector inputConfigs = {&inputConfig}; - - PipelineManager::GetInstance()->mPluginCntMap["inputs"][InputFile::sName] = 1; - APSARA_TEST_EQUAL(false, PipelineManager::GetInstance()->CheckIfGlobalSingletonInputLoaded(inputConfigs)); - } - { // test singleton input not loaded - auto inputConfig = Json::Value(); - inputConfig["Type"] = InputNetworkSecurity::sName; - std::vector inputConfigs = {&inputConfig}; - - PipelineManager::GetInstance()->mPluginCntMap["inputs"][InputNetworkSecurity::sName] = 0; - APSARA_TEST_EQUAL(false, PipelineManager::GetInstance()->CheckIfGlobalSingletonInputLoaded(inputConfigs)); - } - { // test singleton input loaded - auto inputConfig = Json::Value(); - inputConfig["Type"] = InputNetworkSecurity::sName; - std::vector inputConfigs = {&inputConfig}; - - PipelineManager::GetInstance()->mPluginCntMap["inputs"][InputNetworkSecurity::sName] = 1; - APSARA_TEST_EQUAL(true, PipelineManager::GetInstance()->CheckIfGlobalSingletonInputLoaded(inputConfigs)); - } -} - UNIT_TEST_CASE(PipelineManagerUnittest, TestPipelineManagement) -UNIT_TEST_CASE(PipelineManagerUnittest, TestCheckIfGlobalSingletonInputLoaded) } // namespace logtail From 9dcbc5e7f89901edd75164a30e5fdc4908562798 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 3 Dec 2024 00:07:20 +0800 Subject: [PATCH 03/15] fix --- core/pipeline/PipelineManager.cpp | 1 - core/pipeline/plugin/PluginRegistry.h | 2 -- .../unittest/config/CommonConfigProviderUnittest.cpp | 12 ++++++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 0855869938..9fba29b8d1 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -96,7 +96,6 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { ConfigFeedbackStatus::FAILED); continue; } - LOG_INFO(sLogger, ("pipeline building for existing config succeeded", "stop the old pipeline and start the new one")("config", config.mName)); diff --git a/core/pipeline/plugin/PluginRegistry.h b/core/pipeline/plugin/PluginRegistry.h index 9a094e16fa..ddfa3a7bd8 100644 --- a/core/pipeline/plugin/PluginRegistry.h +++ b/core/pipeline/plugin/PluginRegistry.h @@ -55,8 +55,6 @@ class PluginRegistry { bool IsValidNativeProcessorPlugin(const std::string& name) const; bool IsValidNativeFlusherPlugin(const std::string& name) const; bool IsGlobalSingletonInputPlugin(const std::string& name) const; - bool IsGlobalSingletonProcessorPlugin(const std::string& name) const; - bool IsGlobalSingletonFlusherPlugin(const std::string& name) const; private: enum PluginCat { INPUT_PLUGIN, PROCESSOR_PLUGIN, FLUSHER_PLUGIN }; diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index b2928709d1..d7552886b4 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -436,22 +436,22 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); APSARA_TEST_TRUE(!pipelineConfigDiff.first.IsEmpty()); APSARA_TEST_EQUAL(2U, pipelineConfigDiff.first.mAdded.size()); - APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[0].mName, "config1"); - APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[1].mName, + APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[0].mName, LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); + APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[1].mName, "config1"); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames().size(), 2); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], "config1"); + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], "config1"); // 再次处理 pipelineconfig pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); APSARA_TEST_TRUE(pipelineConfigDiff.first.IsEmpty()); APSARA_TEST_TRUE(pipelineConfigDiff.first.mAdded.empty()); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames().size(), 2); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], "config1"); + APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()); - APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[1], "config1"); APSARA_TEST_EQUAL(provider.mInstanceConfigInfoMap.size(), 2); From 973cbe9459cf6c2d511089c00342fb292660c70e Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 3 Dec 2024 16:44:05 +0800 Subject: [PATCH 04/15] fix --- core/config/ConfigDiff.h | 2 + core/config/PipelineConfig.cpp | 7 +- core/config/PipelineConfig.h | 7 +- core/config/watcher/PipelineConfigWatcher.cpp | 261 ++++++++++-------- core/config/watcher/PipelineConfigWatcher.h | 37 ++- .../config/PipelineConfigWatcherUnittest.cpp | 219 +++------------ 6 files changed, 211 insertions(+), 322 deletions(-) diff --git a/core/config/ConfigDiff.h b/core/config/ConfigDiff.h index 38fa7ba00a..5bd9c8d205 100644 --- a/core/config/ConfigDiff.h +++ b/core/config/ConfigDiff.h @@ -38,4 +38,6 @@ using PipelineConfigDiff = ConfigDiff; using TaskConfigDiff = ConfigDiff; using InstanceConfigDiff = ConfigDiff; +enum ConfigDiffEnum { Added, Modified, Removed, Unchanged }; + } // namespace logtail diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index e0642b6ee1..1d86a97927 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -191,6 +191,9 @@ bool PipelineConfig::Parse() { mRegion); } const string pluginType = it->asString(); + if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(pluginType)) { + mSingletonInput = pluginType; + } if (i == 0) { if (PluginRegistry::GetInstance()->IsValidGoPlugin(pluginType)) { mHasGoInput = true; @@ -238,10 +241,10 @@ bool PipelineConfig::Parse() { } } // TODO: remove these special restrictions - if (hasFileInput && (*mDetail)["inputs"].size() > 1) { + if ((hasFileInput || !mSingletonInput.empty()) && (*mDetail)["inputs"].size() > 1) { PARAM_ERROR_RETURN(sLogger, alarm, - "more than 1 input_file or input_container_stdio plugin is given", + "more than 1 input_file or input_container_stdio or global singleton plugin is given", noModule, mName, mProject, diff --git a/core/config/PipelineConfig.h b/core/config/PipelineConfig.h index 7d845126e9..d4e896daf6 100644 --- a/core/config/PipelineConfig.h +++ b/core/config/PipelineConfig.h @@ -32,6 +32,7 @@ struct PipelineConfig { uint32_t mCreateTime = 0; const Json::Value* mGlobal = nullptr; std::vector mInputs; + std::string mSingletonInput; std::vector mProcessors; std::vector mAggregators; std::vector mFlushers; @@ -49,7 +50,8 @@ struct PipelineConfig { std::string mLogstore; std::string mRegion; - PipelineConfig(const std::string& name, std::unique_ptr&& detail) : mName(name), mDetail(std::move(detail)) {} + PipelineConfig(const std::string& name, std::unique_ptr&& detail) + : mName(name), mDetail(std::move(detail)) {} bool Parse(); @@ -67,7 +69,8 @@ struct PipelineConfig { // bool IsProcessRunnerInvolved() const { // // 长期过渡使用，待C++部分的时序聚合能力与Go持平后恢复下面的正式版 // return !(mHasGoInput && !mHasNativeProcessor); - // // return !(mHasGoInput && !mHasNativeProcessor && (mHasGoProcessor || (mHasGoFlusher && !mHasNativeFlusher))); + // // return !(mHasGoInput && !mHasNativeProcessor && (mHasGoProcessor || (mHasGoFlusher && + // !mHasNativeFlusher))); // } bool HasGoPlugin() const { return mHasGoFlusher || mHasGoProcessor || mHasGoInput; } diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 1751a55195..dcc0e63149 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -17,12 +17,10 @@ #include #include "common/FileSystemUtil.h" -#include "common/ParamExtractor.h" #include "config/ConfigUtil.h" #include "logger/Logger.h" #include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" -#include "pipeline/plugin/PluginRegistry.h" #include "task_pipeline/TaskPipelineManager.h" using namespace std; @@ -39,10 +37,11 @@ pair PipelineConfigWatcher::CheckConfigDiff( PipelineConfigDiff pDiff; TaskConfigDiff tDiff; unordered_set configSet; + SingletonConfigCache singletonCache; // inner configs - InsertInnerPipelines(pDiff, tDiff, configSet); + InsertInnerPipelines(pDiff, tDiff, configSet, singletonCache); // configs from file - InsertPipelines(pDiff, tDiff, configSet); + InsertPipelines(pDiff, tDiff, configSet, singletonCache); for (const auto& name : mPipelineManager->GetAllConfigNames()) { if (configSet.find(name) == configSet.end()) { @@ -81,12 +80,14 @@ pair PipelineConfigWatcher::CheckConfigDiff( } else { LOG_DEBUG(sLogger, ("config files scan done", "no task update")); } + return make_pair(std::move(pDiff), std::move(tDiff)); } void PipelineConfigWatcher::InsertInnerPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, - unordered_set& configSet) { + unordered_set& configSet, + SingletonConfigCache& singletonCache) { std::map innerPipelines; // self-monitor metric innerPipelines[LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()] @@ -117,7 +118,7 @@ void PipelineConfigWatcher::InsertInnerPipelines(PipelineConfigDiff& pDiff, LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", pipeline.first)); continue; } - if (!CheckAddedConfig(pipeline.first, std::move(detail), pDiff, tDiff)) { + if (!CheckAddedConfig(pipeline.first, std::move(detail), pDiff, tDiff, singletonCache)) { continue; } } else if (pipeline.second != iter->second) { @@ -158,7 +159,7 @@ void PipelineConfigWatcher::InsertInnerPipelines(PipelineConfigDiff& pDiff, } continue; } - if (!CheckModifiedConfig(pipeline.first, std::move(detail), pDiff, tDiff)) { + if (!CheckModifiedConfig(pipeline.first, std::move(detail), pDiff, tDiff, singletonCache)) { continue; } } else { @@ -169,9 +170,8 @@ void PipelineConfigWatcher::InsertInnerPipelines(PipelineConfigDiff& pDiff, void PipelineConfigWatcher::InsertPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, - std::unordered_set& configSet) { - std::unordered_map>> toBeDiffedConfigs; - std::unordered_map singletonConfigs; + std::unordered_set& configSet, + SingletonConfigCache& singletonCache) { for (const auto& dir : mSourceDir) { error_code ec; filesystem::file_status s = filesystem::status(dir, ec); @@ -214,74 +214,87 @@ void PipelineConfigWatcher::InsertPipelines(PipelineConfigDiff& pDiff, } configSet.insert(configName); - unique_ptr detail = make_unique(); - if (!LoadConfigDetailFromFile(path, *detail)) { - continue; + auto iter = mFileInfoMap.find(filepath); + uintmax_t size = filesystem::file_size(path, ec); + filesystem::file_time_type mTime = filesystem::last_write_time(path, ec); + if (iter == mFileInfoMap.end()) { + mFileInfoMap[filepath] = make_pair(size, mTime); + unique_ptr detail = make_unique(); + if (!LoadConfigDetailFromFile(path, *detail)) { + continue; + } + if (!IsConfigEnabled(configName, *detail)) { + LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", configName)); + continue; + } + if (!CheckAddedConfig(configName, std::move(detail), pDiff, tDiff, singletonCache)) { + continue; + } + } else if (iter->second.first != size || iter->second.second != mTime) { + // for config currently running, we leave it untouched if new config is invalid + mFileInfoMap[filepath] = make_pair(size, mTime); + unique_ptr detail = make_unique(); + if (!LoadConfigDetailFromFile(path, *detail)) { + continue; + } + if (!IsConfigEnabled(configName, *detail)) { + switch (GetConfigType(*detail)) { + case ConfigType::Pipeline: + if (mPipelineManager->FindConfigByName(configName)) { + pDiff.mRemoved.push_back(configName); + LOG_INFO(sLogger, + ("existing valid config modified and disabled", + "prepare to stop current running pipeline")("config", configName)); + } else { + LOG_INFO(sLogger, + ("existing invalid config modified and disabled", + "skip current object")("config", configName)); + } + break; + case ConfigType::Task: + if (mTaskPipelineManager->FindPipelineByName(configName)) { + tDiff.mRemoved.push_back(configName); + LOG_INFO(sLogger, + ("existing valid config modified and disabled", + "prepare to stop current running task")("config", configName)); + } else { + LOG_INFO(sLogger, + ("existing invalid config modified and disabled", + "skip current object")("config", configName)); + } + break; + } + continue; + } + if (!CheckModifiedConfig(configName, std::move(detail), pDiff, tDiff, singletonCache)) { + continue; + } + } else { + LOG_DEBUG(sLogger, ("existing config file unchanged", "skip current object")); } - PreCheckConfig(configName, path, std::move(detail), toBeDiffedConfigs, singletonConfigs); } } - - for (auto& config : toBeDiffedConfigs) { - error_code ec; - const filesystem::path& path = config.second.first; - const string& configName = path.stem().string(); - const string& filepath = path.string(); - - auto iter = mFileInfoMap.find(filepath); - uintmax_t size = filesystem::file_size(path, ec); - filesystem::file_time_type mTime = filesystem::last_write_time(path, ec); - unique_ptr detail = std::move(config.second.second); - if (iter == mFileInfoMap.end()) { - mFileInfoMap[filepath] = make_pair(size, mTime); - if (!CheckAddedConfig(configName, std::move(detail), pDiff, tDiff)) { - continue; - } - } else if (iter->second.first != size || iter->second.second != mTime) { - // for config currently running, we leave it untouched if new config is invalid - mFileInfoMap[filepath] = make_pair(size, mTime); - if (!IsConfigEnabled(configName, *detail)) { - switch (GetConfigType(*detail)) { - case ConfigType::Pipeline: - if (mPipelineManager->FindConfigByName(configName)) { - pDiff.mRemoved.push_back(configName); - LOG_INFO(sLogger, - ("existing valid config modified and disabled", - "prepare to stop current running pipeline")("config", configName)); - } else { - LOG_INFO(sLogger, - ("existing invalid config modified and disabled", - "skip current object")("config", configName)); - } - break; - case ConfigType::Task: - if (mTaskPipelineManager->FindPipelineByName(configName)) { - tDiff.mRemoved.push_back(configName); - LOG_INFO(sLogger, - ("existing valid config modified and disabled", - "prepare to stop current running task")("config", configName)); - } else { - LOG_INFO(sLogger, - ("existing invalid config modified and disabled", - "skip current object")("config", configName)); - } - break; - } - continue; - } - if (!CheckModifiedConfig(configName, std::move(detail), pDiff, tDiff)) { - continue; - } + for (const auto& [name, config] : singletonCache) { + if (config->diffEnum == ConfigDiffEnum::Added) { + pDiff.mAdded.push_back(std::move(config->config)); + LOG_INFO(sLogger, + ("new config found and passed topology check", "prepare to build pipeline")("config", + config->config.mName)); } else { - LOG_DEBUG(sLogger, ("existing config file unchanged", "skip current object")); + pDiff.mModified.push_back(std::move(config->config)); + LOG_INFO(sLogger, + ("existing invalid config modified and passed topology check", + "prepare to build pipeline")("config", config->config.mName)); } } } -bool PipelineConfigWatcher::CheckAddedConfig(const string& configName, - unique_ptr&& configDetail, - PipelineConfigDiff& pDiff, - TaskConfigDiff& tDiff) { +bool PipelineConfigWatcher::CheckAddedConfig( + const string& configName, + unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + std::unordered_map>& singletonCache) { switch (GetConfigType(*configDetail)) { case ConfigType::Pipeline: { PipelineConfig config(configName, std::move(configDetail)); @@ -295,9 +308,7 @@ bool PipelineConfigWatcher::CheckAddedConfig(const string& configName, config.mRegion); return false; } - pDiff.mAdded.push_back(std::move(config)); - LOG_INFO(sLogger, - ("new config found and passed topology check", "prepare to build pipeline")("config", configName)); + PushPipelineConfig(std::move(config), ConfigDiffEnum::Added, pDiff, singletonCache); break; } case ConfigType::Task: { @@ -317,10 +328,12 @@ bool PipelineConfigWatcher::CheckAddedConfig(const string& configName, return true; } -bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, - unique_ptr&& configDetail, - PipelineConfigDiff& pDiff, - TaskConfigDiff& tDiff) { +bool PipelineConfigWatcher::CheckModifiedConfig( + const string& configName, + unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + std::unordered_map>& singletonCache) { switch (GetConfigType(*configDetail)) { case ConfigType::Pipeline: { shared_ptr p = mPipelineManager->FindConfigByName(configName); @@ -339,10 +352,7 @@ bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, config.mRegion); return false; } - pDiff.mAdded.push_back(std::move(config)); - LOG_INFO(sLogger, - ("existing invalid config modified and passed topology check", - "prepare to build pipeline")("config", configName)); + PushPipelineConfig(std::move(config), ConfigDiffEnum::Modified, pDiff, singletonCache); } else if (*configDetail != p->GetConfig()) { PipelineConfig config(configName, std::move(configDetail)); if (!config.Parse()) { @@ -358,10 +368,7 @@ bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, config.mRegion); return false; } - pDiff.mModified.push_back(std::move(config)); - LOG_INFO(sLogger, - ("existing valid config modified and passed topology check", - "prepare to rebuild pipeline")("config", configName)); + PushPipelineConfig(std::move(config), ConfigDiffEnum::Modified, pDiff, singletonCache); } else { LOG_DEBUG(sLogger, ("existing valid config file modified, but no change found", "skip current object")); } @@ -410,50 +417,64 @@ bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, return true; } -bool PipelineConfigWatcher::PreCheckConfig(const string& configName, - const filesystem::path& path, - unique_ptr&& configDetail, - std::unordered_map& toBeDiffedConfigs, - std::unordered_map& singletonConfigs) { - // check enabled - if (!IsConfigEnabled(configName, *configDetail)) { - LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", configName)); - return false; - } - // check singleton input, and only check this, others will be checked in Parse() - std::string errorMsg; - uint32_t createTime = 0; - if (!GetOptionalUIntParam(*configDetail, "createTime", createTime, errorMsg)) { - createTime = 0; - } - - std::vector inputTypes; - GetAllInputTypes(*configDetail, inputTypes); - for (const auto& inputType : inputTypes) { - if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(inputType)) { - auto it = singletonConfigs.find(inputType); - if (it != singletonConfigs.end()) { - if (it->second.first < createTime - || (it->second.first == createTime && it->second.second < configName)) { +void PipelineConfigWatcher::PushPipelineConfig(PipelineConfig&& config, + ConfigDiffEnum diffEnum, + PipelineConfigDiff& pDiff, + SingletonConfigCache& singletonCache) { + // singleton input + if (!config.mSingletonInput.empty()) { + if (diffEnum == ConfigDiffEnum::Added || diffEnum == ConfigDiffEnum::Modified) { + auto it = singletonCache.find(config.mSingletonInput); + if (it != singletonCache.end()) { + if (it->second->config.mCreateTime < config.mCreateTime + || (it->second->config.mCreateTime == config.mCreateTime + && it->second->config.mName < config.mName)) { LOG_WARNING(sLogger, ("global singleton plugin found, but another older config or smaller name config " "already exists", - "skip current object")("config", configName)); - return false; + "skip current object")("config", config.mName)("inputType", config.mSingletonInput)); + return; + } + if (mPipelineManager->FindConfigByName(it->second->config.mName)) { + pDiff.mRemoved.push_back(it->second->config.mName); + LOG_WARNING( + sLogger, + ("existing valid config with global singleton plugin, but another older config or smaller " + "name config found", + "prepare to stop current running pipeline")("config", it->second->config.mName)); + } else { + LOG_WARNING(sLogger, + ("global singleton plugin found, but another older config or smaller name config " + "already exists", + "skip current object")("config", it->second->config.mName)("inputType", + config.mSingletonInput)); } - toBeDiffedConfigs.erase(it->second.second); } + auto pipelineConfig = make_shared(std::move(config), diffEnum); + singletonCache[pipelineConfig->config.mSingletonInput] = pipelineConfig; + return; + } else { + LOG_ERROR(sLogger, ("should not reach here", "invalid diff enum")("diff", diffEnum)); } } - for (const auto& inputType : inputTypes) { - if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(inputType)) { - singletonConfigs[inputType] = make_pair(createTime, configName); - } + // no singleton input + switch (diffEnum) { + case ConfigDiffEnum::Added: + pDiff.mAdded.push_back(std::move(config)); + LOG_INFO( + sLogger, + ("new config found and passed topology check", "prepare to build pipeline")("config", config.mName)); + break; + case ConfigDiffEnum::Modified: + pDiff.mModified.push_back(std::move(config)); + LOG_INFO(sLogger, + ("existing invalid config modified and passed topology check", + "prepare to build pipeline")("config", config.mName)); + break; + default: + break; } - toBeDiffedConfigs[configName] = make_pair(path, std::move(configDetail)); - return true; } - } // namespace logtail diff --git a/core/config/watcher/PipelineConfigWatcher.h b/core/config/watcher/PipelineConfigWatcher.h index cb13e93b5c..53caafd681 100644 --- a/core/config/watcher/PipelineConfigWatcher.h +++ b/core/config/watcher/PipelineConfigWatcher.h @@ -26,12 +26,17 @@ namespace logtail { -using ConfigWithPath = std::pair>; -using ConfigPriority = std::pair; - class PipelineManager; class TaskPipelineManager; +struct PipelineConfigWithDiffInfo { + PipelineConfig config; + ConfigDiffEnum diffEnum; + PipelineConfigWithDiffInfo(PipelineConfig&& config, ConfigDiffEnum diffEnum) + : config(std::move(config)), diffEnum(diffEnum) {} +}; +using SingletonConfigCache = std::unordered_map>; + class PipelineConfigWatcher : public ConfigWatcher { public: PipelineConfigWatcher(const PipelineConfigWatcher&) = delete; @@ -52,22 +57,28 @@ class PipelineConfigWatcher : public ConfigWatcher { PipelineConfigWatcher(); ~PipelineConfigWatcher() = default; - void - InsertInnerPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set& configSet); - void InsertPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set& configSet); + void InsertInnerPipelines(PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + std::unordered_set& configSet, + SingletonConfigCache& singletonCache); + void InsertPipelines(PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + std::unordered_set& configSet, + SingletonConfigCache& singletonCache); bool CheckAddedConfig(const std::string& configName, std::unique_ptr&& configDetail, PipelineConfigDiff& pDiff, - TaskConfigDiff& tDiff); + TaskConfigDiff& tDiff, + SingletonConfigCache& singletonCache); bool CheckModifiedConfig(const std::string& configName, std::unique_ptr&& configDetail, PipelineConfigDiff& pDiff, - TaskConfigDiff& tDiff); - bool PreCheckConfig(const std::string& configName, - const std::filesystem::path& path, - std::unique_ptr&& configDetail, - std::unordered_map& toBeDiffedConfigs, - std::unordered_map& singletonConfigs); + TaskConfigDiff& tDiff, + SingletonConfigCache& singletonCache); + void PushPipelineConfig(PipelineConfig&& config, + ConfigDiffEnum diffEnum, + PipelineConfigDiff& pDiff, + SingletonConfigCache& singletonCache); const PipelineManager* mPipelineManager = nullptr; const TaskPipelineManager* mTaskPipelineManager = nullptr; diff --git a/core/unittest/config/PipelineConfigWatcherUnittest.cpp b/core/unittest/config/PipelineConfigWatcherUnittest.cpp index e1fcd49c62..dd27c533cb 100644 --- a/core/unittest/config/PipelineConfigWatcherUnittest.cpp +++ b/core/unittest/config/PipelineConfigWatcherUnittest.cpp @@ -28,8 +28,7 @@ namespace logtail { class PipelineConfigWatcherUnittest : public testing::Test { public: - void TestPreCheckConfig() const; - void TestPreCheckConfigWithInvalidConfig() const; + void TestPushPipelineConfig() const; protected: static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); } @@ -38,11 +37,11 @@ class PipelineConfigWatcherUnittest : public testing::Test { private: }; -void PipelineConfigWatcherUnittest::TestPreCheckConfig() const { +void PipelineConfigWatcherUnittest::TestPushPipelineConfig() const { + PipelineConfigDiff pDiff; unique_ptr configJson; string configStr, errorMsg; - std::unordered_map toBeDiffedConfigs; - std::unordered_map singletonConfigs; + SingletonConfigCache singletonCache; configStr = R"( { @@ -56,9 +55,11 @@ void PipelineConfigWatcherUnittest::TestPreCheckConfig() const { configJson.reset(new Json::Value()); APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); std::string configName1 = "test1"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); + PipelineConfig config1(configName1, std::move(configJson)); + config1.Parse(); + PipelineConfigWatcher::GetInstance()->PushPipelineConfig( + std::move(config1), ConfigDiffEnum::Added, pDiff, singletonCache); + APSARA_TEST_EQUAL_FATAL(1, pDiff.mAdded.size()); configStr = R"( { @@ -72,9 +73,11 @@ void PipelineConfigWatcherUnittest::TestPreCheckConfig() const { configJson.reset(new Json::Value()); APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); std::string configName2 = "test2"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName2, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(2, toBeDiffedConfigs.size()); + PipelineConfig config2(configName2, std::move(configJson)); + config2.Parse(); + PipelineConfigWatcher::GetInstance()->PushPipelineConfig( + std::move(config2), ConfigDiffEnum::Added, pDiff, singletonCache); + APSARA_TEST_EQUAL_FATAL(2, pDiff.mAdded.size()); // case: singleton input configStr = R"( @@ -90,10 +93,12 @@ void PipelineConfigWatcherUnittest::TestPreCheckConfig() const { configJson.reset(new Json::Value()); APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); std::string configName3 = "test3"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName3, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); - APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName3); + PipelineConfig config3(configName3, std::move(configJson)); + config3.Parse(); + APSARA_TEST_EQUAL_FATAL("input_network_observer", config3.mSingletonInput); + PipelineConfigWatcher::GetInstance()->PushPipelineConfig( + std::move(config3), ConfigDiffEnum::Added, pDiff, singletonCache); + APSARA_TEST_EQUAL_FATAL(singletonCache["input_network_observer"]->config.mName, configName3); // case: compare by create time configStr = R"( @@ -108,10 +113,12 @@ void PipelineConfigWatcherUnittest::TestPreCheckConfig() const { configJson.reset(new Json::Value()); APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); std::string configName4 = "test4"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName4, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); - APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName4); + PipelineConfig config4(configName4, std::move(configJson)); + config4.Parse(); + APSARA_TEST_EQUAL_FATAL("input_network_observer", config4.mSingletonInput); + PipelineConfigWatcher::GetInstance()->PushPipelineConfig( + std::move(config4), ConfigDiffEnum::Added, pDiff, singletonCache); + APSARA_TEST_EQUAL_FATAL(singletonCache["input_network_observer"]->config.mName, configName4); // case: compare by name configStr = R"( @@ -125,175 +132,17 @@ void PipelineConfigWatcherUnittest::TestPreCheckConfig() const { )"; configJson.reset(new Json::Value()); APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName5 = "a-test5"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName5, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); - APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName5); - - // case: load config with singleton input and not singleton input - configStr = R"( - { - "inputs": [ - { - "Type": "input_file" - }, - { - "Type": "input_network_observer" - } - ], - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName6 = "ab-test6"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName6, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(3, toBeDiffedConfigs.size()); - APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName5); - - // case: load config with different singleton input - configStr = R"( - { - "inputs": [ - { - "Type": "input_network_security" - } - ], - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName7 = "test7"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName7, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(4, toBeDiffedConfigs.size()); - APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_security"].second, configName7); - - // case: load config with two singleton input, one valid and one invalid - configStr = R"( - { - "inputs": [ - { - "Type": "input_network_observer" - }, - { - "Type": "input_network_security" - } - ], - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName8 = "a-test8"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName8, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(4, toBeDiffedConfigs.size()); - APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_observer"].second, configName5); - APSARA_TEST_EQUAL_FATAL(singletonConfigs["input_network_security"].second, configName7); + std::string configName5 = "z-test5"; + PipelineConfig config5(configName5, std::move(configJson)); + config5.Parse(); + APSARA_TEST_EQUAL_FATAL("input_network_observer", config5.mSingletonInput); + PipelineConfigWatcher::GetInstance()->PushPipelineConfig( + std::move(config5), ConfigDiffEnum::Added, pDiff, singletonCache); + APSARA_TEST_EQUAL_FATAL(singletonCache["input_network_observer"]->config.mName, configName4); } -void PipelineConfigWatcherUnittest::TestPreCheckConfigWithInvalidConfig() const { - { - unique_ptr configJson; - string configStr, errorMsg; - std::unordered_map toBeDiffedConfigs; - std::unordered_map singletonConfigs; - - configStr = R"( - { - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName1 = "test1"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); - } - { - unique_ptr configJson; - string configStr, errorMsg; - std::unordered_map toBeDiffedConfigs; - std::unordered_map singletonConfigs; - - configStr = R"( - { - "inputs": 1 - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName1 = "test1"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); - } - { - unique_ptr configJson; - string configStr, errorMsg; - std::unordered_map toBeDiffedConfigs; - std::unordered_map singletonConfigs; - - configStr = R"( - { - "inputs": [], - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName1 = "test1"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); - } - { - unique_ptr configJson; - string configStr, errorMsg; - std::unordered_map toBeDiffedConfigs; - std::unordered_map singletonConfigs; - - configStr = R"( - { - "inputs": [ - { - } - ], - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName1 = "test1"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); - } - { - unique_ptr configJson; - string configStr, errorMsg; - std::unordered_map toBeDiffedConfigs; - std::unordered_map singletonConfigs; - - configStr = R"( - { - "inputs": [ - { - "Type": 1 - } - ], - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName1 = "test1"; - PipelineConfigWatcher::GetInstance()->PreCheckConfig( - configName1, std::filesystem::path(), std::move(configJson), toBeDiffedConfigs, singletonConfigs); - APSARA_TEST_EQUAL_FATAL(1, toBeDiffedConfigs.size()); - } -} -UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestPreCheckConfig) -UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestPreCheckConfigWithInvalidConfig) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestPushPipelineConfig) } // namespace logtail From fa4d20f80fb542fba05a4f198413b6e40f44b61e Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 3 Dec 2024 17:05:15 +0800 Subject: [PATCH 05/15] fix --- core/config/PipelineConfig.h | 6 ++---- core/pipeline/PipelineManager.cpp | 5 +++-- core/pipeline/plugin/creator/PluginCreator.h | 2 +- core/pipeline/plugin/instance/InputInstance.h | 8 +++++--- core/plugin/input/InputContainerStdio.h | 1 + core/plugin/input/InputFile.h | 1 + core/plugin/input/InputFileSecurity.h | 2 +- core/plugin/input/InputInternalMetrics.h | 1 - core/plugin/input/InputNetworkObserver.h | 1 + 9 files changed, 15 insertions(+), 12 deletions(-) diff --git a/core/config/PipelineConfig.h b/core/config/PipelineConfig.h index d4e896daf6..458ef3db43 100644 --- a/core/config/PipelineConfig.h +++ b/core/config/PipelineConfig.h @@ -50,8 +50,7 @@ struct PipelineConfig { std::string mLogstore; std::string mRegion; - PipelineConfig(const std::string& name, std::unique_ptr&& detail) - : mName(name), mDetail(std::move(detail)) {} + PipelineConfig(const std::string& name, std::unique_ptr&& detail) : mName(name), mDetail(std::move(detail)) {} bool Parse(); @@ -69,8 +68,7 @@ struct PipelineConfig { // bool IsProcessRunnerInvolved() const { // // 长期过渡使用，待C++部分的时序聚合能力与Go持平后恢复下面的正式版 // return !(mHasGoInput && !mHasNativeProcessor); - // // return !(mHasGoInput && !mHasNativeProcessor && (mHasGoProcessor || (mHasGoFlusher && - // !mHasNativeFlusher))); + // // return !(mHasGoInput && !mHasNativeProcessor && (mHasGoProcessor || (mHasGoFlusher && !mHasNativeFlusher))); // } bool HasGoPlugin() const { return mHasGoFlusher || mHasGoProcessor || mHasGoInput; } diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 9fba29b8d1..9076047293 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -29,7 +29,8 @@ #include "shennong/ShennongManager.h" #endif #include "config/feedbacker/ConfigFeedbackReceiver.h" -#include "plugin/PluginRegistry.h" +#include "pipeline/queue/ProcessQueueManager.h" +#include "pipeline/queue/QueueKeyManager.h" using namespace std; @@ -39,7 +40,7 @@ PipelineManager::PipelineManager() : mInputRunners({ PrometheusInputRunner::GetInstance(), #if defined(__linux__) && !defined(__ANDROID__) - ebpf::eBPFServer::GetInstance(), + ebpf::eBPFServer::GetInstance(), #endif }) { } diff --git a/core/pipeline/plugin/creator/PluginCreator.h b/core/pipeline/plugin/creator/PluginCreator.h index a61d492cdb..6888927186 100644 --- a/core/pipeline/plugin/creator/PluginCreator.h +++ b/core/pipeline/plugin/creator/PluginCreator.h @@ -16,8 +16,8 @@ #pragma once -#include #include +#include #include "pipeline/plugin/instance/PluginInstance.h" diff --git a/core/pipeline/plugin/instance/InputInstance.h b/core/pipeline/plugin/instance/InputInstance.h index 8505ecb130..139f5b554b 100644 --- a/core/pipeline/plugin/instance/InputInstance.h +++ b/core/pipeline/plugin/instance/InputInstance.h @@ -28,12 +28,14 @@ namespace logtail { class InputInstance : public PluginInstance { public: - InputInstance(Input* plugin, const PluginInstance::PluginMeta& pluginMeta) - : PluginInstance(pluginMeta), mPlugin(plugin) {} + InputInstance(Input* plugin, const PluginInstance::PluginMeta& pluginMeta) : PluginInstance(pluginMeta), mPlugin(plugin) {} const std::string& Name() const override { return mPlugin->Name(); } - bool Init(const Json::Value& config, PipelineContext& context, size_t inputIdx, Json::Value& optionalGoPipeline); + bool Init(const Json::Value& config, + PipelineContext& context, + size_t inputIdx, + Json::Value& optionalGoPipeline); bool Start() { return mPlugin->Start(); } bool Stop(bool isPipelineRemoving) { return mPlugin->Stop(isPipelineRemoving); } bool SupportAck() const { return mPlugin->SupportAck(); } diff --git a/core/plugin/input/InputContainerStdio.h b/core/plugin/input/InputContainerStdio.h index 72350ce69e..f5955b4fa6 100644 --- a/core/plugin/input/InputContainerStdio.h +++ b/core/plugin/input/InputContainerStdio.h @@ -23,6 +23,7 @@ #include "file_server/MultilineOptions.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" #include "pipeline/plugin/interface/Input.h" +#include "file_server/reader/FileReaderOptions.h" namespace logtail { diff --git a/core/plugin/input/InputFile.h b/core/plugin/input/InputFile.h index 37ab51a8bb..4d966f1e67 100644 --- a/core/plugin/input/InputFile.h +++ b/core/plugin/input/InputFile.h @@ -23,6 +23,7 @@ #include "file_server/MultilineOptions.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" #include "pipeline/plugin/interface/Input.h" +#include "file_server/reader/FileReaderOptions.h" namespace logtail { diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index 2860c121a5..61b1573dd1 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -19,6 +19,7 @@ #include #include "ebpf/config.h" +#include "pipeline/plugin/interface/Input.h" #include "ebpf/eBPFServer.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" @@ -33,7 +34,6 @@ class InputFileSecurity : public Input { bool Start() override; bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return false; } - ebpf::SecurityOptions mSecurityOptions; PluginMetricManagerPtr mPluginMgr; }; diff --git a/core/plugin/input/InputInternalMetrics.h b/core/plugin/input/InputInternalMetrics.h index c59cf89608..694edf85af 100644 --- a/core/plugin/input/InputInternalMetrics.h +++ b/core/plugin/input/InputInternalMetrics.h @@ -30,7 +30,6 @@ class InputInternalMetrics : public Input { bool Start() override; bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return true; } - private: SelfMonitorMetricRules mSelfMonitorMetricRules; }; diff --git a/core/plugin/input/InputNetworkObserver.h b/core/plugin/input/InputNetworkObserver.h index 3468f64d82..33bf4ab0cc 100644 --- a/core/plugin/input/InputNetworkObserver.h +++ b/core/plugin/input/InputNetworkObserver.h @@ -19,6 +19,7 @@ #include #include "ebpf/config.h" +#include "pipeline/plugin/interface/Input.h" #include "ebpf/include/export.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" From 39028eb4a0e4e7ca56a589caa40b6edbb1f3cc78 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 3 Dec 2024 17:07:02 +0800 Subject: [PATCH 06/15] fix --- core/pipeline/PipelineManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 9076047293..57abcde874 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -40,7 +40,7 @@ PipelineManager::PipelineManager() : mInputRunners({ PrometheusInputRunner::GetInstance(), #if defined(__linux__) && !defined(__ANDROID__) - ebpf::eBPFServer::GetInstance(), + ebpf::eBPFServer::GetInstance(), #endif }) { } From cbebb481eaf2c0e2ff43801a7676f024e4b96083 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 3 Dec 2024 17:08:38 +0800 Subject: [PATCH 07/15] fix --- core/plugin/input/InputFileSecurity.h | 1 + core/unittest/pipeline/PipelineManagerUnittest.cpp | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index 61b1573dd1..79a94e82a4 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -34,6 +34,7 @@ class InputFileSecurity : public Input { bool Start() override; bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return false; } + ebpf::SecurityOptions mSecurityOptions; PluginMetricManagerPtr mPluginMgr; }; diff --git a/core/unittest/pipeline/PipelineManagerUnittest.cpp b/core/unittest/pipeline/PipelineManagerUnittest.cpp index df0c771343..29c56da393 100644 --- a/core/unittest/pipeline/PipelineManagerUnittest.cpp +++ b/core/unittest/pipeline/PipelineManagerUnittest.cpp @@ -14,8 +14,6 @@ #include "pipeline/Pipeline.h" #include "pipeline/PipelineManager.h" -#include "pipeline/plugin/PluginRegistry.h" -#include "plugin/input/InputNetworkSecurity.h" #include "unittest/Unittest.h" using namespace std; @@ -25,10 +23,6 @@ namespace logtail { class PipelineManagerUnittest : public testing::Test { public: void TestPipelineManagement() const; - -protected: - static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); } - static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } }; void PipelineManagerUnittest::TestPipelineManagement() const { From 7ec3d0766957c7e460d7518167522b2b644681a1 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 3 Dec 2024 17:11:28 +0800 Subject: [PATCH 08/15] fix --- core/plugin/input/InputFileSecurity.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index 79a94e82a4..fea0b459fc 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -21,7 +21,7 @@ #include "ebpf/config.h" #include "pipeline/plugin/interface/Input.h" #include "ebpf/eBPFServer.h" -#include "monitor/metric_models/ReentrantMetricsRecord.h" +#include "monitor/PluginMetricManager.h" namespace logtail { @@ -34,7 +34,7 @@ class InputFileSecurity : public Input { bool Start() override; bool Stop(bool isPipelineRemoving) override; bool SupportAck() const override { return false; } - + ebpf::SecurityOptions mSecurityOptions; PluginMetricManagerPtr mPluginMgr; }; From 753d7c021c8f2094a9605934f5fd9e05e333da60 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 3 Dec 2024 17:12:34 +0800 Subject: [PATCH 09/15] fix --- core/plugin/input/InputFileSecurity.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/plugin/input/InputFileSecurity.h b/core/plugin/input/InputFileSecurity.h index fea0b459fc..ebff74842f 100644 --- a/core/plugin/input/InputFileSecurity.h +++ b/core/plugin/input/InputFileSecurity.h @@ -21,7 +21,7 @@ #include "ebpf/config.h" #include "pipeline/plugin/interface/Input.h" #include "ebpf/eBPFServer.h" -#include "monitor/PluginMetricManager.h" +#include "monitor/metric_models/ReentrantMetricsRecord.h" namespace logtail { From a036a707ac0a05a5c17c81f3f7fe37045d9c2481 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 4 Dec 2024 00:26:56 +0800 Subject: [PATCH 10/15] fix --- core/config/PipelineConfig.h | 6 ++- core/config/watcher/PipelineConfigWatcher.cpp | 50 ++++++++++--------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/core/config/PipelineConfig.h b/core/config/PipelineConfig.h index 458ef3db43..d4e896daf6 100644 --- a/core/config/PipelineConfig.h +++ b/core/config/PipelineConfig.h @@ -50,7 +50,8 @@ struct PipelineConfig { std::string mLogstore; std::string mRegion; - PipelineConfig(const std::string& name, std::unique_ptr&& detail) : mName(name), mDetail(std::move(detail)) {} + PipelineConfig(const std::string& name, std::unique_ptr&& detail) + : mName(name), mDetail(std::move(detail)) {} bool Parse(); @@ -68,7 +69,8 @@ struct PipelineConfig { // bool IsProcessRunnerInvolved() const { // // 长期过渡使用，待C++部分的时序聚合能力与Go持平后恢复下面的正式版 // return !(mHasGoInput && !mHasNativeProcessor); - // // return !(mHasGoInput && !mHasNativeProcessor && (mHasGoProcessor || (mHasGoFlusher && !mHasNativeFlusher))); + // // return !(mHasGoInput && !mHasNativeProcessor && (mHasGoProcessor || (mHasGoFlusher && + // !mHasNativeFlusher))); // } bool HasGoPlugin() const { return mHasGoFlusher || mHasGoProcessor || mHasGoInput; } diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 58124abb4f..2d48dd4712 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -44,9 +44,22 @@ pair PipelineConfigWatcher::CheckConfigDiff( SingletonConfigCache singletonCache; // builtin pipeline configs InsertBuiltInPipelines(pDiff, tDiff, configSet, singletonCache); - // file pipeline configs + // file pipeline configs InsertPipelines(pDiff, tDiff, configSet, singletonCache); + for (const auto& [name, config] : singletonCache) { + if (config->diffEnum == ConfigDiffEnum::Added) { + pDiff.mAdded.push_back(std::move(config->config)); + LOG_INFO(sLogger, + ("new config found and passed topology check", "prepare to build pipeline")("config", + config->config.mName)); + } else { + pDiff.mModified.push_back(std::move(config->config)); + LOG_INFO(sLogger, + ("existing invalid config modified and passed topology check", + "prepare to build pipeline")("config", config->config.mName)); + } + } for (const auto& name : mPipelineManager->GetAllConfigNames()) { if (configSet.find(name) == configSet.end()) { pDiff.mRemoved.push_back(name); @@ -89,9 +102,9 @@ pair PipelineConfigWatcher::CheckConfigDiff( } void PipelineConfigWatcher::InsertBuiltInPipelines(PipelineConfigDiff& pDiff, - TaskConfigDiff& tDiff, - unordered_set& configSet, - SingletonConfigCache& singletonCache) { + TaskConfigDiff& tDiff, + unordered_set& configSet, + SingletonConfigCache& singletonCache) { #ifdef __ENTERPRISE__ const std::map& builtInPipelines = EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs(); @@ -281,19 +294,6 @@ void PipelineConfigWatcher::InsertPipelines(PipelineConfigDiff& pDiff, } } } - for (const auto& [name, config] : singletonCache) { - if (config->diffEnum == ConfigDiffEnum::Added) { - pDiff.mAdded.push_back(std::move(config->config)); - LOG_INFO(sLogger, - ("new config found and passed topology check", "prepare to build pipeline")("config", - config->config.mName)); - } else { - pDiff.mModified.push_back(std::move(config->config)); - LOG_INFO(sLogger, - ("existing invalid config modified and passed topology check", - "prepare to build pipeline")("config", config->config.mName)); - } - } } bool PipelineConfigWatcher::CheckAddedConfig( @@ -316,6 +316,8 @@ bool PipelineConfigWatcher::CheckAddedConfig( return false; } PushPipelineConfig(std::move(config), ConfigDiffEnum::Added, pDiff, singletonCache); + LOG_INFO(sLogger, + ("new config found and passed topology check", "prepare to build pipeline")("config", configName)); break; } case ConfigType::Task: { @@ -359,7 +361,10 @@ bool PipelineConfigWatcher::CheckModifiedConfig( config.mRegion); return false; } - PushPipelineConfig(std::move(config), ConfigDiffEnum::Modified, pDiff, singletonCache); + PushPipelineConfig(std::move(config), ConfigDiffEnum::Added, pDiff, singletonCache); + LOG_INFO(sLogger, + ("existing invalid config modified and passed topology check", + "prepare to build pipeline")("config", configName)); } else if (*configDetail != p->GetConfig()) { PipelineConfig config(configName, std::move(configDetail)); if (!config.Parse()) { @@ -376,6 +381,9 @@ bool PipelineConfigWatcher::CheckModifiedConfig( return false; } PushPipelineConfig(std::move(config), ConfigDiffEnum::Modified, pDiff, singletonCache); + LOG_INFO(sLogger, + ("existing valid config modified and passed topology check", + "prepare to rebuild pipeline")("config", configName)); } else { LOG_DEBUG(sLogger, ("existing valid config file modified, but no change found", "skip current object")); } @@ -469,15 +477,9 @@ void PipelineConfigWatcher::PushPipelineConfig(PipelineConfig&& config, switch (diffEnum) { case ConfigDiffEnum::Added: pDiff.mAdded.push_back(std::move(config)); - LOG_INFO( - sLogger, - ("new config found and passed topology check", "prepare to build pipeline")("config", config.mName)); break; case ConfigDiffEnum::Modified: pDiff.mModified.push_back(std::move(config)); - LOG_INFO(sLogger, - ("existing invalid config modified and passed topology check", - "prepare to build pipeline")("config", config.mName)); break; default: break; From 0c41858afa733d3eed7361a736319dc099225026 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 5 Dec 2024 14:00:56 +0800 Subject: [PATCH 11/15] fix --- core/config/ConfigUtil.cpp | 16 - core/config/watcher/PipelineConfigWatcher.cpp | 152 ++- core/config/watcher/PipelineConfigWatcher.h | 8 +- core/pipeline/Pipeline.cpp | 1 + core/pipeline/Pipeline.h | 2 + core/unittest/config/ConfigUpdateUnittest.cpp | 37 +- .../config/PipelineConfigWatcherUnittest.cpp | 1148 +++++++++++++++-- core/unittest/config/PipelineManagerMock.h | 58 + 8 files changed, 1239 insertions(+), 183 deletions(-) create mode 100644 core/unittest/config/PipelineManagerMock.h diff --git a/core/config/ConfigUtil.cpp b/core/config/ConfigUtil.cpp index f60c90cf3b..2c3975f4c7 100644 --- a/core/config/ConfigUtil.cpp +++ b/core/config/ConfigUtil.cpp @@ -80,22 +80,6 @@ bool IsConfigEnabled(const string& name, const Json::Value& detail) { return true; } -void GetAllInputTypes(const Json::Value& detail, std::vector& inputTypes) { - const char* key = "inputs"; - const Json::Value* inputs = detail.find(key, key + strlen(key)); - if (inputs == nullptr || !inputs->isArray()) { - return; - } - for (const Json::Value& input : *inputs) { - const char* typeKey = "Type"; - const Json::Value* type = input.find(typeKey, typeKey + strlen(typeKey)); - if (type == nullptr || !type->isString()) { - continue; - } - inputTypes.push_back(type->asString()); - } -} - ConfigType GetConfigType(const Json::Value& detail) { return detail.isMember("task") ? ConfigType::Task : ConfigType::Pipeline; } diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 2d48dd4712..13740e4401 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -47,19 +47,7 @@ pair PipelineConfigWatcher::CheckConfigDiff( // file pipeline configs InsertPipelines(pDiff, tDiff, configSet, singletonCache); - for (const auto& [name, config] : singletonCache) { - if (config->diffEnum == ConfigDiffEnum::Added) { - pDiff.mAdded.push_back(std::move(config->config)); - LOG_INFO(sLogger, - ("new config found and passed topology check", "prepare to build pipeline")("config", - config->config.mName)); - } else { - pDiff.mModified.push_back(std::move(config->config)); - LOG_INFO(sLogger, - ("existing invalid config modified and passed topology check", - "prepare to build pipeline")("config", config->config.mName)); - } - } + CheckSingletonInput(pDiff, singletonCache); for (const auto& name : mPipelineManager->GetAllConfigNames()) { if (configSet.find(name) == configSet.end()) { pDiff.mRemoved.push_back(name); @@ -291,17 +279,17 @@ void PipelineConfigWatcher::InsertPipelines(PipelineConfigDiff& pDiff, } } else { LOG_DEBUG(sLogger, ("existing config file unchanged", "skip current object")); + CheckUnchangedConfig(configName, path, pDiff, tDiff, singletonCache); } } } } -bool PipelineConfigWatcher::CheckAddedConfig( - const string& configName, - unique_ptr&& configDetail, - PipelineConfigDiff& pDiff, - TaskConfigDiff& tDiff, - std::unordered_map>& singletonCache) { +bool PipelineConfigWatcher::CheckAddedConfig(const string& configName, + unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + SingletonConfigCache& singletonCache) { switch (GetConfigType(*configDetail)) { case ConfigType::Pipeline: { PipelineConfig config(configName, std::move(configDetail)); @@ -337,12 +325,11 @@ bool PipelineConfigWatcher::CheckAddedConfig( return true; } -bool PipelineConfigWatcher::CheckModifiedConfig( - const string& configName, - unique_ptr&& configDetail, - PipelineConfigDiff& pDiff, - TaskConfigDiff& tDiff, - std::unordered_map>& singletonCache) { +bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, + unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + SingletonConfigCache& singletonCache) { switch (GetConfigType(*configDetail)) { case ConfigType::Pipeline: { shared_ptr p = mPipelineManager->FindConfigByName(configName); @@ -361,10 +348,10 @@ bool PipelineConfigWatcher::CheckModifiedConfig( config.mRegion); return false; } - PushPipelineConfig(std::move(config), ConfigDiffEnum::Added, pDiff, singletonCache); LOG_INFO(sLogger, ("existing invalid config modified and passed topology check", "prepare to build pipeline")("config", configName)); + PushPipelineConfig(std::move(config), ConfigDiffEnum::Added, pDiff, singletonCache); } else if (*configDetail != p->GetConfig()) { PipelineConfig config(configName, std::move(configDetail)); if (!config.Parse()) { @@ -380,10 +367,10 @@ bool PipelineConfigWatcher::CheckModifiedConfig( config.mRegion); return false; } - PushPipelineConfig(std::move(config), ConfigDiffEnum::Modified, pDiff, singletonCache); LOG_INFO(sLogger, ("existing valid config modified and passed topology check", "prepare to rebuild pipeline")("config", configName)); + PushPipelineConfig(std::move(config), ConfigDiffEnum::Modified, pDiff, singletonCache); } else { LOG_DEBUG(sLogger, ("existing valid config file modified, but no change found", "skip current object")); } @@ -433,45 +420,51 @@ bool PipelineConfigWatcher::CheckModifiedConfig( } +bool PipelineConfigWatcher::CheckUnchangedConfig(const std::string& configName, + const filesystem::path& path, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + SingletonConfigCache& singletonCache) { + auto pipeline = mPipelineManager->FindConfigByName(configName); + auto task = mTaskPipelineManager->FindPipelineByName(configName).get(); + if (pipeline) { + std::unique_ptr configDetail = make_unique(); + PipelineConfig config(configName, std::move(configDetail)); + config.mCreateTime = pipeline->GetContext().GetCreateTime(); + config.mSingletonInput = pipeline->GetSingletonInput(); + PushPipelineConfig(std::move(config), ConfigDiffEnum::Unchanged, pDiff, singletonCache); + } else if (task) { + return true; + } else { // low priority singleton input in last config update, sort it again + unique_ptr detail = make_unique(); + if (!LoadConfigDetailFromFile(path, *detail)) { + return false; + } + if (!IsConfigEnabled(configName, *detail)) { + LOG_INFO(sLogger, ("unchanged config found and disabled", "skip current object")("config", configName)); + return false; + } + if (!CheckAddedConfig(configName, std::move(detail), pDiff, tDiff, singletonCache)) { + return false; + } + } + return true; +} + void PipelineConfigWatcher::PushPipelineConfig(PipelineConfig&& config, ConfigDiffEnum diffEnum, PipelineConfigDiff& pDiff, SingletonConfigCache& singletonCache) { // singleton input if (!config.mSingletonInput.empty()) { - if (diffEnum == ConfigDiffEnum::Added || diffEnum == ConfigDiffEnum::Modified) { - auto it = singletonCache.find(config.mSingletonInput); - if (it != singletonCache.end()) { - if (it->second->config.mCreateTime < config.mCreateTime - || (it->second->config.mCreateTime == config.mCreateTime - && it->second->config.mName < config.mName)) { - LOG_WARNING(sLogger, - ("global singleton plugin found, but another older config or smaller name config " - "already exists", - "skip current object")("config", config.mName)("inputType", config.mSingletonInput)); - return; - } - if (mPipelineManager->FindConfigByName(it->second->config.mName)) { - pDiff.mRemoved.push_back(it->second->config.mName); - LOG_WARNING( - sLogger, - ("existing valid config with global singleton plugin, but another older config or smaller " - "name config found", - "prepare to stop current running pipeline")("config", it->second->config.mName)); - } else { - LOG_WARNING(sLogger, - ("global singleton plugin found, but another older config or smaller name config " - "already exists", - "skip current object")("config", it->second->config.mName)("inputType", - config.mSingletonInput)); - } - } - auto pipelineConfig = make_shared(std::move(config), diffEnum); - singletonCache[pipelineConfig->config.mSingletonInput] = pipelineConfig; - return; + if (diffEnum == ConfigDiffEnum::Added || diffEnum == ConfigDiffEnum::Modified + || diffEnum == ConfigDiffEnum::Unchanged) { + singletonCache[config.mSingletonInput].push_back( + make_shared(std::move(config), diffEnum)); } else { LOG_ERROR(sLogger, ("should not reach here", "invalid diff enum")("diff", diffEnum)); } + return; } // no singleton input switch (diffEnum) { @@ -486,4 +479,49 @@ void PipelineConfigWatcher::PushPipelineConfig(PipelineConfig&& config, } } +void PipelineConfigWatcher::CheckSingletonInput(PipelineConfigDiff& pDiff, SingletonConfigCache& singletonCache) { + for (auto& [name, configs] : singletonCache) { + std::sort(configs.begin(), + configs.end(), + [](const std::shared_ptr& a, + const std::shared_ptr& b) -> bool { + if (a->config.mCreateTime == b->config.mCreateTime) { + return a->config.mName < b->config.mName; + } + return a->config.mCreateTime < b->config.mCreateTime; + }); + for (size_t i = 0; i < configs.size(); ++i) { + const auto& diffEnum = configs[i]->diffEnum; + const auto& configName = configs[i]->config.mName; + if (i == 0) { + if (diffEnum == ConfigDiffEnum::Added) { + LOG_INFO(sLogger, + ("new config with singleton input found and passed topology check", + "prepare to build pipeline")("config", configName)); + pDiff.mAdded.push_back(std::move(configs[0]->config)); + } else if (diffEnum == ConfigDiffEnum::Modified) { + LOG_INFO(sLogger, + ("existing config with singleton input modified and passed topology check", + "prepare to build pipeline")("config", configName)); + pDiff.mModified.push_back(std::move(configs[0]->config)); + } + } else { + if (diffEnum == ConfigDiffEnum::Modified) { + LOG_WARNING(sLogger, + ("global singleton plugin found, but another older config or smaller name config " + "already exists", + "skip current object")("config", configName)); + pDiff.mRemoved.push_back(configName); + } else if (diffEnum == ConfigDiffEnum::Unchanged) { + LOG_WARNING(sLogger, + ("existing valid config with global singleton plugin, but another older config or " + "smaller name config found", + "prepare to stop current running pipeline")("config", configName)); + pDiff.mRemoved.push_back(configName); + } + } + } + } +} + } // namespace logtail diff --git a/core/config/watcher/PipelineConfigWatcher.h b/core/config/watcher/PipelineConfigWatcher.h index 0f6ea721dc..766b55c512 100644 --- a/core/config/watcher/PipelineConfigWatcher.h +++ b/core/config/watcher/PipelineConfigWatcher.h @@ -35,7 +35,7 @@ struct PipelineConfigWithDiffInfo { PipelineConfigWithDiffInfo(PipelineConfig&& config, ConfigDiffEnum diffEnum) : config(std::move(config)), diffEnum(diffEnum) {} }; -using SingletonConfigCache = std::unordered_map>; +using SingletonConfigCache = std::unordered_map>>; class PipelineConfigWatcher : public ConfigWatcher { public: @@ -75,10 +75,16 @@ class PipelineConfigWatcher : public ConfigWatcher { PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, SingletonConfigCache& singletonCache); + bool CheckUnchangedConfig(const std::string& configName, + const std::filesystem::path& path, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff, + SingletonConfigCache& singletonCache); void PushPipelineConfig(PipelineConfig&& config, ConfigDiffEnum diffEnum, PipelineConfigDiff& pDiff, SingletonConfigCache& singletonCache); + void CheckSingletonInput(PipelineConfigDiff& pDiff, SingletonConfigCache& singletonCache); const PipelineManager* mPipelineManager = nullptr; const TaskPipelineManager* mTaskPipelineManager = nullptr; diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 1f8e4d5868..3ed21f7d2b 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -70,6 +70,7 @@ void AddExtendedGlobalParamToGoPipeline(const Json::Value& extendedParams, Json: bool Pipeline::Init(PipelineConfig&& config) { mName = config.mName; mConfig = std::move(config.mDetail); + mSingletonInput = config.mSingletonInput; mContext.SetConfigName(mName); mContext.SetCreateTime(config.mCreateTime); mContext.SetPipeline(*this); diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index d6a55911c7..bd1794cee7 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -61,6 +61,7 @@ class Pipeline { const std::string& Name() const { return mName; } PipelineContext& GetContext() const { return mContext; } const Json::Value& GetConfig() const { return *mConfig; } + const std::string& GetSingletonInput() const { return mSingletonInput; } const std::vector>& GetFlushers() const { return mFlushers; } bool IsFlushingThroughGoPipeline() const { return !mGoPipelineWithoutInput.isNull(); } const std::unordered_map>& GetPluginStatistics() const { @@ -100,6 +101,7 @@ class Pipeline { mutable PipelineContext mContext; std::unordered_map> mPluginCntMap; std::unique_ptr mConfig; + std::string mSingletonInput; std::atomic_uint16_t mPluginID; std::atomic_int16_t mInProcessCnt; diff --git a/core/unittest/config/ConfigUpdateUnittest.cpp b/core/unittest/config/ConfigUpdateUnittest.cpp index 341f48f00d..6b71e40eb8 100644 --- a/core/unittest/config/ConfigUpdateUnittest.cpp +++ b/core/unittest/config/ConfigUpdateUnittest.cpp @@ -26,48 +26,13 @@ #include "pipeline/plugin/PluginRegistry.h" #include "task_pipeline/TaskPipelineManager.h" #include "unittest/Unittest.h" +#include "unittest/config/PipelineManagerMock.h" #include "unittest/plugin/PluginMock.h" using namespace std; namespace logtail { -class PipelineMock : public Pipeline { -public: - bool Init(PipelineConfig&& config) { - mConfig = std::move(config.mDetail); - WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, - MetricCategory::METRIC_CATEGORY_PIPELINE, - {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mName}}); - mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); - return (*mConfig)["valid"].asBool(); - } -}; - -class PipelineManagerMock : public PipelineManager { -public: - static PipelineManagerMock* GetInstance() { - static PipelineManagerMock instance; - return &instance; - } - - void ClearEnvironment() { - mPipelineNameEntityMap.clear(); - mPluginCntMap.clear(); - } - -private: - shared_ptr BuildPipeline(PipelineConfig&& config) override { - // this should be synchronized with PipelineManager::BuildPipeline, except for the pointer type. - shared_ptr p = make_shared(); - if (!p->Init(std::move(config))) { - return nullptr; - } - return p; - } -}; - class ConfigUpdateUnittest : public testing::Test { public: void OnStartUp() const; diff --git a/core/unittest/config/PipelineConfigWatcherUnittest.cpp b/core/unittest/config/PipelineConfigWatcherUnittest.cpp index dd27c533cb..389dc83bcd 100644 --- a/core/unittest/config/PipelineConfigWatcherUnittest.cpp +++ b/core/unittest/config/PipelineConfigWatcherUnittest.cpp @@ -21,6 +21,7 @@ #include "config/watcher/PipelineConfigWatcher.h" #include "plugin/PluginRegistry.h" #include "unittest/Unittest.h" +#include "unittest/config/PipelineManagerMock.h" using namespace std; @@ -28,121 +29,1122 @@ namespace logtail { class PipelineConfigWatcherUnittest : public testing::Test { public: - void TestPushPipelineConfig() const; + void TestLoadSingletonConfig(); protected: - static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); } + static void SetUpTestCase() { + PluginRegistry::GetInstance()->LoadPlugins(); + PipelineConfigWatcher::GetInstance()->SetPipelineManager(PipelineManagerMock::GetInstance()); + } static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } private: -}; + void PrepareConfig() { + filesystem::create_directories(configDir1); + PipelineConfigWatcher::GetInstance()->AddSource(configDir1.string()); + filesystem::create_directories(configDir2); + PipelineConfigWatcher::GetInstance()->AddSource(configDir2.string()); + } + + void ClearConfig() { + PipelineManagerMock::GetInstance()->ClearEnvironment(); + PipelineConfigWatcher::GetInstance()->ClearEnvironment(); + filesystem::remove_all(configDir1); + filesystem::remove_all(configDir2); + } -void PipelineConfigWatcherUnittest::TestPushPipelineConfig() const { - PipelineConfigDiff pDiff; - unique_ptr configJson; - string configStr, errorMsg; - SingletonConfigCache singletonCache; + filesystem::path configDir1 = "./continuous_pipeline_config1"; + filesystem::path configDir2 = "./continuous_pipeline_config2"; - configStr = R"( + const std::string greaterPriorityConfig = R"( { + "createTime": 1, + "valid": true, "inputs": [ { - "Type": "input_file" + "Type": "input_network_observer" } ], - } - )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName1 = "test1"; - PipelineConfig config1(configName1, std::move(configJson)); - config1.Parse(); - PipelineConfigWatcher::GetInstance()->PushPipelineConfig( - std::move(config1), ConfigDiffEnum::Added, pDiff, singletonCache); - APSARA_TEST_EQUAL_FATAL(1, pDiff.mAdded.size()); - - configStr = R"( - { - "inputs": [ + "flushers": [ { - "Type": "input_file" + "Type": "flusher_sls" } - ], + ] } )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName2 = "test2"; - PipelineConfig config2(configName2, std::move(configJson)); - config2.Parse(); - PipelineConfigWatcher::GetInstance()->PushPipelineConfig( - std::move(config2), ConfigDiffEnum::Added, pDiff, singletonCache); - APSARA_TEST_EQUAL_FATAL(2, pDiff.mAdded.size()); - - // case: singleton input - configStr = R"( + + const std::string lessPriorityConfig = R"( { - "createTime": 123456, + "createTime": 2, + "valid": true, "inputs": [ { "Type": "input_network_observer" } ], + "flushers": [ + { + "Type": "flusher_sls" + } + ] } )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName3 = "test3"; - PipelineConfig config3(configName3, std::move(configJson)); - config3.Parse(); - APSARA_TEST_EQUAL_FATAL("input_network_observer", config3.mSingletonInput); - PipelineConfigWatcher::GetInstance()->PushPipelineConfig( - std::move(config3), ConfigDiffEnum::Added, pDiff, singletonCache); - APSARA_TEST_EQUAL_FATAL(singletonCache["input_network_observer"]->config.mName, configName3); - - // case: compare by create time - configStr = R"( + + const std::string modifiedGreaterPriorityConfig = R"( { + "createTime": 1, + "valid": true, "inputs": [ { "Type": "input_network_observer" } ], + "processors": [], + "flushers": [ + { + "Type": "flusher_sls" + } + ] } )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName4 = "test4"; - PipelineConfig config4(configName4, std::move(configJson)); - config4.Parse(); - APSARA_TEST_EQUAL_FATAL("input_network_observer", config4.mSingletonInput); - PipelineConfigWatcher::GetInstance()->PushPipelineConfig( - std::move(config4), ConfigDiffEnum::Added, pDiff, singletonCache); - APSARA_TEST_EQUAL_FATAL(singletonCache["input_network_observer"]->config.mName, configName4); - - // case: compare by name - configStr = R"( + + const std::string modifiedLessPriorityConfig = R"( { + "createTime": 2, + "valid": true, "inputs": [ { "Type": "input_network_observer" } ], + "processors": [], + "flushers": [ + { + "Type": "flusher_sls" + } + ] } )"; - configJson.reset(new Json::Value()); - APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); - std::string configName5 = "z-test5"; - PipelineConfig config5(configName5, std::move(configJson)); - config5.Parse(); - APSARA_TEST_EQUAL_FATAL("input_network_observer", config5.mSingletonInput); - PipelineConfigWatcher::GetInstance()->PushPipelineConfig( - std::move(config5), ConfigDiffEnum::Added, pDiff, singletonCache); - APSARA_TEST_EQUAL_FATAL(singletonCache["input_network_observer"]->config.mName, configName4); -} +}; + +void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { + // there are 4 kinds of a config: added, modified, removed, unchanged + // there are 4 kinds of priority relationship: first > second, first < second, + // first > second -> first < second, first < second -> first > second + // total case: 4 (first kind) * 4(second kind) * 4(priority) = 64 + { // case: added -> added, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> added, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> added, first > second -> first < second + // should not happen + } + { // case: added -> added, first < second -> first > second + // should not happen + } + { // case: added -> modified, first > second + PrepareConfig(); + ofstream fout(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> modified, first < second + PrepareConfig(); + ofstream fout(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> modified, first > second -> first < second + // should not happen + } + { // case: added -> modified, first < second -> first > second + // should not happen + } + { // case: added -> removed, first > second + PrepareConfig(); + ofstream fout(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> removed, first < second + PrepareConfig(); + ofstream fout(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> removed, first > second -> first < second + // should not happen + } + { // case: added -> removed, first < second -> first > second + // should not happen + } + { // case: added -> unchanged, first > second + PrepareConfig(); + ofstream fout(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> unchanged, first < second + PrepareConfig(); + ofstream fout(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: added -> unchanged, first > second -> first < second + // should not happen + } + { // case: added -> unchanged, first < second -> first > second + // should not happen + } + { // case: modified -> added, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> added, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> added, first > second -> first < second + // should not happen + } + { // case: modified -> added, first < second -> first > second + // should not happen + } + { // case: modified -> modified, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> modified, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> modified, first > second -> first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> modified, first < second -> first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> removed, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> removed, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> removed, first > second -> first < second + // should not happen + } + { // case: modified -> removed, first < second -> first > second + // should not happen + } + { // case: modified -> unchanged, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> unchanged, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> unchanged, first > second -> first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + fout.open(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: modified -> unchanged, first < second -> first > second + PrepareConfig(); + ofstream fout(configDir1 / "test3.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir1 / "test3.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test3", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: removed -> added, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir1 / "test1.json"); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: removed -> added, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir1 / "test1.json"); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: removed -> added, first > second -> first < second + // should not happen + } + { // case: removed -> added, first < second -> first > second + // should not happen + } + { // case: removed -> modified, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + this_thread::sleep_for(chrono::milliseconds(1)); + + filesystem::remove(configDir1 / "test1.json"); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: removed -> modified, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + this_thread::sleep_for(chrono::milliseconds(1)); + + filesystem::remove(configDir1 / "test1.json"); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: removed -> modified, first > second -> first < second + // should not happen + } + { // case: removed -> modified, first < second -> first > second + // should not happen + } + { // case: removed -> removed, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir1 / "test1.json"); + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(0U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + ClearConfig(); + } + { // case: removed -> removed, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir1 / "test1.json"); + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(0U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + ClearConfig(); + } + { // case: removed -> removed, first > second -> first < second + // should not happen + } + { // case: removed -> removed, first < second -> first > second + // should not happen + } + { // case: removed -> unchanged, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir1 / "test1.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: removed -> unchanged, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir1 / "test1.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: removed -> unchanged, first > second -> first < second + // should not happen + } + { // case: removed -> unchanged, first < second -> first > second + // should not happen + } + { // case: unchanged -> added, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> added, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> added, first > second -> first < second + // should not happen + } + { // case: unchanged -> added, first < second -> first > second + // should not happen + } + { // case: unchanged -> modified, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> modified, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedGreaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> modified, first > second -> first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> modified, first < second -> first > second + PrepareConfig(); + ofstream fout(configDir1 / "test3.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + fout.open(configDir2 / "test2.json", ios::trunc); + fout << modifiedLessPriorityConfig; + fout.close(); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test3", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> removed, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> removed, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + filesystem::remove(configDir2 / "test2.json"); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> removed, first > second -> first < second + // should not happen + } + { // case: unchanged -> removed, first < second -> first > second + // should not happen + } + { // case: unchanged -> unchanged, first > second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> unchanged, first < second + PrepareConfig(); + ofstream fout(configDir1 / "test1.json", ios::trunc); + fout << lessPriorityConfig; + fout.close(); + fout.open(configDir2 / "test2.json", ios::trunc); + fout << greaterPriorityConfig; + fout.close(); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + ClearConfig(); + } + { // case: unchanged -> unchanged, first > second -> first < second + // should not happen + } + { // case: unchanged -> unchanged, first < second -> first > second + // should not happen + } +} -UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestPushPipelineConfig) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadSingletonConfig) } // namespace logtail diff --git a/core/unittest/config/PipelineManagerMock.h b/core/unittest/config/PipelineManagerMock.h new file mode 100644 index 0000000000..b25e4f05b9 --- /dev/null +++ b/core/unittest/config/PipelineManagerMock.h @@ -0,0 +1,58 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "pipeline/PipelineManager.h" + +using namespace std; + +namespace logtail { + +class PipelineMock : public Pipeline { +public: + bool Init(PipelineConfig&& config) { + mConfig = std::move(config.mDetail); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, + MetricCategory::METRIC_CATEGORY_PIPELINE, + {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mName}}); + mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); + mSingletonInput = config.mSingletonInput; + mContext.SetCreateTime(config.mCreateTime); + return (*mConfig)["valid"].asBool(); + } +}; + +class PipelineManagerMock : public PipelineManager { +public: + static PipelineManagerMock* GetInstance() { + static PipelineManagerMock instance; + return &instance; + } + + void ClearEnvironment() { + mPipelineNameEntityMap.clear(); + mPluginCntMap.clear(); + } + +private: + shared_ptr BuildPipeline(PipelineConfig&& config) override { + // this should be synchronized with PipelineManager::BuildPipeline, except for the pointer type. + shared_ptr p = make_shared(); + if (!p->Init(std::move(config))) { + return nullptr; + } + return p; + } +}; +} // namespace logtail \ No newline at end of file From dccfddff6d2dd04ed97671d0dc5d212c95a13ea4 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 5 Dec 2024 14:55:35 +0800 Subject: [PATCH 12/15] fix --- core/config/ConfigUtil.h | 1 - core/config/PipelineConfig.cpp | 14 +++- core/config/watcher/PipelineConfigWatcher.cpp | 78 ++++++++++++------- 3 files changed, 63 insertions(+), 30 deletions(-) diff --git a/core/config/ConfigUtil.h b/core/config/ConfigUtil.h index 1d1c708b42..c5fd7d0ae2 100644 --- a/core/config/ConfigUtil.h +++ b/core/config/ConfigUtil.h @@ -31,7 +31,6 @@ bool ParseConfigDetail(const std::string& content, Json::Value& detail, std::string& errorMsg); bool IsConfigEnabled(const std::string& name, const Json::Value& detail); -void GetAllInputTypes(const Json::Value& detail, std::vector& inputTypes); ConfigType GetConfigType(const Json::Value& detail); } // namespace logtail diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index 1d86a97927..223ad46778 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -193,6 +193,16 @@ bool PipelineConfig::Parse() { const string pluginType = it->asString(); if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(pluginType)) { mSingletonInput = pluginType; + if (itr->size() > 1) { + PARAM_ERROR_RETURN(sLogger, + alarm, + "more than 1 input plugin is given when global singleton input plugin is used", + noModule, + mName, + mProject, + mLogstore, + mRegion); + } } if (i == 0) { if (PluginRegistry::GetInstance()->IsValidGoPlugin(pluginType)) { @@ -241,10 +251,10 @@ bool PipelineConfig::Parse() { } } // TODO: remove these special restrictions - if ((hasFileInput || !mSingletonInput.empty()) && (*mDetail)["inputs"].size() > 1) { + if ((hasFileInput) && (*mDetail)["inputs"].size() > 1) { PARAM_ERROR_RETURN(sLogger, alarm, - "more than 1 input_file or input_container_stdio or global singleton plugin is given", + "more than 1 input_file or input_container_stdio is given", noModule, mName, mProject, diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 13740e4401..5946ad5c42 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -169,6 +169,7 @@ void PipelineConfigWatcher::InsertBuiltInPipelines(PipelineConfigDiff& pDiff, } } else { LOG_DEBUG(sLogger, ("existing inner config unchanged", "skip current object")); + CheckUnchangedConfig(pipelineName, path, pDiff, tDiff, singletonCache); } } #else @@ -419,7 +420,6 @@ bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, return true; } - bool PipelineConfigWatcher::CheckUnchangedConfig(const std::string& configName, const filesystem::path& path, PipelineConfigDiff& pDiff, @@ -427,14 +427,14 @@ bool PipelineConfigWatcher::CheckUnchangedConfig(const std::string& configName, SingletonConfigCache& singletonCache) { auto pipeline = mPipelineManager->FindConfigByName(configName); auto task = mTaskPipelineManager->FindPipelineByName(configName).get(); - if (pipeline) { + if (task) { + return true; + } else if (pipeline) { // running pipeline in last config update std::unique_ptr configDetail = make_unique(); PipelineConfig config(configName, std::move(configDetail)); config.mCreateTime = pipeline->GetContext().GetCreateTime(); config.mSingletonInput = pipeline->GetSingletonInput(); PushPipelineConfig(std::move(config), ConfigDiffEnum::Unchanged, pDiff, singletonCache); - } else if (task) { - return true; } else { // low priority singleton input in last config update, sort it again unique_ptr detail = make_unique(); if (!LoadConfigDetailFromFile(path, *detail)) { @@ -444,9 +444,21 @@ bool PipelineConfigWatcher::CheckUnchangedConfig(const std::string& configName, LOG_INFO(sLogger, ("unchanged config found and disabled", "skip current object")("config", configName)); return false; } - if (!CheckAddedConfig(configName, std::move(detail), pDiff, tDiff, singletonCache)) { + PipelineConfig config(configName, std::move(detail)); + if (!config.Parse()) { + LOG_ERROR(sLogger, ("new config found but invalid", "skip current object")("config", configName)); + AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM, + "new config found but invalid: skip current object, config: " + + configName, + config.mProject, + config.mLogstore, + config.mRegion); return false; } + if (!config.mSingletonInput.empty()) { + singletonCache[config.mSingletonInput].push_back( + make_shared(std::move(config), ConfigDiffEnum::Added)); + } } return true; } @@ -494,30 +506,42 @@ void PipelineConfigWatcher::CheckSingletonInput(PipelineConfigDiff& pDiff, Singl const auto& diffEnum = configs[i]->diffEnum; const auto& configName = configs[i]->config.mName; if (i == 0) { - if (diffEnum == ConfigDiffEnum::Added) { - LOG_INFO(sLogger, - ("new config with singleton input found and passed topology check", - "prepare to build pipeline")("config", configName)); - pDiff.mAdded.push_back(std::move(configs[0]->config)); - } else if (diffEnum == ConfigDiffEnum::Modified) { - LOG_INFO(sLogger, - ("existing config with singleton input modified and passed topology check", - "prepare to build pipeline")("config", configName)); - pDiff.mModified.push_back(std::move(configs[0]->config)); + switch (diffEnum) { + // greatest priority config + case ConfigDiffEnum::Added: + LOG_INFO(sLogger, + ("new config with singleton input found and passed topology check", + "prepare to build pipeline")("config", configName)); + pDiff.mAdded.push_back(std::move(configs[0]->config)); + break; + case ConfigDiffEnum::Modified: + LOG_INFO(sLogger, + ("existing config with singleton input modified and passed topology check", + "prepare to build pipeline")("config", configName)); + pDiff.mModified.push_back(std::move(configs[0]->config)); + break; + default: + break; } } else { - if (diffEnum == ConfigDiffEnum::Modified) { - LOG_WARNING(sLogger, - ("global singleton plugin found, but another older config or smaller name config " - "already exists", - "skip current object")("config", configName)); - pDiff.mRemoved.push_back(configName); - } else if (diffEnum == ConfigDiffEnum::Unchanged) { - LOG_WARNING(sLogger, - ("existing valid config with global singleton plugin, but another older config or " - "smaller name config found", - "prepare to stop current running pipeline")("config", configName)); - pDiff.mRemoved.push_back(configName); + // other low priority configs + switch (diffEnum) { + case ConfigDiffEnum::Modified: + LOG_WARNING(sLogger, + ("global singleton plugin found, but another older config or smaller name config " + "already exists", + "skip current object")("config", configName)); + pDiff.mRemoved.push_back(configName); + break; + case ConfigDiffEnum::Unchanged: + LOG_WARNING(sLogger, + ("existing valid config with global singleton plugin, but another older config or " + "smaller name config found", + "prepare to stop current running pipeline")("config", configName)); + pDiff.mRemoved.push_back(configName); + break; + default: + break; } } } From aa63d7821fae70b7be407eb9ac30d80bbbf9125e Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 5 Dec 2024 15:22:03 +0800 Subject: [PATCH 13/15] fix --- core/config/watcher/PipelineConfigWatcher.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 5946ad5c42..8a284ce815 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -169,7 +169,6 @@ void PipelineConfigWatcher::InsertBuiltInPipelines(PipelineConfigDiff& pDiff, } } else { LOG_DEBUG(sLogger, ("existing inner config unchanged", "skip current object")); - CheckUnchangedConfig(pipelineName, path, pDiff, tDiff, singletonCache); } } #else From 3abe1ce1fbe2c67ec00b65f8944884187f6cce00 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Fri, 6 Dec 2024 15:01:24 +0800 Subject: [PATCH 14/15] fix --- core/config/PipelineConfig.cpp | 2 +- .../config/PipelineConfigWatcherUnittest.cpp | 581 +++++++++++++----- 2 files changed, 443 insertions(+), 140 deletions(-) diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index 223ad46778..4a5dfb2e50 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -251,7 +251,7 @@ bool PipelineConfig::Parse() { } } // TODO: remove these special restrictions - if ((hasFileInput) && (*mDetail)["inputs"].size() > 1) { + if (hasFileInput && (*mDetail)["inputs"].size() > 1) { PARAM_ERROR_RETURN(sLogger, alarm, "more than 1 input_file or input_container_stdio is given", diff --git a/core/unittest/config/PipelineConfigWatcherUnittest.cpp b/core/unittest/config/PipelineConfigWatcherUnittest.cpp index 389dc83bcd..56c8c3fdd4 100644 --- a/core/unittest/config/PipelineConfigWatcherUnittest.cpp +++ b/core/unittest/config/PipelineConfigWatcherUnittest.cpp @@ -29,7 +29,10 @@ namespace logtail { class PipelineConfigWatcherUnittest : public testing::Test { public: - void TestLoadSingletonConfig(); + void TestLoadAddedSingletonConfig(); + void TestLoadModifiedSingletonConfig(); + void TestLoadRemovedSingletonConfig(); + void TestLoadUnchangedSingletonConfig(); protected: static void SetUpTestCase() { @@ -125,30 +128,70 @@ class PipelineConfigWatcherUnittest : public testing::Test { ] } )"; + + const std::string otherConfig = R"( + { + "createTime": 3, + "valid": true, + "inputs": [ + { + "Type": "input_process_security" + } + ], + "flushers": [ + { + "Type": "flusher_sls" + } + ] + } + )"; + + const std::string modifiedOtherConfig = R"( + { + "createTime": 3, + "valid": true, + "inputs": [ + { + "Type": "input_process_security" + } + ], + "processors": [], + "flushers": [ + { + "Type": "flusher_sls" + } + ] + } + )"; }; -void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { - // there are 4 kinds of a config: added, modified, removed, unchanged - // there are 4 kinds of priority relationship: first > second, first < second, - // first > second -> first < second, first < second -> first > second - // total case: 4 (first kind) * 4(second kind) * 4(priority) = 64 +// there are 4 kinds of a config: added, modified, removed, unchanged +// there are 4 kinds of priority relationship: first > second, first < second, +// first > second -> first < second, first < second -> first > second +// total case: 4 (first kind) * 4(second kind) * 4(priority) = 64 +void PipelineConfigWatcherUnittest::TestLoadAddedSingletonConfig() { { // case: added -> added, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); - fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: added -> added, first < second @@ -156,18 +199,23 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { ofstream fout(configDir1 / "test1.json", ios::trunc); fout << lessPriorityConfig; fout.close(); - fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: added -> added, first > second -> first < second @@ -181,9 +229,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { ofstream fout(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -192,14 +243,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: added -> modified, first < second @@ -207,9 +264,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { ofstream fout(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -218,14 +278,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: added -> modified, first > second -> first < second @@ -239,22 +305,27 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { ofstream fout(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(1U, allConfigNames.size()); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } { // case: added -> removed, first < second @@ -262,22 +333,27 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { ofstream fout(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir1 / "test1.json", ios::trunc); fout << lessPriorityConfig; fout.close(); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(1U, allConfigNames.size()); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } { // case: added -> removed, first > second -> first < second @@ -291,9 +367,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { ofstream fout(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; @@ -304,8 +383,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: added -> unchanged, first < second @@ -313,9 +395,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { ofstream fout(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir1 / "test1.json", ios::trunc); fout << lessPriorityConfig; @@ -326,8 +411,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: added -> unchanged, first > second -> first < second @@ -336,6 +424,9 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { { // case: added -> unchanged, first < second -> first > second // should not happen } +} + +void PipelineConfigWatcherUnittest::TestLoadModifiedSingletonConfig() { { // case: modified -> added, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); @@ -352,14 +443,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: modified -> added, first < second @@ -378,14 +475,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: modified -> added, first > second -> first < second @@ -402,9 +505,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -413,14 +519,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: modified -> modified, first < second @@ -431,9 +543,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -442,14 +557,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: modified -> modified, first > second -> first < second @@ -460,9 +581,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -471,14 +595,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: modified -> modified, first < second -> first > second @@ -489,9 +619,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -500,14 +633,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: modified -> removed, first > second @@ -518,23 +657,28 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(1U, allConfigNames.size()); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } { // case: modified -> removed, first < second @@ -545,23 +689,28 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(1U, allConfigNames.size()); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } { // case: modified -> removed, first > second -> first < second @@ -578,9 +727,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -592,8 +744,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: modified -> unchanged, first < second @@ -604,9 +759,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -619,8 +777,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: modified -> unchanged, first > second -> first < second @@ -631,9 +792,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test1.json", ios::trunc); @@ -645,8 +809,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: modified -> unchanged, first < second -> first > second @@ -657,9 +824,13 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); fout.open(configDir1 / "test3.json", ios::trunc); fout << greaterPriorityConfig; @@ -670,10 +841,16 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test3", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test3", allConfigNames[1]); ClearConfig(); } +} + +void PipelineConfigWatcherUnittest::TestLoadRemovedSingletonConfig() { { // case: removed -> added, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); @@ -687,14 +864,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: removed -> added, first < second @@ -710,14 +893,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: removed -> added, first > second -> first < second @@ -734,23 +923,32 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); this_thread::sleep_for(chrono::milliseconds(1)); filesystem::remove(configDir1 / "test1.json"); fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: removed -> modified, first < second @@ -761,23 +959,32 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); this_thread::sleep_for(chrono::milliseconds(1)); filesystem::remove(configDir1 / "test1.json"); fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: removed -> modified, first > second -> first < second @@ -794,16 +1001,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); filesystem::remove(configDir1 / "test1.json"); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_EQUAL_FATAL(0U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); @@ -817,16 +1028,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); filesystem::remove(configDir1 / "test1.json"); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_EQUAL_FATAL(0U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); @@ -846,9 +1061,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); filesystem::remove(configDir1 / "test1.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); @@ -857,8 +1075,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: removed -> unchanged, first < second @@ -869,9 +1090,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); filesystem::remove(configDir1 / "test1.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); @@ -880,8 +1104,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: removed -> unchanged, first > second -> first < second @@ -890,6 +1117,9 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { { // case: removed -> unchanged, first < second -> first > second // should not happen } +} + +void PipelineConfigWatcherUnittest::TestLoadUnchangedSingletonConfig() { { // case: unchanged -> added, first > second PrepareConfig(); ofstream fout(configDir1 / "test1.json", ios::trunc); @@ -902,14 +1132,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> added, first < second @@ -924,14 +1160,20 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> added, first > second -> first < second @@ -948,21 +1190,30 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> modified, first < second @@ -973,21 +1224,30 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> modified, first > second -> first < second @@ -998,21 +1258,30 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> modified, first < second -> first > second @@ -1023,21 +1292,30 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << modifiedOtherConfig; + fout.close(); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mModified.size()); APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test3", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test3", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> removed, first > second @@ -1048,19 +1326,24 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(1U, allConfigNames.size()); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } { // case: unchanged -> removed, first < second @@ -1071,19 +1354,24 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); filesystem::remove(configDir2 / "test2.json"); + filesystem::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0, diff.first.mModified.size()); - APSARA_TEST_EQUAL_FATAL(1, diff.first.mRemoved.size()); + APSARA_TEST_EQUAL_FATAL(2, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(1U, allConfigNames.size()); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[0]); ClearConfig(); } { // case: unchanged -> removed, first > second -> first < second @@ -1100,9 +1388,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); @@ -1110,8 +1401,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test1", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test1", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> unchanged, first < second @@ -1122,9 +1416,12 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); + fout.open(configDir2 / "test-other.json", ios::trunc); + fout << otherConfig; + fout.close(); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL_FATAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0, diff.first.mAdded.size()); @@ -1132,8 +1429,11 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { APSARA_TEST_EQUAL_FATAL(0, diff.first.mRemoved.size()); PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); - APSARA_TEST_EQUAL_FATAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - APSARA_TEST_EQUAL_FATAL("test2", PipelineManagerMock::GetInstance()->GetAllConfigNames()[0]); + auto allConfigNames = PipelineManagerMock::GetInstance()->GetAllConfigNames(); + APSARA_TEST_EQUAL_FATAL(2U, allConfigNames.size()); + sort(allConfigNames.begin(), allConfigNames.end()); + APSARA_TEST_EQUAL_FATAL("test-other", allConfigNames[0]); + APSARA_TEST_EQUAL_FATAL("test2", allConfigNames[1]); ClearConfig(); } { // case: unchanged -> unchanged, first > second -> first < second @@ -1144,7 +1444,10 @@ void PipelineConfigWatcherUnittest::TestLoadSingletonConfig() { } } -UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadSingletonConfig) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadAddedSingletonConfig) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadModifiedSingletonConfig) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadRemovedSingletonConfig) +UNIT_TEST_CASE(PipelineConfigWatcherUnittest, TestLoadUnchangedSingletonConfig) } // namespace logtail From 8559f64d1464a4272000bbf06ac5f204887a6967 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 11 Dec 2024 00:59:05 +0800 Subject: [PATCH 15/15] fix --- core/config/PipelineConfig.cpp | 1 + core/config/PipelineConfig.h | 2 +- core/config/watcher/PipelineConfigWatcher.cpp | 8 ++++---- core/pipeline/Pipeline.h | 4 ++-- core/pipeline/plugin/PluginRegistry.cpp | 4 ++-- core/pipeline/plugin/PluginRegistry.h | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index 4a5dfb2e50..4aeea027ca 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -191,6 +191,7 @@ bool PipelineConfig::Parse() { mRegion); } const string pluginType = it->asString(); + // when input is singleton, there should only one input to simpify config load transaction if (PluginRegistry::GetInstance()->IsGlobalSingletonInputPlugin(pluginType)) { mSingletonInput = pluginType; if (itr->size() > 1) { diff --git a/core/config/PipelineConfig.h b/core/config/PipelineConfig.h index d4e896daf6..6a9219cdcd 100644 --- a/core/config/PipelineConfig.h +++ b/core/config/PipelineConfig.h @@ -32,7 +32,7 @@ struct PipelineConfig { uint32_t mCreateTime = 0; const Json::Value* mGlobal = nullptr; std::vector mInputs; - std::string mSingletonInput; + std::optional mSingletonInput; std::vector mProcessors; std::vector mAggregators; std::vector mFlushers; diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index 8a284ce815..fb914c4d55 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -454,8 +454,8 @@ bool PipelineConfigWatcher::CheckUnchangedConfig(const std::string& configName, config.mRegion); return false; } - if (!config.mSingletonInput.empty()) { - singletonCache[config.mSingletonInput].push_back( + if (config.mSingletonInput) { + singletonCache[config.mSingletonInput.value()].push_back( make_shared(std::move(config), ConfigDiffEnum::Added)); } } @@ -467,10 +467,10 @@ void PipelineConfigWatcher::PushPipelineConfig(PipelineConfig&& config, PipelineConfigDiff& pDiff, SingletonConfigCache& singletonCache) { // singleton input - if (!config.mSingletonInput.empty()) { + if (config.mSingletonInput) { if (diffEnum == ConfigDiffEnum::Added || diffEnum == ConfigDiffEnum::Modified || diffEnum == ConfigDiffEnum::Unchanged) { - singletonCache[config.mSingletonInput].push_back( + singletonCache[config.mSingletonInput.value()].push_back( make_shared(std::move(config), diffEnum)); } else { LOG_ERROR(sLogger, ("should not reach here", "invalid diff enum")("diff", diffEnum)); diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index bd1794cee7..29666c68c1 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -61,7 +61,7 @@ class Pipeline { const std::string& Name() const { return mName; } PipelineContext& GetContext() const { return mContext; } const Json::Value& GetConfig() const { return *mConfig; } - const std::string& GetSingletonInput() const { return mSingletonInput; } + const std::optional& GetSingletonInput() const { return mSingletonInput; } const std::vector>& GetFlushers() const { return mFlushers; } bool IsFlushingThroughGoPipeline() const { return !mGoPipelineWithoutInput.isNull(); } const std::unordered_map>& GetPluginStatistics() const { @@ -101,7 +101,7 @@ class Pipeline { mutable PipelineContext mContext; std::unordered_map> mPluginCntMap; std::unique_ptr mConfig; - std::string mSingletonInput; + std::optional mSingletonInput; std::atomic_uint16_t mPluginID; std::atomic_int16_t mInProcessCnt; diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index acb0d82b50..d62648bf27 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -187,8 +187,8 @@ void PluginRegistry::RegisterInputCreator(PluginCreator* creator, bool isSinglet RegisterCreator(INPUT_PLUGIN, creator, isSingleton); } -void PluginRegistry::RegisterProcessorCreator(PluginCreator* creator, bool isSingleton) { - RegisterCreator(PROCESSOR_PLUGIN, creator, isSingleton); +void PluginRegistry::RegisterProcessorCreator(PluginCreator* creator) { + RegisterCreator(PROCESSOR_PLUGIN, creator, false); } void PluginRegistry::RegisterFlusherCreator(PluginCreator* creator, bool isSingleton) { diff --git a/core/pipeline/plugin/PluginRegistry.h b/core/pipeline/plugin/PluginRegistry.h index ddfa3a7bd8..1c552da6dc 100644 --- a/core/pipeline/plugin/PluginRegistry.h +++ b/core/pipeline/plugin/PluginRegistry.h @@ -80,7 +80,7 @@ class PluginRegistry { void LoadStaticPlugins(); void LoadDynamicPlugins(const std::set& plugins); void RegisterInputCreator(PluginCreator* creator, bool isSingleton = false); - void RegisterProcessorCreator(PluginCreator* creator, bool isSingleton = false); + void RegisterProcessorCreator(PluginCreator* creator); void RegisterFlusherCreator(PluginCreator* creator, bool isSingleton = false); PluginCreator* LoadProcessorPlugin(DynamicLibLoader& loader, const std::string pluginType); void RegisterCreator(PluginCat cat, PluginCreator* creator, bool isSingleton);