Merge branch 'main' into feat/prom-stream-process

.devcontainer/Dockerfile

@@ -45,4 +45,4 @@ RUN source /tmp/.env && rm /tmp/.env; \
 
 USER $USERNAME
 
-RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct
+RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct

.github/workflows/benchmark.yaml

@@ -111,9 +111,8 @@ jobs:
           last_commit_message=$(git log -1 --pretty=%B)
           git -c user.name=github-action-benchmark -c [email protected] commit --amend -m "$last_commit_message"
       
-      # TODO: Push gh-pages branch by github-action-bot
-      # - name: Push benchmark result
-        # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages
+      - name: Push benchmark result
+        run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages
 
   result:
     if: github.event.pull_request.merged == true

.github/workflows/build-core-ut.yaml

@@ -76,14 +76,17 @@ jobs:
           WITHOUTGDB: ON
           MAKE_JOBS: 16
           WITHSPL: ON
-          # BUILD_TYPE: Debug # TODO: Uncomment when memory management is refined
+          BUILD_TYPE: Debug
         run: CURRENT_DIR=$(pwd) && sed -i "s|/src|$CURRENT_DIR|g" docker/Dockerfile_build && make core PATH_IN_DOCKER=$(pwd)
 
+      - name: Build Coverage Image
+        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage .
+
       - name: Unit Test
-        run: make unittest_core
+        run: docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd) && make unittest_core"
 
       - name: Unit Test Coverage
-        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*\.pb\.cc\" -e \".*\.pb\.h\" -e \".*unittest.*\" -e \".*sdk.*\" -e \".*logger.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*runner.*\""
+        run: docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*\.pb\.cc\" -e \".*\.pb\.h\" -e \".*unittest.*\" -e \".*sdk.*\" -e \".*logger.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*runner.*\""
 
       - name: Setup Python3.10
         uses: actions/setup-python@v5

README.md

@@ -1,8 +1,8 @@
-# Alibaba iLogtail - Fast and Lightweight Observability Data Collector | 中文用户手册（[GitBook](https://ilogtail.gitbook.io/ilogtail-docs/) | [可观测中文社区](https://open.observability.cn/project/ilogtail/about/)）
+# Alibaba LoongCollector - Fast and Lightweight Observability Data Collector | 中文用户手册（[GitBook](https://ilogtail.gitbook.io/ilogtail-docs/) | [可观测中文社区](https://open.observability.cn/project/ilogtail/about/)）
 
-<img src="https://sls-opensource.oss-us-west-1.aliyuncs.com/ilogtail/ilogtail.svg?versionId=CAEQMxiBgIDEmq.m6BciIDkzNmE2OWU4NzIwZjQ1Y2ZiYmIxZjhiYjMyNmQxZTdi" alt="ilogtail logo" height="150px" align="right" />
+<img src="https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/images/logo/jpg/black-blue.jpg" alt="loongcollector logo" height="100px" align="center" />
 
-iLogtail was born for observable scenarios and has many production-level features such as lightweight, high performance, and automated configuration, which are widely used internally by Alibaba Group and tens of thousands of external Alibaba Cloud customers. You can deploy it in physical machines, Kubernetes and other environments to collect telemetry data, such as logs, traces and metrics.
+LoongCollector was born for observable scenarios and has many production-level features such as lightweight, high performance, and automated configuration, which are widely used internally by Alibaba Group and tens of thousands of external Alibaba Cloud customers. You can deploy it in physical machines, Kubernetes and other environments to collect telemetry data, such as logs, traces and metrics.
 
 [![GitHub contributors](https://img.shields.io/github/contributors/alibaba/ilogtail)](https://github.com/alibaba/ilogtail/contributors)
 [![GitHub stars](https://img.shields.io/github/stars/alibaba/ilogtail)](https://github.com/alibaba/ilogtail/stargazers)
@@ -14,7 +14,7 @@ iLogtail was born for observable scenarios and has many production-level feature
 
 ## Abstract
 
-The core advantages of **iLogtail**:
+The core advantages of **LoongCollector**:
 
 * Support a variety of Logs, Traces, Metrics data collection, and friendly to container and Kubernetes environment support.
 * The resource cost of data collection is quite low, 5-20 times better than similar telemetry data collection Agent performance.
@@ -23,7 +23,7 @@ The core advantages of **iLogtail**:
 * Support configuration remote management and provide a variety of ways, such as SLS console, SDK, K8s Operator, etc.
 * Supports multiple advanced features such as self-monitoring, flow control, resource control, alarms, and statistics collection.
 
-**iLogtail** supports the collection of a variety of telemetry data and transmission to a variety of different backends, such as [SLS observable platform](https://www.aliyun.com/product/sls). The data supported for collection are mainly as follows:
+**LoongCollector** supports the collection of a variety of telemetry data and transmission to a variety of different backends, such as [SLS observable platform](https://www.aliyun.com/product/sls). The data supported for collection are mainly as follows:
 
 * Logs
   * Collect static log files
@@ -45,16 +45,16 @@ The core advantages of **iLogtail**:
 
 ## Quick Start
 
-For the complexity of C++ dependencies, the compilation of iLogtail requires you have docker installed. If you aim to build iLogtail from sources, you can go ahead and start with the following commands.
+For the complexity of C++ dependencies, the compilation of LoongCollector requires you have docker installed. If you aim to build LoongCollector from sources, you can go ahead and start with the following commands.
 
 1. Start with local
 
 ```bash
 make
 cp -r example_config/quick_start/* output
 cd output
-./ilogtail
-# Now, ilogtail is collecting data from output/simple.log and outputing the result to stdout
+./loongcollector
+# Now, LoongCollector is collecting data from output/simple.log and outputing the result to stdout
 ```
 
  HEAD
@@ -93,14 +93,14 @@ There are many ways to contribute:
 You can report bugs, make suggestions or participate in discussions through [Github Issues](https://github.com/alibaba/ilogtail/issues) and [Github Discussions](https://github.com/alibaba/ilogtail/discussions), or contact us with the following ways:
 
 * Bilibili：[阿里云SLS](https://space.bilibili.com/630680534?from=search&seid=2845737427240690794&spm_id_from=333.337.0.0)
-* Zhihu：[iLogtail社区](https://www.zhihu.com/column/c_1533139823409270785)
-* DingTalk：iLogtail社区
+* Zhihu：[LoongCollector社区](https://www.zhihu.com/column/c_1533139823409270785)
+* DingTalk：LoongCollector社区
 
 <img src="https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/images/chatgroup/chatgroup_20240408.png" style="width: 50%; height: 50%" />
 
 ## Our Users
 
-Tens of thousands of companies use iLogtail in Alibaba Cloud, IDC, or other clouds. More details please see [here](https://help.aliyun.com/document_detail/250269.html).
+Tens of thousands of companies use LoongCollector in Alibaba Cloud, IDC, or other clouds. More details please see [here](https://help.aliyun.com/document_detail/250269.html).
 
 ## Licence
 

core/CMakeLists.txt

@@ -114,7 +114,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake)
 
 # Subdirectories (modules). except for common, input, processor, flusher, observer, helper, spl, and provider.
 set(SUB_DIRECTORIES_LIST
-        application app_config checkpoint container_manager metadata logger go_pipeline monitor monitor/metric_constants monitor/profile_sender models
+        application app_config checkpoint container_manager metadata logger go_pipeline monitor monitor/metric_constants monitor/metric_models monitor/profile_sender models
         config config/watcher constants
         pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer
         task_pipeline

core/app_config/AppConfig.cpp

@@ -430,22 +430,6 @@ string GetAgentLogName() {
     }
 }
 
-string GetAgentSnapshotDir() {
-    if (BOOL_FLAG(logtail_mode)) {
-        return GetProcessExecutionDir() + STRING_FLAG(logtail_snapshot_dir);
-    } else {
-        return GetAgentLogDir() + "snapshot";
-    }
-}
-
-string GetAgentStatusLogName() {
-    if (BOOL_FLAG(logtail_mode)) {
-        return "ilogtail_status.LOG";
-    } else {
-        return "loongcollector_status.LOG";
-    }
-}
-
 string GetObserverEbpfHostPath() {
     if (BOOL_FLAG(logtail_mode)) {
         return STRING_FLAG(sls_observer_ebpf_host_path);
@@ -909,7 +893,6 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) {
         mSendRequestConcurrency = confJson["send_request_concurrency"].asInt();
     else
         mSendRequestConcurrency = INT32_FLAG(send_request_concurrency);
-    LogtailMonitor::GetInstance()->UpdateConstMetric("send_request_concurrency", mSendRequestConcurrency);
 
     if (confJson.isMember("process_thread_count") && confJson["process_thread_count"].isInt())
         mProcessThreadCount = confJson["process_thread_count"].asInt();

core/app_config/AppConfig.h

@@ -48,8 +48,6 @@ std::string GetLocalEventDataFileName();
 std::string GetInotifyWatcherDirsDumpFileName();
 std::string GetAgentLoggersPrefix();
 std::string GetAgentLogName();
-std::string GetAgentSnapshotDir();
-std::string GetAgentStatusLogName();
 std::string GetObserverEbpfHostPath();
 std::string GetSendBufferFileNamePrefix();
 std::string GetLegacyUserLocalConfigFilePath();

core/application/Application.cpp

@@ -133,7 +133,6 @@ void Application::Init() {
     const string& configIP = AppConfig::GetInstance()->GetConfigIP();
     if (!configIP.empty()) {
         LoongCollectorMonitor::mIpAddr = configIP;
-        LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_ip", GetHostIp());
     } else if (!interface.empty()) {
         LoongCollectorMonitor::mIpAddr = GetHostIp(interface);
         if (LoongCollectorMonitor::mIpAddr.empty()) {
@@ -151,7 +150,6 @@ void Application::Init() {
     const string& configHostName = AppConfig::GetInstance()->GetConfigHostName();
     if (!configHostName.empty()) {
         LoongCollectorMonitor::mHostname = configHostName;
-        LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_hostname", GetHostName());
     }
 
     GenerateInstanceId();
@@ -198,7 +196,6 @@ void Application::Init() {
 
 void Application::Start() { // GCOVR_EXCL_START
     LoongCollectorMonitor::mStartTime = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S");
-    LogtailMonitor::GetInstance()->UpdateConstMetric("start_time", LoongCollectorMonitor::mStartTime);
 
 #if defined(__ENTERPRISE__) && defined(_MSC_VER)
     InitWindowsSignalObject();

core/common/SafeQueue.h

@@ -74,6 +74,10 @@ class SafeQueue {
         return mQueue.size();
     }
 
+#ifdef APSARA_UNIT_TEST_MAIN
+    void Clear() { std::queue<T>().swap(mQueue); }
+#endif
+
 private:
     std::queue<T> mQueue;
     mutable std::mutex mMux;

core/config/watcher/PipelineConfigWatcher.cpp

@@ -18,6 +18,10 @@
 
 #include "common/FileSystemUtil.h"
 #include "config/ConfigUtil.h"
+#include "config/common_provider/CommonConfigProvider.h"
+#ifdef __ENTERPRISE__
+#include "config/provider/EnterpriseConfigProvider.h"
+#endif
 #include "logger/Logger.h"
 #include "monitor/Monitor.h"
 #include "pipeline/PipelineManager.h"
@@ -37,9 +41,9 @@ pair<PipelineConfigDiff, TaskConfigDiff> PipelineConfigWatcher::CheckConfigDiff(
     PipelineConfigDiff pDiff;
     TaskConfigDiff tDiff;
     unordered_set<string> configSet;
-    // inner configs
-    InsertInnerPipelines(pDiff, tDiff, configSet);
-    // configs from file
+    // builtin pipeline configs
+    InsertBuiltInPipelines(pDiff, tDiff, configSet);
+    // file pipeline configs 
     InsertPipelines(pDiff, tDiff, configSet);
 
     for (const auto& name : mPipelineManager->GetAllConfigNames()) {
@@ -83,87 +87,90 @@ pair<PipelineConfigDiff, TaskConfigDiff> PipelineConfigWatcher::CheckConfigDiff(
     return make_pair(std::move(pDiff), std::move(tDiff));
 }
 
-void PipelineConfigWatcher::InsertInnerPipelines(PipelineConfigDiff& pDiff,
+void PipelineConfigWatcher::InsertBuiltInPipelines(PipelineConfigDiff& pDiff,
                                                  TaskConfigDiff& tDiff,
                                                  unordered_set<string>& configSet) {
-    std::map<std::string, std::string> innerPipelines;
-    // self-monitor metric
-    innerPipelines[LoongCollectorMonitor::GetInnerSelfMonitorMetricPipelineName()]
-        = LoongCollectorMonitor::GetInnerSelfMonitorMetricPipeline();
+#ifdef __ENTERPRISE__
+    const std::map<std::string, std::string>& builtInPipelines
+        = EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs();
 
-    // process
-    for (const auto& pipeline : innerPipelines) {
-        if (configSet.find(pipeline.first) != configSet.end()) {
+    for (const auto& pipeline : builtInPipelines) {
+        const string& pipelineName = pipeline.first;
+        const string& pipleineDetail = pipeline.second;
+        if (configSet.find(pipelineName) != configSet.end()) {
             LOG_WARNING(sLogger,
                         ("more than 1 config with the same name is found", "skip current config")("inner pipeline",
-                                                                                                  pipeline.first));
+                                                                                                  pipelineName));
             continue;
         }
-        configSet.insert(pipeline.first);
+        configSet.insert(pipelineName);
 
         string errorMsg;
-        auto iter = mInnerConfigMap.find(pipeline.first);
+        auto iter = mInnerConfigMap.find(pipelineName);
         if (iter == mInnerConfigMap.end()) {
-            mInnerConfigMap[pipeline.first] = pipeline.second;
+            mInnerConfigMap[pipelineName] = pipleineDetail;
             unique_ptr<Json::Value> detail = make_unique<Json::Value>();
-            if (!ParseConfigDetail(pipeline.second, ".json", *detail, errorMsg)) {
+            if (!ParseConfigDetail(pipleineDetail, ".json", *detail, errorMsg)) {
                 LOG_WARNING(sLogger,
                             ("config format error", "skip current object")("error msg", errorMsg)("inner pipeline",
-                                                                                                  pipeline.first));
+                                                                                                  pipelineName));
                 continue;
             }
-            if (!IsConfigEnabled(pipeline.first, *detail)) {
-                LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", pipeline.first));
+            if (!IsConfigEnabled(pipelineName, *detail)) {
+                LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", pipelineName));
                 continue;
             }
-            if (!CheckAddedConfig(pipeline.first, std::move(detail), pDiff, tDiff)) {
+            if (!CheckAddedConfig(pipelineName, std::move(detail), pDiff, tDiff)) {
                 continue;
             }
-        } else if (pipeline.second != iter->second) {
-            mInnerConfigMap[pipeline.first] = pipeline.second;
+        } else if (pipleineDetail != iter->second) {
+            mInnerConfigMap[pipelineName] = pipleineDetail;
             unique_ptr<Json::Value> detail = make_unique<Json::Value>();
-            if (!ParseConfigDetail(pipeline.second, ".json", *detail, errorMsg)) {
+            if (!ParseConfigDetail(pipleineDetail, ".json", *detail, errorMsg)) {
                 LOG_WARNING(sLogger,
                             ("config format error", "skip current object")("error msg", errorMsg)("inner pipeline",
-                                                                                                  pipeline.first));
+                                                                                                  pipelineName));
                 continue;
             }
-            if (!IsConfigEnabled(pipeline.first, *detail)) {
+            if (!IsConfigEnabled(pipelineName, *detail)) {
                 switch (GetConfigType(*detail)) {
                     case ConfigType::Pipeline:
-                        if (mPipelineManager->FindConfigByName(pipeline.first)) {
-                            pDiff.mRemoved.push_back(pipeline.first);
+                        if (mPipelineManager->FindConfigByName(pipelineName)) {
+                            pDiff.mRemoved.push_back(pipelineName);
                             LOG_INFO(sLogger,
                                      ("existing valid config modified and disabled",
-                                      "prepare to stop current running pipeline")("config", pipeline.first));
+                                      "prepare to stop current running pipeline")("config", pipelineName));
                         } else {
                             LOG_INFO(sLogger,
                                      ("existing invalid config modified and disabled",
-                                      "skip current object")("config", pipeline.first));
+                                      "skip current object")("config", pipelineName));
                         }
                         break;
                     case ConfigType::Task:
-                        if (mTaskPipelineManager->FindPipelineByName(pipeline.first)) {
-                            tDiff.mRemoved.push_back(pipeline.first);
+                        if (mTaskPipelineManager->FindPipelineByName(pipelineName)) {
+                            tDiff.mRemoved.push_back(pipelineName);
                             LOG_INFO(sLogger,
                                      ("existing valid config modified and disabled",
-                                      "prepare to stop current running task")("config", pipeline.first));
+                                      "prepare to stop current running task")("config", pipelineName));
                         } else {
                             LOG_INFO(sLogger,
                                      ("existing invalid config modified and disabled",
-                                      "skip current object")("config", pipeline.first));
+                                      "skip current object")("config", pipelineName));
                         }
                         break;
                 }
                 continue;
             }
-            if (!CheckModifiedConfig(pipeline.first, std::move(detail), pDiff, tDiff)) {
+            if (!CheckModifiedConfig(pipelineName, std::move(detail), pDiff, tDiff)) {
                 continue;
             }
         } else {
             LOG_DEBUG(sLogger, ("existing inner config unchanged", "skip current object"));
         }
     }
+#else
+    return;
+#endif
 }
 
 void PipelineConfigWatcher::InsertPipelines(PipelineConfigDiff& pDiff,

core/config/watcher/PipelineConfigWatcher.h

@@ -46,7 +46,7 @@ class PipelineConfigWatcher : public ConfigWatcher {
     PipelineConfigWatcher();
     ~PipelineConfigWatcher() = default;
 
-    void InsertInnerPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set<std::string>& configSet);
+    void InsertBuiltInPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set<std::string>& configSet);
     void InsertPipelines(PipelineConfigDiff& pDiff, TaskConfigDiff& tDiff, std::unordered_set<std::string>& configSet);
     bool CheckAddedConfig(const std::string& configName,
                           std::unique_ptr<Json::Value>&& configDetail,