Modify the logic that converts the parsing environment to flags. (#2037)

alibaba · Jan 15, 2025 · a605b1c · a605b1c
1 parent 122a3bc
commit a605b1c
Show file tree

Hide file tree

Showing 11 changed files with 487 additions and 74 deletions.
diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp
@@ -17,6 +17,7 @@
 #include <algorithm>
 #include <filesystem>
 #include <iostream>
+#include <unordered_set>
 #include <utility>
 
 #include "boost/filesystem.hpp"
@@ -161,9 +162,9 @@ DEFINE_FLAG_STRING(metrics_report_method,
                    "method to report metrics (default none, means logtail will not report metrics)",
                    "sls");
 
-DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator service", "");
-DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
-DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
+DEFINE_FLAG_STRING(operator_service, "loong collector operator service", "");
+DEFINE_FLAG_INT32(operator_service_port, "loong collector operator service port", 8888);
+DEFINE_FLAG_INT32(k8s_meta_service_port, "loong collector operator service port", 9000);
 DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");
 
 DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
@@ -210,14 +211,21 @@ const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE = 10;
 const uint32_t SLOW_FALL_BACK_FAIL_PERCENTAGE = 40;
 
 std::string AppConfig::sLocalConfigDir = "local";
+
+const std::string LOONGCOLLECTOR_ENV_PREFIX = "LOONG_";
+
+std::string GetLoongcollectorEnv(const std::string& flagName) {
+    return LOONGCOLLECTOR_ENV_PREFIX + ToUpperCaseString(flagName);
+}
+
 void CreateAgentDir() {
     try {
-        const char* value = getenv("logtail_mode");
+        const char* value = getenv("LOGTAIL_MODE");
         if (value != NULL) {
             STRING_FLAG(logtail_mode) = StringTo<bool>(value);
         }
     } catch (const exception& e) {
-        std::cout << "load config from env error, env_name:logtail_mode, error:" << e.what() << std::endl;
+        std::cout << "load config from env error, env_name:LOGTAIL_MODE, error:" << e.what() << std::endl;
     }
     if (BOOL_FLAG(logtail_mode)) {
         return;
@@ -226,7 +234,8 @@ void CreateAgentDir() {
     Json::Value emptyJson;
 #define PROCESSDIRFLAG(flag_name) \
     try { \
-        const char* value = getenv(#flag_name); \
+        const auto env_name = GetLoongcollectorEnv(#flag_name); \
+        const char* value = getenv(env_name.c_str()); \
         if (value != NULL) { \
             STRING_FLAG(flag_name) = StringTo<string>(value); \
         } \
@@ -247,11 +256,11 @@ void CreateAgentDir() {
         } \
     }
 
-    PROCESSDIRFLAG(loongcollector_conf_dir);
-    PROCESSDIRFLAG(loongcollector_log_dir);
-    PROCESSDIRFLAG(loongcollector_data_dir);
-    PROCESSDIRFLAG(loongcollector_run_dir);
-    PROCESSDIRFLAG(loongcollector_third_party_dir);
+    PROCESSDIRFLAG(conf_dir);
+    PROCESSDIRFLAG(logs_dir);
+    PROCESSDIRFLAG(data_dir);
+    PROCESSDIRFLAG(run_dir);
+    PROCESSDIRFLAG(third_party_dir);
 }
 
 std::string GetAgentThirdPartyDir() {
@@ -262,7 +271,7 @@ std::string GetAgentThirdPartyDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = AppConfig::GetInstance()->GetLoongcollectorConfDir();
     } else {
-        dir = STRING_FLAG(loongcollector_third_party_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(third_party_dir) + PATH_SEPARATOR;
     }
     return dir;
 }
@@ -278,7 +287,7 @@ std::string GetAgentLogDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = GetProcessExecutionDir();
     } else {
-        dir = STRING_FLAG(loongcollector_log_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(logs_dir) + PATH_SEPARATOR;
     }
 #endif
     return dir;
@@ -372,7 +381,7 @@ std::string GetAgentDataDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "checkpoint";
     } else {
-        dir = STRING_FLAG(loongcollector_data_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(data_dir) + PATH_SEPARATOR;
     }
 #endif
     if (!CheckExistance(dir)) {
@@ -396,7 +405,7 @@ std::string GetAgentConfDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = GetProcessExecutionDir();
     } else {
-        dir = STRING_FLAG(loongcollector_conf_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(conf_dir) + PATH_SEPARATOR;
     }
 #endif
     return dir;
@@ -413,7 +422,7 @@ std::string GetAgentRunDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = GetProcessExecutionDir();
     } else {
-        dir = STRING_FLAG(loongcollector_run_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(run_dir) + PATH_SEPARATOR;
     }
 #endif
     return dir;
@@ -841,27 +850,6 @@ bool LoadSingleValueEnvConfig(const char* envKey, T& configValue, const T minVal
     return false;
 }
 
-/**
- * @brief 从环境变量加载配置值（如果存在）
- *
- * @tparam T 配置值的类型
- * @param envKey 环境变量的键
- * @param cfgValue 配置值的引用，如果环境变量存在，将被更新
- */
-template <typename T>
-void LoadEnvValueIfExisting(const char* envKey, T& cfgValue) {
-    try {
-        const char* value = getenv(envKey);
-        if (value != NULL) {
-            T val = StringTo<T>(value);
-            cfgValue = val;
-            LOG_INFO(sLogger, ("load config from env", envKey)("value", val));
-        }
-    } catch (const std::exception& e) {
-        LOG_WARNING(sLogger, ("load config from env error", envKey)("error", e.what()));
-    }
-}
-
 void AppConfig::LoadEnvResourceLimit() {
     LoadSingleValueEnvConfig("cpu_usage_limit", mCpuUsageUpLimit, (float)0.4);
     LoadSingleValueEnvConfig("mem_usage_limit", mMemUsageUpLimit, (int64_t)384);
@@ -1457,11 +1445,7 @@ void AppConfig::InitEnvMapping(const std::string& envStr, std::map<std::string,
     }
 }
 void AppConfig::SetConfigFlag(const std::string& flagName, const std::string& value) {
-    static set<string> sIgnoreFlagSet = {"loongcollector_conf_dir",
-                                         "loongcollector_log_dir",
-                                         "loongcollector_data_dir",
-                                         "loongcollector_run_dir",
-                                         "logtail_mode"};
+    static set<string> sIgnoreFlagSet = {"conf_dir", "logs_dir", "data_dir", "run_dir", "logtail_mode"};
     if (sIgnoreFlagSet.find(flagName) != sIgnoreFlagSet.end()) {
         return;
     }
@@ -1508,9 +1492,43 @@ void AppConfig::ParseEnvToFlags() {
         }
     }
 #endif
+    std::unordered_set<std::string> sIgnoreFlagSet = {"buffer_file_path",
+                                                      "check_point_filename",
+                                                      "data_server_port",
+                                                      "host_path_blacklist",
+                                                      "process_thread_count",
+                                                      "send_request_concurrency",
+                                                      "check_point_dump_interval",
+                                                      "check_point_max_count",
+                                                      "enable_root_path_collection",
+                                                      "ilogtail_config",
+                                                      "ilogtail_discard_interval",
+                                                      "default_tail_limit_kb",
+                                                      "logreader_max_rotate_queue_size",
+                                                      "force_release_deleted_file_fd_timeout",
+                                                      "batch_send_interval",
+                                                      "ALIYUN_LOG_FILE_TAGS",
+                                                      "default_container_host_path",
+                                                      "default_max_inotify_watch_num",
+                                                      "enable_full_drain_mode",
+                                                      "ilogtail_discard_old_data",
+                                                      "timeout_interval",
+                                                      "enable_env_ref_in_config",
+                                                      "max_watch_dir_count",
+                                                      "polling_max_stat_count",
+                                                      "polling_max_stat_count_per_config",
+                                                      "polling_max_stat_count_per_dir"};
     for (const auto& iter : envMapping) {
-        const std::string& key = iter.first;
         const std::string& value = iter.second;
+        std::string key = iter.first;
+        // Skip if key is not in ignore set and doesn't start with prefix
+        if (sIgnoreFlagSet.find(key) == sIgnoreFlagSet.end() && !StartWith(key, LOONGCOLLECTOR_ENV_PREFIX)) {
+            continue;
+        }
+        // Convert to lowercase if key has prefix
+        if (StartWith(key, LOONGCOLLECTOR_ENV_PREFIX)) {
+            key = ToLowerCaseString(key.substr(LOONGCOLLECTOR_ENV_PREFIX.size()));
+        }
         SetConfigFlag(key, value);
         // 尝试解析为 double
         char* end;

diff --git a/core/common/LogtailCommonFlags.cpp b/core/common/LogtailCommonFlags.cpp
@@ -122,8 +122,8 @@ DEFINE_FLAG_STRING(default_container_host_path, "", "/logtail_host");
 #endif
 
 // dir
-DEFINE_FLAG_STRING(loongcollector_conf_dir, "loongcollector config dir", "conf");
-DEFINE_FLAG_STRING(loongcollector_log_dir, "loongcollector log dir", "log");
-DEFINE_FLAG_STRING(loongcollector_data_dir, "loongcollector data dir", "data");
-DEFINE_FLAG_STRING(loongcollector_run_dir, "loongcollector run dir", "run");
-DEFINE_FLAG_STRING(loongcollector_third_party_dir, "loongcollector third party dir", "thirdparty");
+DEFINE_FLAG_STRING(conf_dir, "loongcollector config dir", "conf");
+DEFINE_FLAG_STRING(logs_dir, "loongcollector log dir", "log");
+DEFINE_FLAG_STRING(data_dir, "loongcollector data dir", "data");
+DEFINE_FLAG_STRING(run_dir, "loongcollector run dir", "run");
+DEFINE_FLAG_STRING(third_party_dir, "loongcollector third party dir", "thirdparty");
diff --git a/core/common/LogtailCommonFlags.h b/core/common/LogtailCommonFlags.h
@@ -53,8 +53,8 @@ DECLARE_FLAG_INT32(timeout_interval);
 
 DECLARE_FLAG_STRING(default_container_host_path);
 
-DECLARE_FLAG_STRING(loongcollector_conf_dir);
-DECLARE_FLAG_STRING(loongcollector_log_dir);
-DECLARE_FLAG_STRING(loongcollector_data_dir);
-DECLARE_FLAG_STRING(loongcollector_run_dir);
-DECLARE_FLAG_STRING(loongcollector_third_party_dir);
+DECLARE_FLAG_STRING(conf_dir);
+DECLARE_FLAG_STRING(logs_dir);
+DECLARE_FLAG_STRING(data_dir);
+DECLARE_FLAG_STRING(run_dir);
+DECLARE_FLAG_STRING(third_party_dir);
diff --git a/core/metadata/K8sMetadata.h b/core/metadata/K8sMetadata.h
@@ -21,8 +21,8 @@
 #include "common/Flags.h"
 #include "common/LRUCache.h"
 
-DECLARE_FLAG_STRING(loong_collector_operator_service);
-DECLARE_FLAG_INT32(loong_collector_k8s_meta_service_port);
+DECLARE_FLAG_STRING(operator_service);
+DECLARE_FLAG_INT32(k8s_meta_service_port);
 
 namespace logtail {
 
@@ -59,8 +59,8 @@ class K8sMetadata {
     std::string mServiceHost;
     int32_t mServicePort;
     K8sMetadata(size_t cacheSize) : containerCache(cacheSize, 0), ipCache(cacheSize, 0) {
-        mServiceHost = STRING_FLAG(loong_collector_operator_service);
-        mServicePort = INT32_FLAG(loong_collector_k8s_meta_service_port);
+        mServiceHost = STRING_FLAG(operator_service);
+        mServicePort = INT32_FLAG(k8s_meta_service_port);
     }
     K8sMetadata(const K8sMetadata&) = delete;
     K8sMetadata& operator=(const K8sMetadata&) = delete;

diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp
@@ -38,15 +38,15 @@
 
 using namespace std;
 
-DECLARE_FLAG_STRING(loong_collector_operator_service);
-DECLARE_FLAG_INT32(loong_collector_operator_service_port);
+DECLARE_FLAG_STRING(operator_service);
+DECLARE_FLAG_INT32(operator_service_port);
 DECLARE_FLAG_STRING(_pod_name_);
 
 namespace logtail {
 
 PrometheusInputRunner::PrometheusInputRunner()
-    : mServiceHost(STRING_FLAG(loong_collector_operator_service)),
-      mServicePort(INT32_FLAG(loong_collector_operator_service_port)),
+    : mServiceHost(STRING_FLAG(operator_service)),
+      mServicePort(INT32_FLAG(operator_service_port)),
       mPodName(STRING_FLAG(_pod_name_)),
       mEventPool(true),
       mUnRegisterMs(0) {

diff --git a/core/unittest/app_config/AppConfigUnittest.cpp b/core/unittest/app_config/AppConfigUnittest.cpp
@@ -28,12 +28,15 @@ DECLARE_FLAG_STRING(ebpf_converage_config_strategy);
 DECLARE_FLAG_STRING(ebpf_sample_config_strategy);
 DECLARE_FLAG_DOUBLE(ebpf_sample_config_config_rate);
 DECLARE_FLAG_BOOL(logtail_mode);
+DECLARE_FLAG_STRING(host_path_blacklist);
+DECLARE_FLAG_DOUBLE(default_machine_cpu_usage_threshold);
 
 namespace logtail {
 
 class AppConfigUnittest : public ::testing::Test {
 public:
     void TestRecurseParseJsonToFlags();
+    void TestParseEnvToFlags();
 
 private:
     void writeLogtailConfigJSON(const Json::Value& v) {
@@ -168,7 +171,34 @@ void AppConfigUnittest::TestRecurseParseJsonToFlags() {
     APSARA_TEST_EQUAL(INT32_FLAG(ebpf_receive_event_chan_cap), 55);
 }
 
+void AppConfigUnittest::TestParseEnvToFlags() {
+    // 忽略列表中的环境变量，继续可以用小写且允许 LOONG_ 前缀的格式
+    {
+        SetEnv("host_path_blacklist", "test1");
+        AppConfig::GetInstance()->ParseEnvToFlags();
+        APSARA_TEST_EQUAL(STRING_FLAG(host_path_blacklist), "test1");
+        UnsetEnv("host_path_blacklist");
+
+        SetEnv("LOONG_host_path_blacklist", "test2");
+        AppConfig::GetInstance()->ParseEnvToFlags();
+        APSARA_TEST_EQUAL(STRING_FLAG(host_path_blacklist), "test2");
+    }
+    // 不忽略列表中的环境变量，需要为大写,LOONG_ 前缀
+    {
+        SetEnv("default_machine_cpu_usage_threshold", "1");
+        AppConfig::GetInstance()->ParseEnvToFlags();
+        APSARA_TEST_NOT_EQUAL(DOUBLE_FLAG(default_machine_cpu_usage_threshold), 1);
+        APSARA_TEST_EQUAL(DOUBLE_FLAG(default_machine_cpu_usage_threshold), 0.4);
+        UnsetEnv("default_machine_cpu_usage_threshold");
+
+        SetEnv("LOONG_DEFAULT_MACHINE_CPU_USAGE_THRESHOLD", "2");
+        AppConfig::GetInstance()->ParseEnvToFlags();
+        APSARA_TEST_EQUAL(DOUBLE_FLAG(default_machine_cpu_usage_threshold), 2);
+    }
+}
+
 UNIT_TEST_CASE(AppConfigUnittest, TestRecurseParseJsonToFlags);
+UNIT_TEST_CASE(AppConfigUnittest, TestParseEnvToFlags);
 
 } // namespace logtail
 

diff --git a/docs/cn/installation/logtail-mode.md b/docs/cn/installation/logtail-mode.md
@@ -29,7 +29,7 @@ LoongCollector 提供了 Logtail 兼容模式，可以让您在升级到 LoongCo
 **方式二：环境变量**
 
 ```bash
-export logtail_mode=true
+export LOGTAIL_MODE=true
 ./loongcollector
 ```
 
@@ -42,7 +42,7 @@ export logtail_mode=true
 1. 需要给LoongCollector容器添加环境变量：
 
     ```bash
-    logtail_mode=true
+    LOGTAIL_MODE=true
     ```
 
 2. 需要调整LoongCollector挂载路径映射：

diff --git a/docs/cn/installation/loongcollector-dir.md b/docs/cn/installation/loongcollector-dir.md
@@ -127,28 +127,28 @@ inotify日志：`/opt/loongcollector/run/inotify_watcher_dirs`
 
 LoongCollector 提供以下参数用于自定义各类目录位置：
 
-- `loongcollector_conf_dir`: 配置目录
+- 配置目录: gflag为`conf_dir`、环境变量为`LOONG_CONF_DIR`
 
-- `loongcollector_log_dir`: 日志目录
+- 日志目录: gflag为`logs_dir`、环境变量为`LOONG_LOGS_DIR`
 
-- `loongcollector_data_dir`: 数据目录
+- 数据目录: gflag为`data_dir`、环境变量为`LOONG_DATA_DIR`
 
-- `loongcollector_run_dir`: 运行时目录
+- 运行时目录: gflag为`run_dir`、环境变量为`LOONG_RUN_DIR`
 
-- `loongcollector_third_party_dir`: 第三方依赖目录
+- 第三方依赖目录: gflag为`third_party_dir`、环境变量为`LOONG_THIRD_PARTY_DIR`
 
 ### 配置方式
 
 1. 命令行参数:
 
    ```bash
-   ./loongcollector --loongcollector_conf_dir=/custom/path/conf
+   ./loongcollector --conf_dir=/custom/path/conf
    ```
 
 2. 环境变量:
 
    ```bash
-   export loongcollector_conf_dir=/custom/path/conf
+   export LOONG_CONF_DIR=/custom/path/conf
    ./loongcollector
    ```