Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modify the logic that converts the parsing environment to flags. #2037

Merged
merged 18 commits into from
Jan 15, 2025
104 changes: 61 additions & 43 deletions core/app_config/AppConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include <algorithm>
#include <filesystem>
#include <iostream>
#include <unordered_set>
#include <utility>

#include "boost/filesystem.hpp"
Expand Down Expand Up @@ -161,9 +162,9 @@ DEFINE_FLAG_STRING(metrics_report_method,
"method to report metrics (default none, means logtail will not report metrics)",
"sls");

DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator service", "");
DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
DEFINE_FLAG_STRING(operator_service, "loong collector operator service", "");
DEFINE_FLAG_INT32(operator_service_port, "loong collector operator service port", 8888);
DEFINE_FLAG_INT32(k8s_meta_service_port, "loong collector operator service port", 9000);
DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");

DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
Expand Down Expand Up @@ -210,14 +211,21 @@ const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE = 10;
const uint32_t SLOW_FALL_BACK_FAIL_PERCENTAGE = 40;

std::string AppConfig::sLocalConfigDir = "local";

const std::string LOONGCOLLECTOR_ENV_PREFIX = "LOONG_";

std::string GetLoongcollectorEnv(const std::string& flagName) {
return LOONGCOLLECTOR_ENV_PREFIX + ToUpperCaseString(flagName);
}

void CreateAgentDir() {
try {
const char* value = getenv("logtail_mode");
const char* value = getenv("LOGTAIL_MODE");
if (value != NULL) {
STRING_FLAG(logtail_mode) = StringTo<bool>(value);
}
} catch (const exception& e) {
std::cout << "load config from env error, env_name:logtail_mode, error:" << e.what() << std::endl;
std::cout << "load config from env error, env_name:LOGTAIL_MODE, error:" << e.what() << std::endl;
}
if (BOOL_FLAG(logtail_mode)) {
return;
Expand All @@ -226,7 +234,8 @@ void CreateAgentDir() {
Json::Value emptyJson;
#define PROCESSDIRFLAG(flag_name) \
try { \
const char* value = getenv(#flag_name); \
const auto env_name = GetLoongcollectorEnv(#flag_name); \
const char* value = getenv(env_name.c_str()); \
if (value != NULL) { \
STRING_FLAG(flag_name) = StringTo<string>(value); \
} \
Expand All @@ -247,11 +256,11 @@ void CreateAgentDir() {
} \
}

PROCESSDIRFLAG(loongcollector_conf_dir);
PROCESSDIRFLAG(loongcollector_log_dir);
PROCESSDIRFLAG(loongcollector_data_dir);
PROCESSDIRFLAG(loongcollector_run_dir);
PROCESSDIRFLAG(loongcollector_third_party_dir);
PROCESSDIRFLAG(conf_dir);
PROCESSDIRFLAG(logs_dir);
PROCESSDIRFLAG(data_dir);
PROCESSDIRFLAG(run_dir);
PROCESSDIRFLAG(third_party_dir);
}

std::string GetAgentThirdPartyDir() {
Expand All @@ -262,7 +271,7 @@ std::string GetAgentThirdPartyDir() {
if (BOOL_FLAG(logtail_mode)) {
dir = AppConfig::GetInstance()->GetLoongcollectorConfDir();
} else {
dir = STRING_FLAG(loongcollector_third_party_dir) + PATH_SEPARATOR;
dir = STRING_FLAG(third_party_dir) + PATH_SEPARATOR;
}
return dir;
}
Expand All @@ -278,7 +287,7 @@ std::string GetAgentLogDir() {
if (BOOL_FLAG(logtail_mode)) {
dir = GetProcessExecutionDir();
} else {
dir = STRING_FLAG(loongcollector_log_dir) + PATH_SEPARATOR;
dir = STRING_FLAG(logs_dir) + PATH_SEPARATOR;
}
#endif
return dir;
Expand Down Expand Up @@ -372,7 +381,7 @@ std::string GetAgentDataDir() {
if (BOOL_FLAG(logtail_mode)) {
dir = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "checkpoint";
} else {
dir = STRING_FLAG(loongcollector_data_dir) + PATH_SEPARATOR;
dir = STRING_FLAG(data_dir) + PATH_SEPARATOR;
}
#endif
if (!CheckExistance(dir)) {
Expand All @@ -396,7 +405,7 @@ std::string GetAgentConfDir() {
if (BOOL_FLAG(logtail_mode)) {
dir = GetProcessExecutionDir();
} else {
dir = STRING_FLAG(loongcollector_conf_dir) + PATH_SEPARATOR;
dir = STRING_FLAG(conf_dir) + PATH_SEPARATOR;
}
#endif
return dir;
Expand All @@ -413,7 +422,7 @@ std::string GetAgentRunDir() {
if (BOOL_FLAG(logtail_mode)) {
dir = GetProcessExecutionDir();
} else {
dir = STRING_FLAG(loongcollector_run_dir) + PATH_SEPARATOR;
dir = STRING_FLAG(run_dir) + PATH_SEPARATOR;
}
#endif
return dir;
Expand Down Expand Up @@ -841,27 +850,6 @@ bool LoadSingleValueEnvConfig(const char* envKey, T& configValue, const T minVal
return false;
}

/**
* @brief 从环境变量加载配置值(如果存在)
*
* @tparam T 配置值的类型
* @param envKey 环境变量的键
* @param cfgValue 配置值的引用,如果环境变量存在,将被更新
*/
template <typename T>
void LoadEnvValueIfExisting(const char* envKey, T& cfgValue) {
try {
const char* value = getenv(envKey);
if (value != NULL) {
T val = StringTo<T>(value);
cfgValue = val;
LOG_INFO(sLogger, ("load config from env", envKey)("value", val));
}
} catch (const std::exception& e) {
LOG_WARNING(sLogger, ("load config from env error", envKey)("error", e.what()));
}
}

void AppConfig::LoadEnvResourceLimit() {
LoadSingleValueEnvConfig("cpu_usage_limit", mCpuUsageUpLimit, (float)0.4);
LoadSingleValueEnvConfig("mem_usage_limit", mMemUsageUpLimit, (int64_t)384);
Expand Down Expand Up @@ -1457,11 +1445,7 @@ void AppConfig::InitEnvMapping(const std::string& envStr, std::map<std::string,
}
}
void AppConfig::SetConfigFlag(const std::string& flagName, const std::string& value) {
static set<string> sIgnoreFlagSet = {"loongcollector_conf_dir",
"loongcollector_log_dir",
"loongcollector_data_dir",
"loongcollector_run_dir",
"logtail_mode"};
static set<string> sIgnoreFlagSet = {"conf_dir", "logs_dir", "data_dir", "run_dir", "logtail_mode"};
if (sIgnoreFlagSet.find(flagName) != sIgnoreFlagSet.end()) {
return;
}
Expand Down Expand Up @@ -1508,9 +1492,43 @@ void AppConfig::ParseEnvToFlags() {
}
}
#endif
std::unordered_set<std::string> sIgnoreFlagSet = {"buffer_file_path",
"check_point_filename",
"data_server_port",
"host_path_blacklist",
"process_thread_count",
"send_request_concurrency",
"check_point_dump_interval",
"check_point_max_count",
"enable_root_path_collection",
"ilogtail_config",
"ilogtail_discard_interval",
"default_tail_limit_kb",
"logreader_max_rotate_queue_size",
"force_release_deleted_file_fd_timeout",
"batch_send_interval",
"ALIYUN_LOG_FILE_TAGS",
"default_container_host_path",
"default_max_inotify_watch_num",
"enable_full_drain_mode",
"ilogtail_discard_old_data",
"timeout_interval",
"enable_env_ref_in_config",
"max_watch_dir_count",
"polling_max_stat_count",
"polling_max_stat_count_per_config",
"polling_max_stat_count_per_dir"};
for (const auto& iter : envMapping) {
const std::string& key = iter.first;
const std::string& value = iter.second;
std::string key = iter.first;
// Skip if key is not in ignore set and doesn't start with prefix
if (sIgnoreFlagSet.find(key) == sIgnoreFlagSet.end() && !StartWith(key, LOONGCOLLECTOR_ENV_PREFIX)) {
continue;
}
// Convert to lowercase if key has prefix
if (StartWith(key, LOONGCOLLECTOR_ENV_PREFIX)) {
key = ToLowerCaseString(key.substr(LOONGCOLLECTOR_ENV_PREFIX.size()));
}
SetConfigFlag(key, value);
// 尝试解析为 double
char* end;
Expand Down
10 changes: 5 additions & 5 deletions core/common/LogtailCommonFlags.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,8 @@ DEFINE_FLAG_STRING(default_container_host_path, "", "/logtail_host");
#endif

// dir
DEFINE_FLAG_STRING(loongcollector_conf_dir, "loongcollector config dir", "conf");
DEFINE_FLAG_STRING(loongcollector_log_dir, "loongcollector log dir", "log");
DEFINE_FLAG_STRING(loongcollector_data_dir, "loongcollector data dir", "data");
DEFINE_FLAG_STRING(loongcollector_run_dir, "loongcollector run dir", "run");
DEFINE_FLAG_STRING(loongcollector_third_party_dir, "loongcollector third party dir", "thirdparty");
DEFINE_FLAG_STRING(conf_dir, "loongcollector config dir", "conf");
DEFINE_FLAG_STRING(logs_dir, "loongcollector log dir", "log");
DEFINE_FLAG_STRING(data_dir, "loongcollector data dir", "data");
DEFINE_FLAG_STRING(run_dir, "loongcollector run dir", "run");
DEFINE_FLAG_STRING(third_party_dir, "loongcollector third party dir", "thirdparty");
10 changes: 5 additions & 5 deletions core/common/LogtailCommonFlags.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ DECLARE_FLAG_INT32(timeout_interval);

DECLARE_FLAG_STRING(default_container_host_path);

DECLARE_FLAG_STRING(loongcollector_conf_dir);
DECLARE_FLAG_STRING(loongcollector_log_dir);
DECLARE_FLAG_STRING(loongcollector_data_dir);
DECLARE_FLAG_STRING(loongcollector_run_dir);
DECLARE_FLAG_STRING(loongcollector_third_party_dir);
DECLARE_FLAG_STRING(conf_dir);
DECLARE_FLAG_STRING(logs_dir);
DECLARE_FLAG_STRING(data_dir);
DECLARE_FLAG_STRING(run_dir);
DECLARE_FLAG_STRING(third_party_dir);
8 changes: 4 additions & 4 deletions core/metadata/K8sMetadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
#include "common/Flags.h"
#include "common/LRUCache.h"

DECLARE_FLAG_STRING(loong_collector_operator_service);
DECLARE_FLAG_INT32(loong_collector_k8s_meta_service_port);
DECLARE_FLAG_STRING(operator_service);
DECLARE_FLAG_INT32(k8s_meta_service_port);

namespace logtail {

Expand Down Expand Up @@ -59,8 +59,8 @@ class K8sMetadata {
std::string mServiceHost;
int32_t mServicePort;
K8sMetadata(size_t cacheSize) : containerCache(cacheSize, 0), ipCache(cacheSize, 0) {
mServiceHost = STRING_FLAG(loong_collector_operator_service);
mServicePort = INT32_FLAG(loong_collector_k8s_meta_service_port);
mServiceHost = STRING_FLAG(operator_service);
mServicePort = INT32_FLAG(k8s_meta_service_port);
}
K8sMetadata(const K8sMetadata&) = delete;
K8sMetadata& operator=(const K8sMetadata&) = delete;
Expand Down
8 changes: 4 additions & 4 deletions core/prometheus/PrometheusInputRunner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@

using namespace std;

DECLARE_FLAG_STRING(loong_collector_operator_service);
DECLARE_FLAG_INT32(loong_collector_operator_service_port);
DECLARE_FLAG_STRING(operator_service);
DECLARE_FLAG_INT32(operator_service_port);
DECLARE_FLAG_STRING(_pod_name_);

namespace logtail {

PrometheusInputRunner::PrometheusInputRunner()
: mServiceHost(STRING_FLAG(loong_collector_operator_service)),
mServicePort(INT32_FLAG(loong_collector_operator_service_port)),
: mServiceHost(STRING_FLAG(operator_service)),
mServicePort(INT32_FLAG(operator_service_port)),
mPodName(STRING_FLAG(_pod_name_)),
mEventPool(true),
mUnRegisterMs(0) {
Expand Down
30 changes: 30 additions & 0 deletions core/unittest/app_config/AppConfigUnittest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,15 @@ DECLARE_FLAG_STRING(ebpf_converage_config_strategy);
DECLARE_FLAG_STRING(ebpf_sample_config_strategy);
DECLARE_FLAG_DOUBLE(ebpf_sample_config_config_rate);
DECLARE_FLAG_BOOL(logtail_mode);
DECLARE_FLAG_STRING(host_path_blacklist);
DECLARE_FLAG_DOUBLE(default_machine_cpu_usage_threshold);

namespace logtail {

class AppConfigUnittest : public ::testing::Test {
public:
void TestRecurseParseJsonToFlags();
void TestParseEnvToFlags();

private:
void writeLogtailConfigJSON(const Json::Value& v) {
Expand Down Expand Up @@ -168,7 +171,34 @@ void AppConfigUnittest::TestRecurseParseJsonToFlags() {
APSARA_TEST_EQUAL(INT32_FLAG(ebpf_receive_event_chan_cap), 55);
}

void AppConfigUnittest::TestParseEnvToFlags() {
// 忽略列表中的环境变量,继续可以用小写且允许 LOONG_ 前缀的格式
{
SetEnv("host_path_blacklist", "test1");
AppConfig::GetInstance()->ParseEnvToFlags();
APSARA_TEST_EQUAL(STRING_FLAG(host_path_blacklist), "test1");
UnsetEnv("host_path_blacklist");

SetEnv("LOONG_host_path_blacklist", "test2");
quzard marked this conversation as resolved.
Show resolved Hide resolved
AppConfig::GetInstance()->ParseEnvToFlags();
APSARA_TEST_EQUAL(STRING_FLAG(host_path_blacklist), "test2");
}
// 不忽略列表中的环境变量,需要为大写,LOONG_ 前缀
{
SetEnv("default_machine_cpu_usage_threshold", "1");
AppConfig::GetInstance()->ParseEnvToFlags();
APSARA_TEST_NOT_EQUAL(DOUBLE_FLAG(default_machine_cpu_usage_threshold), 1);
APSARA_TEST_EQUAL(DOUBLE_FLAG(default_machine_cpu_usage_threshold), 0.4);
UnsetEnv("default_machine_cpu_usage_threshold");

SetEnv("LOONG_DEFAULT_MACHINE_CPU_USAGE_THRESHOLD", "2");
AppConfig::GetInstance()->ParseEnvToFlags();
APSARA_TEST_EQUAL(DOUBLE_FLAG(default_machine_cpu_usage_threshold), 2);
}
}

UNIT_TEST_CASE(AppConfigUnittest, TestRecurseParseJsonToFlags);
UNIT_TEST_CASE(AppConfigUnittest, TestParseEnvToFlags);

} // namespace logtail

Expand Down
4 changes: 2 additions & 2 deletions docs/cn/installation/logtail-mode.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ LoongCollector 提供了 Logtail 兼容模式,可以让您在升级到 LoongCo
**方式二:环境变量**

```bash
export logtail_mode=true
export LOGTAIL_MODE=true
./loongcollector
```

Expand All @@ -42,7 +42,7 @@ export logtail_mode=true
1. 需要给LoongCollector容器添加环境变量:

```bash
logtail_mode=true
LOGTAIL_MODE=true
```

2. 需要调整LoongCollector挂载路径映射:
Expand Down
14 changes: 7 additions & 7 deletions docs/cn/installation/loongcollector-dir.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,28 +127,28 @@ inotify日志:`/opt/loongcollector/run/inotify_watcher_dirs`

LoongCollector 提供以下参数用于自定义各类目录位置:

- `loongcollector_conf_dir`: 配置目录
- 配置目录: gflag为`conf_dir`、环境变量为`LOONG_CONF_DIR`

- `loongcollector_log_dir`: 日志目录
- 日志目录: gflag为`logs_dir`、环境变量为`LOONG_LOGS_DIR`

- `loongcollector_data_dir`: 数据目录
- 数据目录: gflag为`data_dir`、环境变量为`LOONG_DATA_DIR`

- `loongcollector_run_dir`: 运行时目录
- 运行时目录: gflag为`run_dir`、环境变量为`LOONG_RUN_DIR`

- `loongcollector_third_party_dir`: 第三方依赖目录
- 第三方依赖目录: gflag为`third_party_dir`、环境变量为`LOONG_THIRD_PARTY_DIR`

### 配置方式

1. 命令行参数:

```bash
./loongcollector --loongcollector_conf_dir=/custom/path/conf
./loongcollector --conf_dir=/custom/path/conf
```

2. 环境变量:

```bash
export loongcollector_conf_dir=/custom/path/conf
export LOONG_CONF_DIR=/custom/path/conf
./loongcollector
```

Expand Down
Loading
Loading