Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into feat/event_pool
Browse files Browse the repository at this point in the history
  • Loading branch information
henryzhx8 committed Oct 26, 2024
2 parents 67d2f1c + 19c681f commit 7e6c469
Show file tree
Hide file tree
Showing 310 changed files with 9,436 additions and 37,126 deletions.
118 changes: 118 additions & 0 deletions .github/workflows/benchmark.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
# Copyright 2021 iLogtail Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Benchmark

on:
pull_request:
branches:
- main
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
types:
- closed

jobs:
CI:
if: github.event.pull_request.merged == true
runs-on: ${{ matrix.runner }}
timeout-minutes: 60
strategy:
matrix:
go-version: [ 1.19.10 ]
runner: [ ubuntu-latest ]
fail-fast: true
permissions:
contents: write
steps:
# Clean up space to prevent action from running out of disk space.
- name: clean
if: matrix.runner == 'ubuntu-latest'
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo -E apt-get -qq autoremove --purge
sudo -E apt-get -qq clean
- name: Check disk space
run: |
df -hT $PWD
- name: Set up Go ${{ matrix.go-version }}
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}

- name: Check out code
uses: actions/checkout@v2
with:
submodules: true

- name: Update Docker-compose to v2
run: |
sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
- name: System environment
run: |
uname -r
docker --version
go version
- name: Run benchmark
env:
BUILD_LOGTAIL_UT: OFF
WITHOUTGDB: ON
run: |
make benchmark
git stash
- name: Store benchmark result iLogtail
uses: benchmark-action/github-action-benchmark@v1
with:
name: benchmark
tool: "customSmallerIsBetter"
output-file-path: "test/benchmark/report/ilogtail_statistic_all.json"
auto-push: false
summary-always: true

- name: Store benchmark result other agents
run: |
git checkout gh-pages
mv test/benchmark/report/records_all.json dev/bench/records_all.js
sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js
git add dev/bench/records_all.js
last_commit_message=$(git log -1 --pretty=%B)
git -c user.name=github-action-benchmark -c [email protected] commit --amend -m "$last_commit_message"
# TODO: Push gh-pages branch by github-action-bot
# - name: Push benchmark result
# run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages

result:
if: github.event.pull_request.merged == true
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ CI ]
steps:
- name: Build Result
run: echo "Just to make the GitHub merge button green"
2 changes: 1 addition & 1 deletion .github/workflows/build-core-ut.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ jobs:
run: make unittest_core

- name: Unit Test Coverage
run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*sdk.*\" -e \".*observer.*\" -e \".*logger.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*protobuf.*\" -e \".*runner.*\""
run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*sdk.*\" -e \".*logger.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*protobuf.*\" -e \".*runner.*\""

- name: Setup Python3.10
uses: actions/setup-python@v5
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/static-check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ on:
branches:
- main
- 1.*
- benchmark
jobs:
CI:
env:
Expand Down
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
.devcontainer/

### GoLand
.idea
**/.idea/**

### MacOS
.DS_Store
Expand Down Expand Up @@ -55,7 +55,8 @@ _deps
# Custom
/build/
core/build/
core/protobuf/sls/*.pb.*
core/protobuf/config_server/*/*.pb.*
core/protobuf/*/*.pb.*
core/common/Version.cpp
!/Makefile
# Enterprise
Expand Down
11 changes: 9 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,12 @@ e2e-core: clean gocdocker e2edocker
./scripts/e2e.sh e2e core

.PHONY: e2e-performance
e2e-performance: clean docker gocdocker
e2e-performance: clean docker
./scripts/e2e.sh e2e performance

.PHONY: unittest_e2e_engine
unittest_e2e_engine: clean gocdocker
cd test && go test $$(go list ./... | grep -Ev "engine|e2e") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
cd test && go test $$(go list ./... | grep -Ev "engine|e2e|benchmark") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready

.PHONY: unittest_plugin
unittest_plugin: clean import_plugins
Expand All @@ -210,6 +210,13 @@ unittest_pluginmanager: clean import_plugins
go test $$(go list ./...|grep -Ev "telegraf|external|envconfig"| grep -E "plugin_main|pluginmanager") -coverprofile .coretestCoverage.txt
mv ./plugins/input/prometheus/input_prometheus.go.bak ./plugins/input/prometheus/input_prometheus.go

# benchmark
.PHONY: benchmark
benchmark: clean e2edocker
./scripts/e2e.sh benchmark performance
./scripts/benchmark_collect_result.sh


.PHONY: all
all: clean import_plugins
./scripts/gen_build_scripts.sh all "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" "$(OUT_DIR)" "$(DOCKER_BUILD_EXPORT_GO_ENVS)" "$(DOCKER_BUILD_COPY_GIT_CONFIGS)" "$(PLUGINS_CONFIG_FILE)" "$(GO_MOD_FILE)"
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Alibaba iLogtail - Fast and Lightweight Observability Data Collector | [中文用户手册](https://ilogtail.gitbook.io/ilogtail-docs/)
# Alibaba iLogtail - Fast and Lightweight Observability Data Collector | 中文用户手册[GitBook](https://ilogtail.gitbook.io/ilogtail-docs/) | [可观测中文社区](https://open.observability.cn/project/ilogtail/about/)

<img src="https://sls-opensource.oss-us-west-1.aliyuncs.com/ilogtail/ilogtail.svg?versionId=CAEQMxiBgIDEmq.m6BciIDkzNmE2OWU4NzIwZjQ1Y2ZiYmIxZjhiYjMyNmQxZTdi" alt="ilogtail logo" height="150px" align="right" />

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ message AgentAttributes {
bytes version = 1; // Agent's version
bytes ip = 2; // Agent's ip
bytes hostname = 3; // Agent's hostname
bytes hostid = 4; // Agent's hostid https://opentelemetry.io/docs/specs/semconv/attributes-registry/host/
map<string, bytes> extras = 100; // Agent's other attributes
// before 100 (inclusive) are reserved for future official fields
}
Expand Down
9 changes: 2 additions & 7 deletions core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,11 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake)

# Subdirectories (modules). except for common, input, processor, flusher, observer, helper, spl, and provider.
set(SUB_DIRECTORIES_LIST
application app_config checkpoint container_manager logger go_pipeline monitor monitor/metric_constants profile_sender models
application app_config checkpoint container_manager metadata logger go_pipeline monitor monitor/metric_constants profile_sender models
config config/watcher
instance_config
pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer
runner runner/sink/http
protobuf/sls
protobuf/sls protobuf/models
file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling
prometheus prometheus/labels prometheus/schedulers prometheus/async
ebpf ebpf/observer ebpf/security ebpf/handler
Expand All @@ -141,7 +140,6 @@ endif()
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories("/opt/logtail_spl/include")
if (LINUX)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/observer)
if (WITHSPL)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/spl)
endif()
Expand Down Expand Up @@ -177,9 +175,6 @@ elseif(UNIX)
file(GLOB REMOVE_EVENT_LISTENER_SOURCES file_server/event_listener/*_Windows.cpp file_server/event_listener/*_Windows.h)
list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${REMOVE_EVENT_LISTENER_SOURCES})
if (LINUX)
# observer
file(GLOB_RECURSE APPEND_OBSERVER_SOURCES observer/*)
list(APPEND FRAMEWORK_SOURCE_FILES ${APPEND_OBSERVER_SOURCES})
if (WITHSPL)
set(SRC_FILES ${PLUGIN_SOURCE_FILES_SPL})
endif()
Expand Down
42 changes: 31 additions & 11 deletions core/app_config/AppConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ DEFINE_FLAG_STRING(metrics_report_method,

DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator service", "");
DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");

DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
Expand Down Expand Up @@ -499,6 +500,14 @@ string GetFileTagsDir() {
}
}

string GetPipelineConfigDir() {
if (BOOL_FLAG(logtail_mode)) {
return "config";
} else {
return "pipeline_config";
}
}

AppConfig::AppConfig() {
LOG_INFO(sLogger, ("AppConfig AppConfig", "success"));
SetIlogtailConfigJson("");
Expand Down Expand Up @@ -609,7 +618,7 @@ void AppConfig::LoadAppConfig(const std::string& ilogtailConfigFile) {
if (BOOL_FLAG(logtail_mode)) {
loadAppConfigLogtailMode(ilogtailConfigFile);
} else {
std::string confDir = GetAgentConfDir();
std::string confDir = GetAgentConfDir();
SetLoongcollectorConfDir(AbsolutePath(confDir, mProcessExecutionDir));
}
// 加载本地instanceconfig
Expand All @@ -627,7 +636,6 @@ void AppConfig::LoadAppConfig(const std::string& ilogtailConfigFile) {
}

void AppConfig::loadAppConfigLogtailMode(const std::string& ilogtailConfigFile) {

Json::Value confJson(Json::objectValue);
std::string newConfDir;

Expand Down Expand Up @@ -1354,8 +1362,11 @@ void AppConfig::InitEnvMapping(const std::string& envStr, std::map<std::string,
}
}
void AppConfig::SetConfigFlag(const std::string& flagName, const std::string& value) {
static set<string> sIgnoreFlagSet
= {"loongcollector_conf_dir", "loongcollector_log_dir", "loongcollector_data_dir", "loongcollector_run_dir", "logtail_mode"};
static set<string> sIgnoreFlagSet = {"loongcollector_conf_dir",
"loongcollector_log_dir",
"loongcollector_data_dir",
"loongcollector_run_dir",
"logtail_mode"};
if (sIgnoreFlagSet.find(flagName) != sIgnoreFlagSet.end()) {
return;
}
Expand Down Expand Up @@ -1402,7 +1413,7 @@ void AppConfig::ParseEnvToFlags() {
}
}
#endif
for (const auto & iter : envMapping) {
for (const auto& iter : envMapping) {
const std::string& key = iter.first;
const std::string& value = iter.second;
SetConfigFlag(key, value);
Expand Down Expand Up @@ -1479,8 +1490,8 @@ void AppConfig::ReadFlagsFromMap(const std::unordered_map<std::string, std::stri
* - 记录无法转换的值
*/
void AppConfig::RecurseParseJsonToFlags(const Json::Value& confJson, std::string prefix) {
const static unordered_set<string> sIgnoreKeySet = {"data_server_list"};
const static unordered_set<string> sForceKeySet = {"config_server_address_list"};
const static unordered_set<string> sIgnoreKeySet = {"data_server_list", "legacy_data_server_list"};
const static unordered_set<string> sForceKeySet = {"config_server_address_list", "config_server_list"};
for (auto name : confJson.getMemberNames()) {
auto jsonvalue = confJson[name];
string fullName;
Expand Down Expand Up @@ -1704,7 +1715,10 @@ void AppConfig::UpdateFileTags() {
return;
}

void AppConfig::MergeJson(Json::Value& mainConfJson, const Json::Value& subConfJson, std::unordered_map<std::string, std::string>& keyToConfigName, const std::string& configName) {
void AppConfig::MergeJson(Json::Value& mainConfJson,
const Json::Value& subConfJson,
std::unordered_map<std::string, std::string>& keyToConfigName,
const std::string& configName) {
for (const auto& subkey : subConfJson.getMemberNames()) {
mainConfJson[subkey] = subConfJson[subkey];
keyToConfigName[subkey] = configName;
Expand All @@ -1718,9 +1732,15 @@ void AppConfig::LoadInstanceConfig(const std::map<std::string, std::shared_ptr<I
mRemoteInstanceConfigKeyToConfigName.clear();
for (const auto& config : instanceConfig) {
if (EndWith(config.second->mDirName, AppConfig::sLocalConfigDir)) {
MergeJson(localInstanceConfig, config.second->GetConfig(), mLocalInstanceConfigKeyToConfigName, config.second->mDirName+"/"+config.second->mConfigName);
MergeJson(localInstanceConfig,
config.second->GetConfig(),
mLocalInstanceConfigKeyToConfigName,
config.second->mDirName + "/" + config.second->mConfigName);
} else {
MergeJson(remoteInstanceConfig, config.second->GetConfig(), mRemoteInstanceConfigKeyToConfigName, config.second->mDirName+"/"+config.second->mConfigName);
MergeJson(remoteInstanceConfig,
config.second->GetConfig(),
mRemoteInstanceConfigKeyToConfigName,
config.second->mDirName + "/" + config.second->mConfigName);
}
}
if (localInstanceConfig != mLocalInstanceConfig || mRemoteInstanceConfig != remoteInstanceConfig) {
Expand Down Expand Up @@ -1759,7 +1779,7 @@ void AppConfig::RegisterCallback(const std::string& key, std::function<bool()>*
mCallbacks[key] = callback;
}

template<typename T>
template <typename T>
T AppConfig::MergeConfig(const T& defaultValue,
const T& currentValue,
const std::string& name,
Expand Down
8 changes: 6 additions & 2 deletions core/app_config/AppConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ std::string GetProfileSnapshotDumpFileName();
std::string GetObserverEbpfHostPath();
std::string GetSendBufferFileNamePrefix();
std::string GetLegacyUserLocalConfigFilePath();
std::string GetExactlyOnceCheckpoint();
std::string GetExactlyOnceCheckpoint();
std::string GetPipelineConfigDir();

template <class T>
class DoubleBuffer {
Expand Down Expand Up @@ -224,7 +225,10 @@ class AppConfig {
*/
void CheckAndAdjustParameters();
void MergeJson(Json::Value& mainConfJson, const Json::Value& subConfJson);
void MergeJson(Json::Value& mainConfJson, const Json::Value& subConfJson, std::unordered_map<std::string, std::string>& keyToConfigName, const std::string& configName);
void MergeJson(Json::Value& mainConfJson,
const Json::Value& subConfJson,
std::unordered_map<std::string, std::string>& keyToConfigName,
const std::string& configName);
/**
* @brief Load *.json from config.d dir
*
Expand Down
Loading

0 comments on commit 7e6c469

Please sign in to comment.