Merge branch 'main' into feat/promTextParser

alibaba · Jul 22, 2024 · ded3f8c · ded3f8c
2 parents ac92537 + c8385ca
commit ded3f8c
Show file tree

Hide file tree

Showing 113 changed files with 11,201 additions and 908 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -28,6 +28,18 @@ RUN wget http://mirrors.ustc.edu.cn/gnu/libc/glibc-2.18.tar.gz && \
     cd ../ && \
     rm -fr glibc-2.18*
 
+# install python3.8
+RUN cd /opt && curl -O https://cdn.npmmirror.com/binaries/python/3.8.12/Python-3.8.12.tgz && \
+    tar -zxvf Python-3.8.12.tgz && cd Python-3.8.12 && \
+    mkdir /usr/local/python3 && \
+    ./configure --prefix=/usr/local/python3 && \
+    make clean && make && make install && \
+    cp /usr/local/python3/bin/python3.8 /usr/bin/python3
+# install gcovr
+RUN python3 -m pip install --upgrade pip
+RUN cp /usr/local/python3/bin/pip3 /usr/bin/pip3 && pip3 install gcovr==7.0
+RUN cp /usr/local/python3/bin/gcovr /usr/bin/gcovr
+
 # Create the user
 COPY .env /tmp/.env
 RUN source /tmp/.env && rm /tmp/.env; \
@@ -43,6 +55,4 @@ RUN source /tmp/.env && rm /tmp/.env; \
     chown -R $USERNAME:$GROUPNAME /opt $(eval echo ~$USERNAME); \
     chmod -R 755 $(eval echo ~$USERNAME);
 
-USER $USERNAME
-
 RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct
diff --git a/.github/workflows/build-core-ut.yaml b/.github/workflows/build-core-ut.yaml
@@ -17,13 +17,13 @@ name: Build Core Unit Test
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -51,7 +51,7 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
-          
+
       - name: Check disk space
         run: |
           df -hT $PWD
@@ -65,21 +65,33 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: true
+          fetch-depth: 2
 
       - name: Build Unit Test
         env:
           BUILD_LOGTAIL: OFF
           BUILD_LOGTAIL_UT: ON
-          ENABLE_COMPATIBLE_MODE: ON
+          ENABLE_COMPATIBLE_MODE: OFF
           ENABLE_STATIC_LINK_CRT: ON
           WITHOUTGDB: ON
-          MAKE_JOBS: 8
+          MAKE_JOBS: 16
           # BUILD_TYPE: Debug # TODO: Uncomment when memory management is refined
-        run: make core
+        run: CURRENT_DIR=$(pwd) && sed -i "s|/src|$CURRENT_DIR|g" docker/Dockerfile_build && make core PATH_IN_DOCKER=$(pwd)
 
       - name: Unit Test
         run: make unittest_core
 
+      - name: Unit Test Coverage
+        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*log_pb.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""
+
+      - name: Setup Python3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Report code coverage
+        run: python3 tools/coverage-diff/main.py core/coverage.txt
+
   result:
     runs-on: arc-runner-set-ilogtail
     timeout-minutes: 90

diff --git a/.github/workflows/build-core.yaml b/.github/workflows/build-core.yaml
@@ -17,13 +17,13 @@ name: Build Core
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -37,7 +37,7 @@ jobs:
       matrix:
         go-version: [1.19]
         # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
-        runner: [ arc-runner-set-ilogtail ]
+        runner: [arc-runner-set-ilogtail]
       fail-fast: true
     steps:
       - name: prepare ubuntu environment
@@ -60,7 +60,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
-          
+
       - name: Check out code
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -17,13 +17,13 @@ name: Build
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -37,7 +37,7 @@ jobs:
       matrix:
         go-version: [1.19]
         # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
-        runner: [ arc-runner-set-ilogtail ]
+        runner: [arc-runner-set-ilogtail]
       fail-fast: true
     steps:
       - name: prepare ubuntu environment
@@ -51,7 +51,7 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
-          
+
       - name: Check disk space
         run: |
           df -hT $PWD
@@ -60,7 +60,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
-          
+
       - name: Check out code
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/e2e-framework.yaml b/.github/workflows/e2e-framework.yaml
@@ -17,13 +17,13 @@ name: E2E Core Test
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -34,8 +34,8 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        go-version: [ 1.19.10 ]
-        runner: [ ubuntu-latest ]
+        go-version: [1.19.10]
+        runner: [ubuntu-latest]
       fail-fast: true
     steps:
       # Clean up space to prevent action from running out of disk space.
@@ -81,7 +81,7 @@ jobs:
   result:
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    needs: [ CI ]
+    needs: [CI]
     steps:
       - name: Build Result
         run: echo "Just to make the GitHub merge button green"
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -17,13 +17,13 @@ name: E2E Test
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -34,8 +34,8 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        go-version: [ 1.19.10 ]
-        runner: [ ubuntu-latest ]
+        go-version: [1.19.10]
+        runner: [ubuntu-latest]
       fail-fast: true
     steps:
       # Clean up space to prevent action from running out of disk space.
@@ -83,7 +83,7 @@ jobs:
   result:
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    needs: [ CI ]
+    needs: [CI]
     steps:
       - name: Build Result
         run: echo "Just to make the GitHub merge button green"
diff --git a/Makefile b/Makefile
@@ -20,6 +20,7 @@ BUILD_REPOSITORY ?= aliyun/ilogtail_build
 GENERATED_HOME ?= generated_files
 PLUGINS_CONFIG_FILE ?= "plugins.yml,external_plugins.yml"
 GO_MOD_FILE ?= go.mod
+PATH_IN_DOCER ?= /src
 DOCKER_BUILD_EXPORT_GO_ENVS ?= true
 DOCKER_BUILD_COPY_GIT_CONFIGS ?= true
 DOCKER_BUILD_USE_BUILDKIT ?=
@@ -119,7 +120,7 @@ lint-e2e: clean tools
 
 .PHONY: core
 core: clean import_plugins
-	./scripts/gen_build_scripts.sh core $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
+	./scripts/gen_build_scripts.sh core $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE) $(PATH_IN_DOCKER)
 	./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
 	./$(GENERATED_HOME)/gen_copy_docker.sh
 

diff --git a/config_server/protocol/v2/agent.proto b/config_server/protocol/v2/agent.proto
@@ -70,7 +70,7 @@ enum RequestFlags {
 // API: /Agent/Heartbeat
 
 // Agent sends requests to the ConfigServer to get config updates and receive commands.
-message HearbeatRequest {
+message HeartbeatRequest {
     bytes request_id = 1;
     uint64 sequence_num = 2;                    // Increment every request, for server to check sync status
     uint64 capabilities = 3;                    // Bitmask of flags defined by AgentCapabilities enum
@@ -85,7 +85,7 @@ message HearbeatRequest {
     repeated CommandInfo custom_commands = 12;  // Information about command history
     uint64 flags = 13;                          // Predefined command flag
     bytes opaque = 14;                          // Opaque data for extension
-    // before 100 (inclusive) are reserved for future official fields}
+    // before 100 (inclusive) are reserved for future official fields
 }
 
 // Define Config's detail