Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

speedup CI UT job #1606

Merged
merged 8 commits into from
Jul 18, 2024
Merged

speedup CI UT job #1606

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1662648
Split the original UT CI job into two separate jobs: one with SPL and…
alph00 Jul 11, 2024
5b11b0a
fix
alph00 Jul 16, 2024
7f453c4
fix
alph00 Jul 16, 2024
20ae021
fix
alph00 Jul 17, 2024
e9e7621
fix
alph00 Jul 17, 2024
1ccc757
fix
alph00 Jul 17, 2024
e2c3f13
fix
alph00 Jul 17, 2024
8fcc0aa
fix
alph00 Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/build-core-ut.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,15 @@ jobs:
ENABLE_COMPATIBLE_MODE: OFF
ENABLE_STATIC_LINK_CRT: ON
WITHOUTGDB: ON
MAKE_JOBS: 8
MAKE_JOBS: 16
# BUILD_TYPE: Debug # TODO: Uncomment when memory management is refined
run: CURRENT_DIR=$(pwd) && sed -i "s|/src|$CURRENT_DIR|g" docker/Dockerfile_build && make core PATH_IN_DOCKER=$(pwd)

- name: Unit Test
run: make unittest_core

- name: Unit Test Coverage
run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt"
run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*log_pb.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""

- name: Setup Python3.10
uses: actions/setup-python@v5
Expand Down
18 changes: 9 additions & 9 deletions .github/workflows/build-core.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: Build Core
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -37,7 +37,7 @@ jobs:
matrix:
go-version: [1.19]
# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
runner: [ arc-runner-set-ilogtail ]
runner: [arc-runner-set-ilogtail]
fail-fast: true
steps:
- name: prepare ubuntu environment
Expand All @@ -60,7 +60,7 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}

- name: Check out code
uses: actions/checkout@v4
with:
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: Build
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -37,7 +37,7 @@ jobs:
matrix:
go-version: [1.19]
# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
runner: [ arc-runner-set-ilogtail ]
runner: [arc-runner-set-ilogtail]
fail-fast: true
steps:
- name: prepare ubuntu environment
Expand All @@ -51,7 +51,7 @@ jobs:
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo -E apt-get -qq autoremove --purge
sudo -E apt-get -qq clean

- name: Check disk space
run: |
df -hT $PWD
Expand All @@ -60,7 +60,7 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}

- name: Check out code
uses: actions/checkout@v4
with:
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/e2e-framework.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: E2E Core Test
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -34,8 +34,8 @@ jobs:
timeout-minutes: 60
strategy:
matrix:
go-version: [ 1.19.10 ]
runner: [ ubuntu-latest ]
go-version: [1.19.10]
runner: [ubuntu-latest]
fail-fast: true
steps:
# Clean up space to prevent action from running out of disk space.
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:
result:
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ CI ]
needs: [CI]
steps:
- name: Build Result
run: echo "Just to make the GitHub merge button green"
20 changes: 10 additions & 10 deletions .github/workflows/e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: E2E Test
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -34,8 +34,8 @@ jobs:
timeout-minutes: 60
strategy:
matrix:
go-version: [ 1.19.10 ]
runner: [ ubuntu-latest ]
go-version: [1.19.10]
runner: [ubuntu-latest]
fail-fast: true
steps:
# Clean up space to prevent action from running out of disk space.
Expand Down Expand Up @@ -83,7 +83,7 @@ jobs:
result:
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ CI ]
needs: [CI]
steps:
- name: Build Result
run: echo "Just to make the GitHub merge button green"
83 changes: 45 additions & 38 deletions core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,17 @@ option(ENABLE_ENTERPRISE "enable enterprise feature")
cmake_dependent_option(ENABLE_COMPATIBLE_MODE "Build Logtail in compatible mode (for low version Linux)" OFF "LINUX" OFF)
cmake_dependent_option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT statically" OFF "LINUX" OFF)
option(WITHOUTGDB "Build Logtail without gdb")
option(WITHOUTSPL "Build Logtail and UT without SPL")
option(WITHSPL "Build Logtail and UT with SPL" ON)
option(BUILD_LOGTAIL_UT "Build unit test for Logtail")

if (BUILD_LOGTAIL_SHARED_LIBRARY AND WITHSPL)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BUILD_LOGTAIL_SHARED_LIBRARY 注释说明下应用场景

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

image BUILD_LOGTAIL_SHARED_LIBRARY 是用于生成动态库的开关，是原来就有的

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

代码注释，不是评论注释。说明下什么场景会用

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已修改

message(FATEL_ERROR, "Generating logtail shared library is not supported to be linked with SPL. WITHSPL should be set OFF.")
return()
endif()

set(UT_LINK OFF)
set(UT_LINK_NEED_SPL OFF)
alph00 marked this conversation as resolved.
Show resolved Hide resolved

if (ENABLE_ENTERPRISE)
message(STATUS "Enable Enterprise Feature.")
add_definitions(-D__ENTERPRISE__)
Expand All @@ -41,7 +49,7 @@ else ()
include(${CMAKE_CURRENT_SOURCE_DIR}/options.cmake)
endif ()

if (WITHOUTSPL)
if (NOT WITHSPL)
add_definitions(-D__EXCLUDE_SPL__)
endif()

Expand Down Expand Up @@ -92,7 +100,8 @@ endif ()
include(${CMAKE_CURRENT_SOURCE_DIR}/utils.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/links.cmake)
set(SUBDIR_SOURCE_FILES "")
set(SUBDIR_SOURCE_FILES_CORE "")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个最终应该叫PLUGIN_SOURCE_FILES，只包括input、processor和flusher目录下的文件

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已修改为PLUGIN_SOURCE_FILES_CORE

set(SUBDIR_SOURCE_FILES_SPL "")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SPL_SOURCE_FILES

Copy link
Collaborator Author

@alph00 alph00 Jul 17, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已修改为PLUGIN_SOURCE_FILES_SPL

include(${CMAKE_CURRENT_SOURCE_DIR}/common/common.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/common/links.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/input/input.cmake)
Expand Down Expand Up @@ -127,7 +136,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories("/opt/logtail_spl/include")
if (LINUX)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/observer)
if (NOT WITHOUTSPL)
if (WITHSPL)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/spl)
endif()
endif()
Expand All @@ -153,7 +162,6 @@ if(MSVC)
# remove linux event listener
file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Linux.cpp event_listener/*_Linux.h)
list(REMOVE_ITEM SOURCE_FILES_LIST ${REMOVE_EVENT_LISTENER_SOURCES})
set(WINDOWS_SOURCE_FILES ${SOURCE_FILES_LIST} ${SUBDIR_SOURCE_FILES})
elseif(UNIX)
# remove windows event listener
file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Windows.cpp event_listener/*_Windows.h)
Expand All @@ -162,13 +170,16 @@ elseif(UNIX)
# observer
file(GLOB_RECURSE APPEND_OBSERVER_SOURCES observer/*)
list(APPEND SOURCE_FILES_LIST ${APPEND_OBSERVER_SOURCES})
if (WITHSPL)
set(SRC_FILES ${SUBDIR_SOURCE_FILES_SPL})
endif()
endif()
set(UNIX_SOURCE_FILES ${SOURCE_FILES_LIST} ${SUBDIR_SOURCE_FILES})
endif()
set(SRC_FILES ${SRC_FILES} ${SOURCE_FILES_LIST} ${SUBDIR_SOURCE_FILES_CORE})
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SOURCE_FILES_LIST能换个名字吗？比如CORE_SOURCE_FILES

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已改为 FRAMEWORK_SOURCE_FILES


# Generate SPL library.
if (LINUX)
if (NOT WITHOUTSPL)
if (WITHSPL)
add_subdirectory(spl)
endif()
endif()
Expand All @@ -177,47 +188,35 @@ endif()
if (BUILD_LOGTAIL)
if (ENABLE_ENTERPRISE)
if (UNIX)
add_executable(${LOGTAIL_TARGET} enterprise_logtail.cpp ${UNIX_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} enterprise_logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_executable(${LOGTAIL_TARGET} enterprise_logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} enterprise_logtail_windows.cpp ${SRC_FILES})
endif ()
else ()
if (UNIX)
add_executable(${LOGTAIL_TARGET} logtail.cpp ${UNIX_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_executable(${LOGTAIL_TARGET} logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} logtail_windows.cpp ${SRC_FILES})
endif ()
endif()
endif()

if (BUILD_LOGTAIL_SHARED_LIBRARY)
if (NOT WITHOUTSPL)
message(ERROR, "generating logtail shared library is not supported with SPL.")
else()
if (ENABLE_ENTERPRISE)
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail.cpp ${UNIX_SOURCE_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
endif ()
else ()
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED logtail.cpp ${UNIX_SOURCE_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
endif ()
endif()
if (BUILD_LOGTAIL_SHARED_LIBRARY)
if (ENABLE_ENTERPRISE)
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail_windows.cpp ${SRC_FILES})
endif ()
else ()
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED logtail_windows.cpp ${SRC_FILES})
endif ()
endif()
endif ()

# Logtail UT.
if (BUILD_LOGTAIL_UT)
message(STATUS "Build unittest.")
include(CTest)
enable_testing()
add_subdirectory(unittest)
endif ()

# Generate independent libraries.
# add_subdirectory(helper)
add_subdirectory(go_pipeline)
Expand All @@ -226,8 +225,16 @@ add_subdirectory(common)
# Link libraries.
if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY)
input_link(${LOGTAIL_TARGET})
processor_link(${LOGTAIL_TARGET})
processor_link(${LOGTAIL_TARGET} ${UT_LINK} ${UT_LINK_NEED_SPL})
alph00 marked this conversation as resolved.
Show resolved Hide resolved
flusher_link(${LOGTAIL_TARGET})
all_link(${LOGTAIL_TARGET})
common_link(${LOGTAIL_TARGET})
endif()
endif()

# Logtail UT.
if (BUILD_LOGTAIL_UT)
message(STATUS "Build unittest.")
include(CTest)
enable_testing()
add_subdirectory(unittest)
endif ()
2 changes: 1 addition & 1 deletion core/common/common.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,4 @@ elseif(UNIX)
endif()

# Set source files to parent
set(SUBDIR_SOURCE_FILES ${SUBDIR_SOURCE_FILES} ${THIS_SOURCE_FILES_LIST})
set(SUBDIR_SOURCE_FILES_CORE ${SUBDIR_SOURCE_FILES_CORE} ${THIS_SOURCE_FILES_LIST})
12 changes: 2 additions & 10 deletions core/dependencies.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,11 +242,7 @@ macro(link_lz4 target_name)
if (lz4_${LINK_OPTION_SUFFIX})
target_link_libraries(${target_name} "${lz4_${LINK_OPTION_SUFFIX}}")
elseif (UNIX)
if (WITHOUTSPL OR BUILD_LOGTAIL_SHARED_LIBRARY)
target_link_libraries(${target_name} "${lz4_${LIBRARY_DIR_SUFFIX}}/liblz4.so")
else ()
target_link_libraries(${target_name} "${lz4_${LIBRARY_DIR_SUFFIX}}/liblz4.a")
endif ()
target_link_libraries(${target_name} "${lz4_${LIBRARY_DIR_SUFFIX}}/liblz4.so")
elseif (MSVC)
target_link_libraries(${target_name}
debug "liblz4_staticd"
Expand Down Expand Up @@ -310,11 +306,7 @@ macro(link_unwind target_name)
elseif (ANDROID)
# target_link_libraries(${target_name} "${unwind_${LIBRARY_DIR_SUFFIX}}/libunwindstack.a")
elseif (UNIX)
if (WITHOUTSPL OR BUILD_LOGTAIL_SHARED_LIBRARY)
target_link_libraries(${target_name} "${unwind_${LIBRARY_DIR_SUFFIX}}/libunwind.so")
else ()
target_link_libraries(${target_name} "${unwind_${LIBRARY_DIR_SUFFIX}}/libunwind.a")
endif ()
target_link_libraries(${target_name} "${unwind_${LIBRARY_DIR_SUFFIX}}/libunwind.so")
elseif (MSVC)
target_link_libraries(${target_name}
debug "breakpad_commond.lib"
Expand Down
4 changes: 2 additions & 2 deletions core/flusher/flusher.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
include_directories(flusher)

# Add source files
file(GLOB THIS_SOURCE_FILES ${CMAKE_SOURCE_DIR}/flusher/*.c ${CMAKE_SOURCE_DIR}/flusher/*.cc ${CMAKE_SOURCE_DIR}/flusher/*.cpp ${CMAKE_SOURCE_DIR}/flusher/*.h)
file(GLOB THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/flusher/*.c ${CMAKE_SOURCE_DIR}/flusher/*.cc ${CMAKE_SOURCE_DIR}/flusher/*.cpp ${CMAKE_SOURCE_DIR}/flusher/*.h)

# Set source files to parent
set(SUBDIR_SOURCE_FILES ${SUBDIR_SOURCE_FILES} ${THIS_SOURCE_FILES})
set(SUBDIR_SOURCE_FILES_CORE ${SUBDIR_SOURCE_FILES_CORE} ${THIS_SOURCE_FILES_LIST})
2 changes: 1 addition & 1 deletion core/input/input.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,4 @@ elseif(UNIX)
endif()

# Set source files to parent
set(SUBDIR_SOURCE_FILES ${SUBDIR_SOURCE_FILES} ${THIS_SOURCE_FILES_LIST})
set(SUBDIR_SOURCE_FILES_CORE ${SUBDIR_SOURCE_FILES_CORE} ${THIS_SOURCE_FILES_LIST})
Loading
Loading