From e181d4bc328c91cc8d1e3760193b0871b4fa2ac9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 15 Mar 2024 15:08:38 +0900 Subject: [PATCH 1/8] workflows: windows: Translate AppVeyor's unit testing flow to GHA Signed-off-by: Hiroshi Hatake --- .../workflows/call-windows-unit-tests.yaml | 180 ++++++++++++++++++ .github/workflows/pr-windows-build.yaml | 11 ++ 2 files changed, 191 insertions(+) create mode 100644 .github/workflows/call-windows-unit-tests.yaml diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml new file mode 100644 index 00000000000..5bcd23ba4c5 --- /dev/null +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -0,0 +1,180 @@ +--- +name: Reusable workflow to run unit tests on Windows packages (only for x86 and x64) + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + unstable: + description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add. + type: string + required: false + default: '' + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +jobs: + call-windows-unit-test-get-meta: + name: Determine build info + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + armSupported: ${{ steps.armcheck.outputs.armSupported }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Determine if we are doing a build with ARM support + id: armcheck + # Check for new contents from https://github.com/fluent/fluent-bit/pull/6621 + run: | + if grep -q "winarm64" CMakeLists.txt ; then + echo "armSupported=true" >> $GITHUB_OUTPUT + else + echo "armSupported=false" >> $GITHUB_OUTPUT + fi + shell: bash + + call-build-windows-unit-test: + runs-on: windows-latest + environment: ${{ inputs.environment }} + needs: + - call-windows-unit-test-get-meta + strategy: + fail-fast: false + matrix: + config: + - name: "Windows 32bit" + arch: x86 + openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static + cmake_additional_opt: "" + vcpkg_triplet: x86-windows-static + - name: "Windows 64bit" + arch: x64 + openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static + cmake_additional_opt: "" + vcpkg_triplet: x64-windows-static + permissions: + contents: read + # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures. + env: + PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin;C:\vcpkg; + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Get dependencies + run: | + Invoke-WebRequest -OutFile winflexbison.zip $env:WINFLEXBISON + Expand-Archive winflexbison.zip -Destination C:\WinFlexBison + Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe + Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe + echo "C:\WinFlexBison" | Out-File -FilePath $env:GITHUB_PATH -Append + env: + WINFLEXBISON: https://github.com/lexxmark/winflexbison/releases/download/v2.5.22/win_flex_bison-2.5.22.zip + shell: pwsh + + - name: Set up with Developer Command Prompt for Microsoft Visual C++ + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.config.arch }} + + - name: Get gzip command w/ chocolatey + uses: crazy-max/ghaction-chocolatey@v3 + with: + args: install gzip -y + + # http://man7.org/linux/man-pages/man1/date.1.html + - name: Get Date + id: get-date + run: | + echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore cached packages of vcpkg + id: cache-unit-test-vcpkg-sources + uses: actions/cache/restore@v4 + with: + path: | + C:\vcpkg\packages + key: ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg-${{ steps.get-date.outputs.date }} + restore-keys: | + ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg- + enableCrossOsArchive: false + + - name: Build openssl with vcpkg + run: | + C:\vcpkg\vcpkg install --recurse openssl --triplet ${{ matrix.config.vcpkg_triplet }} + shell: cmd + + - name: Build libyaml with vcpkg + run: | + C:\vcpkg\vcpkg install --recurse libyaml --triplet ${{ matrix.config.vcpkg_triplet }} + shell: cmd + + - name: Save packages of vcpkg + id: save-vcpkg-sources + uses: actions/cache/save@v4 + with: + path: | + C:\vcpkg\packages + key: ${{ steps.cache-unit-test-vcpkg-sources.outputs.cache-primary-key }} + enableCrossOsArchive: false + + - name: run unit-test for Fluent Bit packages (only for x86 and x64) + run: | + cmake -G "NMake Makefiles" ` + -D FLB_TESTS_INTERNAL=On ` + -D FLB_NIGHTLY_BUILD='${{ inputs.unstable }}' ` + -D OPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ` + ${{ matrix.config.cmake_additional_opt }} ` + -D FLB_LIBYAML_DIR=C:\vcpkg\packages\libyaml_${{ matrix.config.vcpkg_triplet }} ` + -D FLB_WITHOUT_flb-rt-out_elasticsearch=On ` + -D FLB_WITHOUT_flb-rt-out_td=On ` + -D FLB_WITHOUT_flb-rt-out_forward=On ` + -D FLB_WITHOUT_flb-rt-in_disk=On ` + -D FLB_WITHOUT_flb-rt-in_proc=On ` + -D FLB_WITHOUT_flb-it-parser=On ` + -D FLB_WITHOUT_flb-it-unit_sizes=On ` + -D FLB_WITHOUT_flb-it-network=On ` + -D FLB_WITHOUT_flb-it-pack=On ` + -D FLB_WITHOUT_flb-it-signv4=On ` + -D FLB_WITHOUT_flb-it-aws_credentials=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_ec2=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_http=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_profile=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_sts=On ` + -D FLB_WITHOUT_flb-it-aws_util=On ` + -D FLB_WITHOUT_flb-it-input_chunk=On ` + ../ + cmake --build . + ctest --build-run-dir $PWD --output-on-failure + shell: pwsh + working-directory: build \ No newline at end of file diff --git a/.github/workflows/pr-windows-build.yaml b/.github/workflows/pr-windows-build.yaml index 7080471c159..4b77a11fd4e 100644 --- a/.github/workflows/pr-windows-build.yaml +++ b/.github/workflows/pr-windows-build.yaml @@ -38,3 +38,14 @@ jobs: environment: pr secrets: token: ${{ secrets.GITHUB_TOKEN }} + + run-windows-unit-tests: + needs: + - pr-windows-build + uses: ./.github/workflows/call-windows-unit-tests.yaml + with: + version: ${{ github.sha }} + ref: ${{ github.sha }} + environment: pr + secrets: + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 6d2a0e31bd29818248386cbbeafaa2b959f1d37c Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 15 Mar 2024 15:52:06 +0900 Subject: [PATCH 2/8] internal: test: file: Handle newline character difference In Windows, the newline(\n) characters in internal test data are replaced with \r\n by git settings: core.autocrlf true. This settings accicdentally converting the newlines. This commit handles the converted characters. Signed-off-by: Hiroshi Hatake --- tests/internal/file.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/internal/file.c b/tests/internal/file.c index 974c266873e..139cbb00532 100644 --- a/tests/internal/file.c +++ b/tests/internal/file.c @@ -8,6 +8,9 @@ #define TEXT_FILE FLB_TESTS_DATA_PATH "/data/file/text_file.txt" #define EMPTY_FILE FLB_TESTS_DATA_PATH "/data/file/empty_file.txt" +#define NEWLINE "\n" +#define CRNEWLINE "\r\n" + static void check_equals(flb_sds_t result, const char *expected) { size_t expected_len = strlen(expected); @@ -23,7 +26,13 @@ static void check_equals(flb_sds_t result, const char *expected) static void test_file_read_text_file() { flb_sds_t result = flb_file_read(TEXT_FILE); - check_equals(result, "Some text file\n\nline 3\n\nline 5\n"); + /* In Windows, \n is replaced with \r\n by git settings. */ + if (strstr(result, "\r\n") != NULL) { + check_equals(result, "Some text file\r\n\r\nline 3\r\n\r\nline 5\r\n"); + } + else { + check_equals(result, "Some text file\n\nline 3\n\nline 5\n"); + } flb_sds_destroy(result); } From ab1369c867cca8faecf8fcaa3f7f9f06c2aace1e Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Fri, 15 Mar 2024 11:03:33 +0000 Subject: [PATCH 3/8] workflows: update to split tests Signed-off-by: Patrick Stephens --- .../workflows/call-windows-unit-tests.yaml | 54 ++++++------------- .github/workflows/pr-windows-build.yaml | 2 +- 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml index 5bcd23ba4c5..3b6c90c6c72 100644 --- a/.github/workflows/call-windows-unit-tests.yaml +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -25,46 +25,11 @@ on: token: description: The Github token or similar to authenticate with. required: true - bucket: - description: The name of the S3 (US-East) bucket to push packages into. - required: false - access_key_id: - description: The S3 access key id for the bucket. - required: false - secret_access_key: - description: The S3 secret access key for the bucket. - required: false jobs: - call-windows-unit-test-get-meta: - name: Determine build info - runs-on: ubuntu-latest - permissions: - contents: read - outputs: - armSupported: ${{ steps.armcheck.outputs.armSupported }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - - - name: Determine if we are doing a build with ARM support - id: armcheck - # Check for new contents from https://github.com/fluent/fluent-bit/pull/6621 - run: | - if grep -q "winarm64" CMakeLists.txt ; then - echo "armSupported=true" >> $GITHUB_OUTPUT - else - echo "armSupported=false" >> $GITHUB_OUTPUT - fi - shell: bash - call-build-windows-unit-test: runs-on: windows-latest environment: ${{ inputs.environment }} - needs: - - call-windows-unit-test-get-meta strategy: fail-fast: false matrix: @@ -148,7 +113,7 @@ jobs: key: ${{ steps.cache-unit-test-vcpkg-sources.outputs.cache-primary-key }} enableCrossOsArchive: false - - name: run unit-test for Fluent Bit packages (only for x86 and x64) + - name: Build unit-test for Fluent Bit packages (only for x86 and x64) run: | cmake -G "NMake Makefiles" ` -D FLB_TESTS_INTERNAL=On ` @@ -175,6 +140,19 @@ jobs: -D FLB_WITHOUT_flb-it-input_chunk=On ` ../ cmake --build . - ctest --build-run-dir $PWD --output-on-failure shell: pwsh - working-directory: build \ No newline at end of file + working-directory: build + + - name: Upload unit test binaries + uses: actions/upload-artifact@v4 + with: + name: windows-unit-tests-${{ matrix.config.arch }} + path: | + build/**/*.exe + if-no-files-found: error + + - name: Build unit-test for Fluent Bit packages (only for x86 and x64) + run: | + ctest --build-run-dir $PWD --output-on-failure + shell: pwsh + working-directory: build diff --git a/.github/workflows/pr-windows-build.yaml b/.github/workflows/pr-windows-build.yaml index 4b77a11fd4e..110d85fda64 100644 --- a/.github/workflows/pr-windows-build.yaml +++ b/.github/workflows/pr-windows-build.yaml @@ -48,4 +48,4 @@ jobs: ref: ${{ github.sha }} environment: pr secrets: - token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + token: ${{ secrets.GITHUB_TOKEN }} From 905ed714746da8d1b1a9668314b1fb8cdf30b9e7 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 15 Mar 2024 20:12:00 +0900 Subject: [PATCH 4/8] workflows: windows: Add quotes for $PWD Signed-off-by: Hiroshi Hatake --- .github/workflows/call-windows-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml index 3b6c90c6c72..a18a358e263 100644 --- a/.github/workflows/call-windows-unit-tests.yaml +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -153,6 +153,6 @@ jobs: - name: Build unit-test for Fluent Bit packages (only for x86 and x64) run: | - ctest --build-run-dir $PWD --output-on-failure + ctest --build-run-dir "$PWD" --output-on-failure shell: pwsh working-directory: build From da69e70afd47536bdaa55e3156611d0cbbc8ead4 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 15 Mar 2024 20:18:32 +0900 Subject: [PATCH 5/8] test: internal: file: Remove needless lines Signed-off-by: Hiroshi Hatake --- tests/internal/file.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/internal/file.c b/tests/internal/file.c index 139cbb00532..3f01487bd12 100644 --- a/tests/internal/file.c +++ b/tests/internal/file.c @@ -8,9 +8,6 @@ #define TEXT_FILE FLB_TESTS_DATA_PATH "/data/file/text_file.txt" #define EMPTY_FILE FLB_TESTS_DATA_PATH "/data/file/empty_file.txt" -#define NEWLINE "\n" -#define CRNEWLINE "\r\n" - static void check_equals(flb_sds_t result, const char *expected) { size_t expected_len = strlen(expected); From c3b7a8baed56a1ffdffd408836061bcc2e1c30d9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 15 Mar 2024 20:35:56 +0900 Subject: [PATCH 6/8] workflows: windows: Display dependent DLLs by dumpbin Signed-off-by: Hiroshi Hatake --- .github/workflows/call-windows-unit-tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml index a18a358e263..aa3fb23adf0 100644 --- a/.github/workflows/call-windows-unit-tests.yaml +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -151,6 +151,10 @@ jobs: build/**/*.exe if-no-files-found: error + - name: Display dependencies w/ dumpbin + run: | + dumpbin /dependents .\bin\fluent-bit.exe + - name: Build unit-test for Fluent Bit packages (only for x86 and x64) run: | ctest --build-run-dir "$PWD" --output-on-failure From 052f11570365c6396eedcb711e37123c1f822ac1 Mon Sep 17 00:00:00 2001 From: Pat Date: Fri, 15 Mar 2024 16:53:30 +0200 Subject: [PATCH 7/8] Update .github/workflows/call-windows-unit-tests.yaml Signed-off-by: Pat Signed-off-by: Hiroshi Hatake --- .github/workflows/call-windows-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml index aa3fb23adf0..9e7a324e95b 100644 --- a/.github/workflows/call-windows-unit-tests.yaml +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -153,7 +153,7 @@ jobs: - name: Display dependencies w/ dumpbin run: | - dumpbin /dependents .\bin\fluent-bit.exe + dumpbin /dependents .build\bin\fluent-bit.exe - name: Build unit-test for Fluent Bit packages (only for x86 and x64) run: | From 79a17932cfe00c8dc40c293398345c1c991c9c1c Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Sat, 16 Mar 2024 09:32:13 +0900 Subject: [PATCH 8/8] workflows: windows: Specify working directory for dumpbin step Signed-off-by: Hiroshi Hatake --- .github/workflows/call-windows-unit-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml index 9e7a324e95b..c24fedc0861 100644 --- a/.github/workflows/call-windows-unit-tests.yaml +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -153,7 +153,8 @@ jobs: - name: Display dependencies w/ dumpbin run: | - dumpbin /dependents .build\bin\fluent-bit.exe + dumpbin /dependents .\bin\fluent-bit.exe + working-directory: build - name: Build unit-test for Fluent Bit packages (only for x86 and x64) run: |