Merge branch 'fluent:master' into fix/timeout-issue

.github/workflows/cron-scorecards-analysis.yaml

@@ -3,8 +3,10 @@
 # https://openssf.org/blog/2022/01/19/reducing-security-risks-in-open-source-software-at-scale-scorecards-launches-v4/
 name: Scorecards supply-chain security
 on:
-  # Only the default branch is supported.
-  branch_protection_rule:
+  push:
+    # Only the default branch is supported.
+    branches:
+    - main
   schedule:
     # Weekly on Saturdays.
     - cron: '30 1 * * 6'
@@ -31,29 +33,20 @@ jobs:
       - name: "Run analysis"
         uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46
         with:
-          results_file: scorecard-results.sarif
+          results_file: results.sarif
           results_format: sarif
-          # (Optional) fine-grained personal access token. Uncomment the `repo_token` line below if:
-          # - you want to enable the Branch-Protection check on a *public* repository, or
-          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-fine-grained-pat-optional.
-          repo_token: ${{ secrets.SCORECARD_TOKEN }}
-          #
-          # Publish the results for public repositories to enable scorecard badges. For more details, see
-          # https://github.com/ossf/scorecard-action#publishing-results.
-          # For private repositories, `publish_results` will automatically be set to `false`, regardless
-          # of the value entered here.
           publish_results: true
 
       - name: "Upload artifact"
         uses: actions/upload-artifact@v4
         with:
           name: SARIF file
-          path: scorecard-results.sarif
+          path: results.sarif
           retention-days: 7
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: scorecard-results.sarif
+          sarif_file: results.sarif
           category: ossf-scorecard

.github/workflows/cron-unstable-build.yaml

@@ -13,8 +13,7 @@ on:
   # Run nightly build at this time, bit of trial and error but this seems good.
   schedule:
     - cron: "0 6 * * *" # master build
-    - cron: "0 12 * * *" # 2.2 build
-    - cron: "0 18 * * *" # 2.1 build
+    - cron: "0 12 * * *" # 3.0 build
 
 # We do not want a new unstable build to run whilst we are releasing the current unstable build.
 concurrency: unstable-build-release
@@ -52,16 +51,10 @@ jobs:
           echo "cron_branch=master" >> $GITHUB_ENV
         shell: bash
 
-      - name: 2.2 run
+      - name: 3.0 run
         if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *'
         run: |
-          echo "cron_branch=2.2" >> $GITHUB_ENV
-        shell: bash
-
-      - name: 2.1 run
-        if: github.event_name == 'schedule' && github.event.schedule=='0 18 * * *'
-        run: |
-          echo "cron_branch=2.1" >> $GITHUB_ENV
+          echo "cron_branch=3.0" >> $GITHUB_ENV
         shell: bash
 
       - name: Output the branch to use

.github/workflows/pr-compile-check.yaml

@@ -30,3 +30,46 @@ jobs:
         push: false
         load: false
         provenance: false
+
+  # Sanity check for compilation using system libraries
+  pr-compile-system-libs:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        flb_option:
+          - "-DFLB_PREFER_SYSTEM_LIBS=On"
+        compiler:
+          - gcc
+          - clang
+    steps:
+      - name: Setup environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev
+          sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+
+      - name: Checkout Fluent Bit code
+        uses: actions/checkout@v4
+
+      - name: ${{ matrix.compiler }} - ${{ matrix.flb_option }}
+        run: |
+          export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+          echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+          cmake $GLOBAL_OPTS $FLB_OPT  ../
+          make -j $nparallel
+        working-directory: build
+        env:
+          CC: ${{ matrix.compiler }}
+          CXX: ${{ matrix.compiler }}
+          FLB_OPT: ${{ matrix.flb_option }}
+          GLOBAL_OPTS: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
+
+      - name: Display dependencies w/ ldd
+        run: |
+          ldd ./bin/fluent-bit
+        working-directory: build

.github/workflows/staging-release.yaml

@@ -504,6 +504,7 @@ jobs:
   staging-release-images-latest-tags:
     # Only update latest tags for 3.1 releases
     if: startsWith(github.event.inputs.version, '3.1')
+    # if: startsWith(github.event.inputs.version, '4.0')
     name: Release latest Linux container images
     runs-on: ubuntu-latest
     needs:
@@ -804,6 +805,7 @@ jobs:
           make_latest: false
 
       - name: Release 3.1 and latest
+        # TODO: change to 3.1 branch once 4.0 series is ready
         uses: softprops/action-gh-release@v2
         if: startsWith(inputs.version, '3.1')
         with:
@@ -814,6 +816,17 @@ jobs:
           tag_name: v${{ inputs.version }}
           make_latest: true
 
+      # - name: Release 4.0 and latest
+      #   uses: softprops/action-gh-release@v2
+      #   if: startsWith(inputs.version, '4.0')
+      #   with:
+      #     body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
+      #     draft: false
+      #     generate_release_notes: true
+      #     name: "Fluent Bit ${{ inputs.version }}"
+      #     tag_name: v${{ inputs.version }}
+      #     make_latest: true
+
   staging-release-windows-checksums:
     name: Get Windows checksums for new release
     runs-on: ubuntu-22.04
@@ -903,15 +916,24 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: fluent/fluent-bit-docs
+          ref: 3.0
           token: ${{ secrets.GH_PA_TOKEN }}
 
       - name: Release 3.1 and latest
+        # TODO: change to 3.1 branch once 4.0 series is ready
         if: startsWith(inputs.version, '3.1')
         uses: actions/checkout@v4
         with:
           repository: fluent/fluent-bit-docs
           token: ${{ secrets.GH_PA_TOKEN }}
 
+      # - name: Release 4.0 and latest
+      #   if: startsWith(inputs.version, '4.0')
+      #   uses: actions/checkout@v4
+      #   with:
+      #     repository: fluent/fluent-bit-docs
+      #     token: ${{ secrets.GH_PA_TOKEN }}
+
       - name: Ensure we have the script we need
         run: |
           if [[ ! -f update-release-version-docs.sh ]] ; then
@@ -935,7 +957,7 @@ jobs:
 
       - name: Raise docs PR
         id: cpr
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: 'release: update to v${{ inputs.version }}'
           signoff: true
@@ -984,15 +1006,23 @@ jobs:
         with:
           ref: 2.2
 
-      - name: Release 3.0 not latest
+      - name: Release 3.0
         if: startsWith(inputs.version, '3.0')
         uses: actions/checkout@v4
         with:
           ref: 3.0
 
-      - name: Release 3.1 latest
+      - name: Release 3.1
         if: startsWith(inputs.version, '3.1')
         uses: actions/checkout@v4
+        with:
+          ref: 3.1
+
+      - name: Release 4.0
+        if: startsWith(inputs.version, '4.0')
+        uses: actions/checkout@v4
+        with:
+          ref: master
 
       # Get the new version to use
       - name: 'Get next minor version'
@@ -1011,7 +1041,7 @@ jobs:
 
       - name: Raise FB PR to update version
         id: cpr
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: 'release: update to ${{ steps.semvers.outputs.patch }}'
           signoff: true

.github/workflows/unit-tests.yaml

@@ -110,6 +110,113 @@ jobs:
           CXX: g++
           FLB_OPT: ${{ matrix.flb_option }}
 
+  run-aarch64-unit-tests:
+    # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
+    runs-on: ${{(github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb' || 'ubuntu-latest' }}
+    permissions:
+      contents: read
+    # We chain this after Linux one as there are costs for actuated workers
+    needs:
+      - run-ubuntu-unit-tests
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - name: "Aarch64 actuated testing"
+            flb_option: "-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1"
+            omit_option: "-DFLB_WITHOUT_flb-it-utils=1 -DFLB_WITHOUT_flb-it-pack=1"
+            global_option: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
+            unit_test_option: "-DFLB_TESTS_INTERNAL=On"
+            compiler: gcc
+    steps:
+      - name: Checkout Fluent Bit code
+        uses: actions/checkout@v4
+
+      - name: Set up Actuated mirror
+        if: github.repository == 'fluent/fluent-bit'
+        uses: self-actuated/hub-mirror@master
+
+      - name: Setup environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-9 g++-9 clang-12 cmake flex bison libsystemd-dev gcovr libyaml-dev
+          sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
+
+      - name: Build and test with actuated runners
+        run: |
+            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+            sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+            sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90
+
+            export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+            export FLB_OPTION="${{ matrix.config.flb_option }}"
+            export FLB_OMIT_OPTION="${{ matrix.config.omit_option }}"
+            export GLOBAL_OPTION="${{ matrix.config.global_option }}"
+            export FLB_UNIT_TEST_OPTION="${{ matrix.config.unit_test_option }}"
+            export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}"
+
+            echo "CC = ${{ matrix.config.compiler }}, CXX = ${{ matrix.config.compiler }}, FLB_OPT = $FLB_OPT"
+
+            cmake ${FLB_OPT}  ../
+            make -j $nparallel
+            ctest -j $nparallel --build-run-dir . --output-on-failure
+        working-directory: build
+        env:
+          CC: ${{ matrix.config.compiler }}
+          CXX: ${{ matrix.config.compiler }}
+
+  run-qemu-ubuntu-unit-tests:
+    # We chain this after Linux one as there are CPU time costs for QEMU emulation
+    needs:
+      - run-ubuntu-unit-tests
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - s390x
+    steps:
+      - name: Checkout Fluent Bit code
+        uses: actions/checkout@v4
+
+      - name: Prepare and build with QEMU ${{ matrix.arch }}
+        uses: uraimo/run-on-arch-action@v2
+        id: build-and-test-on-qemu
+        with:
+          arch: ${{ matrix.arch }}
+          distro: ubuntu20.04
+          shell: /bin/bash
+          dockerRunArgs: |
+            --volume "/var/lib/dbus/machine-id:/var/lib/dbus/machine-id"
+            --volume "/etc/machine-id:/etc/machine-id"
+          install: |
+            apt-get update
+            apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev #libsystemd-dev
+            ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+
+            update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
+            update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
+            update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+          run: |
+            cd build
+            export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+            export FLB_OPTION="-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1"
+            export FLB_OMIT_OPTION=""
+            export GLOBAL_OPTION="-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
+            export FLB_UNIT_TEST_OPTION="-DFLB_TESTS_INTERNAL=On"
+            export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}"
+            export CC=gcc
+            export CXX=gcc
+
+            echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
+
+            cmake ${FLB_OPT}  ../
+            make -j $nparallel
+            ctest -j $nparallel --build-run-dir . --output-on-failure
+
+
   # Required check looks at this so do not remove
   run-all-unit-tests:
     if: always()
@@ -120,6 +227,8 @@ jobs:
     needs:
       - run-macos-unit-tests
       - run-ubuntu-unit-tests
+      - run-aarch64-unit-tests
+      - run-qemu-ubuntu-unit-tests
     steps:
       - name: Check build matrix status
         # Ignore MacOS failures