Merge branch 'master' into patch-1

Signed-off-by: George Cheng <[email protected]>

.github/workflows/cron-scorecards-analysis.yaml

@@ -3,8 +3,10 @@
 # https://openssf.org/blog/2022/01/19/reducing-security-risks-in-open-source-software-at-scale-scorecards-launches-v4/
 name: Scorecards supply-chain security
 on:
-  # Only the default branch is supported.
-  branch_protection_rule:
+  push:
+    # Only the default branch is supported.
+    branches:
+    - main
   schedule:
     # Weekly on Saturdays.
     - cron: '30 1 * * 6'
@@ -29,31 +31,22 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534
+        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46
         with:
-          results_file: scorecard-results.sarif
+          results_file: results.sarif
           results_format: sarif
-          # (Optional) fine-grained personal access token. Uncomment the `repo_token` line below if:
-          # - you want to enable the Branch-Protection check on a *public* repository, or
-          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-fine-grained-pat-optional.
-          repo_token: ${{ secrets.SCORECARD_TOKEN }}
-          #
-          # Publish the results for public repositories to enable scorecard badges. For more details, see
-          # https://github.com/ossf/scorecard-action#publishing-results.
-          # For private repositories, `publish_results` will automatically be set to `false`, regardless
-          # of the value entered here.
           publish_results: true
 
       - name: "Upload artifact"
         uses: actions/upload-artifact@v4
         with:
           name: SARIF file
-          path: scorecard-results.sarif
+          path: results.sarif
           retention-days: 7
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: scorecard-results.sarif
+          sarif_file: results.sarif
           category: ossf-scorecard

.github/workflows/cron-unstable-build.yaml

@@ -13,8 +13,8 @@ on:
   # Run nightly build at this time, bit of trial and error but this seems good.
   schedule:
     - cron: "0 6 * * *" # master build
-    - cron: "0 12 * * *" # 2.2 build
-    - cron: "0 18 * * *" # 2.1 build
+    - cron: "0 12 * * *" # 3.0 build
+    - cron: "0 18 * * *" # 3.1 build
 
 # We do not want a new unstable build to run whilst we are releasing the current unstable build.
 concurrency: unstable-build-release
@@ -52,16 +52,16 @@ jobs:
           echo "cron_branch=master" >> $GITHUB_ENV
         shell: bash
 
-      - name: 2.2 run
+      - name: 3.0 run
         if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *'
         run: |
-          echo "cron_branch=2.2" >> $GITHUB_ENV
+          echo "cron_branch=3.0" >> $GITHUB_ENV
         shell: bash
 
-      - name: 2.1 run
+      - name: 3.1 run
         if: github.event_name == 'schedule' && github.event.schedule=='0 18 * * *'
         run: |
-          echo "cron_branch=2.1" >> $GITHUB_ENV
+          echo "cron_branch=3.1" >> $GITHUB_ENV
         shell: bash
 
       - name: Output the branch to use

.github/workflows/pr-compile-check.yaml

@@ -30,3 +30,51 @@ jobs:
         push: false
         load: false
         provenance: false
+
+  # Sanity check for compilation using system libraries
+  pr-compile-system-libs:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        flb_option:
+          - "-DFLB_PREFER_SYSTEM_LIBS=On"
+        compiler:
+          - gcc
+          - clang
+    steps:
+      - name: Setup environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \
+            libnghttp2-dev libjemalloc-dev
+          sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+          mkdir -p /tmp/libbacktrace/build && \
+            curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \
+            tar --strip-components=1 -xzC /tmp/libbacktrace/ && \
+            pushd /tmp/libbacktrace/build && ../configure && make && sudo make install && popd
+
+      - name: Checkout Fluent Bit code
+        uses: actions/checkout@v4
+
+      - name: ${{ matrix.compiler }} - ${{ matrix.flb_option }}
+        run: |
+          export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+          echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+          cmake $GLOBAL_OPTS $FLB_OPT  ../
+          make -j $nparallel
+        working-directory: build
+        env:
+          CC: ${{ matrix.compiler }}
+          CXX: ${{ matrix.compiler }}
+          FLB_OPT: ${{ matrix.flb_option }}
+          GLOBAL_OPTS: "-DFLB_JEMALLOC=On -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
+
+      - name: Display dependencies w/ ldd
+        run: |
+          ldd ./bin/fluent-bit
+        working-directory: build

.github/workflows/staging-release.yaml

@@ -502,8 +502,8 @@ jobs:
           TAG: ${{ steps.get-tag.outputs.tag }}
 
   staging-release-images-latest-tags:
-    # Only update latest tags for 3.1 releases
-    if: startsWith(github.event.inputs.version, '3.1')
+    # Only update latest tags for 3.2 releases
+    if: startsWith(github.event.inputs.version, '3.2')
     name: Release latest Linux container images
     runs-on: ubuntu-latest
     needs:
@@ -803,9 +803,21 @@ jobs:
           target_commitish: '3.0'
           make_latest: false
 
-      - name: Release 3.1 and latest
+      - name: Release 3.1 - not latest
         uses: softprops/action-gh-release@v2
         if: startsWith(inputs.version, '3.1')
+        with:
+          body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
+          draft: false
+          generate_release_notes: true
+          name: "Fluent Bit ${{ inputs.version }}"
+          tag_name: v${{ inputs.version }}
+          target_commitish: '3.1'
+          make_latest: false
+
+      - name: Release 3.2 and latest
+        uses: softprops/action-gh-release@v2
+        if: startsWith(inputs.version, '3.2')
         with:
           body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
           draft: false
@@ -903,11 +915,20 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: fluent/fluent-bit-docs
+          ref: 3.0
           token: ${{ secrets.GH_PA_TOKEN }}
 
-      - name: Release 3.1 and latest
+      - name: Release 3.1 - not latest
         if: startsWith(inputs.version, '3.1')
         uses: actions/checkout@v4
+        with:
+          repository: fluent/fluent-bit-docs
+          ref: 3.1
+          token: ${{ secrets.GH_PA_TOKEN }}
+
+      - name: Release 3.2 and latest
+        if: startsWith(inputs.version, '3.2')
+        uses: actions/checkout@v4
         with:
           repository: fluent/fluent-bit-docs
           token: ${{ secrets.GH_PA_TOKEN }}
@@ -935,7 +956,7 @@ jobs:
 
       - name: Raise docs PR
         id: cpr
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: 'release: update to v${{ inputs.version }}'
           signoff: true
@@ -984,15 +1005,23 @@ jobs:
         with:
           ref: 2.2
 
-      - name: Release 3.0 not latest
+      - name: Release 3.0
         if: startsWith(inputs.version, '3.0')
         uses: actions/checkout@v4
         with:
           ref: 3.0
 
-      - name: Release 3.1 latest
+      - name: Release 3.1
         if: startsWith(inputs.version, '3.1')
         uses: actions/checkout@v4
+        with:
+          ref: 3.1
+
+      - name: Release 3.2
+        if: startsWith(inputs.version, '3.2')
+        uses: actions/checkout@v4
+        with:
+          ref: master
 
       # Get the new version to use
       - name: 'Get next minor version'
@@ -1011,7 +1040,7 @@ jobs:
 
       - name: Raise FB PR to update version
         id: cpr
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: 'release: update to ${{ steps.semvers.outputs.patch }}'
           signoff: true

.github/workflows/unit-tests.yaml

@@ -42,6 +42,8 @@ jobs:
           - "-DFLB_COVERAGE=On"
           - "-DFLB_SANITIZE_MEMORY=On"
           - "-DFLB_SANITIZE_THREAD=On"
+          - "-DFLB_SIMD=On"
+          - "-DFLB_SIMD=Off"
         compiler:
           - gcc
           - clang
@@ -54,7 +56,7 @@ jobs:
       - name: Setup environment
         run: |
           sudo apt-get update
-          sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev
+          sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common
           sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
 
       - uses: actions/checkout@v4
@@ -110,6 +112,112 @@ jobs:
           CXX: g++
           FLB_OPT: ${{ matrix.flb_option }}
 
+  run-aarch64-unit-tests:
+    # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
+    runs-on: ${{(github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb' || 'ubuntu-latest' }}
+    permissions:
+      contents: read
+    # We chain this after Linux one as there are costs for actuated workers
+    needs:
+      - run-ubuntu-unit-tests
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - name: "Aarch64 actuated testing"
+            flb_option: "-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1"
+            omit_option: ""
+            global_option: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
+            unit_test_option: "-DFLB_TESTS_INTERNAL=On"
+            compiler: gcc
+    steps:
+      - name: Checkout Fluent Bit code
+        uses: actions/checkout@v4
+
+      - name: Set up Actuated mirror
+        if: github.repository == 'fluent/fluent-bit'
+        uses: self-actuated/hub-mirror@master
+
+      - name: Setup environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-9 g++-9 clang-12 cmake flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common
+          sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
+
+      - name: Build and test with actuated runners
+        run: |
+            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+            sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+            sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90
+
+            export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+            export FLB_OPTION="${{ matrix.config.flb_option }}"
+            export FLB_OMIT_OPTION="${{ matrix.config.omit_option }}"
+            export GLOBAL_OPTION="${{ matrix.config.global_option }}"
+            export FLB_UNIT_TEST_OPTION="${{ matrix.config.unit_test_option }}"
+            export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}"
+
+            echo "CC = ${{ matrix.config.compiler }}, CXX = ${{ matrix.config.compiler }}, FLB_OPT = $FLB_OPT"
+
+            cmake ${FLB_OPT}  ../
+            make -j $nparallel
+            ctest -j $nparallel --build-run-dir . --output-on-failure
+        working-directory: build
+        env:
+          CC: ${{ matrix.config.compiler }}
+          CXX: ${{ matrix.config.compiler }}
+
+  run-qemu-ubuntu-unit-tests:
+    # We chain this after Linux one as there are CPU time costs for QEMU emulation
+    needs:
+      - run-ubuntu-unit-tests
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - s390x
+          - riscv64
+    steps:
+      - name: Checkout Fluent Bit code
+        uses: actions/checkout@v4
+
+      - name: Prepare and build with QEMU ${{ matrix.arch }}
+        uses: uraimo/run-on-arch-action@v2
+        id: build-and-test-on-qemu
+        with:
+          arch: ${{ matrix.arch }}
+          distro: ubuntu22.04
+          shell: /bin/bash
+          dockerRunArgs: |
+            --volume "/var/lib/dbus/machine-id:/var/lib/dbus/machine-id"
+            --volume "/etc/machine-id:/etc/machine-id"
+          install: |
+            apt-get update
+            apt-get install -y gcc-12 g++-12 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common
+
+            update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90
+            update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90
+          run: |
+            cd build
+            export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+            export FLB_OPTION="-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1"
+            export FLB_OMIT_OPTION=""
+            export GLOBAL_OPTION="-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
+            export FLB_UNIT_TEST_OPTION="-DFLB_TESTS_INTERNAL=On"
+            export FLB_OPT="${FLB_OPTION} ${GLOBAL_OPTION} ${FLB_UNIT_TEST_OPTION} ${FLB_OMIT_OPTION}"
+            export CC=gcc
+            export CXX=gcc
+
+            echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
+
+            cmake ${FLB_OPT}  ../
+            make -j $nparallel
+            ctest -j $nparallel --build-run-dir . --output-on-failure
+
+
   # Required check looks at this so do not remove
   run-all-unit-tests:
     if: always()
@@ -120,6 +228,8 @@ jobs:
     needs:
       - run-macos-unit-tests
       - run-ubuntu-unit-tests
+      - run-aarch64-unit-tests
+      - run-qemu-ubuntu-unit-tests
     steps:
       - name: Check build matrix status
         # Ignore MacOS failures

.gitignore

@@ -28,7 +28,7 @@ workflow/
 *.key
 *.log
 *.tar.gz
-
+*.o
 # clangd files
 .cache/
 compile_commands.json
@@ -40,4 +40,4 @@ workflow/
 # examples
 examples/wasi_serde_json/target/
 # WASM test data
-tests/runtime/wasm/go/*.wasm
+tests/runtime/wasm/go/*.wasm