diff --git a/.github/workflows/build-legacy-branch.yaml b/.github/workflows/build-legacy-branch.yaml index 2bca2a719ca..65bfa4bb385 100644 --- a/.github/workflows/build-legacy-branch.yaml +++ b/.github/workflows/build-legacy-branch.yaml @@ -80,7 +80,7 @@ jobs: - name: Build the legacy x86_64 debug image if: matrix.arch == 'amd64' - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./Dockerfile.x86_64.debug context: . @@ -102,7 +102,7 @@ jobs: raw,${{ matrix.suffix }}-${{ inputs.ref }} - name: Build the legacy ${{ matrix.arch }} image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./Dockerfile.${{ matrix.suffix }} context: . diff --git a/.github/workflows/call-build-images.yaml b/.github/workflows/call-build-images.yaml index 507dc405873..1330b94abf0 100644 --- a/.github/workflows/call-build-images.yaml +++ b/.github/workflows/call-build-images.yaml @@ -62,7 +62,7 @@ jobs: # E.g. if we build version 1.9.2 we want to tag with 1.9.2 and 1.9. - name: Determine major version tag id: determine-major-version - uses: frabert/replace-string-action@v2.4 + uses: frabert/replace-string-action@v2.5 with: pattern: '^(\d+\.\d+).*$' string: ${{ inputs.version }} @@ -114,7 +114,7 @@ jobs: - name: Build the production images id: build_push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./dockerfiles/Dockerfile context: . @@ -141,7 +141,7 @@ jobs: - name: Build the debug multi-arch images id: debug_build_push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./dockerfiles/Dockerfile context: . @@ -181,7 +181,7 @@ jobs: shell: bash - name: Upload the schema - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: ./fluent-bit-schema*.json name: fluent-bit-schema-${{ inputs.version }} @@ -314,7 +314,7 @@ jobs: docker push ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }} # We cannot use this action as it requires privileged mode - # uses: docker/build-push-action@v5 + # uses: docker/build-push-action@v6 # with: # file: ./dockerfiles/Dockerfile.windows # context: . diff --git a/.github/workflows/call-build-linux-packages.yaml b/.github/workflows/call-build-linux-packages.yaml index afe84807333..7a79b382d32 100644 --- a/.github/workflows/call-build-linux-packages.yaml +++ b/.github/workflows/call-build-linux-packages.yaml @@ -78,7 +78,7 @@ jobs: SOURCE_FILENAME_PREFIX: source-${{ inputs.version }} - name: Upload the source artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: source-${{ inputs.version }} path: source-packages/* @@ -132,7 +132,7 @@ jobs: - name: Replace all special characters with dashes id: formatted_distro - run: + run: | output=${INPUT//[\/]/-} echo "$INPUT --> $output" echo "replaced=$output" >> "$GITHUB_OUTPUT" @@ -151,7 +151,7 @@ jobs: working-directory: packaging - name: Upload the ${{ steps.formatted_distro.outputs.replaced }} artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: packages-${{ inputs.version }}-${{ steps.formatted_distro.outputs.replaced }} path: packaging/packages/ diff --git a/.github/workflows/call-build-macos.yaml b/.github/workflows/call-build-macos.yaml index c97ee83866d..5aa8f5f8cf7 100644 --- a/.github/workflows/call-build-macos.yaml +++ b/.github/workflows/call-build-macos.yaml @@ -64,12 +64,21 @@ jobs: call-build-macos-package: if: needs.call-build-macos-legacy-check.outputs.build-type == 'modern' - runs-on: macos-latest + runs-on: ${{ matrix.config.runner }} environment: ${{ inputs.environment }} needs: - call-build-macos-legacy-check permissions: contents: read + strategy: + fail-fast: false + matrix: + config: + - name: "Normal macOS-latest runner (Intel)" + runner: macos-12 + - name: "Apple Silicon macOS runner" + runner: macos-14 + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -90,9 +99,9 @@ jobs: working-directory: build - name: Upload build packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: macos-packages + name: macos-packages on ${{ matrix.config.runner }} path: | build/fluent-bit-*-apple* build/fluent-bit-*-intel* @@ -108,6 +117,15 @@ jobs: - call-build-macos-package permissions: contents: read + strategy: + fail-fast: false + matrix: + config: + - name: "Normal macOS-latest package (Intel)" + os: macos-12 + - name: "Apple Silicon macOS package" + os: macos-14 + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -116,9 +134,9 @@ jobs: - name: Download all artefacts continue-on-error: true - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: macos-packages + name: macos-packages on ${{ matrix.config.os }} path: artifacts/ - name: Push MacOS packages to S3 diff --git a/.github/workflows/call-build-windows.yaml b/.github/workflows/call-build-windows.yaml index 7adc229a7ac..3b501b3d8b1 100644 --- a/.github/workflows/call-build-windows.yaml +++ b/.github/workflows/call-build-windows.yaml @@ -1,6 +1,16 @@ --- name: Reusable workflow to build Windows packages optionally into S3 bucket +# +# If you change dependencies etc here, please also check and update +# the other Windows build resources: +# +# - DEVELOPER_GUIDE.md "Windows" section +# - appveyor.yml +# - .github/workflows/call-build-windows.yaml +# - dockerfiles/Dockerfile.windows +# + on: workflow_call: inputs: @@ -98,7 +108,7 @@ jobs: - name: Get dependencies run: | - Invoke-WebRequest -O winflexbison.zip $env:WINFLEXBISON + Invoke-WebRequest -OutFile winflexbison.zip $env:WINFLEXBISON Expand-Archive winflexbison.zip -Destination C:\WinFlexBison Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe @@ -126,7 +136,7 @@ jobs: - name: Restore cached packages of vcpkg id: cache-vcpkg-sources - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: | C:\vcpkg\packages @@ -147,7 +157,7 @@ jobs: - name: Save packages of vcpkg id: save-vcpkg-sources - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: | C:\vcpkg\packages @@ -168,9 +178,9 @@ jobs: - name: Upload build packages # Skip upload if we skipped build. if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: windows-packages + name: windows-packages-${{ matrix.config.arch }} path: | build/*-bit-*.exe build/*-bit-*.msi @@ -195,10 +205,10 @@ jobs: ref: master - name: Download all artefacts - continue-on-error: true - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: windows-packages + pattern: windows-packages-* + merge-multiple: true path: artifacts/ - name: Set up Windows checksums diff --git a/.github/workflows/call-integration-image-build.yaml b/.github/workflows/call-integration-image-build.yaml index 1644883c0d4..65a47fc4d58 100644 --- a/.github/workflows/call-integration-image-build.yaml +++ b/.github/workflows/call-integration-image-build.yaml @@ -62,7 +62,7 @@ jobs: raw,${{ inputs.image-tag }} - name: Build the AMD64 image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./dockerfiles/Dockerfile context: . @@ -74,21 +74,6 @@ jobs: push: true load: false - - name: Upload the image just in case as an artefact - run: | - docker pull $IMAGE - docker save --output /tmp/pr-image.tar $IMAGE - env: - IMAGE: ${{ steps.meta.outputs.tags }} - shell: bash - - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: pr-${{ github.event.pull_request.number }}-image - path: /tmp/pr-image.tar - if-no-files-found: error - - name: Extract metadata from Github id: meta-debug uses: docker/metadata-action@v5 @@ -98,7 +83,7 @@ jobs: raw,${{ inputs.image-tag }}-debug - name: Build the AMD64 debug image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./dockerfiles/Dockerfile context: . diff --git a/.github/workflows/call-run-integration-test.yaml b/.github/workflows/call-run-integration-test.yaml index facff7fe4f9..a01287c2d28 100644 --- a/.github/workflows/call-run-integration-test.yaml +++ b/.github/workflows/call-run-integration-test.yaml @@ -57,12 +57,12 @@ jobs: cli_config_credentials_token: ${{ secrets.terraform_api_token }} - id: 'auth' - uses: 'google-github-actions/auth@v1' + uses: 'google-github-actions/auth@v2' with: credentials_json: ${{ secrets.gcp-service-account-key }} - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v1' + uses: 'google-github-actions/setup-gcloud@v2' - name: Replace terraform variables. run: | @@ -192,18 +192,18 @@ jobs: bats-version: 1.9.0 - name: Create k8s Kind Cluster - uses: helm/kind-action@v1.8.0 + uses: helm/kind-action@v1.10.0 with: node_image: kindest/node:${{ matrix.k8s-release }} cluster_name: kind - name: Set up Helm - uses: azure/setup-helm@v3.5 + uses: azure/setup-helm@v4 with: version: v3.8.1 - name: Set up Kubectl - uses: azure/setup-kubectl@v3.2 + uses: azure/setup-kubectl@v4 - name: Run tests timeout-minutes: 60 @@ -240,12 +240,12 @@ jobs: repository: fluent/fluent-bit-ci - if: matrix.cloud == 'gke' - uses: 'google-github-actions/auth@v1' + uses: 'google-github-actions/auth@v2' with: credentials_json: ${{ secrets.gcp-service-account-key }} - if: matrix.cloud == 'gke' - uses: 'google-github-actions/setup-gcloud@v1' + uses: 'google-github-actions/setup-gcloud@v2' with: install_components: 'gke-gcloud-auth-plugin' @@ -255,16 +255,16 @@ jobs: bats-version: 1.9.0 - name: Set up Helm - uses: azure/setup-helm@v3.5 + uses: azure/setup-helm@v4 with: version: v3.8.1 - name: Set up Kubectl - uses: azure/setup-kubectl@v3.2 + uses: azure/setup-kubectl@v4 - name: Get the GKE Kubeconfig if: matrix.cloud == 'gke' - uses: 'google-github-actions/get-gke-credentials@v1' + uses: 'google-github-actions/get-gke-credentials@v2' with: cluster_name: ${{ needs.call-run-terraform-setup.outputs.gke-cluster-name }} location: ${{ needs.call-run-terraform-setup.outputs.gke-cluster-zone }} diff --git a/.github/workflows/call-test-images.yaml b/.github/workflows/call-test-images.yaml index 31c3ca14dd5..5a8978d890e 100644 --- a/.github/workflows/call-test-images.yaml +++ b/.github/workflows/call-test-images.yaml @@ -184,15 +184,15 @@ jobs: ref: ${{ inputs.ref }} - name: Create k8s Kind Cluster - uses: helm/kind-action@v1.8.0 + uses: helm/kind-action@v1.10.0 - name: Set up Helm - uses: azure/setup-helm@v3.5 + uses: azure/setup-helm@v4 with: version: v3.6.3 - name: Set up Kubectl - uses: azure/setup-kubectl@v3.2 + uses: azure/setup-kubectl@v4 - name: Test the HTTP server is responding timeout-minutes: 5 diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml new file mode 100644 index 00000000000..c24fedc0861 --- /dev/null +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -0,0 +1,163 @@ +--- +name: Reusable workflow to run unit tests on Windows packages (only for x86 and x64) + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + unstable: + description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add. + type: string + required: false + default: '' + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + +jobs: + call-build-windows-unit-test: + runs-on: windows-latest + environment: ${{ inputs.environment }} + strategy: + fail-fast: false + matrix: + config: + - name: "Windows 32bit" + arch: x86 + openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static + cmake_additional_opt: "" + vcpkg_triplet: x86-windows-static + - name: "Windows 64bit" + arch: x64 + openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static + cmake_additional_opt: "" + vcpkg_triplet: x64-windows-static + permissions: + contents: read + # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures. + env: + PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin;C:\vcpkg; + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Get dependencies + run: | + Invoke-WebRequest -OutFile winflexbison.zip $env:WINFLEXBISON + Expand-Archive winflexbison.zip -Destination C:\WinFlexBison + Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe + Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe + echo "C:\WinFlexBison" | Out-File -FilePath $env:GITHUB_PATH -Append + env: + WINFLEXBISON: https://github.com/lexxmark/winflexbison/releases/download/v2.5.22/win_flex_bison-2.5.22.zip + shell: pwsh + + - name: Set up with Developer Command Prompt for Microsoft Visual C++ + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.config.arch }} + + - name: Get gzip command w/ chocolatey + uses: crazy-max/ghaction-chocolatey@v3 + with: + args: install gzip -y + + # http://man7.org/linux/man-pages/man1/date.1.html + - name: Get Date + id: get-date + run: | + echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore cached packages of vcpkg + id: cache-unit-test-vcpkg-sources + uses: actions/cache/restore@v4 + with: + path: | + C:\vcpkg\packages + key: ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg-${{ steps.get-date.outputs.date }} + restore-keys: | + ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg- + enableCrossOsArchive: false + + - name: Build openssl with vcpkg + run: | + C:\vcpkg\vcpkg install --recurse openssl --triplet ${{ matrix.config.vcpkg_triplet }} + shell: cmd + + - name: Build libyaml with vcpkg + run: | + C:\vcpkg\vcpkg install --recurse libyaml --triplet ${{ matrix.config.vcpkg_triplet }} + shell: cmd + + - name: Save packages of vcpkg + id: save-vcpkg-sources + uses: actions/cache/save@v4 + with: + path: | + C:\vcpkg\packages + key: ${{ steps.cache-unit-test-vcpkg-sources.outputs.cache-primary-key }} + enableCrossOsArchive: false + + - name: Build unit-test for Fluent Bit packages (only for x86 and x64) + run: | + cmake -G "NMake Makefiles" ` + -D FLB_TESTS_INTERNAL=On ` + -D FLB_NIGHTLY_BUILD='${{ inputs.unstable }}' ` + -D OPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ` + ${{ matrix.config.cmake_additional_opt }} ` + -D FLB_LIBYAML_DIR=C:\vcpkg\packages\libyaml_${{ matrix.config.vcpkg_triplet }} ` + -D FLB_WITHOUT_flb-rt-out_elasticsearch=On ` + -D FLB_WITHOUT_flb-rt-out_td=On ` + -D FLB_WITHOUT_flb-rt-out_forward=On ` + -D FLB_WITHOUT_flb-rt-in_disk=On ` + -D FLB_WITHOUT_flb-rt-in_proc=On ` + -D FLB_WITHOUT_flb-it-parser=On ` + -D FLB_WITHOUT_flb-it-unit_sizes=On ` + -D FLB_WITHOUT_flb-it-network=On ` + -D FLB_WITHOUT_flb-it-pack=On ` + -D FLB_WITHOUT_flb-it-signv4=On ` + -D FLB_WITHOUT_flb-it-aws_credentials=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_ec2=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_http=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_profile=On ` + -D FLB_WITHOUT_flb-it-aws_credentials_sts=On ` + -D FLB_WITHOUT_flb-it-aws_util=On ` + -D FLB_WITHOUT_flb-it-input_chunk=On ` + ../ + cmake --build . + shell: pwsh + working-directory: build + + - name: Upload unit test binaries + uses: actions/upload-artifact@v4 + with: + name: windows-unit-tests-${{ matrix.config.arch }} + path: | + build/**/*.exe + if-no-files-found: error + + - name: Display dependencies w/ dumpbin + run: | + dumpbin /dependents .\bin\fluent-bit.exe + working-directory: build + + - name: Build unit-test for Fluent Bit packages (only for x86 and x64) + run: | + ctest --build-run-dir "$PWD" --output-on-failure + shell: pwsh + working-directory: build diff --git a/.github/workflows/cron-scorecards-analysis.yaml b/.github/workflows/cron-scorecards-analysis.yaml index c19a2c59739..20b31d8b112 100644 --- a/.github/workflows/cron-scorecards-analysis.yaml +++ b/.github/workflows/cron-scorecards-analysis.yaml @@ -29,7 +29,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 + uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 with: results_file: scorecard-results.sarif results_format: sarif @@ -45,7 +45,7 @@ jobs: publish_results: true - name: "Upload artifact" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SARIF file path: scorecard-results.sarif @@ -53,7 +53,7 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: scorecard-results.sarif category: ossf-scorecard diff --git a/.github/workflows/cron-stale.yaml b/.github/workflows/cron-stale.yaml index 523685c5085..6d7e0e91e60 100644 --- a/.github/workflows/cron-stale.yaml +++ b/.github/workflows/cron-stale.yaml @@ -8,7 +8,7 @@ jobs: name: Mark stale runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days. Maintainers can add the `exempt-stale` label.' diff --git a/.github/workflows/cron-trivy.yaml b/.github/workflows/cron-trivy.yaml index a126681946f..92545149176 100644 --- a/.github/workflows/cron-trivy.yaml +++ b/.github/workflows/cron-trivy.yaml @@ -72,7 +72,7 @@ jobs: format: table - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: trivy-results-${{ matrix.local_tag }}.sarif category: ${{ matrix.arch }} container @@ -80,7 +80,7 @@ jobs: # In case we need to analyse the uploaded files for some reason. - name: Detain results for debug if needed - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: trivy-results-${{ matrix.local_tag }}.sarif path: trivy-results-${{ matrix.local_tag }}.sarif diff --git a/.github/workflows/cron-unstable-build.yaml b/.github/workflows/cron-unstable-build.yaml index 76fdb5ce658..37d9f17b9c8 100644 --- a/.github/workflows/cron-unstable-build.yaml +++ b/.github/workflows/cron-unstable-build.yaml @@ -13,7 +13,8 @@ on: # Run nightly build at this time, bit of trial and error but this seems good. schedule: - cron: "0 6 * * *" # master build - - cron: "0 12 * * *" # 2.1 build + - cron: "0 12 * * *" # 2.2 build + - cron: "0 18 * * *" # 2.1 build # We do not want a new unstable build to run whilst we are releasing the current unstable build. concurrency: unstable-build-release @@ -36,7 +37,7 @@ jobs: run: echo "date=$(date '+%Y-%m-%d-%H_%M_%S')" >> $GITHUB_OUTPUT - name: Debug event output - uses: hmarr/debug-action@v2 + uses: hmarr/debug-action@v3 # Now we need to determine which branch to build - name: Manual run - get branch @@ -51,8 +52,14 @@ jobs: echo "cron_branch=master" >> $GITHUB_ENV shell: bash - - name: 2.1 run + - name: 2.2 run if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *' + run: | + echo "cron_branch=2.2" >> $GITHUB_ENV + shell: bash + + - name: 2.1 run + if: github.event_name == 'schedule' && github.event.schedule=='0 18 * * *' run: | echo "cron_branch=2.1" >> $GITHUB_ENV shell: bash @@ -160,7 +167,7 @@ jobs: - name: Download all artefacts continue-on-error: true - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts/ diff --git a/.github/workflows/pr-closed-docker.yaml b/.github/workflows/pr-closed-docker.yaml index 628f5e92ca5..4d2a374b3e0 100644 --- a/.github/workflows/pr-closed-docker.yaml +++ b/.github/workflows/pr-closed-docker.yaml @@ -12,7 +12,7 @@ jobs: # We may need a specific token here with `packages:admin` privileges which is not available to GITHUB_TOKEN packages: write steps: - - uses: vlaurin/action-ghcr-prune@v0.5.0 + - uses: vlaurin/action-ghcr-prune@v0.6.0 with: organization: fluent container: fluent-bit/pr-${{ github.event.number }} diff --git a/.github/workflows/pr-compile-check.yaml b/.github/workflows/pr-compile-check.yaml index d9f3dc92ab5..329c57071b9 100644 --- a/.github/workflows/pr-compile-check.yaml +++ b/.github/workflows/pr-compile-check.yaml @@ -22,7 +22,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Attempt to build current source for CentOS 7 - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: ./dockerfiles/Dockerfile.centos7 diff --git a/.github/workflows/pr-fuzz.yaml b/.github/workflows/pr-fuzz.yaml index 79371ed5dab..c8c4b5afb7f 100644 --- a/.github/workflows/pr-fuzz.yaml +++ b/.github/workflows/pr-fuzz.yaml @@ -26,14 +26,14 @@ jobs: language: c output-sarif: true - name: Upload Crash - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() && steps.build.outcome == 'success' with: name: artifacts path: ./out/artifacts - name: Upload Sarif if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: # Path to SARIF file relative to the root of the repository sarif_file: cifuzz-sarif/results.sarif diff --git a/.github/workflows/pr-image-tests.yaml b/.github/workflows/pr-image-tests.yaml index 2cdc362200e..90bffde637d 100644 --- a/.github/workflows/pr-image-tests.yaml +++ b/.github/workflows/pr-image-tests.yaml @@ -36,7 +36,7 @@ jobs: - name: Build the multi-arch images id: build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./dockerfiles/Dockerfile context: . @@ -55,7 +55,7 @@ jobs: shell: bash - name: Build the debug multi-arch images - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: file: ./dockerfiles/Dockerfile context: . diff --git a/.github/workflows/pr-package-tests.yaml b/.github/workflows/pr-package-tests.yaml index b461ac4dac1..2f249a5f770 100644 --- a/.github/workflows/pr-package-tests.yaml +++ b/.github/workflows/pr-package-tests.yaml @@ -32,7 +32,7 @@ jobs: run: echo "date=$(date '+%Y-%m-%d-%H_%M_%S')" >> $GITHUB_OUTPUT - name: Debug event output - uses: hmarr/debug-action@v2 + uses: hmarr/debug-action@v3 pr-package-test-build-generate-matrix: name: PR - packages build matrix diff --git a/.github/workflows/pr-perf-test.yaml b/.github/workflows/pr-perf-test.yaml index e94f1cbf31e..de50d7f3578 100644 --- a/.github/workflows/pr-perf-test.yaml +++ b/.github/workflows/pr-perf-test.yaml @@ -32,7 +32,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 - name: Upload plots to CML run: | diff --git a/.github/workflows/pr-windows-build.yaml b/.github/workflows/pr-windows-build.yaml index 2831122fe33..110d85fda64 100644 --- a/.github/workflows/pr-windows-build.yaml +++ b/.github/workflows/pr-windows-build.yaml @@ -1,5 +1,18 @@ name: PR - Windows checks + +# +# Test PRs on Windows +# +# This won't run automatically on PRs from untrusted repos, it must be approved +# manually. If PR authors want to run it themselves, they should enable running +# actions on their fork, then invoke it on their branch via their forked repo's +# Actions tab. +# + on: + # Enable invocation via Github repo Actions tab. Having this in the repo + # allows people with github forks to run this job on their own branches to + # build Windows branches conveniently. See DEVELOPER_GUIDE.md. workflow_dispatch: pull_request: @@ -25,3 +38,14 @@ jobs: environment: pr secrets: token: ${{ secrets.GITHUB_TOKEN }} + + run-windows-unit-tests: + needs: + - pr-windows-build + uses: ./.github/workflows/call-windows-unit-tests.yaml + with: + version: ${{ github.sha }} + ref: ${{ github.sha }} + environment: pr + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/staging-build.yaml b/.github/workflows/staging-build.yaml index 2bf6524fe56..a409dd337ac 100644 --- a/.github/workflows/staging-build.yaml +++ b/.github/workflows/staging-build.yaml @@ -61,7 +61,7 @@ jobs: INPUT_VERSION: ${{ inputs.version || github.ref_name }} # String the 'v' prefix for tags. - - uses: frabert/replace-string-action@v2.4 + - uses: frabert/replace-string-action@v2.5 id: formatted_version with: pattern: '[v]*(.*)$' @@ -94,7 +94,7 @@ jobs: - name: Download the schema generated by call-build-images # We may have no schema so ignore that failure continue-on-error: true - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: fluent-bit-schema-${{ needs.staging-build-get-meta.outputs.version }} path: artifacts/ diff --git a/.github/workflows/staging-release.yaml b/.github/workflows/staging-release.yaml index ca8fd7535bf..58d2660c935 100644 --- a/.github/workflows/staging-release.yaml +++ b/.github/workflows/staging-release.yaml @@ -67,14 +67,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - # Check we can download the AppVeyor build which confirms it matches the version to release as well as being a successful build - - name: Get Appveyor binaries - run: | - ./packaging/appveyor-download.sh - shell: bash - env: - TAG: v${{ github.event.inputs.version }} - staging-release-generate-package-matrix: name: Get package matrix runs-on: ubuntu-latest @@ -341,11 +333,6 @@ jobs: - staging-release-version-check permissions: contents: read - outputs: - windows-exe32-hash: ${{ steps.windows.outputs.WIN_32_EXE_HASH }} - windows-zip32-hash: ${{ steps.windows.outputs.WIN_32_ZIP_HASH }} - windows-exe64-hash: ${{ steps.windows.outputs.WIN_64_EXE_HASH }} - windows-zip64-hash: ${{ steps.windows.outputs.WIN_64_ZIP_HASH }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -361,14 +348,6 @@ jobs: AWS_REGION: "us-east-1" shell: bash - - name: Get Appveyor binaries - run: | - ./packaging/appveyor-download.sh - shell: bash - env: - TAG: v${{ github.event.inputs.version }} - OUTPUT_DIR: appveyor - - name: Move components from staging and setup run: | ./packaging/update-source-packages.sh @@ -392,25 +371,6 @@ jobs: AWS_REGION: "us-east-1" shell: bash - - name: Provide output for documentation PR - id: windows - # do not fail the build for this - continue-on-error: true - run: | - ls -l $BASE_DIR/ - export WIN_32_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}') - export WIN_32_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}') - export WIN_64_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}') - export WIN_64_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}') - set | grep WIN_ - echo WIN_32_EXE_HASH="$WIN_32_EXE_HASH" >> $GITHUB_OUTPUT - echo WIN_32_ZIP_HASH="$WIN_32_ZIP_HASH" >> $GITHUB_OUTPUT - echo WIN_64_EXE_HASH="$WIN_64_EXE_HASH" >> $GITHUB_OUTPUT - echo WIN_64_ZIP_HASH="$WIN_64_ZIP_HASH" >> $GITHUB_OUTPUT - shell: bash - env: - BASE_DIR: release/${{ needs.staging-release-version-check.outputs.major-version }} - # Simple skopeo copy jobs to transfer image from staging to release registry with optional GPG key signing. # Unfortunately skopeo currently does not support Cosign: https://github.com/containers/skopeo/issues/1533 staging-release-images: @@ -459,7 +419,7 @@ jobs: shell: bash - name: Promote container images from staging to GHCR.io - if: ${{ startsWith(github.event.inputs.version, '2.0') || startsWith(github.event.inputs.version, '2.1') || ! startsWith(matrix.tag, 'latest') }} + if: ${{ startsWith(github.event.inputs.version, '2.') || startsWith(github.event.inputs.version, '3.') || ! startsWith(matrix.tag, 'latest') }} run: | docker run --rm \ quay.io/skopeo/stable:latest \ @@ -542,8 +502,8 @@ jobs: TAG: ${{ steps.get-tag.outputs.tag }} staging-release-images-latest-tags: - # Only update latest tags for 2.2 releases - if: startsWith(github.event.inputs.version, '2.2') + # Only update latest tags for 3.1 releases + if: startsWith(github.event.inputs.version, '3.1') name: Release latest Linux container images runs-on: ubuntu-latest needs: @@ -808,7 +768,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Release 2.0 - not latest - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 if: startsWith(inputs.version, '2.0') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -817,9 +777,10 @@ jobs: name: "Fluent Bit ${{ inputs.version }}" tag_name: v${{ inputs.version }} target_commitish: '2.0' + make_latest: false - name: Release 2.1 - not latest - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 if: startsWith(inputs.version, '2.1') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" @@ -828,22 +789,86 @@ jobs: name: "Fluent Bit ${{ inputs.version }}" tag_name: v${{ inputs.version }} target_commitish: '2.1' + make_latest: false - - name: Release 2.2 and latest - uses: softprops/action-gh-release@v1 - if: startsWith(inputs.version, '2.2') + - name: Release 3.0 - not latest + uses: softprops/action-gh-release@v2 + if: startsWith(inputs.version, '3.0') + with: + body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" + draft: false + generate_release_notes: true + name: "Fluent Bit ${{ inputs.version }}" + tag_name: v${{ inputs.version }} + target_commitish: '3.0' + make_latest: false + + - name: Release 3.1 and latest + uses: softprops/action-gh-release@v2 + if: startsWith(inputs.version, '3.1') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" draft: false generate_release_notes: true name: "Fluent Bit ${{ inputs.version }}" tag_name: v${{ inputs.version }} + make_latest: true + + staging-release-windows-checksums: + name: Get Windows checksums for new release + runs-on: ubuntu-22.04 + environment: release + needs: + - staging-release-update-non-linux-s3 + permissions: + contents: none + outputs: + windows-exe32-hash: ${{ steps.hashes.outputs.WIN_32_EXE_HASH }} + windows-zip32-hash: ${{ steps.hashes.outputs.WIN_32_ZIP_HASH }} + windows-exe64-hash: ${{ steps.hashes.outputs.WIN_64_EXE_HASH }} + windows-zip64-hash: ${{ steps.hashes.outputs.WIN_64_ZIP_HASH }} + windows-arm-exe64-hash: ${{ steps.hashes.outputs.WIN_64_ARM_EXE_HASH }} + windows-arm-zip64-hash: ${{ steps.hashes.outputs.WIN_64_ARM_ZIP_HASH }} + steps: + - name: Sync release Windows directory to get checksums + run: + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/windows" ./ --exclude "*" --include "*.sha256" + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + + - name: Provide output for documentation PR + id: hashes + # do not fail the build for this + continue-on-error: true + run: | + ls -l + export WIN_32_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}') + export WIN_32_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}') + export WIN_64_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}') + export WIN_64_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}') + if [[ -f "./fluent-bit-${{ inputs.version }}-winarm64.exe.sha256" ]]; then + export WIN_64_ARM_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-winarm64.exe.sha256"|awk '{print $1}') + fi + if [[ -f "./fluent-bit-${{ inputs.version }}-winarm64.zip.sha256" ]]; then + export WIN_64_ARM_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-winarm64.zip.sha256"|awk '{print $1}') + fi + set | grep WIN_ + echo WIN_32_EXE_HASH="$WIN_32_EXE_HASH" >> $GITHUB_OUTPUT + echo WIN_32_ZIP_HASH="$WIN_32_ZIP_HASH" >> $GITHUB_OUTPUT + echo WIN_64_EXE_HASH="$WIN_64_EXE_HASH" >> $GITHUB_OUTPUT + echo WIN_64_ZIP_HASH="$WIN_64_ZIP_HASH" >> $GITHUB_OUTPUT + echo WIN_64_ARM_EXE_HASH="$WIN_64_ARM_EXE_HASH" >> $GITHUB_OUTPUT + echo WIN_64_ARM_ZIP_HASH="$WIN_64_ARM_ZIP_HASH" >> $GITHUB_OUTPUT + shell: bash staging-release-create-docs-pr: name: Create docs updates for new release needs: - staging-release-images - - staging-release-source-s3 + - staging-release-windows-checksums permissions: contents: none environment: release @@ -865,9 +890,24 @@ jobs: ref: 2.1 token: ${{ secrets.GH_PA_TOKEN }} - - name: Release 2.2 and latest + - name: Release 2.2 - not latest if: startsWith(inputs.version, '2.2') uses: actions/checkout@v4 + with: + repository: fluent/fluent-bit-docs + ref: 2.2 + token: ${{ secrets.GH_PA_TOKEN }} + + - name: Release 3.0 - not latest + if: startsWith(inputs.version, '3.0') + uses: actions/checkout@v4 + with: + repository: fluent/fluent-bit-docs + token: ${{ secrets.GH_PA_TOKEN }} + + - name: Release 3.1 and latest + if: startsWith(inputs.version, '3.1') + uses: actions/checkout@v4 with: repository: fluent/fluent-bit-docs token: ${{ secrets.GH_PA_TOKEN }} @@ -886,14 +926,16 @@ jobs: shell: bash env: NEW_VERSION: ${{ inputs.version }} - WIN_32_EXE_HASH: ${{ needs.staging-release-source-s3.outputs.windows-exe32-hash }} - WIN_32_ZIP_HASH: ${{ needs.staging-release-source-s3.outputs.windows-zip32-hash }} - WIN_64_EXE_HASH: ${{ needs.staging-release-source-s3.outputs.windows-exe64-hash }} - WIN_64_ZIP_HASH: ${{ needs.staging-release-source-s3.outputs.windows-zip64-hash }} + WIN_32_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-exe32-hash }} + WIN_32_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-zip32-hash }} + WIN_64_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-exe64-hash }} + WIN_64_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-zip64-hash }} + WIN_64_ARM_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-arm-exe64-hash }} + WIN_64_ARM_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-arm-zip64-hash }} - name: Raise docs PR id: cpr - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 with: commit-message: 'release: update to v${{ inputs.version }}' signoff: true @@ -936,9 +978,21 @@ jobs: with: ref: 2.1 - - name: Release 2.2 and latest + - name: Release 2.2 if: startsWith(inputs.version, '2.2') uses: actions/checkout@v4 + with: + ref: 2.2 + + - name: Release 3.0 not latest + if: startsWith(inputs.version, '3.0') + uses: actions/checkout@v4 + with: + ref: 3.0 + + - name: Release 3.1 latest + if: startsWith(inputs.version, '3.1') + uses: actions/checkout@v4 # Get the new version to use - name: 'Get next minor version' @@ -957,7 +1011,7 @@ jobs: - name: Raise FB PR to update version id: cpr - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 with: commit-message: 'release: update to ${{ steps.semvers.outputs.patch }}' signoff: true diff --git a/.github/workflows/update-dockerhub.yaml b/.github/workflows/update-dockerhub.yaml index cdd5dbe16de..a7b01b13e5a 100644 --- a/.github/workflows/update-dockerhub.yaml +++ b/.github/workflows/update-dockerhub.yaml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v3 + uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 38f963a60e8..0e2a880b277 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,9 @@ cmake_policy(SET CMP0069 NEW) set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) # Fluent Bit Version -set(FLB_VERSION_MAJOR 2) -set(FLB_VERSION_MINOR 2) -set(FLB_VERSION_PATCH 1) +set(FLB_VERSION_MAJOR 3) +set(FLB_VERSION_MINOR 1) +set(FLB_VERSION_PATCH 0) set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}") set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -159,132 +159,8 @@ option(FLB_CUSTOM_CALYPTIA "Enable Calyptia Support" Yes) # Config formats option(FLB_CONFIG_YAML "Enable YAML config format" Yes) -# Built-in Plugins -option(FLB_IN_CPU "Enable CPU input plugin" Yes) -option(FLB_IN_THERMAL "Enable Thermal plugin" Yes) -option(FLB_IN_DISK "Enable Disk input plugin" Yes) -option(FLB_IN_DOCKER "Enable Docker input plugin" Yes) -option(FLB_IN_DOCKER_EVENTS "Enable Docker events input plugin" Yes) -option(FLB_IN_EXEC "Enable Exec input plugin" Yes) -option(FLB_IN_EXEC_WASI "Enable Exec WASI input plugin" Yes) -option(FLB_IN_EVENT_TEST "Enable Events test plugin" Yes) -option(FLB_IN_EVENT_TYPE "Enable event type plugin" Yes) -option(FLB_IN_FLUENTBIT_METRICS "Enable Fluent Bit metrics plugin" Yes) -option(FLB_IN_FORWARD "Enable Forward input plugin" Yes) -option(FLB_IN_HEALTH "Enable Health input plugin" Yes) -option(FLB_IN_HTTP "Enable HTTP input plugin" Yes) -option(FLB_IN_MEM "Enable Memory input plugin" Yes) -option(FLB_IN_KUBERNETES_EVENTS "Enable Kubernetes Events plugin" Yes) -option(FLB_IN_KAFKA "Enable Kafka input plugin" Yes) -option(FLB_IN_KMSG "Enable Kernel log input plugin" Yes) -option(FLB_IN_LIB "Enable library mode input plugin" Yes) -option(FLB_IN_RANDOM "Enable random input plugin" Yes) -option(FLB_IN_SERIAL "Enable Serial input plugin" Yes) -option(FLB_IN_STDIN "Enable Standard input plugin" Yes) -option(FLB_IN_SYSLOG "Enable Syslog input plugin" Yes) -option(FLB_IN_TAIL "Enable Tail input plugin" Yes) -option(FLB_IN_UDP "Enable UDP input plugin" Yes) -option(FLB_IN_TCP "Enable TCP input plugin" Yes) -option(FLB_IN_UNIX_SOCKET "Enable Unix socket input plugin" No) -option(FLB_IN_MQTT "Enable MQTT Broker input plugin" Yes) -option(FLB_IN_HEAD "Enable Head input plugin" Yes) -option(FLB_IN_PROC "Enable Process input plugin" Yes) -option(FLB_IN_SYSTEMD "Enable Systemd input plugin" Yes) -option(FLB_IN_DUMMY "Enable Dummy input plugin" Yes) -option(FLB_IN_NGINX_EXPORTER_METRICS "Enable Nginx Metrics input plugin" Yes) -option(FLB_IN_NETIF "Enable NetworkIF input plugin" Yes) -option(FLB_IN_WINLOG "Enable Windows Log input plugin" No) -option(FLB_IN_WINSTAT "Enable Windows Stat input plugin" No) -option(FLB_IN_WINEVTLOG "Enable Windows EvtLog input plugin" No) -option(FLB_IN_COLLECTD "Enable Collectd input plugin" Yes) -option(FLB_IN_PROMETHEUS_SCRAPE "Enable Promeheus Scrape input plugin" Yes) -option(FLB_IN_STATSD "Enable StatsD input plugin" Yes) -option(FLB_IN_EVENT_TEST "Enable event test plugin" No) -option(FLB_IN_STORAGE_BACKLOG "Enable storage backlog input plugin" Yes) -option(FLB_IN_EMITTER "Enable emitter input plugin" Yes) -option(FLB_IN_NODE_EXPORTER_METRICS "Enable node exporter metrics input plugin" Yes) -option(FLB_IN_WINDOWS_EXPORTER_METRICS "Enable windows exporter metrics input plugin" Yes) -option(FLB_IN_PODMAN_METRICS "Enable Podman Metrics input plugin" Yes) -option(FLB_IN_OPENTELEMETRY "Enable OpenTelemetry input plugin" Yes) -option(FLB_IN_ELASTICSEARCH "Enable Elasticsearch (Bulk API) input plugin" Yes) -option(FLB_IN_CALYPTIA_FLEET "Enable Calyptia Fleet input plugin" Yes) -option(FLB_IN_SPLUNK "Enable Splunk HTTP HEC input plugin" Yes) -option(FLB_IN_PROCESS_EXPORTER_METRICS "Enable process exporter metrics input plugin" Yes) -option(FLB_OUT_AZURE "Enable Azure output plugin" Yes) -option(FLB_OUT_AZURE_BLOB "Enable Azure output plugin" Yes) -option(FLB_OUT_AZURE_LOGS_INGESTION "Enable Azure Logs Ingestion output plugin" Yes) -option(FLB_OUT_AZURE_KUSTO "Enable Azure Kusto output plugin" Yes) -option(FLB_OUT_BIGQUERY "Enable BigQuery output plugin" Yes) -option(FLB_OUT_CALYPTIA "Enable Calyptia monitoring plugin" Yes) -option(FLB_OUT_COUNTER "Enable Counter output plugin" Yes) -option(FLB_OUT_DATADOG "Enable DataDog output plugin" Yes) -option(FLB_OUT_ES "Enable Elasticsearch output plugin" Yes) -option(FLB_OUT_EXIT "Enable Exit output plugin" Yes) -option(FLB_OUT_FORWARD "Enable Forward output plugin" Yes) -option(FLB_OUT_GELF "Enable GELF output plugin" Yes) -option(FLB_OUT_HTTP "Enable HTTP output plugin" Yes) -option(FLB_OUT_INFLUXDB "Enable InfluxDB output plugin" Yes) -option(FLB_OUT_NATS "Enable NATS output plugin" Yes) -option(FLB_OUT_NRLOGS "Enable New Relic output plugin" Yes) -option(FLB_OUT_OPENSEARCH "Enable OpenSearch output plugin" Yes) -option(FLB_OUT_TCP "Enable TCP output plugin" Yes) -option(FLB_OUT_UDP "Enable UDP output plugin" Yes) -option(FLB_OUT_PLOT "Enable Plot output plugin" Yes) -option(FLB_OUT_FILE "Enable file output plugin" Yes) -option(FLB_OUT_TD "Enable Treasure Data output plugin" Yes) -option(FLB_OUT_RETRY "Enable Retry test output plugin" No) -option(FLB_OUT_PGSQL "Enable PostgreSQL output plugin" No) -option(FLB_OUT_SKYWALKING "Enable Apache SkyWalking output plugin" Yes) -option(FLB_OUT_SLACK "Enable Slack output plugin" Yes) -option(FLB_OUT_SPLUNK "Enable Splunk output plugin" Yes) -option(FLB_OUT_STACKDRIVER "Enable Stackdriver output plugin" Yes) -option(FLB_OUT_STDOUT "Enable STDOUT output plugin" Yes) -option(FLB_OUT_SYSLOG "Enable Syslog output plugin" Yes) -option(FLB_OUT_LIB "Enable library mode output plugin" Yes) -option(FLB_OUT_NULL "Enable dev null output plugin" Yes) -option(FLB_OUT_FLOWCOUNTER "Enable flowcount output plugin" Yes) -option(FLB_OUT_LOGDNA "Enable LogDNA output plugin" Yes) -option(FLB_OUT_LOKI "Enable Loki output plugin" Yes) -option(FLB_OUT_KAFKA "Enable Kafka output plugin" Yes) -option(FLB_OUT_KAFKA_REST "Enable Kafka Rest output plugin" Yes) -option(FLB_OUT_CLOUDWATCH_LOGS "Enable AWS CloudWatch output plugin" Yes) -option(FLB_OUT_KINESIS_FIREHOSE "Enable AWS Firehose output plugin" Yes) -option(FLB_OUT_KINESIS_STREAMS "Enable AWS Kinesis output plugin" Yes) -option(FLB_OUT_OPENTELEMETRY "Enable OpenTelemetry plugin" Yes) -option(FLB_OUT_PROMETHEUS_EXPORTER "Enable Prometheus exporter plugin" Yes) -option(FLB_OUT_PROMETHEUS_REMOTE_WRITE "Enable Prometheus remote write plugin" Yes) -option(FLB_OUT_S3 "Enable AWS S3 output plugin" Yes) -option(FLB_OUT_VIVO_EXPORTER "Enabel Vivo exporter output plugin" Yes) -option(FLB_OUT_WEBSOCKET "Enable Websocket output plugin" Yes) -option(FLB_OUT_ORACLE_LOG_ANALYTICS "Enable Oracle Cloud Infrastructure Logging analytics plugin" Yes) -option(FLB_OUT_CHRONICLE "Enable Google Chronicle output plugin" Yes) -option(FLB_FILTER_ALTER_SIZE "Enable alter_size filter" Yes) -option(FLB_FILTER_AWS "Enable aws filter" Yes) -option(FLB_FILTER_ECS "Enable AWS ECS filter" Yes) -option(FLB_FILTER_CHECKLIST "Enable checklist filter" Yes) -option(FLB_FILTER_EXPECT "Enable expect filter" Yes) -option(FLB_FILTER_GREP "Enable grep filter" Yes) -option(FLB_FILTER_MODIFY "Enable modify filter" Yes) -option(FLB_FILTER_STDOUT "Enable stdout filter" Yes) -option(FLB_FILTER_SYSINFO "Enable sysinfo filter" Yes) -option(FLB_FILTER_PARSER "Enable parser filter" Yes) -option(FLB_FILTER_KUBERNETES "Enable kubernetes filter" Yes) -option(FLB_FILTER_REWRITE_TAG "Enable tag rewrite filter" Yes) -option(FLB_FILTER_THROTTLE "Enable throttle filter" Yes) -option(FLB_FILTER_THROTTLE_SIZE "Enable throttle size filter" No) -option(FLB_FILTER_TYPE_CONVERTER "Enable type converter filter" Yes) -option(FLB_FILTER_MULTILINE "Enable multiline filter" Yes) -option(FLB_FILTER_NEST "Enable nest filter" Yes) -option(FLB_FILTER_LOG_TO_METRICS "Enable log-derived metrics filter" Yes) -option(FLB_FILTER_LUA "Enable Lua scripting filter" Yes) -option(FLB_FILTER_LUA_USE_MPACK "Enable mpack on the lua filter" No) -option(FLB_FILTER_RECORD_MODIFIER "Enable record_modifier filter" Yes) -option(FLB_FILTER_TENSORFLOW "Enable tensorflow filter" No) -option(FLB_FILTER_GEOIP2 "Enable geoip2 filter" Yes) -option(FLB_FILTER_NIGHTFALL "Enable Nightfall filter" Yes) -option(FLB_FILTER_WASM "Enable WASM filter" Yes) -option(FLB_PROCESSOR_LABELS "Enable metrics label manipulation processor" Yes) -option(FLB_PROCESSOR_ATTRIBUTES "Enable atributes manipulation processor" Yes) +# List of plugins available and defaults for each option +include(cmake/plugins_options.cmake) if(DEFINED FLB_NIGHTLY_BUILD AND NOT "${FLB_NIGHTLY_BUILD}" STREQUAL "") FLB_DEFINITION_VAL(FLB_NIGHTLY_BUILD ${FLB_NIGHTLY_BUILD}) @@ -439,10 +315,10 @@ endif() # Enable Debug symbols if specified if(MSVC) set(CMAKE_BUILD_TYPE None) # Avoid flag conflicts (See CMakeList.txt:L18) -elseif(FLB_DEBUG) - set(CMAKE_BUILD_TYPE "Debug") elseif(FLB_RELEASE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") +elseif(FLB_DEBUG) + set(CMAKE_BUILD_TYPE "Debug") endif() if(FLB_IPO STREQUAL "On" OR (FLB_IPO STREQUAL "ReleaseOnly" AND FLB_RELEASE)) @@ -563,6 +439,15 @@ add_subdirectory(${FLB_PATH_LIB_CMETRICS} EXCLUDE_FROM_ALL) # CTraces add_subdirectory(${FLB_PATH_LIB_CTRACES} EXCLUDE_FROM_ALL) +# Nghttp2 options +FLB_OPTION(ENABLE_LIB_ONLY ON) +FLB_OPTION(ENABLE_STATIC_LIB ON) +FLB_OPTION(ENABLE_SHARED_LIB OFF) + +FLB_DEFINITION(NGHTTP2_STATICLIB) + +add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL) + # C-Ares (DNS library) FLB_OPTION(CARES_STATIC ON) FLB_OPTION(CARES_SHARED OFF) @@ -621,6 +506,12 @@ if(FLB_TLS) if(OPENSSL_FOUND) FLB_DEFINITION(FLB_HAVE_OPENSSL) endif() + + if (FLB_SYSTEM_WINDOWS AND NOT(OPENSSL_FOUND)) + # win32 builds w/o openssl will fail later so we might as well catch it + # early instead. + MESSAGE(FATAL_ERROR "OpenSSL required on Windows, see DEVELOPER_GUIDE.md") + endif() endif() # Metrics @@ -919,6 +810,16 @@ else() set(AUTOCONF_HOST_OPT "--host=${GNU_HOST}") endif() +if(CMAKE_GENERATOR MATCHES "Ninja") + if(FLB_SYSTEM_WINDOWS) + MESSAGE(FATAL_ERROR "Building with Ninja is not supported on Windows") + else() + set(EXTERNAL_BUILD_TOOL "make") + endif() +else() + set(EXTERNAL_BUILD_TOOL "$(MAKE)") +endif() + # Memory Allocator # ================ if(FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux") @@ -934,15 +835,17 @@ if(FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux") message(STATUS "jemalloc configure: ${FLB_JEMALLOC_OPTIONS_LIST}") # Link to Jemalloc as an external dependency + set(FLB_LIBJEMALLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a") ExternalProject_Add(jemalloc SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0 CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0/configure ${AUTOCONF_HOST_OPT} "${FLB_JEMALLOC_OPTIONS_LIST}" --prefix= CFLAGS=-std=gnu99\ -Wall\ -pipe\ -g3\ -O3\ -funroll-loops - BUILD_COMMAND $(MAKE) + BUILD_COMMAND ${EXTERNAL_BUILD_TOOL} + BUILD_BYPRODUCTS ${FLB_LIBJEMALLOC_PATH} INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/ - INSTALL_COMMAND $(MAKE) install_lib_static install_include) + INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} install_lib_static install_include) add_library(libjemalloc STATIC IMPORTED GLOBAL) - set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a") + set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION ${FLB_LIBJEMALLOC_PATH}) add_dependencies(libjemalloc jemalloc) include_directories("${CMAKE_BINARY_DIR}/include/") else() @@ -960,14 +863,16 @@ if(FLB_BACKTRACE) else() set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}") endif() + set(FLB_LIBBACKTRACE_PATH "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a") ExternalProject_Add(backtrace SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/ CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/configure ${AUTOCONF_HOST_OPT} --prefix= --enable-shared=no --enable-static=yes - BUILD_COMMAND $(MAKE) - INSTALL_COMMAND $(MAKE) DESTDIR= install + BUILD_COMMAND ${EXTERNAL_BUILD_TOOL} + BUILD_BYPRODUCTS ${FLB_LIBBACKTRACE_PATH} + INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} DESTDIR= install ) add_library(libbacktrace STATIC IMPORTED GLOBAL) - set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a") + set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION ${FLB_LIBBACKTRACE_PATH}) add_dependencies(libbacktrace backtrace) include_directories("${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/include/") endif() diff --git a/CODEOWNERS b/CODEOWNERS index 98504a4f78c..2b5b404a281 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -16,7 +16,7 @@ /CMakeLists.txt @fujimotos @niedbalski @patrick-stephens @celalettin1286 /cmake/ @fujimotos @niedbalski @patrick-stephens @celalettin1286 -# CI +# CI # ------------------------- /.github/ @niedbalski @patrick-stephens @celalettin1286 /appveyor.yml @niedbalski @patrick-stephens @celalettin1286 @@ -65,7 +65,7 @@ /plugins/out_pgsql @sxd /plugins/out_stackdriver @braydonk @igorpeshansky @qingling128 -# AWS Plugins +# AWS Plugins /plugins/out_s3 @pettitwesley /plugins/out_cloudwatch_logs @pettitwesley /plugins/out_kinesis_firehose @pettitwesley @@ -86,6 +86,7 @@ # Google test code # -------------- /tests/runtime/out_stackdriver.c @braydonk @igorpeshansky @qingling128 +/tests/runtime/data/stackdriver @braydonk @igorpeshansky @qingling128 # Devcontainer /.devcontainer @patrick-stephens @niedbalski @edsiper diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f937e93ac7..fb22c665579 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -188,7 +188,7 @@ When you commit your local changes in your repository (before to push to Github) ``` /* Fluent Bit * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors + * Copyright (C) 2015-2024 The Fluent Bit Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index f58cd0136d3..734471ea61e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -27,6 +27,7 @@ changes to Fluent Bit. - [Output](#output) - [Config Maps](#config-maps) - [Testing](#testing) + - [Building and Testing on Windows](#building-and-testing-on-windows) - [Valgrind](#valgrind) - [Need more help?](#need-more-help) @@ -593,6 +594,8 @@ Note that Fluent Bit uses Cmake 3 and on some systems you may need to invoke it To set up and build your environment, please refer to the packaging containers for a dependency list: . +See [Building and Testing on Windows](#building-and-testing-on-windows) for Windows instructions. + A simple container-based script [`run_code_analysis.sh`](./run_code_analysis.sh) is provided to run unit tests using . ```shell @@ -646,6 +649,88 @@ Test sds_printf... [ OK ] SUCCESS: All unit tests have passed. ``` +### Building and testing on Windows + +Fluent Bit is built with MSVC and CMake on Windows. + +Windows builds of Fluent Bit override the set of enabled plugins. See +[`cmake/windows-setup.cmake`](./cmake/windows-setup.cmake) for the override +list and other Windows-specific build rules. + +#### Using a Github Action + +For developers without convenient access to a Windows build machine the easiest +way to build a branch of Fluent Bit on Windows is to fork the `fluent-bit` +repository and enable workflow runs. The +[`.github/workflows/pr-windows-build.yaml`](.github/workflows/pr-windows-build.yaml) +workflow can then be invoked in your branch via the Actions tab on your forked +repository to build your patch. + +This workflow will not run automatically on pull requests opened against the +`fluent/fluent-bit` repository so it's best you run it yourself in your fork. + +The resulting binaries are uploaded as a Github workflow artifact which can be +found on the workflow run page. + +At time of writing this workflow *does not run any of the Fluent Bit test suite* +- so it only checks that the target branch can be compiled. To run tests, a local +build will be required. + +#### Using Docker for Windows + +For Windows users with Hyper-V capable machines the simplest way to build a +branch of Fluent Bit is to use +[Docker Desktop for Windows](https://docs.docker.com/desktop/install/windows-install/) +to build +[`./dockerfiles/Dockerfile.windows`](./dockerfiles/Dockerfile.windows). + +This method does not require manual installation of any build dependencies. + +The Dockerfile build does *not* run any Fluent Bit test suites. To run tests you +will need to use a different method. + +Most OS virtualisation tools and virtual machine hosting services do not enable +nested virtualisation, so it is generally not possible to use this method on +virtualized windows machines. + +#### Locally on a Windows machine + +Local compilation on a Windows machine takes the longest to set up, but is the +most convenient for iterating development work. + +At time of writing this is the only way to run the Fluent Bit tests on Windows. + +In lieu of full Windows dev environment setup instructions, see these CI automation +resources for how to set up a Windows build of fluent-bit: + +* [`dockerfiles/Dockerfile.windows`](./dockerfiles/Dockerfile.windows) - only build-able using Docker for Windows +* [`appveyor.yml`](./appveyor.yml) +* [`.github/workflows/call-build-windows.yaml`](.github/workflows/call-build-windows.yaml) - github automation that runs the build on a Windows worker. + +The dependencies must be present: + +* Microsoft Visual Studio C/C++ toolchain. The CI automation uses MSVC 2019 at time of writing. MSVC Community Edition works fine. +* [CMake](https://cmake.org/) 3.x on the `PATH` +* A build of [OpenSSL](https://www.openssl.org/) as static libraries, pointed to by the `-DOPENSSL_ROOT_DIR` CMake variable. The CI automation uses [Chocolatey](https://chocolatey.org/) to `choco install -y openssl`. +* `flex.exe` and `bison.exe` must be present on the `PATH`. The CI automation uses https://github.com/lexxmark/winflexbison. + +Assuming that `cmake` is on the `PATH`, Visual Studio is installed, +WinFlexBison is present at `%SYSTEMDRIVE%\WinFlexBison` and OpenSSL is present +at `%PROGRAMFILES%\OpenSSL-Win64`, the following Powershell commands should +produce a basic Fluent Bit build: + +```powershell +$env:PATH += "$env:SystemDrive\WinFlexBison" +mkdir build +cd build +cmake -DOPENSSL_ROOT_DIR="$env:ProgramFiles\OpenSSL-Win64" -S .. -B . +cmake --build . +``` + +The build output will be `bin\Debug\fluent-bit.exe`. + +If in doubt, check the CI and build automation files referenced above for specifics. + ### Valgrind [Valgrind](https://valgrind.org/) is a tool that will help you detect and diagnose memory issues in your code. It will check for memory leaks and invalid memory accesses. diff --git a/README.md b/README.md index 030e7c7c535..a6ba08be9e8 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ |-------------------|--------------------| | Unit Tests (master) | [![CI/Unit Tests](https://github.com/fluent/fluent-bit/actions/workflows/unit-tests.yaml/badge.svg?branch=master)](https://github.com/fluent/fluent-bit/actions/workflows/unit-tests.yaml) | | Integration Tests (master) | [![CI/Integration Tests](https://github.com/fluent/fluent-bit/actions/workflows/master-integration-test.yaml/badge.svg)](https://github.com/fluent/fluent-bit/actions/workflows/master-integration-test.yaml)| +| Arm builds | Arm CI sponsored by Actuated | | Latest release | [![CI/Build](https://github.com/fluent/fluent-bit/actions/workflows/staging-release.yaml/badge.svg)](https://github.com/fluent/fluent-bit/actions/workflows/staging-release.yaml)| ## About diff --git a/SECURITY.md b/SECURITY.md index a846c05407e..19925725e17 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,8 +5,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.x | :white_check_mark: | -| < 2.0 | :x: | +| 3.x | :white_check_mark: | +| < 3.0 | :x: | ## Reporting a Vulnerability diff --git a/appveyor.yml b/appveyor.yml index b13947a70dd..bdf29ece94a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,15 @@ version: v1-winbuild-{build} +# +# If you change dependencies etc here, please also check and update +# the other Windows build resources: +# +# - DEVELOPER_GUIDE.md "Windows" section +# - appveyor.yml +# - .github/workflows/call-build-windows.yaml +# - dockerfiles/Dockerfile.windows +# + image: Visual Studio 2019 platform: diff --git a/cmake/headers.cmake b/cmake/headers.cmake old mode 100644 new mode 100755 index 3e814066c40..45a1394ca7f --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -46,6 +46,10 @@ include_directories( ${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/ ${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/monkey/ + + ${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2 + ${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2/lib/includes/ + ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/ ) if(FLB_IN_KAFKA OR FLB_OUT_KAFKA) diff --git a/cmake/libraries.cmake b/cmake/libraries.cmake index 832d2da6e4a..aaa1e293db3 100644 --- a/cmake/libraries.cmake +++ b/cmake/libraries.cmake @@ -7,20 +7,20 @@ set(FLB_PATH_LIB_CTRACES "lib/ctraces") set(FLB_PATH_LIB_CO "lib/flb_libco") set(FLB_PATH_LIB_RBTREE "lib/rbtree") set(FLB_PATH_LIB_MSGPACK "lib/msgpack-c") +set(FLB_PATH_LIB_NGHTTP2 "lib/nghttp2") set(FLB_PATH_LIB_AVRO "lib/avro") set(FLB_PATH_LIB_CHUNKIO "lib/chunkio") set(FLB_PATH_LIB_LUAJIT "lib/luajit-3065c9") set(FLB_PATH_LIB_MONKEY "lib/monkey") set(FLB_PATH_LIB_JSMN "lib/jsmn") -set(FLB_PATH_LIB_MBEDTLS "lib/mbedtls-2.28.0") -set(FLB_PATH_LIB_SQLITE "lib/sqlite-amalgamation-3390300") +set(FLB_PATH_LIB_SQLITE "lib/sqlite-amalgamation-3450200") set(FLB_PATH_LIB_JANSSON "lib/jansson-e23f558") set(FLB_PATH_LIB_ONIGMO "lib/onigmo") -set(FLB_PATH_LIB_MPACK "lib/mpack-amalgamation-1.1") +set(FLB_PATH_LIB_MPACK "lib/mpack-amalgamation-1.1.1") set(FLB_PATH_LIB_MINIZ "lib/miniz") set(FLB_PATH_LIB_TUTF8E "lib/tutf8e") -set(FLB_PATH_LIB_CARES "lib/c-ares-1.19.1") +set(FLB_PATH_LIB_CARES "lib/c-ares-1.31.0") set(FLB_PATH_LIB_SNAPPY "lib/snappy-fef67ac") -set(FLB_PATH_LIB_RDKAFKA "lib/librdkafka-2.1.0") +set(FLB_PATH_LIB_RDKAFKA "lib/librdkafka-2.3.0") set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb") -set(FLB_PATH_LIB_WASM_MICRO_RUNTIME "lib/wasm-micro-runtime-WAMR-1.2.2") +set(FLB_PATH_LIB_WASM_MICRO_RUNTIME "lib/wasm-micro-runtime-WAMR-1.3.0") diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake new file mode 100644 index 00000000000..5d08e505fff --- /dev/null +++ b/cmake/plugins_options.cmake @@ -0,0 +1,144 @@ +macro(DEFINE_OPTION option_name description default_value) + set(temp_value ${default_value}) + if(FLB_MINIMAL) + set(temp_value OFF) + endif() + option(${option_name} "${description}" ${temp_value}) +endmacro() + +# Add the FLB_MINIMAL option +option(FLB_MINIMAL "Enable minimal build configuration" No) + +# Inputs (sources, data collectors) +# ================================= +DEFINE_OPTION(FLB_IN_CALYPTIA_FLEET "Enable Calyptia Fleet input plugin" ON) +DEFINE_OPTION(FLB_IN_COLLECTD "Enable Collectd input plugin" ON) +DEFINE_OPTION(FLB_IN_CPU "Enable CPU input plugin" ON) +DEFINE_OPTION(FLB_IN_DISK "Enable Disk input plugin" ON) +DEFINE_OPTION(FLB_IN_DOCKER "Enable Docker input plugin" ON) +DEFINE_OPTION(FLB_IN_DOCKER_EVENTS "Enable Docker events input plugin" ON) +DEFINE_OPTION(FLB_IN_DUMMY "Enable Dummy input plugin" ON) +DEFINE_OPTION(FLB_IN_ELASTICSEARCH "Enable Elasticsearch (Bulk API) input plugin" ON) +DEFINE_OPTION(FLB_IN_EMITTER "Enable emitter input plugin" ON) +DEFINE_OPTION(FLB_IN_EVENT_TEST "Enable event test plugin" OFF) +DEFINE_OPTION(FLB_IN_EVENT_TYPE "Enable event type plugin" ON) +DEFINE_OPTION(FLB_IN_EXEC "Enable Exec input plugin" ON) +DEFINE_OPTION(FLB_IN_EXEC_WASI "Enable Exec WASI input plugin" ON) +DEFINE_OPTION(FLB_IN_FLUENTBIT_METRICS "Enable Fluent Bit metrics plugin" ON) +DEFINE_OPTION(FLB_IN_FORWARD "Enable Forward input plugin" ON) +DEFINE_OPTION(FLB_IN_HEAD "Enable Head input plugin" ON) +DEFINE_OPTION(FLB_IN_HEALTH "Enable Health input plugin" ON) +DEFINE_OPTION(FLB_IN_HTTP "Enable HTTP input plugin" ON) +DEFINE_OPTION(FLB_IN_KAFKA "Enable Kafka input plugin" ON) +DEFINE_OPTION(FLB_IN_KMSG "Enable Kernel log input plugin" ON) +DEFINE_OPTION(FLB_IN_KUBERNETES_EVENTS "Enable Kubernetes Events plugin" ON) +DEFINE_OPTION(FLB_IN_LIB "Enable library mode input plugin" ON) +DEFINE_OPTION(FLB_IN_MEM "Enable Memory input plugin" ON) +DEFINE_OPTION(FLB_IN_MQTT "Enable MQTT Broker input plugin" ON) +DEFINE_OPTION(FLB_IN_NETIF "Enable NetworkIF input plugin" ON) +DEFINE_OPTION(FLB_IN_NGINX_EXPORTER_METRICS "Enable Nginx Metrics input plugin" ON) +DEFINE_OPTION(FLB_IN_NODE_EXPORTER_METRICS "Enable node exporter metrics input plugin" ON) +DEFINE_OPTION(FLB_IN_OPENTELEMETRY "Enable OpenTelemetry input plugin" ON) +DEFINE_OPTION(FLB_IN_PODMAN_METRICS "Enable Podman Metrics input plugin" ON) +DEFINE_OPTION(FLB_IN_PROCESS_EXPORTER_METRICS "Enable process exporter metrics input plugin" ON) +DEFINE_OPTION(FLB_IN_PROC "Enable Process input plugin" ON) +DEFINE_OPTION(FLB_IN_PROMETHEUS_REMOTE_WRITE "Enable prometheus remote write input plugin" ON) +DEFINE_OPTION(FLB_IN_PROMETHEUS_SCRAPE "Enable Prometheus Scrape input plugin" ON) +DEFINE_OPTION(FLB_IN_RANDOM "Enable random input plugin" ON) +DEFINE_OPTION(FLB_IN_SERIAL "Enable Serial input plugin" ON) +DEFINE_OPTION(FLB_IN_SPLUNK "Enable Splunk HTTP HEC input plugin" ON) +DEFINE_OPTION(FLB_IN_STDIN "Enable Standard input plugin" ON) +DEFINE_OPTION(FLB_IN_STORAGE_BACKLOG "Enable storage backlog input plugin" ON) +DEFINE_OPTION(FLB_IN_SYSLOG "Enable Syslog input plugin" ON) +DEFINE_OPTION(FLB_IN_SYSTEMD "Enable Systemd input plugin" ON) +DEFINE_OPTION(FLB_IN_TAIL "Enable Tail input plugin" ON) +DEFINE_OPTION(FLB_IN_TCP "Enable TCP input plugin" ON) +DEFINE_OPTION(FLB_IN_THERMAL "Enable Thermal plugin" ON) +DEFINE_OPTION(FLB_IN_UDP "Enable UDP input plugin" ON) +DEFINE_OPTION(FLB_IN_UNIX_SOCKET "Enable Unix socket input plugin" OFF) +DEFINE_OPTION(FLB_IN_WINLOG "Enable Windows Log input plugin" OFF) +DEFINE_OPTION(FLB_IN_WINDOWS_EXPORTER_METRICS "Enable windows exporter metrics input plugin" ON) +DEFINE_OPTION(FLB_IN_WINEVTLOG "Enable Windows EvtLog input plugin" OFF) +DEFINE_OPTION(FLB_IN_WINSTAT "Enable Windows Stat input plugin" OFF) + +# Processors +# ========== +DEFINE_OPTION(FLB_PROCESSOR_CONTENT_MODIFIER "Enable content modifier processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_LABELS "Enable metrics label manipulation processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_METRICS_SELECTOR "Enable metrics selector processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_SQL "Enable SQL processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE "Enable OpenTelemetry envelope processor" ON) + +# Filters +# ======= +DEFINE_OPTION(FLB_FILTER_ALTER_SIZE "Enable alter_size filter" ON) +DEFINE_OPTION(FLB_FILTER_AWS "Enable aws filter" ON) +DEFINE_OPTION(FLB_FILTER_CHECKLIST "Enable checklist filter" ON) +DEFINE_OPTION(FLB_FILTER_ECS "Enable AWS ECS filter" ON) +DEFINE_OPTION(FLB_FILTER_EXPECT "Enable expect filter" ON) +DEFINE_OPTION(FLB_FILTER_GEOIP2 "Enable geoip2 filter" ON) +DEFINE_OPTION(FLB_FILTER_GREP "Enable grep filter" ON) +DEFINE_OPTION(FLB_FILTER_KUBERNETES "Enable kubernetes filter" ON) +DEFINE_OPTION(FLB_FILTER_LOG_TO_METRICS "Enable log-derived metrics filter" ON) +DEFINE_OPTION(FLB_FILTER_LUA "Enable Lua scripting filter" ON) +DEFINE_OPTION(FLB_FILTER_LUA_USE_MPACK "Enable mpack on the lua filter" OFF) +DEFINE_OPTION(FLB_FILTER_MODIFY "Enable modify filter" ON) +DEFINE_OPTION(FLB_FILTER_MULTILINE "Enable multiline filter" ON) +DEFINE_OPTION(FLB_FILTER_NEST "Enable nest filter" ON) +DEFINE_OPTION(FLB_FILTER_NIGHTFALL "Enable Nightfall filter" ON) +DEFINE_OPTION(FLB_FILTER_PARSER "Enable parser filter" ON) +DEFINE_OPTION(FLB_FILTER_REWRITE_TAG "Enable tag rewrite filter" ON) +DEFINE_OPTION(FLB_FILTER_STDOUT "Enable stdout filter" ON) +DEFINE_OPTION(FLB_FILTER_SYSINFO "Enable sysinfo filter" ON) +DEFINE_OPTION(FLB_FILTER_THROTTLE "Enable throttle filter" ON) +DEFINE_OPTION(FLB_FILTER_THROTTLE_SIZE "Enable throttle size filter" OFF) +DEFINE_OPTION(FLB_FILTER_TYPE_CONVERTER "Enable type converter filter" ON) +DEFINE_OPTION(FLB_FILTER_WASM "Enable WASM filter" ON) + +# Outputs (destinations) +# ====================== +DEFINE_OPTION(FLB_OUT_AZURE "Enable Azure output plugin" ON) +DEFINE_OPTION(FLB_OUT_AZURE_BLOB "Enable Azure output plugin" ON) +DEFINE_OPTION(FLB_OUT_AZURE_KUSTO "Enable Azure Kusto output plugin" ON) +DEFINE_OPTION(FLB_OUT_AZURE_LOGS_INGESTION "Enable Azure Logs Ingestion output plugin" ON) +DEFINE_OPTION(FLB_OUT_BIGQUERY "Enable BigQuery output plugin" ON) +DEFINE_OPTION(FLB_OUT_CALYPTIA "Enable Calyptia monitoring plugin" ON) +DEFINE_OPTION(FLB_OUT_CHRONICLE "Enable Google Chronicle output plugin" ON) +DEFINE_OPTION(FLB_OUT_CLOUDWATCH_LOGS "Enable AWS CloudWatch output plugin" ON) +DEFINE_OPTION(FLB_OUT_COUNTER "Enable Counter output plugin" ON) +DEFINE_OPTION(FLB_OUT_DATADOG "Enable DataDog output plugin" ON) +DEFINE_OPTION(FLB_OUT_ES "Enable Elasticsearch output plugin" ON) +DEFINE_OPTION(FLB_OUT_EXIT "Enable Exit output plugin" ON) +DEFINE_OPTION(FLB_OUT_FILE "Enable file output plugin" ON) +DEFINE_OPTION(FLB_OUT_FLOWCOUNTER "Enable flowcount output plugin" ON) +DEFINE_OPTION(FLB_OUT_FORWARD "Enable Forward output plugin" ON) +DEFINE_OPTION(FLB_OUT_GELF "Enable GELF output plugin" ON) +DEFINE_OPTION(FLB_OUT_HTTP "Enable HTTP output plugin" ON) +DEFINE_OPTION(FLB_OUT_INFLUXDB "Enable InfluxDB output plugin" ON) +DEFINE_OPTION(FLB_OUT_KAFKA "Enable Kafka output plugin" ON) +DEFINE_OPTION(FLB_OUT_KAFKA_REST "Enable Kafka Rest output plugin" ON) +DEFINE_OPTION(FLB_OUT_KINESIS_FIREHOSE "Enable AWS Firehose output plugin" ON) +DEFINE_OPTION(FLB_OUT_KINESIS_STREAMS "Enable AWS Kinesis output plugin" ON) +DEFINE_OPTION(FLB_OUT_LOGDNA "Enable LogDNA output plugin" ON) +DEFINE_OPTION(FLB_OUT_LOKI "Enable Loki output plugin" ON) +DEFINE_OPTION(FLB_OUT_NULL "Enable dev null output plugin" ON) +DEFINE_OPTION(FLB_OUT_OPENSEARCH "Enable OpenSearch output plugin" ON) +DEFINE_OPTION(FLB_OUT_OPENTELEMETRY "Enable OpenTelemetry plugin" ON) +DEFINE_OPTION(FLB_OUT_ORACLE_LOG_ANALYTICS "Enable Oracle Cloud Infrastructure Logging analytics plugin" ON) +DEFINE_OPTION(FLB_OUT_PGSQL "Enable PostgreSQL output plugin" OFF) +DEFINE_OPTION(FLB_OUT_PLOT "Enable Plot output plugin" ON) +DEFINE_OPTION(FLB_OUT_PROMETHEUS_EXPORTER "Enable Prometheus exporter plugin" ON) +DEFINE_OPTION(FLB_OUT_PROMETHEUS_REMOTE_WRITE "Enable Prometheus remote write plugin" ON) +DEFINE_OPTION(FLB_OUT_RETRY "Enable Retry test output plugin" OFF) +DEFINE_OPTION(FLB_OUT_S3 "Enable AWS S3 output plugin" ON) +DEFINE_OPTION(FLB_OUT_SKYWALKING "Enable Apache SkyWalking output plugin" ON) +DEFINE_OPTION(FLB_OUT_SLACK "Enable Slack output plugin" ON) +DEFINE_OPTION(FLB_OUT_SPLUNK "Enable Splunk output plugin" ON) +DEFINE_OPTION(FLB_OUT_STACKDRIVER "Enable Stackdriver output plugin" ON) +DEFINE_OPTION(FLB_OUT_STDOUT "Enable STDOUT output plugin" ON) +DEFINE_OPTION(FLB_OUT_SYSLOG "Enable Syslog output plugin" ON) +DEFINE_OPTION(FLB_OUT_TD "Enable Treasure Data output plugin" ON) +DEFINE_OPTION(FLB_OUT_TCP "Enable TCP output plugin" ON) +DEFINE_OPTION(FLB_OUT_UDP "Enable UDP output plugin" ON) +DEFINE_OPTION(FLB_OUT_VIVO_EXPORTER "Enable Vivo exporter output plugin" ON) +DEFINE_OPTION(FLB_OUT_WEBSOCKET "Enable Websocket output plugin" ON) diff --git a/cmake/windows-setup.cmake b/cmake/windows-setup.cmake index fa67cb0d872..60b67bc1e99 100644 --- a/cmake/windows-setup.cmake +++ b/cmake/windows-setup.cmake @@ -30,7 +30,7 @@ if(FLB_WINDOWS_DEFAULTS) # ============= set(FLB_IN_CPU No) set(FLB_IN_DISK No) - set(FLB_IN_EXEC No) + set(FLB_IN_EXEC Yes) set(FLB_IN_EXEC_WASI No) set(FLB_IN_FORWARD Yes) set(FLB_IN_HEALTH No) @@ -61,6 +61,7 @@ if(FLB_WINDOWS_DEFAULTS) set(FLB_IN_PODMAN_METRICS No) set(FLB_IN_ELASTICSEARCH Yes) set(FLB_IN_SPLUNK Yes) + set(FLB_IN_PROMETHEUS_REMOTE_WRITE Yes) # OUTPUT plugins # ============== @@ -119,9 +120,10 @@ find_package(FLEX) find_package(BISON) if (NOT (${FLEX_FOUND} AND ${BISON_FOUND})) - message(STATUS "flex and bison not found. Disable stream_processor building.") - set(FLB_STREAM_PROCESSOR No) - set(FLB_RECORD_ACCESSOR No) + # The build will fail later if flex and bison are missing, so there's no + # point attempting to continue. There's no test cover for windows builds + # without FLB_PARSER anyway. + message(FATAL_ERROR "flex and bison not found, see DEVELOPER_GUIDE.md") endif() if (MSVC) diff --git a/conf/parsers.conf b/conf/parsers.conf index 71706545d0a..ec1b11bf910 100644 --- a/conf/parsers.conf +++ b/conf/parsers.conf @@ -38,6 +38,10 @@ Time_Key time Time_Format %d/%b/%Y:%H:%M:%S %z +[PARSER] + Name logfmt + Format logfmt + [PARSER] Name docker Format json @@ -124,3 +128,11 @@ Name kube-custom Format regex Regex (?[^.]+)?\.?(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$ + +[PARSER] + # Examples: TCP: https://rubular.com/r/Q8YY6fHqlqwGI0 UDP: https://rubular.com/r/B0ID69H9FvN0tp + Name kmsg-netfilter-log + Format regex + Regex ^\<(?[0-9]{1,5})\>1 (?