diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 95460910e22d..2ac4a8911a11 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -22,6 +22,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Auditbeat Mandatory Testing" key: "auditbeat-mandatory-tests" @@ -34,7 +38,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -53,7 +57,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -71,7 +75,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -91,7 +95,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -112,7 +116,7 @@ steps: GOX_FLAGS: "-arch amd64" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -128,14 +132,14 @@ steps: steps: - label: ":linux: Auditbeat Ubuntu Integration Tests" key: "auditbeat-extended-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat mage build integTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -149,14 +153,14 @@ steps: - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests" key: "auditbeat-extended-arm64-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat mage build integTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -177,7 +181,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -198,7 +202,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -218,7 +222,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -241,7 +245,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -262,7 +266,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -283,7 +287,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml deleted file mode 100644 index 4707707e07cf..000000000000 --- a/.buildkite/buildkite.yml +++ /dev/null @@ -1,44 +0,0 @@ -projects: - - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "heartbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/heartbeat" - - "x-pack/libbeat" - - "x-pack/metricbeat" - - "x-pack/osquerybeat" - - "x-pack/packetbeat" - - "x-pack/winlogbeat" - -## Changeset macros that are defined here and used in each specific 3.0 pipeline. -changeset: - ci: - - "^Jenkinsfile" - - "^\\.ci/scripts/.*" - oss: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - xpack: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - - "^x-pack/libbeat/.*" - -disabled: - when: - labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. - - skip-ci - draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index ebb25ca39664..87f1925a3722 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -22,17 +22,21 @@ env: K8S_VERSION: "v1.29.0" ASDF_KIND_VERSION: "0.20.0" + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: - - label: ":ubuntu: Ubuntu Unit Tests" + - label: ":ubuntu: Filebeat Unit Tests" command: | cd filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -42,15 +46,15 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Unit Tests" + context: "filebeat: Linux x86_64 Unit Tests" - - label: ":ubuntu: Ubuntu Go Integration Tests" + - label: ":ubuntu: Filebeat Go Integration Tests" command: | cd filebeat mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -60,15 +64,15 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Go Integration Tests" + context: "filebeat: Filebeat Go Integration Tests" - - label: ":ubuntu: Ubuntu Python Integration Tests" + - label: ":ubuntu: Filebeat Python Integration Tests" command: | cd filebeat mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -80,14 +84,14 @@ steps: - github_commit_status: context: "filebeat: Python Integration Tests" - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: Filebeat Windows 2016 Unit Tests" key: "windows-2016-unit-tests" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -101,14 +105,14 @@ steps: - github_commit_status: context: "filebeat: Windows 2016 Unit Tests" - - label: ":windows: Windows 2022 Unit Tests" + - label: ":windows: Filebeat Windows 2022 Unit Tests" key: "windows-2022-unit-tests" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -124,8 +128,10 @@ steps: - group: "Filebeat Extended Tests" key: "filebeat-extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + steps: - - label: ":mac: MacOS x64_64 Unit Tests" + - label: ":mac: Filebeat macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -135,7 +141,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -144,9 +150,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended MacOS Unit Tests" + context: "filebeat: macOS x86_64 Unit Tests" - - label: ":mac: MacOS arm64 Unit Tests" + - label: ":mac: Filebeat macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -156,7 +162,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -165,9 +171,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended MacOS ARM Unit Tests" + context: "filebeat: macOS arm64 Unit Tests" - - label: ":linux: Ubuntu ARM Unit Tests" + - label: ":linux: Filebeat arm64 Unit Tests" key: "extended-arm64-unit-test" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: | @@ -175,7 +181,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -185,20 +191,21 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Ubuntu ARM Unit Tests" + context: "filebeat: Linux arm64 Unit Tests" - - group: "Extended Windows Tests" + - group: "Filebeat Extended Windows Tests" key: "filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: Filebeat Windows 2019 Unit Tests" key: "windows-extended-2019" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -210,16 +217,16 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 2019 Unit Tests" + context: "filebeat: Windows 2019 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" + - label: ":windows: Filebeat Windows 11 Unit Tests" key: "windows-extended-11" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -231,16 +238,16 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 11 Unit Tests" + context: "filebeat: Windows 11 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: Filebeat Windows 10 Unit Tests" key: "windows-extended-10" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -252,7 +259,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 10 Unit Tests" + context: "filebeat: Windows 10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -264,7 +271,7 @@ steps: - group: "Filebeat Packaging" key: "packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: Filebeat Packaging Linux" key: "packaging-linux" command: | cd filebeat @@ -283,9 +290,9 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: - context: "filebeat: Packaging" + context: "filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: Filebeat Packaging arm64" key: "packaging-arm" command: | cd filebeat @@ -303,4 +310,4 @@ steps: PACKAGES: "docker" notify: - github_commit_status: - context: "filebeat: Packaging ARM" + context: "filebeat: Packaging arm64" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 54cfe6f0ac7e..dc27b45f1ac1 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -21,6 +21,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Heartbeat Mandatory Testing" key: "heartbeat-mandatory-tests" @@ -32,7 +36,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -50,7 +54,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -69,7 +73,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -89,7 +93,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -103,12 +107,12 @@ steps: context: "Heartbeat: Win-2022 Unit Tests" - label: ":ubuntu: Heartbeat Go Integration Tests" - command: | + command: | cd heartbeat mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -126,7 +130,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -150,7 +154,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -163,7 +167,6 @@ steps: - group: "Heartbeat Extended Testing MacOS" key: "heartbeat-extended-tests-macos" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - steps: - label: ":mac: Heartbeat MacOS Unit Tests" key: "macos-extended" @@ -174,7 +177,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -194,7 +197,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -212,12 +215,12 @@ steps: steps: - label: ":windows: Heartbeat Win-2019 Unit Tests" key: "heartbeat-win-extended-2019" - command: | + command: | Set-Location -Path heartbeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -237,7 +240,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -257,7 +260,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 1631b7dc0a1e..9a7228c5b2e3 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then source .buildkite/env-scripts/env.sh if [[ -z "${GO_VERSION-""}" ]]; then export GO_VERSION=$(cat "${WORKSPACE}/.go-version") diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index bc77712c330b..fd485279858e 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -11,6 +11,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Mandatory Tests" key: "mandatory-tests" @@ -23,7 +27,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -43,7 +47,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -63,7 +67,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -83,7 +87,7 @@ steps: make crosscompile retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -103,7 +107,7 @@ steps: make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -125,7 +129,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index d0cece3a5900..ed9fb14f3d42 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -25,6 +25,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Metricbeat Mandatory Tests" key: "metricbeat-mandatory-tests" @@ -34,7 +38,7 @@ steps: command: "cd metricbeat && mage build unitTest" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -53,7 +57,7 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet metricbeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" # TODO move this section to base image / pre-command hook echo "~~~ Installing kind" @@ -67,7 +71,7 @@ steps: cd metricbeat && mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -86,7 +90,7 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" # TODO move this section to base image / pre-command hook echo "~~~ Installing kind" @@ -100,7 +104,7 @@ steps: cd metricbeat && mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -117,7 +121,7 @@ steps: command: "make -C metricbeat crosscompile" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -136,7 +140,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -157,7 +161,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -182,7 +186,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -203,7 +207,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -224,7 +228,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -250,7 +254,7 @@ steps: cd metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -270,7 +274,7 @@ steps: cd metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -281,7 +285,6 @@ steps: - github_commit_status: context: "metricbeat: Extended MacOS arm64 Unit Tests" - - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index a0d139d303b6..c9a5c9f06f2a 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -20,6 +20,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "packetbeat Mandatory Tests" key: "packetbeat-mandatory-tests" @@ -30,7 +34,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -48,7 +52,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" @@ -66,7 +70,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -86,7 +90,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -110,7 +114,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -131,7 +135,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -152,7 +156,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -179,7 +183,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -192,7 +196,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -200,7 +204,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -217,7 +221,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -229,7 +233,6 @@ steps: - github_commit_status: context: "packetbeat: Extended Ubuntu ARM Unit Tests" - - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 8608be3a936a..55affae41289 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,13 +5,13 @@ "pipelineSlug": "beats", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test .*$", - "always_trigger_comment_regex": "^/test .*$", - "skip_ci_labels": [ ], + "trigger_comment_regex": "^/(?:test)", + "always_trigger_comment_regex": "^/(?:test)", + "skip_ci_labels": ["skip-ci"], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": [ ] @@ -21,7 +21,7 @@ "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, diff --git a/.buildkite/pytest.ini b/.buildkite/pytest.ini deleted file mode 100644 index 3eff7473d9f0..000000000000 --- a/.buildkite/pytest.ini +++ /dev/null @@ -1,11 +0,0 @@ -[pytest] -junit_family=xunit1 - -addopts = --strict-markers -markers = - load: Load tests - tag(name): Tag tests with Go-like semantics - -# Ignore setup and teardown for the timeout -#timeout_func_only = True - diff --git a/.buildkite/scripts/changesets.psm1 b/.buildkite/scripts/changesets.psm1 index 15fabd3eba5d..10e4d31a8b61 100644 --- a/.buildkite/scripts/changesets.psm1 +++ b/.buildkite/scripts/changesets.psm1 @@ -56,17 +56,9 @@ function DefineModuleFromTheChangeSet($projectPath) { } } - # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved - if(!$changedModules) { - if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') { - $Env:MODULE = "aws" - } - else { - $Env:MODULE = "kubernetes" - } - } - else { - # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export - $Env:MODULE = $changedModules + if ($changedModules) { + $env:MODULE = $changedModules + Write-Output "~~~ Set env var MODULE to [$env:MODULE]" + Write-Output "~~~ Resuming commands" } } diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh index 3b7ed5d7ec00..5c6fd6c7b0d8 100644 --- a/.buildkite/scripts/changesets.sh +++ b/.buildkite/scripts/changesets.sh @@ -68,13 +68,10 @@ defineModuleFromTheChangeSet() { fi done - if [[ -z "$changed_modules" ]]; then # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - export MODULE="aws" - else - export MODULE="kubernetes" - fi - else - export MODULE="${changed_modules}" # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export + # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993 + if [[ ! -z $changed_modules ]]; then + export MODULE="${changed_modules}" + echo "~~~ Set env var MODULE to [$MODULE]" + echo "~~~ Resuming commands" fi } diff --git a/.buildkite/scripts/run_dynamic_pipeline_tests.sh b/.buildkite/scripts/run_dynamic_pipeline_tests.sh deleted file mode 100755 index 8eb72d7a96b1..000000000000 --- a/.buildkite/scripts/run_dynamic_pipeline_tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed -# this will allow us to fail fast, if e.g. a PR has broken the generator - -set -euo pipefail - -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -pipeline_generator_changeset=( - "^.buildkite/pipeline.py" - "^*/buildkite.yml" -) - -if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then - echo "~~~ Skipping pipeline generator tests" - exit -fi - -echo "~~~ Running pipeline generator tests" - -python3 -mpip install --quiet "pytest" -pushd .buildkite -pytest . -popd diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 6351af6f5a6b..56f8d7257d60 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -44,9 +44,6 @@ exportVars() { if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then exportVars - export RACE_DETECTOR="true" - export TEST_COVERAGE="true" - export DOCKER_PULL="0" export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/setup_cloud_env.sh old mode 100755 new mode 100644 similarity index 92% rename from .buildkite/scripts/cloud_tests.sh rename to .buildkite/scripts/setup_cloud_env.sh index c8f3737e13e3..3c23d71573f1 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/setup_cloud_env.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -euo pipefail - REPO_DIR=$(pwd) teardown() { @@ -31,7 +30,6 @@ tf_cleanup() { trap 'teardown' EXIT # Prepare the cloud resources using Terraform -#startCloudTestEnv "${MODULE_DIR}" echo "~~~ Loading creds" set +o xtrace export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY @@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci" export TF_VAR_REPO="${REPO}" terraform init && terraform apply -auto-approve cd - - -# Run tests -echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" -cd "${BEATS_PROJECT_NAME}" -mage build test diff --git a/.buildkite/test_pipeline.py b/.buildkite/test_pipeline.py deleted file mode 100644 index 75fd949ccc8e..000000000000 --- a/.buildkite/test_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -import os - -import pytest -import pipeline - - -@pytest.fixture -def ubuntu2204_aws_agent(): - return { - "command": "fake-cmd", - "platform": "platform-ingest-beats-ubuntu-2204-aarch64", - "provider": "aws" - } - - -@pytest.fixture() -def fake_simple_group(): - return { - "unitTest": { - "command": "fake-cmd", - "platform": "family/platform-ingest-beats-ubuntu-2204", - }, - "integrationTest": { - "command": "fake-integration", - "platform": "family/platform-ingest-beats-ubuntu-2204", - "env": { - "FOO": "BAR", - }, - }, - } - - -def test_fetch_stage(ubuntu2204_aws_agent): - step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category") - assert step.create_entity() == { - "label": "fake test", - "command": ["cd fake", "fake-cmd"], - "notify": [ - { - "github_commit_status": { - "context": "Fake: test", - } - } - ], - "agents": { - "provider": "aws", - "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64", - "instanceType": "t4g.large", - }, - "artifact_paths": [ - "fake/build/*.xml", - "fake/build/*.json", - ], - } - - -def test_fetch_group(fake_simple_group): - group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing") - assert len(group.steps) == 2 - for step in group.steps: - assert "testing" == step.category - assert "gcp" == step.agent.provider - - assert group.steps[1].env.get("FOO") == "BAR" - - -def test_is_pr(): - os.environ["BUILDKITE_PULL_REQUEST"] = "1234" - assert pipeline.is_pr() is True - os.environ["BUILDKITE_PULL_REQUEST"] = "false" - assert pipeline.is_pr() is False diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index b32391bd2e79..c598224438fe 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -16,6 +16,12 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + # See docker.go. Sets --pull to docker-compose + DOCKER_PULL: 0 + steps: - group: "Winlogbeat Mandatory Tests" key: "winlogbeat-mandatory-tests" @@ -26,7 +32,7 @@ steps: command: "make -C winlogbeat crosscompile" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -45,7 +51,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -66,7 +72,7 @@ steps: key: "mandatory-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -87,7 +93,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -113,7 +119,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -134,7 +140,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 81c1f99ad861..2e13a0d9b959 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -22,6 +22,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/auditbeat Mandatory Tests" key: "x-pack-auditbeat-mandatory-tests" @@ -33,12 +37,12 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/auditbeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/auditbeat mage update build test retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -57,7 +61,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" @@ -76,7 +80,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -97,7 +101,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -122,7 +126,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -143,7 +147,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -164,7 +168,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -190,7 +194,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -209,7 +213,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -230,7 +234,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 322caff662aa..415c3947874a 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -7,21 +7,16 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" - IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" - IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" - IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" - IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/dockerlogbeat Mandatory Tests" key: "xpack-dockerlogbeat-mandatory-tests" @@ -33,7 +28,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -52,14 +47,12 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/dockerlogbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/dockerlogbeat mage goIntegTest - env: - MODULE: $MODULE retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 22b6432fd66c..93b3c2d536ed 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-filebeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -21,18 +21,22 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/filebeat Mandatory Tests" key: "x-pack-filebeat-mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" + - label: ":ubuntu: x-pack/filebeat Unit Tests" key: "x-pack-filebeat-mandatory-linux-unit-test" command: | cd x-pack/filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -42,20 +46,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Ubuntu Unit Tests" + context: "x-pack/filebeat: Unit Tests" - - label: ":go: Go (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/filebeat Go Integration Tests" key: "x-pack-filebeat-mandatory-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" - cd x-pack/filebeat && mage goIntegTest + cd x-pack/filebeat + mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -65,20 +65,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Go (MODULE) Integration Tests" + context: "x-pack/filebeat: Go Integration Tests" - - label: ":python: Python (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/filebeat Python Integration Tests" key: "x-pack-filebeat-mandatory-python-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - cd x-pack/filebeat && mage pythonIntegTest + cd x-pack/filebeat + mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -88,16 +84,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Python (MODULE) Integration Tests" + context: "x-pack/filebeat: Python Integration Tests" - - label: ":windows: Windows 2022 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2022 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -109,16 +105,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2022 Unit Tests" + context: "x-pack/filebeat: Win-2022 Unit Tests" - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -130,39 +126,21 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2016 Unit Tests" - - - label: ":linux: Ubuntu ARM Unit Tests" - key: "x-pack-filebeat-mandatory-linux-arm-unit-test" - command: | - cd x-pack/filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Ubuntu ARM Unit Tests" + context: "x-pack/filebeat: Win-2016 Unit Tests" - group: "x-pack/filebeat Extended Windows Tests" key: "x-pack-filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -174,16 +152,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2019 Unit Tests" + context: "x-pack/filebeat: Win-2019 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: x-pack/filebeat Win 10 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -195,16 +173,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 10 Unit Tests" + context: "x-pack/filebeat: Win 10 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" + - label: ":windows: x-pack/filebeat Win 11 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -216,20 +194,39 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 11 Unit Tests" + context: "x-pack/filebeat: Win 11 Unit Tests" - group: "x-pack/filebeat Extended Tests" key: "x-pack-filebeat-extended-tests" + steps: - - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + key: "x-pack-filebeat-mandatory-linux-arm-unit-test" + command: | + cd x-pack/filebeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Linux arm64 Unit Tests" + + - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat && mage build unitTest + cd x-pack/filebeat + mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -238,18 +235,20 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: MacOS x86_64 Unit Tests" + context: "x-pack/filebeat: macOS x86_64 Unit Tests" - - label: ":mac: MacOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests" + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat && mage build unitTest + cd x-pack/filebeat + mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -258,23 +257,23 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: MacOS arm64 Unit Tests" + context: "x-pack/filebeat: macOS arm64 Unit Tests" - - label: ":linux: Cloud (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - - .buildkite/scripts/cloud_tests.sh + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: @@ -288,26 +287,28 @@ steps: - github_commit_status: context: "x-pack/filebeat: Cloud (MODULE) Tests" - - label: ":linux: Cloud AWS (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test-aws" - skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425" + skip: "Skipping due to elastic/beats#36425" + # https://github.com/elastic/beats/issues/36425 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/cloud_tests.sh + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/filebeat/build/*.xml" @@ -327,7 +328,7 @@ steps: - group: "x-pack/filebeat Packaging" key: "x-pack-filebeat-packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: x-pack/filebeat Packaging Linux" key: "packaging-linux" command: | cd x-pack/filebeat @@ -348,7 +349,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: x-pack/filebeat Packaging arm64" key: "packaging-arm" command: | cd x-pack/filebeat @@ -359,11 +360,11 @@ steps: timeout_in_minutes: 20 agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" notify: - github_commit_status: - context: "x-pack/filebeat: Packaging Linux ARM" + context: "x-pack/filebeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index a20f454b6aa2..3a833f369e24 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -26,6 +26,10 @@ env: ASDF_MAGE_VERSION: 1.15.0 ASDF_NODEJS_VERSION: 18.17.1 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/heartbeat Mandatory Tests" key: "x-pack-heartbeat-mandatory-tests" @@ -41,7 +45,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -64,7 +68,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -84,7 +88,7 @@ steps: mage build test retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -107,7 +111,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -134,7 +138,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -156,7 +160,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -176,7 +180,7 @@ steps: mage build test retry: automatic: - - limit: 3 + - limit: 3 key: "extended-win-2019-unit-tests" agents: provider: "gcp" @@ -206,7 +210,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -227,7 +231,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 6bf456f6d83d..6c26e9614df2 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -17,6 +17,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/libbeat Mandatory Tests" key: "x-pack-libbeat-mandatory-tests" @@ -28,7 +32,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -47,7 +51,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -66,7 +70,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -85,7 +89,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -106,7 +110,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -131,7 +135,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -152,7 +156,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -173,7 +177,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -198,7 +202,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 11ff663e2fd4..4156850f7ae8 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-metricbeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -21,6 +21,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/metricbeat Mandatory Tests" key: "x-pack-metricbeat-mandatory-tests" @@ -32,7 +36,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -51,11 +55,11 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/metricbeat && mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -74,11 +78,11 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/metricbeat && mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -97,7 +101,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -118,7 +122,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -143,7 +147,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -164,7 +168,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -185,7 +189,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -204,14 +208,14 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/ steps: - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh cd x-pack/metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -223,15 +227,16 @@ steps: context: "x-pack/metricbeat: MacOS x86_64 Unit Tests" - label: ":mac: MacOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh cd x-pack/metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -244,22 +249,21 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test" - skip: "doesn't belong in a stage in Jenkins, thus skipped" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 + skip: "Not a stage on Jenkins & elastic/ingest-dev#3170" + # https://github.com/elastic/ingest-dev/issues/3170 + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - - .buildkite/scripts/cloud_tests.sh + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: @@ -275,26 +279,27 @@ steps: - label: ":linux: Cloud AWS (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test-aws" - skip: "https://github.com/elastic/beats/issues/36425" - # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3170" + # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3170 + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/cloud_tests.sh + + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/metricbeat/build/*.xml" @@ -346,7 +351,7 @@ steps: timeout_in_minutes: 20 agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 8812d680c4c7..3200162f7ef5 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -10,7 +10,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -21,6 +20,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/osquerybeat Mandatory Tests" key: "x-pack-osquerybeat-mandatory-tests" @@ -32,7 +35,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -51,7 +54,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -70,7 +73,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -91,7 +94,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -116,7 +119,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -137,7 +140,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -158,7 +161,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -183,7 +186,7 @@ steps: cd x-pack/osquerybeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -201,7 +204,7 @@ steps: cd x-pack/osquerybeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 451bcee84ba8..80bf88843d76 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -20,6 +20,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/packetbeat Mandatory Tests" key: "x-pack-packetbeat-mandatory-tests" @@ -31,7 +35,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -50,7 +54,7 @@ steps: mage systemTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -69,7 +73,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" @@ -88,7 +92,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -109,7 +113,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -131,7 +135,7 @@ steps: mage systemTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -156,7 +160,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -177,7 +181,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -198,7 +202,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -220,7 +224,7 @@ steps: mage systemTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -246,7 +250,7 @@ steps: if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" @@ -271,7 +275,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -291,7 +295,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index a96cf4803fe1..b69aec324985 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -14,24 +14,25 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/Winlogbeat Mandatory Tests" key: "x-pack-winlogbeat-mandatory-tests" steps: - - label: ":windows: x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" command: | Import-Module ./.buildkite/scripts/changesets.psm1 defineModuleFromTheChangeSet 'x-pack/winlogbeat' - Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE" + Write-Output "~~~ Running tests" Set-Location -Path x-pack/winlogbeat mage build unitTest - env: - MODULE: $MODULE retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -52,7 +53,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -73,7 +74,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -99,7 +100,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -120,7 +121,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -141,7 +142,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" diff --git a/.github/workflows/macos-auditbeat.yml b/.github/workflows/macos-auditbeat.yml deleted file mode 100644 index 994ca6dbebc0..000000000000 --- a/.github/workflows/macos-auditbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: auditbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-auditbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'auditbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-filebeat.yml b/.github/workflows/macos-filebeat.yml deleted file mode 100644 index 6b43f5bb6c66..000000000000 --- a/.github/workflows/macos-filebeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: filebeat - -on: - pull_request: - paths: - - '.github/workflows/macos-filebeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'filebeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-heartbeat.yml b/.github/workflows/macos-heartbeat.yml deleted file mode 100644 index c8e346a44026..000000000000 --- a/.github/workflows/macos-heartbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: heartbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-heartbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'heartbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-metricbeat.yml b/.github/workflows/macos-metricbeat.yml deleted file mode 100644 index 59a225e16015..000000000000 --- a/.github/workflows/macos-metricbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: metricbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-metricbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'metricbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && echo "See https://github.com/elastic/beats/issues/29038" - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-packetbeat.yml b/.github/workflows/macos-packetbeat.yml deleted file mode 100644 index be5dc7377e6a..000000000000 --- a/.github/workflows/macos-packetbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: packetbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-packetbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'packetbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-auditbeat.yml b/.github/workflows/macos-xpack-auditbeat.yml deleted file mode 100644 index 3adcb46f6da0..000000000000 --- a/.github/workflows/macos-xpack-auditbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: x-pack-auditbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-auditbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/auditbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-filebeat.yml b/.github/workflows/macos-xpack-filebeat.yml deleted file mode 100644 index 936c0913fa46..000000000000 --- a/.github/workflows/macos-xpack-filebeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: x-pack-filebeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-filebeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/filebeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-functionbeat.yml b/.github/workflows/macos-xpack-functionbeat.yml deleted file mode 100644 index 26a3e311c922..000000000000 --- a/.github/workflows/macos-xpack-functionbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: x-pack-functionbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-functionbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/functionbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-heartbeat.yml b/.github/workflows/macos-xpack-heartbeat.yml deleted file mode 100644 index 8a0c6c1897d2..000000000000 --- a/.github/workflows/macos-xpack-heartbeat.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: x-pack-heartbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-heartbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/heartbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: | - go install github.com/magefile/mage - cd ${{ env.BEAT_MODULE }} && npm install -g @elastic/synthetics - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-metricbeat.yml b/.github/workflows/macos-xpack-metricbeat.yml deleted file mode 100644 index 38f40b051bcf..000000000000 --- a/.github/workflows/macos-xpack-metricbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: x-pack-metricbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-metricbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/metricbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-osquerybeat.yml b/.github/workflows/macos-xpack-osquerybeat.yml deleted file mode 100644 index 1b3be3e31484..000000000000 --- a/.github/workflows/macos-xpack-osquerybeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: x-pack-osquerybeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-osquerybeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/osquerybeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-packetbeat.yml b/.github/workflows/macos-xpack-packetbeat.yml deleted file mode 100644 index 90d9f77e2691..000000000000 --- a/.github/workflows/macos-xpack-packetbeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: x-pack-packetbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-packetbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/packetbeat' - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.mergify.yml b/.mergify.yml index 9a4af7509035..196ece28630c 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,7 +1,7 @@ queue_rules: - name: default conditions: - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats pull_request_rules: - name: self-assign PRs conditions: @@ -57,7 +57,7 @@ pull_request_rules: - name: automatic approval for automated pull requests with bump updates conditions: - author=apmmachine - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^testing/environments/snapshot.*\.yml$ actions: @@ -66,7 +66,7 @@ pull_request_rules: message: Automatically approving mergify - name: automatic squash and merge with success checks and the files matching the regex ^testing/environments/snapshot* are modified. conditions: - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^testing/environments/snapshot.*\.yml$ - "#approved-reviews-by>=1" @@ -99,7 +99,7 @@ pull_request_rules: - name: automatic approval for mergify pull requests with changes in bump-rules conditions: - author=mergify[bot] - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^\.mergify\.yml$ - head~=^add-backport-next.* @@ -109,7 +109,7 @@ pull_request_rules: message: Automatically approving mergify - name: automatic squash and merge with success checks and the files matching the regex ^.mergify.yml is modified. conditions: - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^\.mergify\.yml$ - head~=^add-backport-next.* diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 837de7b80259..8b3afec342c7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -167,6 +167,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Winlogbeat* +- Use fixed size buffer at first pass for event parsing, improving throughput {issue}39530[39530] {pull}39544[39544] *Functionbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 9733b7b74c94..a93f61a0e222 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -200,16 +200,6 @@ COMMIT=${env.GIT_BASE_COMMIT} VERSION=${env.VERSION}-SNAPSHOT""") archiveArtifacts artifacts: 'packaging.properties' } - cleanup { - // Required to enable the flaky test reporting with GitHub. Workspace exists since the post/always runs earlier - dir("${BASE_DIR}"){ - notifyBuildResult(prComment: true, - slackComment: true, - analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: getFlakyBranch()), - githubIssue: isGitHubIssueEnabled(), - githubLabels: 'Team:Elastic-Agent-Data-Plane') - } - } } } diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml deleted file mode 100644 index 2abf9d684075..000000000000 --- a/auditbeat/buildkite.yml +++ /dev/null @@ -1,56 +0,0 @@ -when: - changeset: ## when PR contains any of those entries in the changeset - - "auditbeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - # mandatory stage - it runs always for: - # - branches/tags - # - on PRs - # - GitHub comment /test auditbeat - # - GitHub label auditbeat - mandatory: - # NOTE: stage name should be unique! - unitTest: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - crosscompile: - command: "make crosscompile" - platform: "family/platform-ingest-beats-ubuntu-2204" - env: - GOX_FLAGS: "-arch amd64" - unitTest-rhel-9: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-rhel-9" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - # optional stage - it runs on: - # - branches/tags - # - on PRs if: - # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest - # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... - extended: - # NOTE: stage name should be unique! - integTest: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - integTest-arm: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-macos: - command: "mage build unitTest" - platform: "generic-13-ventura-x64" - provider: "orka" - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml deleted file mode 100644 index 3fcabc5f1ceb..000000000000 --- a/filebeat/buildkite.yml +++ /dev/null @@ -1,60 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "filebeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - mandatory: - unitTest: - command: "mage unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - goIntegTest: - command: "mage goIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - pythonIntegTest: - command: "mage pythonIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - - extended_win: - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2019" - unitTest-windows-11: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-11" - unitTest-windows-10: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-10" - extended: - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" # move this inside the platform leaf - when: - comments: - - "/test filebeat for arm" - labels: - - "arm" - parameters: - - "armTest" - branches: true ## for all the branches - tags: true ## for all the tags - unitTest-macos: - command: ".buildkite/filebeat/scripts/unit-tests.sh" - platform: "generic-13-ventura-x64" - provider: "orka" - when: - comments: - - "/test filebeat for macos" - labels: - - "macOS" - parameters: - - "macosTest" - tags: true ## for all the tags diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go index e6502d384fae..9c1cf8254ace 100644 --- a/winlogbeat/sys/wineventlog/format_message.go +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -75,23 +75,39 @@ func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID valuesPtr = &values[0] } - // Determine the buffer size needed (given in WCHARs). - var bufferUsed uint32 - err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. - return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) - } + // best guess render buffer size, 16KB, to avoid rendering message twice in most cases + const bestGuessRenderBufferSize = 1 << 14 + + // EvtFormatMessage operates with WCHAR buffer, assuming the size of the buffer in characters. + // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage + var bufferNeeded uint32 + bufferSize := uint32(bestGuessRenderBufferSize / 2) // Get a buffer from the pool and adjust its length. bb := sys.NewPooledByteBuffer() defer bb.Free() - // The documentation for EventFormatMessage specifies that the buffer is - // requested "in characters", and the buffer itself is LPWSTR, meaning the - // characters are WCHAR so double the value. - // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage - bb.Reserve(int(bufferUsed * 2)) + bb.Reserve(int(bufferSize * 2)) + + err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) + switch err { //nolint:errorlint // This is an errno or nil. + case nil: // OK + return sys.UTF16BytesToString(bb.Bytes()) + + // Ignore some errors so it can tolerate missing or mismatched parameter values. + case windows.ERROR_EVT_UNRESOLVED_VALUE_INSERT, + windows.ERROR_EVT_UNRESOLVED_PARAMETER_INSERT, + windows.ERROR_EVT_MAX_INSERTS_REACHED: + return sys.UTF16BytesToString(bb.Bytes()) + + case windows.ERROR_INSUFFICIENT_BUFFER: + bb.Reserve(int(bufferNeeded * 2)) + bufferSize = bufferNeeded + + default: + return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) + } - err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferUsed, bb.PtrAt(0), &bufferUsed) + err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) switch err { //nolint:errorlint // This is an errno or nil. case nil: // OK diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index 6b4abfaf5d1f..22495f6bda2e 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -239,15 +239,9 @@ func RenderEvent( // Only a single string is returned when rendering XML. err = FormatEventString(EvtFormatMessageXml, - eventHandle, providerName, EvtHandle(publisherHandle), lang, out) + eventHandle, providerName, EvtHandle(publisherHandle), lang, renderBuf, out) // Recover by rendering the XML without the RenderingInfo (message string). if err != nil { - // Do not try to recover from InsufficientBufferErrors because these - // can be retried with a larger buffer. - if errors.Is(err, sys.InsufficientBufferError{}) { - return err - } - err = RenderEventXML(eventHandle, renderBuf, out) } @@ -256,8 +250,8 @@ func RenderEvent( // Message reads the event data associated with the EvtHandle and renders // and returns the message only. -func Message(h EvtHandle, buf []byte, pubHandleProvider func(string) sys.MessageFiles) (message string, err error) { - providerName, err := evtRenderProviderName(buf, h) +func Message(h EvtHandle, renderBuf []byte, pubHandleProvider func(string) sys.MessageFiles) (message string, err error) { + providerName, err := evtRenderProviderName(renderBuf, h) if err != nil { return "", err } @@ -386,12 +380,15 @@ func Close(h EvtHandle) error { // publisherHandle is a handle to the publisher's metadata as provided by // EvtOpenPublisherMetadata. // lang is the language ID. +// renderBuf is a scratch buffer to render the message, if not provided or of +// insufficient size then a buffer from a system pool will be used func FormatEventString( messageFlag EvtFormatMessageFlag, eventHandle EvtHandle, publisher string, publisherHandle EvtHandle, lang uint32, + renderBuf []byte, out io.Writer, ) error { // Open a publisher handle if one was not provided. @@ -405,29 +402,42 @@ func FormatEventString( defer _EvtClose(ph) //nolint:errcheck // This is just a resource release. } - // Determine the buffer size needed (given in WCHARs). - var bufferUsed uint32 - err := _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. + var bufferPtr *byte + if renderBuf != nil { + bufferPtr = &renderBuf[0] + } + + // EvtFormatMessage operates with WCHAR buffer, assuming the size of the buffer in characters. + // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage + var bufferNeeded uint32 + bufferSize := uint32(len(renderBuf) / 2) + + err := _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferSize, bufferPtr, &bufferNeeded) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. return fmt.Errorf("failed in EvtFormatMessage: %w", err) + } else if err == nil { + // Windows API returns a null terminated WCHAR C-style string in the buffer. bufferNeeded applies + // only when ERROR_INSUFFICIENT_BUFFER is returned. Luckily the UTF16ToUTF8Bytes/UTF16ToString + // functions stop at null termination. Note, as signaled in a comment at the end of this function, + // this behavior is bad for EvtFormatMessageKeyword as then the API returns a list of null terminated + // strings in the buffer (it's fine for now as we don't use this parameter value). + return common.UTF16ToUTF8Bytes(renderBuf, out) } // Get a buffer from the pool and adjust its length. bb := sys.NewPooledByteBuffer() defer bb.Free() - // The documentation for EvtFormatMessage specifies that the buffer is - // requested "in characters", and the buffer itself is LPWSTR, meaning the - // characters are WCHAR so double the value. - // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage - bb.Reserve(int(bufferUsed * 2)) - err = _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferUsed, bb.PtrAt(0), &bufferUsed) + bb.Reserve(int(bufferNeeded * 2)) + bufferSize = bufferNeeded + + err = _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) if err != nil { return fmt.Errorf("failed in EvtFormatMessage: %w", err) } // This assumes there is only a single string value to read. This will - // not work to read keys (when messageFlag == EvtFormatMessageKeyword). + // not work to read keys (when messageFlag == EvtFormatMessageKeyword) return common.UTF16ToUTF8Bytes(bb.Bytes(), out) }