diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 95460910e22d..2ac4a8911a11 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -22,6 +22,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Auditbeat Mandatory Testing" key: "auditbeat-mandatory-tests" @@ -34,7 +38,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -53,7 +57,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -71,7 +75,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -91,7 +95,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -112,7 +116,7 @@ steps: GOX_FLAGS: "-arch amd64" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -128,14 +132,14 @@ steps: steps: - label: ":linux: Auditbeat Ubuntu Integration Tests" key: "auditbeat-extended-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat mage build integTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -149,14 +153,14 @@ steps: - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests" key: "auditbeat-extended-arm64-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat mage build integTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -177,7 +181,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -198,7 +202,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -218,7 +222,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -241,7 +245,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -262,7 +266,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -283,7 +287,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml deleted file mode 100644 index 4707707e07cf..000000000000 --- a/.buildkite/buildkite.yml +++ /dev/null @@ -1,44 +0,0 @@ -projects: - - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "heartbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/heartbeat" - - "x-pack/libbeat" - - "x-pack/metricbeat" - - "x-pack/osquerybeat" - - "x-pack/packetbeat" - - "x-pack/winlogbeat" - -## Changeset macros that are defined here and used in each specific 3.0 pipeline. -changeset: - ci: - - "^Jenkinsfile" - - "^\\.ci/scripts/.*" - oss: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - xpack: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - - "^x-pack/libbeat/.*" - -disabled: - when: - labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. - - skip-ci - draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml index bd1a493b8854..c0005309457e 100644 --- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -3,6 +3,10 @@ env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + MODULE: "kubernetes" + + # Other deps + ASDF_KIND_VERSION: "0.20.0" steps: - group: "Deploy/K8S" @@ -10,22 +14,29 @@ steps: steps: - label: "Checks" - command: ".buildkite/deploy/kubernetes/scripts/make.sh" + command: | + set -euo pipefail + make -C deploy/kubernetes all + make check-no-changes agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-checks" + context: "deploy/k8s checks" - label: "K8S Test/K8S version: v1.29.0" key: "k8s-test-129" env: K8S_VERSION: "v1.29.0" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -35,15 +46,19 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.29.0" + context: "deploy/k8s test v1.29.0" - label: "K8S Test/K8S version: v1.28.0" key: "k8s-test-128" env: K8S_VERSION: "v1.28.0" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -53,15 +68,19 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.28.0" + context: "deploy/k8s test v1.28.0" - label: "K8S Test/K8S version: v1.27.3" key: "k8s-test-1273" env: K8S_VERSION: "v1.27.3" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -71,15 +90,19 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.27.3" + context: "deploy/k8s test v1.27.3" - label: "K8S Test/K8S version: v1.26.6" key: "k8s-test-1266" env: K8S_VERSION: "v1.26.6" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -89,4 +112,4 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.26.6" + context: "deploy/k8s test v1.26.6" diff --git a/.buildkite/deploy/kubernetes/scripts/install-kind.sh b/.buildkite/deploy/kubernetes/scripts/install-kind.sh deleted file mode 100755 index 8c399d2de376..000000000000 --- a/.buildkite/deploy/kubernetes/scripts/install-kind.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MSG="environment variable missing." -KIND_VERSION=${KIND_VERSION:?$MSG} -KIND_BINARY="${BIN}/kind" - -if command -v kind -then - set +e - echo "Found Kind. Checking version.." - FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}') - if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ] - then - echo "--- Versions match. No need to install Kind. Exiting." - exit 0 - fi - set -e -fi - -echo "UNMET DEP: Installing Kind" - -OS=$(uname -s| tr '[:upper:]' '[:lower:]') -ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') -if [ "${ARCH}" == "aarch64" ] ; then - ARCH_SUFFIX=arm64 -else - ARCH_SUFFIX=amd64 -fi - -if curl -sSLo "${KIND_BINARY}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then - chmod +x "${KIND_BINARY}" - echo "Kind installed: ${KIND_VERSION}" -else - echo "Something bad with the download, let's delete the corrupted binary" - if [ -e "${KIND_BINARY}" ] ; then - rm "${KIND_BINARY}" - fi - exit 1 -fi diff --git a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh deleted file mode 100755 index 7f6c75bf3b92..000000000000 --- a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MSG="parameter missing." -K8S_VERSION=${K8S_VERSION:?$MSG} -KUBECTL_BINARY="${BIN}/kubectl" - -if command -v kubectl -then - set +e - echo "Found kubectl. Checking version.." - FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}') - if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ] - then - echo "Kubectl Versions match. No need to install kubectl. Exiting." - exit 0 - fi - set -e -fi - -echo "UNMET DEP: Installing kubectl" - -OS=$(uname -s| tr '[:upper:]' '[:lower:]') -ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') -if [ "${ARCH}" == "aarch64" ] ; then - ARCH_SUFFIX=arm64 -else - ARCH_SUFFIX=amd64 -fi - -if curl -sSLo "${KUBECTL_BINARY}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then - chmod +x "${KUBECTL_BINARY}" - echo "Current K8S Version: ${K8S_VERSION}" - echo "Kubectl installed: ${KUBECTL_BINARY}" -else - echo "--- Something bad with the download, let's delete the corrupted binary" - if [ -e "${KUBECTL_BINARY}" ] ; then - rm "${KUBECTL_BINARY}" - fi - exit 1 -fi - diff --git a/.buildkite/deploy/kubernetes/scripts/make.sh b/.buildkite/deploy/kubernetes/scripts/make.sh deleted file mode 100755 index 4c9120a2d901..000000000000 --- a/.buildkite/deploy/kubernetes/scripts/make.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Checking K8S" -make -C deploy/kubernetes all -make check-no-changes diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh index 3656318bd648..3a4418a18f93 100755 --- a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh +++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh @@ -4,11 +4,15 @@ set -euo pipefail source .buildkite/env-scripts/util.sh +export KUBECONFIG="${WORKSPACE}/kubecfg" +export BIN="${WORKSPACE}/bin" +add_bin_path + echo "--- Installing kind & kubectl" -retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh -retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh +asdf plugin add kind +asdf install kind $ASDF_KIND_VERSION -echo "--- Setting up kind" +echo "~~~ Setting up kind" max_retries=3 timeout=5 retries=0 diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index b30b26c3e8a4..58624e300e69 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -12,7 +12,6 @@ WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" PLATFORM_TYPE="$(uname)" -TMP_FOLDER="tmp.${REPO}" SNAPSHOT="true" PYTEST_ADDOPTS="" OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" @@ -38,7 +37,6 @@ export WORKSPACE export BIN export HW_TYPE export PLATFORM_TYPE -export TMP_FOLDER export SNAPSHOT export PYTEST_ADDOPTS export OSS_MODULE_PATTERN diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh old mode 100644 new mode 100755 diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index ebb25ca39664..87f1925a3722 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -22,17 +22,21 @@ env: K8S_VERSION: "v1.29.0" ASDF_KIND_VERSION: "0.20.0" + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: - - label: ":ubuntu: Ubuntu Unit Tests" + - label: ":ubuntu: Filebeat Unit Tests" command: | cd filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -42,15 +46,15 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Unit Tests" + context: "filebeat: Linux x86_64 Unit Tests" - - label: ":ubuntu: Ubuntu Go Integration Tests" + - label: ":ubuntu: Filebeat Go Integration Tests" command: | cd filebeat mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -60,15 +64,15 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Go Integration Tests" + context: "filebeat: Filebeat Go Integration Tests" - - label: ":ubuntu: Ubuntu Python Integration Tests" + - label: ":ubuntu: Filebeat Python Integration Tests" command: | cd filebeat mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -80,14 +84,14 @@ steps: - github_commit_status: context: "filebeat: Python Integration Tests" - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: Filebeat Windows 2016 Unit Tests" key: "windows-2016-unit-tests" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -101,14 +105,14 @@ steps: - github_commit_status: context: "filebeat: Windows 2016 Unit Tests" - - label: ":windows: Windows 2022 Unit Tests" + - label: ":windows: Filebeat Windows 2022 Unit Tests" key: "windows-2022-unit-tests" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -124,8 +128,10 @@ steps: - group: "Filebeat Extended Tests" key: "filebeat-extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + steps: - - label: ":mac: MacOS x64_64 Unit Tests" + - label: ":mac: Filebeat macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -135,7 +141,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -144,9 +150,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended MacOS Unit Tests" + context: "filebeat: macOS x86_64 Unit Tests" - - label: ":mac: MacOS arm64 Unit Tests" + - label: ":mac: Filebeat macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -156,7 +162,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -165,9 +171,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended MacOS ARM Unit Tests" + context: "filebeat: macOS arm64 Unit Tests" - - label: ":linux: Ubuntu ARM Unit Tests" + - label: ":linux: Filebeat arm64 Unit Tests" key: "extended-arm64-unit-test" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: | @@ -175,7 +181,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -185,20 +191,21 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Ubuntu ARM Unit Tests" + context: "filebeat: Linux arm64 Unit Tests" - - group: "Extended Windows Tests" + - group: "Filebeat Extended Windows Tests" key: "filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: Filebeat Windows 2019 Unit Tests" key: "windows-extended-2019" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -210,16 +217,16 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 2019 Unit Tests" + context: "filebeat: Windows 2019 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" + - label: ":windows: Filebeat Windows 11 Unit Tests" key: "windows-extended-11" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -231,16 +238,16 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 11 Unit Tests" + context: "filebeat: Windows 11 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: Filebeat Windows 10 Unit Tests" key: "windows-extended-10" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -252,7 +259,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 10 Unit Tests" + context: "filebeat: Windows 10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -264,7 +271,7 @@ steps: - group: "Filebeat Packaging" key: "packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: Filebeat Packaging Linux" key: "packaging-linux" command: | cd filebeat @@ -283,9 +290,9 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: - context: "filebeat: Packaging" + context: "filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: Filebeat Packaging arm64" key: "packaging-arm" command: | cd filebeat @@ -303,4 +310,4 @@ steps: PACKAGES: "docker" notify: - github_commit_status: - context: "filebeat: Packaging ARM" + context: "filebeat: Packaging arm64" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 54cfe6f0ac7e..dc27b45f1ac1 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -21,6 +21,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Heartbeat Mandatory Testing" key: "heartbeat-mandatory-tests" @@ -32,7 +36,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -50,7 +54,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -69,7 +73,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -89,7 +93,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -103,12 +107,12 @@ steps: context: "Heartbeat: Win-2022 Unit Tests" - label: ":ubuntu: Heartbeat Go Integration Tests" - command: | + command: | cd heartbeat mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -126,7 +130,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -150,7 +154,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -163,7 +167,6 @@ steps: - group: "Heartbeat Extended Testing MacOS" key: "heartbeat-extended-tests-macos" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - steps: - label: ":mac: Heartbeat MacOS Unit Tests" key: "macos-extended" @@ -174,7 +177,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -194,7 +197,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -212,12 +215,12 @@ steps: steps: - label: ":windows: Heartbeat Win-2019 Unit Tests" key: "heartbeat-win-extended-2019" - command: | + command: | Set-Location -Path heartbeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -237,7 +240,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -257,7 +260,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 870ece78925e..9a7228c5b2e3 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then source .buildkite/env-scripts/env.sh if [[ -z "${GO_VERSION-""}" ]]; then export GO_VERSION=$(cat "${WORKSPACE}/.go-version") @@ -91,9 +91,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then - source .buildkite/env-scripts/env.sh - if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then - .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh - fi -fi diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh index 8ef932725c6e..b441991e6feb 100755 --- a/.buildkite/hooks/scripts/util.sh +++ b/.buildkite/hooks/scripts/util.sh @@ -27,8 +27,8 @@ google_cloud_logout_active_account() { cleanup() { if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then echo "Deleting temporary files..." - if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then - rm -rf "${BIN}/${TMP_FOLDER}.*" + if [[ -n "${BIN:-}" ]]; then + rm -rf "${BIN}" fi echo "Done." fi diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index bc77712c330b..fd485279858e 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -11,6 +11,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Mandatory Tests" key: "mandatory-tests" @@ -23,7 +27,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -43,7 +47,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -63,7 +67,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -83,7 +87,7 @@ steps: make crosscompile retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -103,7 +107,7 @@ steps: make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -125,7 +129,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index d0cece3a5900..ed9fb14f3d42 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -25,6 +25,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "Metricbeat Mandatory Tests" key: "metricbeat-mandatory-tests" @@ -34,7 +38,7 @@ steps: command: "cd metricbeat && mage build unitTest" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -53,7 +57,7 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet metricbeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" # TODO move this section to base image / pre-command hook echo "~~~ Installing kind" @@ -67,7 +71,7 @@ steps: cd metricbeat && mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -86,7 +90,7 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" # TODO move this section to base image / pre-command hook echo "~~~ Installing kind" @@ -100,7 +104,7 @@ steps: cd metricbeat && mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -117,7 +121,7 @@ steps: command: "make -C metricbeat crosscompile" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -136,7 +140,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -157,7 +161,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -182,7 +186,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -203,7 +207,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -224,7 +228,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -250,7 +254,7 @@ steps: cd metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -270,7 +274,7 @@ steps: cd metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -281,7 +285,6 @@ steps: - github_commit_status: context: "metricbeat: Extended MacOS arm64 Unit Tests" - - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index a0d139d303b6..c9a5c9f06f2a 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -20,6 +20,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "packetbeat Mandatory Tests" key: "packetbeat-mandatory-tests" @@ -30,7 +34,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -48,7 +52,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" @@ -66,7 +70,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -86,7 +90,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -110,7 +114,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -131,7 +135,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -152,7 +156,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -179,7 +183,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -192,7 +196,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -200,7 +204,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -217,7 +221,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -229,7 +233,6 @@ steps: - github_commit_status: context: "packetbeat: Extended Ubuntu ARM Unit Tests" - - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py deleted file mode 100755 index 95530b896284..000000000000 --- a/.buildkite/pipeline.py +++ /dev/null @@ -1,431 +0,0 @@ -#!/usr/bin/env python3 -from typing import Any -from ruamel.yaml import YAML -import os -import subprocess -import fnmatch -import sys - - -class Agent: - """Buildkite Agent object""" - - def __init__(self, image: str, provider: str): - self.image: str = image - self.provider: str = provider - - def create_entity(self): - raise NotImplementedError("Not implemented yet") - - -class AWSAgent(Agent): - """AWS Agent object""" - - def __init__(self, image: str, instance_type: str = None): - super().__init__(image, "aws") - if instance_type is None: - self.instance_type: str = "t4g.large" - else: - self.instance_type = instance_type - - def create_entity(self) -> dict[str, str]: - return { - "provider": self.provider, - "imagePrefix": self.image, - "instanceType": self.instance_type, - } - - -class GCPAgent(Agent): - """GCP Agent object""" - - def __init__(self, image: str): - super().__init__(image, "gcp") - - def create_entity(self) -> dict[str, str]: - return { - "provider": self.provider, - "image": self.image, - } - - -class OrkaAgent(Agent): - """Orka Agent object""" - - def __init__(self, image: str): - super().__init__(image, "orka") - - def create_entity(self) -> dict[str, str]: - return { - "provider": self.provider, - "imagePrefix": self.image, - } - - -class Step: - """Buildkite Step object""" - - def __init__( - self, - name: str, - project: str, - category: str, - agent: Agent, - definition: dict[str, Any], - ): - self.command = definition.get("command", "") - self.env = definition.get("env", {}) - self.agent: Agent = agent - self.name: str = name - self.project: str = project - self.category: str = category - self.comment = "/test " + self.project + " " + self.name - self.label = self.name - - def __lt__(self, other): - return self.name < other.name - - def step_command(self) -> list[str]: - commands = [ - f"cd {self.project}", - self.command, - ] - return commands - - def create_entity(self) -> dict[str, Any]: - data = { - "label": f"{self.project} {self.name}", - "command": self.step_command(), - "notify": [ - { - "github_commit_status": { - "context": f"{self.project.title()}: {self.name}", - } - } - ], - "agents": self.agent.create_entity(), - "artifact_paths": [ - f"{self.project}/build/*.xml", - f"{self.project}/build/*.json", - ], - } - if self.env: - data["env"] = self.env - return data - - -class Group: - """Buildkite Group object""" - - def __init__(self, project: str, category: str, steps: list[Step]): - self.project: str = project - self.category: str = category - self.steps: list[Step] = steps - - def __lt__(self, other): - return self.project < other.project - - def create_entity(self) -> dict[str, Any]: - if len(self.steps) == 0: - return {} - - data = { - "group": f"{self.project} {self.category}", - "key": f"{self.project}-{self.category}", - "steps": [step.create_entity() for step in self.steps], - } - - return data - - -class GitHelper: - def __init__(self): - self.files: list[str] = [] - - def get_pr_changeset(self) -> list[str]: - base_branch = os.getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main") - diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)] - result = subprocess.run(diff_command, stdout=subprocess.PIPE) - if result.returncode == 0: - self.files = result.stdout.decode().splitlines() - else: - print(f"Detecting changed files failed, exiting [{result.returncode}]") - exit(result.returncode) - return self.files - - -class BuildkitePipeline: - """Buildkite Pipeline object""" - - def __init__(self, groups: list[Group] = None): - if groups is None: - groups = [] - self.groups: list[Group] = groups - - def create_entity(self): - data = {"steps": [group.create_entity() for group in self.groups]} - return data - - -def is_pr() -> bool: - return os.getenv("BUILDKITE_PULL_REQUEST") != "false" - - -def group_comment(group: Group) -> bool: - comment = os.getenv("GITHUB_PR_TRIGGER_COMMENT") - if comment: - # the comment should be a subset of the values - # in .buildkite/pull-requests.json - # TODO: change /test - comment_prefix = "buildkite test" - if group.category == "mandatory": - # i.e: /test filebeat - return comment_prefix + " " + group.project in comment - else: - # i.e: test filebeat extended - return ( - comment_prefix + " " + group.project + " " + group.category in comment - ) - - -def filter_files_by_glob(files, patterns: list[str]): - for pattern in patterns: - # TODO: Support glob extended patterns: ^ and etc. - # Now it supports only linux glob syntax - if fnmatch.filter(files, pattern): - return True - return False - - -def is_in_pr_changeset( - project_changeset_filters: list[str], changeset: list[str] -) -> bool: - return filter_files_by_glob(changeset, project_changeset_filters) - - -def is_group_enabled( - group: Group, changeset_filters: list[str], changeset: list[str] -) -> bool: - if not is_pr(): - return True - - if ( - is_pr() - and is_in_pr_changeset(changeset_filters, changeset) - and group.category.startswith("mandatory") - ): - return True - - return group_comment(group) - - -def fetch_stage(name: str, stage, project: str, category: str) -> Step: - """Create a step given the yaml object.""" - - agent: Agent = None - if ("provider" not in stage) or stage["provider"] == "gcp": - agent = GCPAgent(image=stage["platform"]) - elif stage["provider"] == "aws": - agent = AWSAgent( - image=stage["platform"], - ) - elif stage["provider"] == "orka": - agent = OrkaAgent(image=stage["platform"]) - - return Step( - category=category, name=name, agent=agent, project=project, definition=stage - ) - - -def fetch_group(stages, project: str, category: str) -> Group: - """Create a group given the yaml object.""" - - steps = [] - - for stage in stages: - steps.append( - fetch_stage( - category=category, name=stage, project=project, stage=stages[stage] - ) - ) - - return Group(project=project, category=category, steps=steps) - - -def fetch_pr_pipeline(yaml: YAML) -> list[Group]: - git_helper = GitHelper() - changeset = git_helper.get_pr_changeset() - groups: list[Group] = [] - doc = pipeline_loader(yaml) - for project in doc["projects"]: - project_file = os.path.join(project, "buildkite.yml") - if not os.path.isfile(project_file): - continue - project_obj = project_loader(yaml, project_file) - group = fetch_group( - stages=project_obj["stages"]["mandatory"], - project=project, - category="mandatory", - ) - - if is_group_enabled(group, project_obj["when"]["changeset"], changeset): - groups.append(group) - - group = fetch_group( - stages=project_obj["stages"]["extended"], - project=project, - category="extended", - ) - - if is_group_enabled(group, project_obj["when"]["changeset"], changeset): - groups.append(group) - - # TODO: improve this merging lists - all_groups = [] - for group in groups: - all_groups.append(group) - - return all_groups - - -class PRComment: - command: str - group: str - project: str - step: str - - def __init__(self, comment: str): - words = comment.split() - self.command = words.pop(0) if words else "" - self.project = words.pop(0) if words else "" - self.group = words.pop(0) if words else "" - self.step = words.pop(0) if words else "" - - -# A comment like "/test filebeat extended" -# Returns a group of steps corresponding to the comment -def fetch_pr_comment_group_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: - groups = [] - doc = pipeline_loader(yaml) - if comment.project in doc["projects"]: - project_file = os.path.join(comment.project, "buildkite.yml") - if not os.path.isfile(project_file): - raise FileNotFoundError( - "buildkite.yml not found in: " + "{}".format(comment.project) - ) - project_obj = project_loader(yaml, project_file) - if not project_obj["stages"][comment.group]: - raise ValueError( - "Group not found in {} buildkite.yml: {}".format( - comment.project, comment.group - ) - ) - - group = fetch_group( - stages=project_obj["stages"][comment.group], - project=comment.project, - category="mandatory", - ) - groups.append(group) - - return groups - - -# A comment like "/test filebeat extended unitTest-macos" -def fetch_pr_comment_step_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: - groups = [] - doc = pipeline_loader(yaml) - if comment.project in doc["projects"]: - project_file = os.path.join(comment.project, "buildkite.yml") - if not os.path.isfile(project_file): - raise FileNotFoundError( - "buildkite.yml not found in: " + "{}".format(comment.project) - ) - project_obj = project_loader(yaml, project_file) - if not project_obj["stages"][comment.group]: - raise ValueError( - "Group not found in {} buildkite.yml: {}".format( - comment.project, comment.group - ) - ) - group = fetch_group( - stages=project_obj["stages"][comment.group], - project=comment.project, - category="mandatory", - ) - - filtered_steps = list( - filter(lambda step: step.name == comment.step, group.steps) - ) - - if not filtered_steps: - raise ValueError( - "Step {} not found in {} buildkite.yml".format( - comment.step, comment.project - ) - ) - group.steps = filtered_steps - groups.append(group) - - return groups - - -def pr_comment_pipeline(pr_comment: PRComment, yaml: YAML) -> list[Group]: - - if pr_comment.command == "/test": - - # A comment like "/test" for a PR - # We rerun the PR pipeline - if not pr_comment.group: - return fetch_pr_pipeline(yaml) - - # A comment like "/test filebeat" - # We don't know what group to run hence raise an error - if pr_comment.project and not pr_comment.group: - raise ValueError( - "Specify group or/and step for {}".format(pr_comment.project) - ) - - # A comment like "/test filebeat extended" - # We rerun the filebeat extended pipeline for the PR - if pr_comment.group and not pr_comment.step: - return fetch_pr_comment_group_pipeline(pr_comment, yaml) - - # A comment like "/test filebeat extended unitTest-macos" - if pr_comment.step: - return fetch_pr_comment_step_pipeline(pr_comment, yaml) - - -# TODO: validate unique stages! -def main() -> None: - yaml = YAML(typ="safe") - all_groups = [] - if is_pr(): - if os.getenv("GITHUB_PR_TRIGGER_COMMENT"): - comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) - all_groups = pr_comment_pipeline(comment, yaml) - else: - all_groups = fetch_pr_pipeline(yaml) - # TODO what to load when not in PR - - # Produce the dynamic pipeline - print( - "# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json" - ) - yaml.dump(BuildkitePipeline(all_groups).create_entity(), sys.stdout) - - -def pipeline_loader(yaml: YAML = YAML(typ="safe")): - with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - return yaml.load(file) - - -def project_loader(yaml: YAML = YAML(typ="safe"), project_file: str = ""): - with open(project_file, "r", encoding="utf8") as project_fp: - return yaml.load(project_fp) - - -if __name__ == "__main__": - - # pylint: disable=E1120 - main() diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index f35d996c7901..55affae41289 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipelineSlug": "beats", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ "mergify[bot]" ], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, @@ -16,28 +16,12 @@ "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": [ ] }, - { - "enabled": true, - "pipelineSlug": "beats-packetbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test packetbeat$", - "always_trigger_comment_regex": "^/test packetbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, { "enabled": true, "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, diff --git a/.buildkite/pytest.ini b/.buildkite/pytest.ini deleted file mode 100644 index 3eff7473d9f0..000000000000 --- a/.buildkite/pytest.ini +++ /dev/null @@ -1,11 +0,0 @@ -[pytest] -junit_family=xunit1 - -addopts = --strict-markers -markers = - load: Load tests - tag(name): Tag tests with Go-like semantics - -# Ignore setup and teardown for the timeout -#timeout_func_only = True - diff --git a/.buildkite/scripts/changesets.psm1 b/.buildkite/scripts/changesets.psm1 index 15fabd3eba5d..10e4d31a8b61 100644 --- a/.buildkite/scripts/changesets.psm1 +++ b/.buildkite/scripts/changesets.psm1 @@ -56,17 +56,9 @@ function DefineModuleFromTheChangeSet($projectPath) { } } - # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved - if(!$changedModules) { - if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') { - $Env:MODULE = "aws" - } - else { - $Env:MODULE = "kubernetes" - } - } - else { - # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export - $Env:MODULE = $changedModules + if ($changedModules) { + $env:MODULE = $changedModules + Write-Output "~~~ Set env var MODULE to [$env:MODULE]" + Write-Output "~~~ Resuming commands" } } diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh index 3b7ed5d7ec00..5c6fd6c7b0d8 100644 --- a/.buildkite/scripts/changesets.sh +++ b/.buildkite/scripts/changesets.sh @@ -68,13 +68,10 @@ defineModuleFromTheChangeSet() { fi done - if [[ -z "$changed_modules" ]]; then # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - export MODULE="aws" - else - export MODULE="kubernetes" - fi - else - export MODULE="${changed_modules}" # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export + # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993 + if [[ ! -z $changed_modules ]]; then + export MODULE="${changed_modules}" + echo "~~~ Set env var MODULE to [$MODULE]" + echo "~~~ Resuming commands" fi } diff --git a/.buildkite/scripts/run_dynamic_pipeline_tests.sh b/.buildkite/scripts/run_dynamic_pipeline_tests.sh deleted file mode 100755 index 8eb72d7a96b1..000000000000 --- a/.buildkite/scripts/run_dynamic_pipeline_tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed -# this will allow us to fail fast, if e.g. a PR has broken the generator - -set -euo pipefail - -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -pipeline_generator_changeset=( - "^.buildkite/pipeline.py" - "^*/buildkite.yml" -) - -if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then - echo "~~~ Skipping pipeline generator tests" - exit -fi - -echo "~~~ Running pipeline generator tests" - -python3 -mpip install --quiet "pytest" -pushd .buildkite -pytest . -popd diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 383ff6ff56b6..56f8d7257d60 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -2,20 +2,21 @@ set -euo pipefail +WORKSPACE=${WORKSPACE:-"$(pwd)"} +GO_VERSION=$(cat .go-version) + export REPO="beats" export DOCKER_REGISTRY="docker.elastic.co" export SETUP_GVM_VERSION="v0.5.1" export DOCKER_COMPOSE_VERSION="1.21.0" export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" + export ASDF_NODEJS_VERSION="18.17.1" export AWS_REGION="eu-central-1" -WORKSPACE=${WORKSPACE:-"$(pwd)"} export WORKSPACE -GO_VERSION=$(cat .go-version) export GO_VERSION - exportVars() { local platform_type="$(uname)" local arch_type="$(uname -m)" @@ -43,9 +44,6 @@ exportVars() { if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then exportVars - export RACE_DETECTOR="true" - export TEST_COVERAGE="true" - export DOCKER_PULL="0" export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/setup_cloud_env.sh old mode 100755 new mode 100644 similarity index 92% rename from .buildkite/scripts/cloud_tests.sh rename to .buildkite/scripts/setup_cloud_env.sh index c8f3737e13e3..3c23d71573f1 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/setup_cloud_env.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -euo pipefail - REPO_DIR=$(pwd) teardown() { @@ -31,7 +30,6 @@ tf_cleanup() { trap 'teardown' EXIT # Prepare the cloud resources using Terraform -#startCloudTestEnv "${MODULE_DIR}" echo "~~~ Loading creds" set +o xtrace export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY @@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci" export TF_VAR_REPO="${REPO}" terraform init && terraform apply -auto-approve cd - - -# Run tests -echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" -cd "${BEATS_PROJECT_NAME}" -mage build test diff --git a/.buildkite/test_pipeline.py b/.buildkite/test_pipeline.py deleted file mode 100644 index 75fd949ccc8e..000000000000 --- a/.buildkite/test_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -import os - -import pytest -import pipeline - - -@pytest.fixture -def ubuntu2204_aws_agent(): - return { - "command": "fake-cmd", - "platform": "platform-ingest-beats-ubuntu-2204-aarch64", - "provider": "aws" - } - - -@pytest.fixture() -def fake_simple_group(): - return { - "unitTest": { - "command": "fake-cmd", - "platform": "family/platform-ingest-beats-ubuntu-2204", - }, - "integrationTest": { - "command": "fake-integration", - "platform": "family/platform-ingest-beats-ubuntu-2204", - "env": { - "FOO": "BAR", - }, - }, - } - - -def test_fetch_stage(ubuntu2204_aws_agent): - step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category") - assert step.create_entity() == { - "label": "fake test", - "command": ["cd fake", "fake-cmd"], - "notify": [ - { - "github_commit_status": { - "context": "Fake: test", - } - } - ], - "agents": { - "provider": "aws", - "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64", - "instanceType": "t4g.large", - }, - "artifact_paths": [ - "fake/build/*.xml", - "fake/build/*.json", - ], - } - - -def test_fetch_group(fake_simple_group): - group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing") - assert len(group.steps) == 2 - for step in group.steps: - assert "testing" == step.category - assert "gcp" == step.agent.provider - - assert group.steps[1].env.get("FOO") == "BAR" - - -def test_is_pr(): - os.environ["BUILDKITE_PULL_REQUEST"] = "1234" - assert pipeline.is_pr() is True - os.environ["BUILDKITE_PULL_REQUEST"] = "false" - assert pipeline.is_pr() is False diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index b32391bd2e79..c598224438fe 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -16,6 +16,12 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + # See docker.go. Sets --pull to docker-compose + DOCKER_PULL: 0 + steps: - group: "Winlogbeat Mandatory Tests" key: "winlogbeat-mandatory-tests" @@ -26,7 +32,7 @@ steps: command: "make -C winlogbeat crosscompile" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -45,7 +51,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -66,7 +72,7 @@ steps: key: "mandatory-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -87,7 +93,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -113,7 +119,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -134,7 +140,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 81c1f99ad861..2e13a0d9b959 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -22,6 +22,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/auditbeat Mandatory Tests" key: "x-pack-auditbeat-mandatory-tests" @@ -33,12 +37,12 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/auditbeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/auditbeat mage update build test retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -57,7 +61,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" @@ -76,7 +80,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -97,7 +101,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -122,7 +126,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -143,7 +147,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -164,7 +168,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -190,7 +194,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -209,7 +213,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -230,7 +234,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 322caff662aa..415c3947874a 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -7,21 +7,16 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" - IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" - IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" - IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" - IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/dockerlogbeat Mandatory Tests" key: "xpack-dockerlogbeat-mandatory-tests" @@ -33,7 +28,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -52,14 +47,12 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/dockerlogbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/dockerlogbeat mage goIntegTest - env: - MODULE: $MODULE retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 22b6432fd66c..93b3c2d536ed 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-filebeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -21,18 +21,22 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/filebeat Mandatory Tests" key: "x-pack-filebeat-mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" + - label: ":ubuntu: x-pack/filebeat Unit Tests" key: "x-pack-filebeat-mandatory-linux-unit-test" command: | cd x-pack/filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -42,20 +46,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Ubuntu Unit Tests" + context: "x-pack/filebeat: Unit Tests" - - label: ":go: Go (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/filebeat Go Integration Tests" key: "x-pack-filebeat-mandatory-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" - cd x-pack/filebeat && mage goIntegTest + cd x-pack/filebeat + mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -65,20 +65,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Go (MODULE) Integration Tests" + context: "x-pack/filebeat: Go Integration Tests" - - label: ":python: Python (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/filebeat Python Integration Tests" key: "x-pack-filebeat-mandatory-python-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - cd x-pack/filebeat && mage pythonIntegTest + cd x-pack/filebeat + mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -88,16 +84,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Python (MODULE) Integration Tests" + context: "x-pack/filebeat: Python Integration Tests" - - label: ":windows: Windows 2022 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2022 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -109,16 +105,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2022 Unit Tests" + context: "x-pack/filebeat: Win-2022 Unit Tests" - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -130,39 +126,21 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2016 Unit Tests" - - - label: ":linux: Ubuntu ARM Unit Tests" - key: "x-pack-filebeat-mandatory-linux-arm-unit-test" - command: | - cd x-pack/filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Ubuntu ARM Unit Tests" + context: "x-pack/filebeat: Win-2016 Unit Tests" - group: "x-pack/filebeat Extended Windows Tests" key: "x-pack-filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -174,16 +152,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2019 Unit Tests" + context: "x-pack/filebeat: Win-2019 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: x-pack/filebeat Win 10 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -195,16 +173,16 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 10 Unit Tests" + context: "x-pack/filebeat: Win 10 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" + - label: ":windows: x-pack/filebeat Win 11 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest key: "x-pack-filebeat-extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -216,20 +194,39 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 11 Unit Tests" + context: "x-pack/filebeat: Win 11 Unit Tests" - group: "x-pack/filebeat Extended Tests" key: "x-pack-filebeat-extended-tests" + steps: - - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + key: "x-pack-filebeat-mandatory-linux-arm-unit-test" + command: | + cd x-pack/filebeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Linux arm64 Unit Tests" + + - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat && mage build unitTest + cd x-pack/filebeat + mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -238,18 +235,20 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: MacOS x86_64 Unit Tests" + context: "x-pack/filebeat: macOS x86_64 Unit Tests" - - label: ":mac: MacOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests" + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat && mage build unitTest + cd x-pack/filebeat + mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -258,23 +257,23 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: MacOS arm64 Unit Tests" + context: "x-pack/filebeat: macOS arm64 Unit Tests" - - label: ":linux: Cloud (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - - .buildkite/scripts/cloud_tests.sh + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: @@ -288,26 +287,28 @@ steps: - github_commit_status: context: "x-pack/filebeat: Cloud (MODULE) Tests" - - label: ":linux: Cloud AWS (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test-aws" - skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425" + skip: "Skipping due to elastic/beats#36425" + # https://github.com/elastic/beats/issues/36425 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/cloud_tests.sh + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/filebeat/build/*.xml" @@ -327,7 +328,7 @@ steps: - group: "x-pack/filebeat Packaging" key: "x-pack-filebeat-packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: x-pack/filebeat Packaging Linux" key: "packaging-linux" command: | cd x-pack/filebeat @@ -348,7 +349,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: x-pack/filebeat Packaging arm64" key: "packaging-arm" command: | cd x-pack/filebeat @@ -359,11 +360,11 @@ steps: timeout_in_minutes: 20 agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" notify: - github_commit_status: - context: "x-pack/filebeat: Packaging Linux ARM" + context: "x-pack/filebeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index a20f454b6aa2..3a833f369e24 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -26,6 +26,10 @@ env: ASDF_MAGE_VERSION: 1.15.0 ASDF_NODEJS_VERSION: 18.17.1 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/heartbeat Mandatory Tests" key: "x-pack-heartbeat-mandatory-tests" @@ -41,7 +45,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -64,7 +68,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -84,7 +88,7 @@ steps: mage build test retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -107,7 +111,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -134,7 +138,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -156,7 +160,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -176,7 +180,7 @@ steps: mage build test retry: automatic: - - limit: 3 + - limit: 3 key: "extended-win-2019-unit-tests" agents: provider: "gcp" @@ -206,7 +210,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -227,7 +231,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 6bf456f6d83d..6c26e9614df2 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -17,6 +17,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/libbeat Mandatory Tests" key: "x-pack-libbeat-mandatory-tests" @@ -28,7 +32,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -47,7 +51,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -66,7 +70,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -85,7 +89,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -106,7 +110,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -131,7 +135,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -152,7 +156,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -173,7 +177,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -198,7 +202,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 11ff663e2fd4..4156850f7ae8 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-metricbeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -21,6 +21,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/metricbeat Mandatory Tests" key: "x-pack-metricbeat-mandatory-tests" @@ -32,7 +36,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -51,11 +55,11 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Will run tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/metricbeat && mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -74,11 +78,11 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" + echo "~~~ Running tests" cd x-pack/metricbeat && mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -97,7 +101,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -118,7 +122,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -143,7 +147,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -164,7 +168,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -185,7 +189,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -204,14 +208,14 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/ steps: - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh cd x-pack/metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -223,15 +227,16 @@ steps: context: "x-pack/metricbeat: MacOS x86_64 Unit Tests" - label: ":mac: MacOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh cd x-pack/metricbeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -244,22 +249,21 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test" - skip: "doesn't belong in a stage in Jenkins, thus skipped" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 + skip: "Not a stage on Jenkins & elastic/ingest-dev#3170" + # https://github.com/elastic/ingest-dev/issues/3170 + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - - .buildkite/scripts/cloud_tests.sh + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: @@ -275,26 +279,27 @@ steps: - label: ":linux: Cloud AWS (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test-aws" - skip: "https://github.com/elastic/beats/issues/36425" - # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3170" + # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3170 + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/cloud_tests.sh + + echo "~~~ Running tests" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/metricbeat/build/*.xml" @@ -346,7 +351,7 @@ steps: timeout_in_minutes: 20 agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index c86a9568fb1d..219bfe5910dd 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -10,7 +10,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -21,6 +20,10 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/osquerybeat Mandatory Tests" key: "x-pack-osquerybeat-mandatory-tests" @@ -32,7 +35,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -51,7 +54,7 @@ steps: mage goIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -70,7 +73,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -91,7 +94,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -116,7 +119,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -137,7 +140,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -158,7 +161,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -183,7 +186,7 @@ steps: cd x-pack/osquerybeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -201,7 +204,7 @@ steps: cd x-pack/osquerybeat && mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 451bcee84ba8..80bf88843d76 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -20,6 +20,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/packetbeat Mandatory Tests" key: "x-pack-packetbeat-mandatory-tests" @@ -31,7 +35,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -50,7 +54,7 @@ steps: mage systemTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -69,7 +73,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" @@ -88,7 +92,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -109,7 +113,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -131,7 +135,7 @@ steps: mage systemTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -156,7 +160,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -177,7 +181,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -198,7 +202,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -220,7 +224,7 @@ steps: mage systemTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -246,7 +250,7 @@ steps: if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" @@ -271,7 +275,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -291,7 +295,7 @@ steps: mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index a96cf4803fe1..b69aec324985 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -14,24 +14,25 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + # Unit tests + RACE_DETECTOR: "true" + TEST_COVERAGE: "true" + steps: - group: "x-pack/Winlogbeat Mandatory Tests" key: "x-pack-winlogbeat-mandatory-tests" steps: - - label: ":windows: x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" command: | Import-Module ./.buildkite/scripts/changesets.psm1 defineModuleFromTheChangeSet 'x-pack/winlogbeat' - Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE" + Write-Output "~~~ Running tests" Set-Location -Path x-pack/winlogbeat mage build unitTest - env: - MODULE: $MODULE retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -52,7 +53,7 @@ steps: key: "mandatory-win-2016-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -73,7 +74,7 @@ steps: key: "mandatory-win-2022-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -99,7 +100,7 @@ steps: key: "extended-win-10-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -120,7 +121,7 @@ steps: key: "extended-win-11-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -141,7 +142,7 @@ steps: key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2019}" diff --git a/.github/workflows/macos-auditbeat.yml b/.github/workflows/macos-auditbeat.yml deleted file mode 100644 index 39c97c8b7193..000000000000 --- a/.github/workflows/macos-auditbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: auditbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-auditbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'auditbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-filebeat.yml b/.github/workflows/macos-filebeat.yml deleted file mode 100644 index 513b87be316c..000000000000 --- a/.github/workflows/macos-filebeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: filebeat - -on: - pull_request: - paths: - - '.github/workflows/macos-filebeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'filebeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-heartbeat.yml b/.github/workflows/macos-heartbeat.yml deleted file mode 100644 index b707e9c7d429..000000000000 --- a/.github/workflows/macos-heartbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: heartbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-heartbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'heartbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-metricbeat.yml b/.github/workflows/macos-metricbeat.yml deleted file mode 100644 index 0f37cfb937b2..000000000000 --- a/.github/workflows/macos-metricbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: metricbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-metricbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'metricbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && echo "See https://github.com/elastic/beats/issues/29038" - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-packetbeat.yml b/.github/workflows/macos-packetbeat.yml deleted file mode 100644 index bebbc5eed90e..000000000000 --- a/.github/workflows/macos-packetbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: packetbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-packetbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'packetbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-auditbeat.yml b/.github/workflows/macos-xpack-auditbeat.yml deleted file mode 100644 index e0484908a9e2..000000000000 --- a/.github/workflows/macos-xpack-auditbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: x-pack-auditbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-auditbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/auditbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-filebeat.yml b/.github/workflows/macos-xpack-filebeat.yml deleted file mode 100644 index 93950c24b572..000000000000 --- a/.github/workflows/macos-xpack-filebeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: x-pack-filebeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-filebeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/filebeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-functionbeat.yml b/.github/workflows/macos-xpack-functionbeat.yml deleted file mode 100644 index 430d8834bb4a..000000000000 --- a/.github/workflows/macos-xpack-functionbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: x-pack-functionbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-functionbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/functionbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-heartbeat.yml b/.github/workflows/macos-xpack-heartbeat.yml deleted file mode 100644 index 9c4995ce20d8..000000000000 --- a/.github/workflows/macos-xpack-heartbeat.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: x-pack-heartbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-heartbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/heartbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: | - go install github.com/magefile/mage - cd ${{ env.BEAT_MODULE }} && npm install -g @elastic/synthetics - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-metricbeat.yml b/.github/workflows/macos-xpack-metricbeat.yml deleted file mode 100644 index 2d2531cd2866..000000000000 --- a/.github/workflows/macos-xpack-metricbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: x-pack-metricbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-metricbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/metricbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-osquerybeat.yml b/.github/workflows/macos-xpack-osquerybeat.yml deleted file mode 100644 index 7678df4b2f2d..000000000000 --- a/.github/workflows/macos-xpack-osquerybeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: x-pack-osquerybeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-osquerybeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/osquerybeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/macos-xpack-packetbeat.yml b/.github/workflows/macos-xpack-packetbeat.yml deleted file mode 100644 index 8167486eb0c9..000000000000 --- a/.github/workflows/macos-xpack-packetbeat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: x-pack-packetbeat - -on: - pull_request: - paths: - - '.github/workflows/macos-xpack-packetbeat.yml' - push: - branches: - - main - - 7.1* - - 8.* - -env: - BEAT_MODULE: 'x-pack/packetbeat' - -permissions: - contents: read - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: cd ${{ env.BEAT_MODULE }} && mage build - - name: Run test - run: cd ${{ env.BEAT_MODULE }} && mage unitTest - - uses: v1v/otel-upload-test-artifact-action@v2 - if: always() - continue-on-error: true - with: - jobName: "macos" - stepName: "Run test" - path: "${{ env.BEAT_MODULE }}/build/TEST*.xml" - type: "junit" - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.mergify.yml b/.mergify.yml index 5ab78c46dada..8bde93185805 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,7 +1,7 @@ queue_rules: - name: default conditions: - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats pull_request_rules: - name: self-assign PRs conditions: @@ -57,7 +57,7 @@ pull_request_rules: - name: automatic approval for automated pull requests with bump updates conditions: - author=apmmachine - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^testing/environments/snapshot.*\.yml$ actions: @@ -66,7 +66,7 @@ pull_request_rules: message: Automatically approving mergify - name: automatic squash and merge with success checks and the files matching the regex ^testing/environments/snapshot* are modified. conditions: - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^testing/environments/snapshot.*\.yml$ - "#approved-reviews-by>=1" @@ -99,7 +99,7 @@ pull_request_rules: - name: automatic approval for mergify pull requests with changes in bump-rules conditions: - author=mergify[bot] - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^\.mergify\.yml$ - head~=^add-backport-next.* @@ -109,7 +109,7 @@ pull_request_rules: message: Automatically approving mergify - name: automatic squash and merge with success checks and the files matching the regex ^.mergify.yml is modified. conditions: - - check-success=beats-ci/pr-merge + - check-success=buildkite/beats - label=automation - files~=^\.mergify\.yml$ - head~=^add-backport-next.* diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0e1ef392c892..bd1770d198ca 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -91,6 +91,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Change cache processor documentation from `write_period` to `write_interval`. {pull}38561[38561] - Fix cache processor expiries heap cleanup on partial file writes. {pull}38561[38561] - Fix cache processor expiries infinite growth when large a large TTL is used and recurring keys are cached. {pull}38561[38561] +- Fix parsing of RFC 3164 process IDs in syslog processor. {issue}38947[38947] {pull}38982[38982] *Auditbeat* - Set field types to correctly match ECS in sessionmd processor {issue}38955[38955] {pull}38994[38994] @@ -153,6 +154,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix setuid root when running under cgroups v2. {pull}37794[37794] - Adjust State loader to only retry when response code status is 5xx {pull}37981[37981] - Reset prctl dumpable flag after cap drop. {pull}38269[38269] +- Redact synthexec cmd output. {pull}39535[39535] *Heartbeat* @@ -173,6 +175,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Winlogbeat* +- Fix error handling in perfmon metrics. {issue}38140[38140] {pull}39404[39404] *Elastic Logging Plugin* @@ -297,6 +300,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Winlogbeat* +- Use fixed size buffer at first pass for event parsing, improving throughput {issue}39530[39530] {pull}39544[39544] *Functionbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 23a4f3798dfe..04df94f4b256 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -200,16 +200,6 @@ COMMIT=${env.GIT_BASE_COMMIT} VERSION=${env.VERSION}-SNAPSHOT""") archiveArtifacts artifacts: 'packaging.properties' } - cleanup { - // Required to enable the flaky test reporting with GitHub. Workspace exists since the post/always runs earlier - dir("${BASE_DIR}"){ - notifyBuildResult(prComment: true, - slackComment: true, - analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: getFlakyBranch()), - githubIssue: isGitHubIssueEnabled(), - githubLabels: 'Team:Elastic-Agent-Data-Plane') - } - } } } diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml deleted file mode 100644 index 2abf9d684075..000000000000 --- a/auditbeat/buildkite.yml +++ /dev/null @@ -1,56 +0,0 @@ -when: - changeset: ## when PR contains any of those entries in the changeset - - "auditbeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - # mandatory stage - it runs always for: - # - branches/tags - # - on PRs - # - GitHub comment /test auditbeat - # - GitHub label auditbeat - mandatory: - # NOTE: stage name should be unique! - unitTest: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - crosscompile: - command: "make crosscompile" - platform: "family/platform-ingest-beats-ubuntu-2204" - env: - GOX_FLAGS: "-arch amd64" - unitTest-rhel-9: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-rhel-9" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - # optional stage - it runs on: - # - branches/tags - # - on PRs if: - # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest - # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... - extended: - # NOTE: stage name should be unique! - integTest: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - integTest-arm: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-macos: - command: "mage build unitTest" - platform: "generic-13-ventura-x64" - provider: "orka" - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml deleted file mode 100644 index 3fcabc5f1ceb..000000000000 --- a/filebeat/buildkite.yml +++ /dev/null @@ -1,60 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "filebeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - mandatory: - unitTest: - command: "mage unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - goIntegTest: - command: "mage goIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - pythonIntegTest: - command: "mage pythonIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - - extended_win: - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2019" - unitTest-windows-11: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-11" - unitTest-windows-10: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-10" - extended: - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" # move this inside the platform leaf - when: - comments: - - "/test filebeat for arm" - labels: - - "arm" - parameters: - - "armTest" - branches: true ## for all the branches - tags: true ## for all the tags - unitTest-macos: - command: ".buildkite/filebeat/scripts/unit-tests.sh" - platform: "generic-13-ventura-x64" - provider: "orka" - when: - comments: - - "/test filebeat for macos" - labels: - - "macOS" - parameters: - - "macosTest" - tags: true ## for all the tags diff --git a/libbeat/reader/syslog/parser/rfc3164.rl b/libbeat/reader/syslog/parser/rfc3164.rl index 0dac77314040..709c049eb06e 100644 --- a/libbeat/reader/syslog/parser/rfc3164.rl +++ b/libbeat/reader/syslog/parser/rfc3164.rl @@ -16,7 +16,7 @@ hostname = graph+ >tok %set_hostname; tag = (print -- [ :\[])+ >tok %set_tag; - content_value = print+ >tok %set_content; + content_value = digit+ >tok %set_content; content = '[' content_value ']'; msg = (tag content? ':' sp)? any+ >tok %set_msg; }%% diff --git a/libbeat/reader/syslog/rfc3164_gen.go b/libbeat/reader/syslog/rfc3164_gen.go index 852ec066f597..b4c4146ba82b 100644 --- a/libbeat/reader/syslog/rfc3164_gen.go +++ b/libbeat/reader/syslog/rfc3164_gen.go @@ -80,10 +80,6 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { goto st_case_29 case 30: goto st_case_30 - case 31: - goto st_case_31 - case 32: - goto st_case_32 case 7: goto st_case_7 case 8: @@ -317,7 +313,7 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { goto _test_eof28 } st_case_28: - if 32 <= data[p] && data[p] <= 126 { + if 48 <= data[p] && data[p] <= 57 { goto tr37 } goto st24 @@ -334,7 +330,7 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { if data[p] == 93 { goto tr39 } - if 32 <= data[p] && data[p] <= 126 { + if 48 <= data[p] && data[p] <= 57 { goto st29 } goto st24 @@ -342,56 +338,16 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { m.setContent(data[tok:p]) - goto st30 - tr42: - - m.setContent(data[tok:p]) - - tok = p - goto st30 st30: if p++; p == pe { goto _test_eof30 } st_case_30: - switch data[p] { - case 58: - goto st31 - case 93: - goto tr39 - } - if 32 <= data[p] && data[p] <= 126 { - goto st29 - } - goto st24 - st31: - if p++; p == pe { - goto _test_eof31 - } - st_case_31: - switch data[p] { - case 32: - goto st32 - case 93: - goto tr39 - } - if 33 <= data[p] && data[p] <= 126 { - goto st29 + if data[p] == 58 { + goto st26 } goto st24 - st32: - if p++; p == pe { - goto _test_eof32 - } - st_case_32: - if data[p] == 93 { - goto tr42 - } - if 32 <= data[p] && data[p] <= 126 { - goto tr37 - } - goto tr11 st7: if p++; p == pe { goto _test_eof7 @@ -816,12 +772,6 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { _test_eof30: cs = 30 goto _test_eof - _test_eof31: - cs = 31 - goto _test_eof - _test_eof32: - cs = 32 - goto _test_eof _test_eof7: cs = 7 goto _test_eof @@ -879,7 +829,7 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { } if p == eof { switch cs { - case 24, 25, 26, 27, 28, 29, 30, 31, 32: + case 24, 25, 26, 27, 28, 29, 30: m.setMsg(data[tok:p]) diff --git a/libbeat/reader/syslog/rfc3164_test.go b/libbeat/reader/syslog/rfc3164_test.go index d1c75fe574eb..a2c80e8c263f 100644 --- a/libbeat/reader/syslog/rfc3164_test.go +++ b/libbeat/reader/syslog/rfc3164_test.go @@ -88,6 +88,19 @@ func TestParseRFC3164(t *testing.T) { msg: "message", }, }, + "ok-procid-with-square-brackets-msg": { + in: "<114>Apr 12 13:30:01 aaaaaa001.adm.domain aaaaaa001[25259]: my.some.domain 10.11.12.13 - USERNAME [12/Apr/2024:13:29:59.993 +0200] /skodas \"GET /skodas/group/pod-documentation/aaa HTTP/1.1\" 301 301 290bytes 1 10327", + want: message{ + timestamp: mustParseTime(time.Stamp, "Apr 12 13:30:01", time.Local), + priority: 114, + facility: 14, + severity: 2, + hostname: "aaaaaa001.adm.domain", + process: "aaaaaa001", + pid: "25259", + msg: "my.some.domain 10.11.12.13 - USERNAME [12/Apr/2024:13:29:59.993 +0200] /skodas \"GET /skodas/group/pod-documentation/aaa HTTP/1.1\" 301 301 290bytes 1 10327", + }, + }, "err-pri-not-a-number": { in: "Oct 11 22:14:15 test-host this is the message", want: message{ diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go index 9add5c03896a..0391266e65a5 100644 --- a/metricbeat/module/windows/perfmon/data.go +++ b/metricbeat/module/windows/perfmon/data.go @@ -20,6 +20,7 @@ package perfmon import ( + "errors" "fmt" "regexp" "strconv" @@ -48,7 +49,7 @@ func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Eve // The counter has a negative value or the counter was successfully found, but the data returned is not valid. // This error can occur if the counter value is less than the previous value. (Because counter values always increment, the counter value rolls over to zero when it reaches its maximum value.) // This is not an error that stops the application from running successfully and a positive counter value should be retrieved in the later calls. - if val.Err.Error == pdh.PDH_CALC_NEGATIVE_VALUE || val.Err.Error == pdh.PDH_INVALID_DATA { + if errors.Is(val.Err.Error, pdh.PDH_CALC_NEGATIVE_VALUE) || errors.Is(val.Err.Error, pdh.PDH_INVALID_DATA) { re.log.Debugw("Counter value retrieval returned", "error", val.Err.Error, "cstatus", pdh.PdhErrno(val.Err.CStatus), logp.Namespace("perfmon"), "query", counterPath) continue @@ -69,7 +70,9 @@ func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Eve if _, ok := eventMap[eventKey]; !ok { eventMap[eventKey] = &mb.Event{ MetricSetFields: mapstr.M{}, - Error: fmt.Errorf("failed on query=%v: %w", counterPath, val.Err.Error), + } + if val.Err.Error != nil { + eventMap[eventKey].Error = fmt.Errorf("failed on query=%v: %w", counterPath, val.Err.Error) } if val.Instance != "" { // will ignore instance index @@ -93,9 +96,11 @@ func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Eve } } // Write the values into the map. - var events []mb.Event + events := make([]mb.Event, len(eventMap)) + iter := 0 for _, val := range eventMap { - events = append(events, *val) + events[iter] = *val + iter++ } return events } @@ -111,7 +116,7 @@ func (re *Reader) groupToSingleEvent(counters map[string][]pdh.CounterValue) mb. // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if val.Err.Error != nil { - if val.Err.Error == pdh.PDH_CALC_NEGATIVE_VALUE || val.Err.Error == pdh.PDH_INVALID_DATA { + if errors.Is(val.Err.Error, pdh.PDH_CALC_NEGATIVE_VALUE) || errors.Is(val.Err.Error, pdh.PDH_INVALID_DATA) { re.log.Debugw("Counter value retrieval returned", "error", val.Err.Error, "cstatus", pdh.PdhErrno(val.Err.CStatus), logp.Namespace("perfmon"), "query", counterPath) continue diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go index 2e9f15e2de7e..9c4691216b34 100644 --- a/metricbeat/module/windows/perfmon/data_test.go +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -28,6 +28,67 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +func TestGroupErrors(t *testing.T) { + reader := Reader{ + config: Config{ + GroupMeasurements: true, + }, + query: pdh.Query{}, + log: nil, + counters: []PerfCounter{ + { + QueryField: "datagrams_sent_per_sec", + QueryName: `\UDPv4\Datagrams Sent/sec`, + Format: "float", + ObjectName: "UDPv4", + ObjectField: "object", + ChildQueries: []string{`\UDPv4\Datagrams Sent/sec`}, + }, + { + QueryField: "%_processor_time", + QueryName: `\Processor Information(_Total)\% Processor Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "_Total", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(_Total)\% Processor Time`}, + }, + { + QueryField: "current_disk_queue_length", + QueryName: `\PhysicalDisk(_Total)\Current Disk Queue Length`, + Format: "float", + ObjectName: "PhysicalDisk", + ObjectField: "object", + InstanceName: "_Total", + InstanceField: "instance", + ChildQueries: []string{`\PhysicalDisk(_Total)\Current Disk Queue Length`}, + }, + }, + } + + counters := map[string][]pdh.CounterValue{ + `\UDPv4\Datagrams Sent/sec`: { + {Instance: "", Measurement: 23}, + }, + `\Processor Information(_Total)\% Processor Time`: { + {Instance: "_Total", Measurement: 11}, + }, + `\PhysicalDisk(_Total)\Current Disk Queue Length`: { + {Instance: "_Total", Measurement: 20}, + }, + } + + events := reader.groupToEvents(counters) + assert.NotNil(t, events) + assert.Equal(t, 3, len(events)) + + for _, event := range events { + assert.NoError(t, event.Error) + } + +} + func TestGroupToEvents(t *testing.T) { reader := Reader{ config: Config{ diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 065dfb5b4bdd..50d0fbdc813c 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-6d69ee76-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-526ba7a5-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.14.0-6d69ee76-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-526ba7a5-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.14.0-6d69ee76-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-526ba7a5-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go index e6502d384fae..9c1cf8254ace 100644 --- a/winlogbeat/sys/wineventlog/format_message.go +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -75,23 +75,39 @@ func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID valuesPtr = &values[0] } - // Determine the buffer size needed (given in WCHARs). - var bufferUsed uint32 - err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. - return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) - } + // best guess render buffer size, 16KB, to avoid rendering message twice in most cases + const bestGuessRenderBufferSize = 1 << 14 + + // EvtFormatMessage operates with WCHAR buffer, assuming the size of the buffer in characters. + // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage + var bufferNeeded uint32 + bufferSize := uint32(bestGuessRenderBufferSize / 2) // Get a buffer from the pool and adjust its length. bb := sys.NewPooledByteBuffer() defer bb.Free() - // The documentation for EventFormatMessage specifies that the buffer is - // requested "in characters", and the buffer itself is LPWSTR, meaning the - // characters are WCHAR so double the value. - // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage - bb.Reserve(int(bufferUsed * 2)) + bb.Reserve(int(bufferSize * 2)) + + err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) + switch err { //nolint:errorlint // This is an errno or nil. + case nil: // OK + return sys.UTF16BytesToString(bb.Bytes()) + + // Ignore some errors so it can tolerate missing or mismatched parameter values. + case windows.ERROR_EVT_UNRESOLVED_VALUE_INSERT, + windows.ERROR_EVT_UNRESOLVED_PARAMETER_INSERT, + windows.ERROR_EVT_MAX_INSERTS_REACHED: + return sys.UTF16BytesToString(bb.Bytes()) + + case windows.ERROR_INSUFFICIENT_BUFFER: + bb.Reserve(int(bufferNeeded * 2)) + bufferSize = bufferNeeded + + default: + return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) + } - err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferUsed, bb.PtrAt(0), &bufferUsed) + err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) switch err { //nolint:errorlint // This is an errno or nil. case nil: // OK diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index 6b4abfaf5d1f..22495f6bda2e 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -239,15 +239,9 @@ func RenderEvent( // Only a single string is returned when rendering XML. err = FormatEventString(EvtFormatMessageXml, - eventHandle, providerName, EvtHandle(publisherHandle), lang, out) + eventHandle, providerName, EvtHandle(publisherHandle), lang, renderBuf, out) // Recover by rendering the XML without the RenderingInfo (message string). if err != nil { - // Do not try to recover from InsufficientBufferErrors because these - // can be retried with a larger buffer. - if errors.Is(err, sys.InsufficientBufferError{}) { - return err - } - err = RenderEventXML(eventHandle, renderBuf, out) } @@ -256,8 +250,8 @@ func RenderEvent( // Message reads the event data associated with the EvtHandle and renders // and returns the message only. -func Message(h EvtHandle, buf []byte, pubHandleProvider func(string) sys.MessageFiles) (message string, err error) { - providerName, err := evtRenderProviderName(buf, h) +func Message(h EvtHandle, renderBuf []byte, pubHandleProvider func(string) sys.MessageFiles) (message string, err error) { + providerName, err := evtRenderProviderName(renderBuf, h) if err != nil { return "", err } @@ -386,12 +380,15 @@ func Close(h EvtHandle) error { // publisherHandle is a handle to the publisher's metadata as provided by // EvtOpenPublisherMetadata. // lang is the language ID. +// renderBuf is a scratch buffer to render the message, if not provided or of +// insufficient size then a buffer from a system pool will be used func FormatEventString( messageFlag EvtFormatMessageFlag, eventHandle EvtHandle, publisher string, publisherHandle EvtHandle, lang uint32, + renderBuf []byte, out io.Writer, ) error { // Open a publisher handle if one was not provided. @@ -405,29 +402,42 @@ func FormatEventString( defer _EvtClose(ph) //nolint:errcheck // This is just a resource release. } - // Determine the buffer size needed (given in WCHARs). - var bufferUsed uint32 - err := _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. + var bufferPtr *byte + if renderBuf != nil { + bufferPtr = &renderBuf[0] + } + + // EvtFormatMessage operates with WCHAR buffer, assuming the size of the buffer in characters. + // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage + var bufferNeeded uint32 + bufferSize := uint32(len(renderBuf) / 2) + + err := _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferSize, bufferPtr, &bufferNeeded) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. return fmt.Errorf("failed in EvtFormatMessage: %w", err) + } else if err == nil { + // Windows API returns a null terminated WCHAR C-style string in the buffer. bufferNeeded applies + // only when ERROR_INSUFFICIENT_BUFFER is returned. Luckily the UTF16ToUTF8Bytes/UTF16ToString + // functions stop at null termination. Note, as signaled in a comment at the end of this function, + // this behavior is bad for EvtFormatMessageKeyword as then the API returns a list of null terminated + // strings in the buffer (it's fine for now as we don't use this parameter value). + return common.UTF16ToUTF8Bytes(renderBuf, out) } // Get a buffer from the pool and adjust its length. bb := sys.NewPooledByteBuffer() defer bb.Free() - // The documentation for EvtFormatMessage specifies that the buffer is - // requested "in characters", and the buffer itself is LPWSTR, meaning the - // characters are WCHAR so double the value. - // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage - bb.Reserve(int(bufferUsed * 2)) - err = _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferUsed, bb.PtrAt(0), &bufferUsed) + bb.Reserve(int(bufferNeeded * 2)) + bufferSize = bufferNeeded + + err = _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) if err != nil { return fmt.Errorf("failed in EvtFormatMessage: %w", err) } // This assumes there is only a single string value to read. This will - // not work to read keys (when messageFlag == EvtFormatMessageKeyword). + // not work to read keys (when messageFlag == EvtFormatMessageKeyword) return common.UTF16ToUTF8Bytes(bb.Bytes(), out) } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthcmd.go b/x-pack/heartbeat/monitors/browser/synthexec/synthcmd.go new file mode 100644 index 000000000000..ce6b8610a419 --- /dev/null +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthcmd.go @@ -0,0 +1,44 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin || synthetics + +package synthexec + +import ( + "fmt" + "os/exec" + "strings" +) + +// Variant of exec.command with redacted params and playwright options, +// which might contain sensitive information. +type SynthCmd struct { + *exec.Cmd +} + +func (cmd *SynthCmd) String() string { + b := new(strings.Builder) + b.WriteString(cmd.Path) + for i := 1; i < len(cmd.Args); i++ { + b.WriteString(" ") + a := cmd.Args[i] + switch a { + case "--params": + fallthrough + case "--playwright-options": + b.WriteString(fmt.Sprintf("%s { REDACTED }", a)) + i++ + default: + b.WriteString(a) + } + } + + return b.String() +} + +// Formatter override redacting params +func (cmd SynthCmd) Format(f fmt.State, verb rune) { + + f.Write([]byte(cmd.String())) +} diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthcmd_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthcmd_test.go new file mode 100644 index 000000000000..3c8057904133 --- /dev/null +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthcmd_test.go @@ -0,0 +1,66 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux || synthetics + +package synthexec + +import ( + "fmt" + "io" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSynthCmdStringOutput(t *testing.T) { + tests := []struct { + name string + stringer func(cmd SynthCmd) string + }{ + { + name: "fmt.Sprintf", + stringer: func(cmd SynthCmd) string { + return fmt.Sprintf("%s", cmd) + }, + }, + { + name: "fmt.Println", + stringer: func(cmd SynthCmd) string { + r, w, err := os.Pipe() + assert.NoError(t, err) + fmt.Fprint(w, cmd) + w.Close() + defer r.Close() + + o, err := io.ReadAll(r) + assert.NoError(t, err) + + return string(o) + }, + }, + { + name: "cmd.String()", + stringer: func(cmd SynthCmd) string { + return cmd.String() + }, + }, + } + + redacted := []string{"secret", "mysecrettoken", "auth", "mysecretauth"} + cmd := SynthCmd{ + exec.Command("/nil", "--params", "{'secret':'mysecrettoken'}", "--playwright-options", "{'auth':'mysecretauth'}"), + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.stringer(cmd) + for _, r := range redacted { + assert.NotContains(t, s, r) + } + }) + } +} diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go index 32f127de98c3..3c0139557f5f 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go @@ -41,7 +41,7 @@ type FilterJourneyConfig struct { // This is practically just used by synthexec_unix.go to add Sysprocattrs // It's still nice for devs to be able to test browser monitors on OSX // where these are unsupported -var platformCmdMutate func(*exec.Cmd) = func(*exec.Cmd) {} +var platformCmdMutate func(*SynthCmd) = func(*SynthCmd) {} var SynthexecTimeout struct{} @@ -57,13 +57,13 @@ func ProjectJob(ctx context.Context, projectPath string, params mapstr.M, filter return startCmdJob(ctx, cmdFactory, nil, params, filterJourneys, fields), nil } -func projectCommandFactory(projectPath string, args ...string) (func() *exec.Cmd, error) { +func projectCommandFactory(projectPath string, args ...string) (func() *SynthCmd, error) { npmRoot, err := getNpmRoot(projectPath) if err != nil { return nil, err } - newCmd := func() *exec.Cmd { + newCmd := func() *SynthCmd { bin := filepath.Join(npmRoot, "node_modules/.bin/elastic-synthetics") // Always put the project path first to prevent conflation with variadic args! // See https://github.com/tj/commander.js/blob/master/docs/options-taking-varying-arguments.md @@ -71,7 +71,7 @@ func projectCommandFactory(projectPath string, args ...string) (func() *exec.Cmd // to the end. cmd := exec.Command(bin, append([]string{projectPath}, args...)...) cmd.Dir = npmRoot - return cmd + return &SynthCmd{cmd} } return newCmd, nil @@ -79,8 +79,8 @@ func projectCommandFactory(projectPath string, args ...string) (func() *exec.Cmd // InlineJourneyJob returns a job that runs the given source as a single journey. func InlineJourneyJob(ctx context.Context, script string, params mapstr.M, fields stdfields.StdMonitorFields, extraArgs ...string) jobs.Job { - newCmd := func() *exec.Cmd { - return exec.Command("elastic-synthetics", append(extraArgs, "--inline")...) //nolint:gosec // we are safely building a command here, users can add args at their own risk + newCmd := func() *SynthCmd { + return &SynthCmd{exec.Command("elastic-synthetics", append(extraArgs, "--inline")...)} //nolint:gosec // we are safely building a command here, users can add args at their own risk } return startCmdJob(ctx, newCmd, &script, params, FilterJourneyConfig{}, fields) @@ -89,7 +89,7 @@ func InlineJourneyJob(ctx context.Context, script string, params mapstr.M, field // startCmdJob adapts commands into a heartbeat job. This is a little awkward given that the command's output is // available via a sequence of events in the multiplexer, while heartbeat jobs are tail recursive continuations. // Here, we adapt one to the other, where each recursive job pulls another item off the chan until none are left. -func startCmdJob(ctx context.Context, newCmd func() *exec.Cmd, stdinStr *string, params mapstr.M, filterJourneys FilterJourneyConfig, sFields stdfields.StdMonitorFields) jobs.Job { +func startCmdJob(ctx context.Context, newCmd func() *SynthCmd, stdinStr *string, params mapstr.M, filterJourneys FilterJourneyConfig, sFields stdfields.StdMonitorFields) jobs.Job { return func(event *beat.Event) ([]jobs.Job, error) { senr := newStreamEnricher(sFields) mpx, err := runCmd(ctx, newCmd(), stdinStr, params, filterJourneys) @@ -124,7 +124,7 @@ func readResultsJob(ctx context.Context, synthEvents <-chan *SynthEvent, enrich // the params var as a CLI argument. func runCmd( ctx context.Context, - cmd *exec.Cmd, + cmd *SynthCmd, stdinStr *string, params mapstr.M, filterJourneys FilterJourneyConfig, @@ -151,12 +151,9 @@ func runCmd( cmd.Args = append(cmd.Args, "--match", filterJourneys.Match) } - // Variant of the command with no params, which could contain sensitive stuff - loggableCmd := exec.Command(cmd.Path, cmd.Args...) //nolint:gosec // we are safely building a command here... if len(params) > 0 { paramsBytes, _ := json.Marshal(params) cmd.Args = append(cmd.Args, "--params", string(paramsBytes)) - loggableCmd.Args = append(loggableCmd.Args, "--params", fmt.Sprintf("\"{%d hidden params}\"", len(params))) } // We need to pass both files in here otherwise we get a broken pipe, even @@ -166,10 +163,10 @@ func runCmd( // see the docs for ExtraFiles in https://golang.org/pkg/os/exec/#Cmd cmd.Args = append(cmd.Args, "--outfd", "3") - logp.Info("Running command: %s in directory: '%s'", loggableCmd.String(), cmd.Dir) + logp.L().Info("Running command: %s in directory: '%s'", cmd, cmd.Dir) if stdinStr != nil { - logp.Debug(debugSelector, "Using stdin str %s", *stdinStr) + logp.L().Debug(debugSelector, "Using stdin str %s", *stdinStr) cmd.Stdin = strings.NewReader(*stdinStr) } @@ -184,7 +181,7 @@ func runCmd( go func() { err := scanToSynthEvents(stdoutPipe, stdoutToSynthEvent, mpx.writeSynthEvent) if err != nil { - logp.Warn("could not scan stdout events from synthetics: %s", err) + logp.L().Warn("could not scan stdout events from synthetics: %s", err) } wg.Done() @@ -198,7 +195,7 @@ func runCmd( go func() { err := scanToSynthEvents(stderrPipe, stderrToSynthEvent, mpx.writeSynthEvent) if err != nil { - logp.Warn("could not scan stderr events from synthetics: %s", err) + logp.L().Warn("could not scan stderr events from synthetics: %s", err) } wg.Done() }() @@ -247,7 +244,7 @@ func runCmd( err = <-cmdStarted if err != nil { - logp.Warn("Could not start command %s: %s", cmd, err) + logp.L().Warn("Could not start command %s: %s", cmd, err) return nil, err } @@ -264,7 +261,7 @@ func runCmd( err := cmd.Process.Kill() if err != nil { - logp.Warn("could not kill synthetics process: %s", err) + logp.L().Warn("could not kill synthetics process: %s", err) } }() @@ -272,19 +269,19 @@ func runCmd( go func() { err := <-cmdDone _ = jsonWriter.Close() - logp.Info("Command has completed(%d): %s", cmd.ProcessState.ExitCode(), loggableCmd.String()) + logp.L().Info("Command has completed(%d): %s", cmd.ProcessState.ExitCode(), cmd) var cmdError *SynthError = nil if err != nil { // err could be generic or it could have been killed by context timeout, log and check context // to decide which error to stream - logp.Warn("Error executing command '%s' (%d): %s", loggableCmd.String(), cmd.ProcessState.ExitCode(), err) + logp.L().Warn("Error executing command '%s' (%d): %s", cmd, cmd.ProcessState.ExitCode(), err) if errors.Is(ctx.Err(), context.DeadlineExceeded) { timeout, _ := ctx.Value(SynthexecTimeout).(time.Duration) - cmdError = ECSErrToSynthError(ecserr.NewCmdTimeoutStatusErr(timeout, loggableCmd.String())) + cmdError = ECSErrToSynthError(ecserr.NewCmdTimeoutStatusErr(timeout, cmd.String())) } else { - cmdError = ECSErrToSynthError(ecserr.NewBadCmdStatusErr(cmd.ProcessState.ExitCode(), loggableCmd.String())) + cmdError = ECSErrToSynthError(ecserr.NewBadCmdStatusErr(cmd.ProcessState.ExitCode(), cmd.String())) } } @@ -315,7 +312,7 @@ func scanToSynthEvents(rdr io.ReadCloser, transform func(bytes []byte, text stri for scanner.Scan() { se, err := transform(scanner.Bytes(), scanner.Text()) if err != nil { - logp.Warn("error parsing line: %s for line: %s", err, scanner.Text()) + logp.L().Warn("error parsing line: %s for line: %s", err, scanner.Text()) continue } if se != nil { @@ -324,7 +321,7 @@ func scanToSynthEvents(rdr io.ReadCloser, transform func(bytes []byte, text stri } if scanner.Err() != nil { - logp.Warn("error scanning synthetics runner results %s", scanner.Err()) + logp.L().Warn("error scanning synthetics runner results %s", scanner.Err()) return scanner.Err() } @@ -337,7 +334,7 @@ var stderrToSynthEvent = lineToSynthEventFactory(Stderr) // lineToSynthEventFactory is a factory that can take a line from the scanner and transform it into a *SynthEvent. func lineToSynthEventFactory(typ string) func(bytes []byte, text string) (res *SynthEvent, err error) { return func(bytes []byte, text string) (res *SynthEvent, err error) { - logp.Info("%s: %s", typ, text) + logp.L().Info("%s: %s", typ, text) return &SynthEvent{ Type: typ, TimestampEpochMicros: float64(time.Now().UnixMicro()), diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go index c90f6083a0dd..266b78ea6115 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go @@ -7,7 +7,6 @@ package synthexec import ( "os" - "os/exec" "golang.org/x/sys/unix" @@ -16,7 +15,7 @@ import ( ) func init() { - platformCmdMutate = func(cmd *exec.Cmd) { + platformCmdMutate = func(cmd *SynthCmd) { logp.L().Warn("invoking node as:", security.NodeChildProcCred, " from: ", os.Getenv("HOME")) // Note that while cmd.SysProcAtr takes a syscall.SysProcAttr object // we are passing in a unix.SysProcAttr object diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go index 762b12358a72..b4b472caf17d 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go @@ -183,7 +183,7 @@ func runAndCollect(t *testing.T, cmd *exec.Cmd, stdinStr string, cmdTimeout time cmd.Dir = filepath.Join(cwd, "testcmd") ctx := context.WithValue(context.TODO(), SynthexecTimeout, cmdTimeout) - mpx, err := runCmd(ctx, cmd, &stdinStr, nil, FilterJourneyConfig{}) + mpx, err := runCmd(ctx, &SynthCmd{cmd}, &stdinStr, nil, FilterJourneyConfig{}) require.NoError(t, err) var synthEvents []*SynthEvent