From fb400c5840b096e28454618103d3f2d956ba8ea2 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 12:46:16 +0000 Subject: [PATCH] Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps (#39211) (#39610) Updated x-pack/filebeat and x-pack/metricbeat pipelines to match same steps & commands as in Jenkins --- .../{cloud_tests.sh => setup_cloud_env.sh} | 7 - .buildkite/x-pack/pipeline.xpack.filebeat.yml | 137 +++++++++--------- .../x-pack/pipeline.xpack.metricbeat.yml | 33 +++-- 3 files changed, 84 insertions(+), 93 deletions(-) rename .buildkite/scripts/{cloud_tests.sh => setup_cloud_env.sh} (92%) mode change 100755 => 100644 diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/setup_cloud_env.sh old mode 100755 new mode 100644 similarity index 92% rename from .buildkite/scripts/cloud_tests.sh rename to .buildkite/scripts/setup_cloud_env.sh index c8f3737e13e3..3c23d71573f1 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/setup_cloud_env.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -euo pipefail - REPO_DIR=$(pwd) teardown() { @@ -31,7 +30,6 @@ tf_cleanup() { trap 'teardown' EXIT # Prepare the cloud resources using Terraform -#startCloudTestEnv "${MODULE_DIR}" echo "~~~ Loading creds" set +o xtrace export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY @@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci" export TF_VAR_REPO="${REPO}" terraform init && terraform apply -auto-approve cd - - -# Run tests -echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" -cd "${BEATS_PROJECT_NAME}" -mage build test diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index f986af28653e..93b3c2d536ed 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-filebeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -29,7 +29,7 @@ steps: - group: "x-pack/filebeat Mandatory Tests" key: "x-pack-filebeat-mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" + - label: ":ubuntu: x-pack/filebeat Unit Tests" key: "x-pack-filebeat-mandatory-linux-unit-test" command: | cd x-pack/filebeat @@ -46,17 +46,13 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Ubuntu Unit Tests" + context: "x-pack/filebeat: Unit Tests" - - label: ":go: Go (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/filebeat Go Integration Tests" key: "x-pack-filebeat-mandatory-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests" - cd x-pack/filebeat && mage goIntegTest + cd x-pack/filebeat + mage goIntegTest retry: automatic: - limit: 3 @@ -69,17 +65,13 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Go (MODULE) Integration Tests" + context: "x-pack/filebeat: Go Integration Tests" - - label: ":python: Python (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/filebeat Python Integration Tests" key: "x-pack-filebeat-mandatory-python-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests" - cd x-pack/filebeat && mage pythonIntegTest + cd x-pack/filebeat + mage pythonIntegTest retry: automatic: - limit: 3 @@ -92,9 +84,9 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Python (MODULE) Integration Tests" + context: "x-pack/filebeat: Python Integration Tests" - - label: ":windows: Windows 2022 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2022 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -113,9 +105,9 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2022 Unit Tests" + context: "x-pack/filebeat: Win-2022 Unit Tests" - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -134,32 +126,14 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2016 Unit Tests" - - - label: ":linux: Ubuntu ARM Unit Tests" - key: "x-pack-filebeat-mandatory-linux-arm-unit-test" - command: | - cd x-pack/filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Ubuntu ARM Unit Tests" + context: "x-pack/filebeat: Win-2016 Unit Tests" - group: "x-pack/filebeat Extended Windows Tests" key: "x-pack-filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -178,9 +152,9 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 2019 Unit Tests" + context: "x-pack/filebeat: Win-2019 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: x-pack/filebeat Win 10 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -199,9 +173,9 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 10 Unit Tests" + context: "x-pack/filebeat: Win 10 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" + - label: ":windows: x-pack/filebeat Win 11 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -220,17 +194,36 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Windows 11 Unit Tests" + context: "x-pack/filebeat: Win 11 Unit Tests" - group: "x-pack/filebeat Extended Tests" key: "x-pack-filebeat-extended-tests" + steps: - - label: ":mac: MacOS x86_64 Unit Tests" + - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + key: "x-pack-filebeat-mandatory-linux-arm-unit-test" + command: | + cd x-pack/filebeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Linux arm64 Unit Tests" + + - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat && mage build unitTest + cd x-pack/filebeat + mage build unitTest retry: automatic: - limit: 3 @@ -242,18 +235,20 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: MacOS x86_64 Unit Tests" + context: "x-pack/filebeat: macOS x86_64 Unit Tests" - - label: ":mac: MacOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" + - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests" + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat && mage build unitTest + cd x-pack/filebeat + mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" @@ -262,9 +257,9 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: MacOS arm64 Unit Tests" + context: "x-pack/filebeat: macOS arm64 Unit Tests" - - label: ":linux: Cloud (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test" if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | @@ -273,12 +268,12 @@ steps: source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat echo "~~~ Running tests" - - .buildkite/scripts/cloud_tests.sh + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: @@ -292,9 +287,10 @@ steps: - github_commit_status: context: "x-pack/filebeat: Cloud (MODULE) Tests" - - label: ":linux: Cloud AWS (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test-aws" - skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425" + skip: "Skipping due to elastic/beats#36425" + # https://github.com/elastic/beats/issues/36425 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail @@ -302,16 +298,17 @@ steps: source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat echo "~~~ Running tests" - .buildkite/scripts/cloud_tests.sh + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/filebeat/build/*.xml" @@ -331,7 +328,7 @@ steps: - group: "x-pack/filebeat Packaging" key: "x-pack-filebeat-packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: x-pack/filebeat Packaging Linux" key: "packaging-linux" command: | cd x-pack/filebeat @@ -352,7 +349,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: x-pack/filebeat Packaging arm64" key: "packaging-arm" command: | cd x-pack/filebeat @@ -363,11 +360,11 @@ steps: timeout_in_minutes: 20 agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" notify: - github_commit_status: - context: "x-pack/filebeat: Packaging Linux ARM" + context: "x-pack/filebeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 701e01c972f0..4156850f7ae8 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-metricbeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -227,7 +227,8 @@ steps: context: "x-pack/metricbeat: MacOS x86_64 Unit Tests" - label: ":mac: MacOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail @@ -248,22 +249,21 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test" - skip: "doesn't belong in a stage in Jenkins, thus skipped" + skip: "Not a stage on Jenkins & elastic/ingest-dev#3170" + # https://github.com/elastic/ingest-dev/issues/3170 if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat echo "~~~ Running tests" - - .buildkite/scripts/cloud_tests.sh + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: @@ -279,26 +279,27 @@ steps: - label: ":linux: Cloud AWS (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test-aws" - skip: "https://github.com/elastic/beats/issues/36425" - # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 + skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3170" + # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3170 if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat + echo "~~~ Running tests" - .buildkite/scripts/cloud_tests.sh + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/metricbeat/build/*.xml" @@ -350,7 +351,7 @@ steps: timeout_in_minutes: 20 agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64"