From 27b4bae916d7cd9fe13b6246424c717901bd3ac1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:36:12 +0300 Subject: [PATCH] move x-pack/packetbeat to static pipeline (#39072) * move x-pack/packetbeat to static pipeline * Update pull-requests.json * change pipeline.yml * update pipeline.xpack.packetbeat.yml * fix typo * fix dependency * fix remarks * Update common.sh * Update common.sh * Update common.sh * Update common.sh * fix remarks * fix remarks * fix remarks --- .buildkite/hooks/pre-command | 1 - .buildkite/pipeline.yml | 27 ++ .buildkite/pull-requests.json | 16 - .buildkite/scripts/common.sh | 10 - .../generate_xpack_packetbeat_pipeline.sh | 285 --------------- .../x-pack/pipeline.xpack.packetbeat.yml | 327 +++++++++++++++--- 6 files changed, 307 insertions(+), 359 deletions(-) delete mode 100644 .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index a014542c9c38..7144331fc284 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -61,7 +61,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=( "beats-metricbeat" "beats-winlogbeat" "beats-winlogbeat" - "beats-xpack-packetbeat" "beats-xpack-winlogbeat" "beats-xpack-dockerlogbeat" "beats-xpack-auditbeat" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7deb787b6afd..00ce04b71d21 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -367,6 +367,33 @@ steps: - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger x-pack/packetbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - x-pack/packetbeat/ + - .buildkite/x-pack/pipeline.xpack.packetbeat.yml + - .buildkite/scripts/ + - .buildkite/hooks/ + # x-pack + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + - x-pack/libbeat/ + config: + trigger: "beats-xpack-packetbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger Winlogbeat" plugins: - monorepo-diff#v1.0.1: diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index c98c1f07578d..b79646acd1bd 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -47,22 +47,6 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-packetbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/packetbeat$", - "always_trigger_comment_regex": "^/test x-pack/packetbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] } \ No newline at end of file diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2aebe4b5638c..9bd5cf93540c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -14,15 +14,12 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" -[ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" -[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" -[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" @@ -35,10 +32,6 @@ xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) -xpack_packetbeat_changeset=( - "^x-pack/packetbeat/.*" - ) - ci_changeset=( "^.buildkite/.*" ) @@ -73,9 +66,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; - "beats-xpack-packetbeat") - BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]} - ;; *) echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh deleted file mode 100644 index 97f37ca734bf..000000000000 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.xpack-packetbeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - - label: ":linux: Ubuntu System Tests" - key: "mandatory-linux-system-test" - command: "cd $BEATS_PROJECT_NAME && mage systemTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu System Tests" - - - label: ":rhel: RHEL9 Unit Tests" - key: "mandatory-rhel9-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2022-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - - - label: ":windows: Windows 2022 System Tests" - skip: "see https://github.com/elastic/beats/issues/38142" - key: "mandatory-win-2022-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - -## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-10-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-11-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-2019-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - - - label: ":windows: Windows 10 System Tests" - skip: "see https://github.com/elastic/beats/issues/38142" - key: "extended-win-10-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" - -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 5e3ce87e2bf4..77fdf2af8483 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -3,7 +3,6 @@ name: "beats-xpack-packetbeat" env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" @@ -18,54 +17,288 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "x-pack/packetbeat Mandatory Tests" + key: "x-pack-packetbeat-mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: | + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Ubuntu Unit Tests" + + - label: ":linux: Ubuntu System Tests" + key: "mandatory-linux-system-test" + command: | + cd x-pack/packetbeat + mage systemTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Ubuntu System Tests" + + - label: ":rhel: RHEL9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: | + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: RHEL9 Unit Tests" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2016 Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2022 Unit Tests" + + - label: ":windows: Windows 2022 System Tests" + key: "mandatory-win-2022-system-tests" + skip: "skipping due to elastic/beats#38142" + command: | + Set-Location -Path x-pack/packetbeat + mage systemTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2022 System Tests" + + - group: "x-pack/packetbeat Extended Windows Tests" + key: "x-pack-packetbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 11 Unit Tests" + + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2019 Unit Tests" + + - label: ":windows: Windows 10 System Tests" + key: "extended-win-10-system-tests" + skip: "skipping due to elastic/beats#38142" + command: | + Set-Location -Path x-pack/packetbeat + mage systemTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2022 System Tests" + + - group: "x-pack/packetbeat Linux arm Extended Tests" + key: "x-pack-packetbeat-extended-linux-arm-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: | + cd x-pack/packetbeat + mage build unitTest + if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Ubuntu ARM Unit Tests" + + - group: "x-pack/packetbeat MacOS Extended Tests" + key: "x-pack-packetbeat-extended-macos-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: MacOS Unit Tests" + + - label: ":mac: MacOS arm64 Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: MacOS arm64 Unit Tests" + + - group: "x-pack/packetbeat Packaging" + key: "x-pack-packetbeat-packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - step: "x-pack-packetbeat-mandatory-tests" + allow_failure: false + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: | + cd x-pack/packetbeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Packaging Linux" - - input: "Input Parameters" - key: "force-run-stages" - fields: - - select: "Packetbeat - run_xpack_packetbeat" - key: "run_xpack_packetbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Packetbeat - run_xpack_packetbeat_macos_tests" - key: "run_xpack_packetbeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Packetbeat - run_xpack_packetbeat_arm_tests" - key: "run_xpack_packetbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - if: "build.source == 'ui'" - - - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic x-pack packetbeat pipeline" - key: "packetbeat-pipeline" - command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: | + cd x-pack/packetbeat + mage package + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Packaging Linux ARM"