diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index e7444e338c9..55085f1401f 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -140,8 +140,6 @@ steps: - label: ":ubuntu: Auditbeat: Crosscompile" command: | make -C auditbeat crosscompile - env: - GOX_FLAGS: "-arch amd64" retry: automatic: - limit: 3 @@ -338,12 +336,12 @@ steps: - group: "Packaging" key: "auditbeat-packaging" + steps: - label: ":ubuntu: Auditbeat: Packaging Linux" key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true command: | set -euo pipefail cd auditbeat @@ -365,7 +363,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true command: | set -euo pipefail cd auditbeat diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh index d5d8c690891..012bf92f918 100755 --- a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh +++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh @@ -2,6 +2,7 @@ set -euo pipefail +WORKSPACE=$(pwd) export KUBECONFIG="${WORKSPACE}/kubecfg" export BIN="${WORKSPACE}/bin" diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 053dc24e080..9dde05ccc81 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -20,6 +20,7 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Integration Tests K8S_VERSION: "v1.29.0" ASDF_KIND_VERSION: "0.20.0" @@ -302,6 +303,8 @@ steps: steps: - label: ":ubuntu: Filebeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd filebeat mage package @@ -315,14 +318,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "filebeat: Packaging Linux" - label: ":ubuntu: Filebeat: Packaging arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd filebeat mage package @@ -334,9 +338,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "filebeat: Packaging Linux arm64" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 8da78502858..27cf5a0d847 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -20,6 +20,7 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + ASDF_NODEJS_VERSION: 18.17.1 # Unit tests RACE_DETECTOR: "true" @@ -324,12 +325,12 @@ steps: - group: "Packaging" key: "heartbeat-packaging" + steps: - label: ":ubuntu: Heartbeat: Packaging Linux" key: "heartbeat-package-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true command: | cd heartbeat mage package @@ -350,7 +351,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true command: | cd heartbeat mage package diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f860e5fcff7..6468046facc 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,29 +6,28 @@ set -euo pipefail # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" -ENABLED_BEATS_PIPELINES_SLUGS=( - "auditbeat" - "heartbeat" - "filebeat" - "beats-metricbeat" - "beats-winlogbeat" - "beats-xpack-winlogbeat" - "beats-xpack-dockerlogbeat" - "beats-xpack-auditbeat" - "beats-xpack-metricbeat" - "beats-xpack-heartbeat" - "deploy-k8s" -) - -for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do - if [[ "$BUILDKITE_PIPELINE_SLUG" == "$slug" ]]; then - source .buildkite/scripts/setenv.sh - break - fi -done - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET fi +CPU_ARCH=$(uname -m) +PLATFORM_TYPE=$(uname) + +if [[ "${CPU_ARCH}" == "x86_64" ]]; then + case "${PLATFORM_TYPE}" in + Linux|Darwin) + export GOX_FLAGS="-arch amd64" + export GO_ARCH_TYPE="amd64" + ;; + MINGW*) + export GOX_FLAGS="-arch 386" + ;; + esac +elif [[ "${CPU_ARCH}" == "aarch64" || "${CPU_ARCH}" == "arm64" ]]; then + export GOX_FLAGS="-arch arm" + export GO_ARCH_TYPE="arm64" +else + echo "Unsupported OS" + exit 1 +fi diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index f48370b0653..0efad780523 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -19,6 +19,7 @@ env: steps: - group: "Check/Update" key: "libbeat-check-update" + steps: - label: "Libbeat: Run check/update" command: | @@ -46,6 +47,7 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" + steps: - label: ":ubuntu: Libbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 9090867c563..c6081bd4c29 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -318,6 +318,8 @@ steps: steps: - label: ":ubuntu: Metricbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd metricbeat mage package @@ -331,14 +333,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "metricbeat: Packaging Linux" - label: ":ubuntu: Metricbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd metricbeat mage package @@ -350,9 +353,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "metricbeat: Packaging Linux arm64" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 91c32e8f27f..9550c7fae92 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -280,6 +280,8 @@ steps: steps: - label: ":ubuntu: Packetbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd packetbeat mage package @@ -293,14 +295,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "packetbeat: Packaging Linux" - label: ":ubuntu: Packetbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd packetbeat mage package @@ -312,9 +315,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "packetbeat: Packaging Linux arm64" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh deleted file mode 100755 index 55e97890481..00000000000 --- a/.buildkite/scripts/common.sh +++ /dev/null @@ -1,481 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -WORKSPACE=${WORKSPACE:-"$(pwd)"} -BIN="${WORKSPACE}/bin" -platform_type="$(uname)" -platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') -arch_type="$(uname -m)" -GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} -GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} -ONLY_DOCS=${ONLY_DOCS:-"true"} -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -# define if needed run the whole pipeline for the particular beat -[ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" -[ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" - -# define if needed run ARM platform-specific tests for the particular beat -[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" - -# define if needed run MacOS platform-specific tests for the particular beat -[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" - -# define if needed run cloud-specific tests for the particular beat -[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" - -ci_changeset=( - "^.buildkite/.*" - ) - -go_mod_changeset=( - "^go.mod" - ) - -oss_changeset=( - "^go.mod" - "^pytest.ini" - "^dev-tools/.*" - "^libbeat/.*" - "^testing/.*" -) - -xpack_changeset=( - "${oss_changeset[@]}" -) - -docs_changeset=( - ".*\\.(asciidoc|md)" - "deploy/kubernetes/.*-kubernetes\\.yaml" - ) - -packaging_changeset=( - "^dev-tools/packaging/.*" - ".go-version" - ) - -case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-xpack-metricbeat") - BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} - ;; - *) - echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." - ;; -esac - -check_and_set_beat_vars() { - local BEATS_PROJECT_NAME=${BEATS_PROJECT_NAME:=""} - if [[ "${BEATS_PROJECT_NAME:=""}" == *"x-pack/"* ]]; then - BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - - BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ - BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels - BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} - TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" - TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" - TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" - TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" - TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_win_tests" - echo "--- Beats project name is $BEATS_XPACK_PROJECT_NAME" - mandatory_changeset=( - "${BEAT_CHANGESET_REFERENCE[@]}" - "${xpack_changeset[@]}" - "${ci_changeset[@]}" - ) - else - BEATS_GH_LABEL=${BEATS_PROJECT_NAME} - TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" - TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" - TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" - TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" - TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_PROJECT_NAME}_win_tests" - echo "--- Beats project name is $BEATS_PROJECT_NAME" - mandatory_changeset=( - "${BEAT_CHANGESET_REFERENCE[@]}" - "${oss_changeset[@]}" - "${ci_changeset[@]}" - ) - fi - BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" - BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" - BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" - BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" - BEATS_GH_WIN_COMMENT="${BEATS_GH_COMMENT} for windows" - BEATS_GH_MACOS_LABEL="macOS" - BEATS_GH_ARM_LABEL="arm" - BEATS_GH_AWS_LABEL="aws" - BEATS_GH_WIN_LABEL="windows" -} - -with_docker_compose() { - local version=$1 - echo "Setting up the Docker-compose environment..." - create_workspace - retry 3 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" - chmod +x ${BIN}/docker-compose - export PATH="${BIN}:${PATH}" - docker-compose version -} - -create_workspace() { - if [[ ! -d "${BIN}" ]]; then - mkdir -p "${BIN}" - fi -} - -add_bin_path() { - echo "Adding PATH to the environment variables..." - create_workspace - export PATH="${BIN}:${PATH}" -} - -check_platform_architeture() { - case "${arch_type}" in - "x86_64") - go_arch_type="amd64" - ;; - "aarch64") - go_arch_type="arm64" - ;; - "arm64") - go_arch_type="arm64" - ;; - *) - echo "The current platform or OS type is unsupported yet" - ;; - esac -} - -with_mage() { - local install_packages=( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "gotest.tools/gotestsum" - ) - create_workspace - for pkg in "${install_packages[@]}"; do - go install "${pkg}@latest" - done - echo "Download modules to local cache" - retry 3 go mod download -} - -with_go() { - echo "Setting up the Go environment..." - create_workspace - check_platform_architeture - retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${go_arch_type}" - chmod +x "${BIN}/gvm" - eval "$(gvm $GO_VERSION)" - go version - which go - local go_path="$(go env GOPATH):$(go env GOPATH)/bin" - export PATH="${go_path}:${PATH}" -} - -checkLinuxType() { - if [ "${platform_type}" == "Linux" ]; then - if grep -q 'ubuntu' /etc/os-release; then - echo "ubuntu" - elif grep -q 'rhel' /etc/os-release; then - echo "rhel" - else - echo "Unsupported Linux" - fi - else - echo "This is not a Linux" - fi -} - -with_python() { - local linuxType="$(checkLinuxType)" - echo "${linuxType}" - if [ "${platform_type}" == "Linux" ]; then - if [ "${linuxType}" = "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y python3-pip python3-venv - elif [ "${linuxType}" = "rhel" ]; then - sudo dnf update -y - sudo dnf install -y python3 python3-pip - pip3 install virtualenv - fi - elif [ "${platform_type}" == "Darwin" ]; then - brew update - pip3 install virtualenv - ulimit -Sn 10000 - fi -} - -with_dependencies() { - local linuxType="$(checkLinuxType)" - echo "${linuxType}" - if [ "${platform_type}" == "Linux" ]; then - if [ "${linuxType}" = "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y libsystemd-dev libpcap-dev librpm-dev - elif [ "${linuxType}" = "rhel" ]; then - # sudo dnf update -y - sudo dnf install -y systemd-devel rpm-devel - wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image - sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image - fi - elif [ "${platform_type}" == "Darwin" ]; then - pip3 install libpcap - fi -} - -config_git() { - if [ -z "$(git config --get user.email)" ]; then - git config --global user.email "beatsmachine@users.noreply.github.com" - git config --global user.name "beatsmachine" - fi -} - -retry() { - local retries=$1 - shift - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - >&2 echo "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - return 0 -} - -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -are_changed_only_paths() { - local patterns=("${@}") - local changed_files=($(git diff --name-only HEAD@{1} HEAD)) - local matched_files=() - for pattern in "${patterns[@]}"; do - local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}")) - if [ "${#matched[@]}" -gt 0 ]; then - matched_files+=("${matched[@]}") - fi - done - if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then - return 0 - fi - return 1 -} - -are_conditions_met_mandatory_tests() { - if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then - return 0 - fi - return 1 -} - -are_conditions_met_macos_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi - fi - return 1 -} - -are_conditions_met_aws_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi - fi - return 1 -} - -are_conditions_met_packaging() { - if are_conditions_met_mandatory_tests; then - if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - return 0 - fi - fi - return 1 -} - -defineModuleFromTheChangeSet() { - # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. - # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. - # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module - local project_path=$1 - local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g') - local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$" - local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - - if [[ "$project_path" == *"x-pack/"* ]]; then - local pattern=("$XPACK_MODULE_PATTERN") - else - local pattern=("$OSS_MODULE_PATTERN") - fi - local changed_modules="" - local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d) - for module_dir in $module_dirs; do - if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then - if [[ -z "$changed_modules" ]]; then - changed_modules=$(basename "$module_dir") - else - changed_modules+=",$(basename "$module_dir")" - fi - fi - done - if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - export MODULE="aws" - else - export MODULE="kubernetes" - fi - else - export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - fi -} - -terraformInit() { - local dir=$1 - echo "Terraform Init on $dir" - pushd "${dir}" > /dev/null - terraform init - popd > /dev/null -} - -withAWS() { - # This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names. - export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY - export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY - export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" -} - -startCloudTestEnv() { - local dir=$1 - withAWS - echo "--- Run docker-compose services for emulated cloud env" - docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d - with_Terraform - terraformInit "$dir" - export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') - export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" - export TF_VAR_CREATED_DATE=$(date +%s) - export TF_VAR_ENVIRONMENT="ci" - export TF_VAR_REPO="${REPO}" - pushd "${dir}" > /dev/null - terraform apply -auto-approve - popd > /dev/null -} - -withNodeJSEnv() { - # HOME="${WORKSPACE}" - local version=$1 - # local nvmPath="${HOME}/.nvm/versions/node/${version}/bin" - echo "Installing nvm" - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - echo "Installing the NodeJs version $version" - nvm install "$version" - # export PATH="${nvmPath}:${PATH}" - nvm use "$version" - node --version -} - -installNodeJsDependencies() { - # Install dependencies to run browsers - if [ "${platform_type}" == "Linux" ]; then - sudo apt-get install -y \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libcups2 \ - libxkbcommon0 \ - libatspi2.0-0 \ - libxcomposite1 \ - libxdamage1 \ - libxfixes3 \ - libxrandr2 \ - libgbm1 \ - libpango-1.0-0 \ - libcairo2 \ - libasound2 - if [ $? -ne 0 ]; then - echo "Error: Failed to install dependencies." - exit 1 - else - echo "Dependencies installed successfully." - fi - elif [ "${platform_type}" == "Darwin" ]; then - echo "TBD" - else - echo "Unsupported platform type." - exit 1 - fi -} - -teardown() { - # Teardown resources after using them - echo "---Terraform Cleanup" - .buildkite/scripts/terraform-cleanup.sh "${MODULE_DIR}" - - echo "---Docker Compose Cleanup" - docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v -} - -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then - unset "$var" - fi - done -} - -if ! are_changed_only_paths "${docs_changeset[@]}" ; then - export ONLY_DOCS="false" - echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." -else - echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." -fi - -if are_paths_changed "${go_mod_changeset[@]}" ; then - export GO_MOD_CHANGES="true" -fi - -if are_paths_changed "${packaging_changeset[@]}" ; then - export PACKAGING_CHANGES="true" -fi - -if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then - # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" -fi - -check_and_set_beat_vars diff --git a/.buildkite/scripts/initCloudEnv.sh b/.buildkite/scripts/initCloudEnv.sh index ac6cb185788..e6c15a15966 100755 --- a/.buildkite/scripts/initCloudEnv.sh +++ b/.buildkite/scripts/initCloudEnv.sh @@ -21,7 +21,7 @@ exportAwsSecrets() { } terraformApply() { - echo "Exporting Terraform Env Vars" + echo "~~~ Exporting Terraform Env Vars" TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') TF_VAR_CREATED_DATE=$(date +%s) export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" @@ -30,10 +30,10 @@ terraformApply() { export TF_VAR_BRANCH export TF_VAR_CREATED_DATE - echo "Terraform Init on $MODULE_DIR" + echo "~~~ Terraform Init on $MODULE_DIR" terraform -chdir="$MODULE_DIR" init - echo "Terraform Apply on $MODULE_DIR" + echo "~~~ Terraform Apply on $MODULE_DIR" terraform -chdir="$MODULE_DIR" apply -auto-approve } diff --git a/.buildkite/scripts/install_macos_tools.sh b/.buildkite/scripts/install_macos_tools.sh index 4572afb4cdc..a29606a922d 100755 --- a/.buildkite/scripts/install_macos_tools.sh +++ b/.buildkite/scripts/install_macos_tools.sh @@ -27,21 +27,6 @@ retry() { return 0 } -define_arch() { - local platform_type="$(uname)" - local arch_type="$(uname -m)" - if [ "${arch_type}" == "x86_64" ]; then - export GOX_FLAGS="-arch amd64" - go_arch_type="amd64" - elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then - export GOX_FLAGS="-arch arm" - go_arch_type="arm64" - else - echo "+++ Unsupported OS archictecture; uname: $platform_type and uname -m: $arch_type" - exit 1 - fi -} - create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -83,8 +68,7 @@ with_mage() { with_go() { echo "Setting up the Go environment..." create_workspace - define_arch - retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE_LOWERCASE}-${go_arch_type}" + retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE_LOWERCASE}-${GO_ARCH_TYPE}" chmod +x "${BIN}/gvm" eval "$(gvm $GO_VERSION)" go version diff --git a/.buildkite/scripts/packaging/package-util.sh b/.buildkite/scripts/packaging/package-util.sh deleted file mode 100755 index 4a50457cc9c..00000000000 --- a/.buildkite/scripts/packaging/package-util.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -is_pr() { - if [[ $BUILDKITE_PULL_REQUEST != false ]]; then - return 0 - else - return 1 - fi -} - -define_tags() { - aliasVersion="${VERSION%.*}${IMG_POSTFIX}" - tags=("${BUILDKITE_COMMIT}") - - if is_pr; then - tags+=("pr-${GITHUB_PR_NUMBER}") - else - tags+=("${SOURCE_TAG}" "${aliasVersion}") - fi -} diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh deleted file mode 100755 index 5744ee0776b..00000000000 --- a/.buildkite/scripts/packaging/package.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/packaging/package-util.sh - -IMG_POSTFIX="-SNAPSHOT" -VARIANTS=("" "-ubi" "-oss") -VERSION="$(make get-version)" -SOURCE_TAG+="${VERSION}${IMG_POSTFIX}" -TARGET="observability-ci/${BEATS_PROJECT_NAME}" - -echo "--- Creating package" -mage -d "${BEATS_PROJECT_NAME}" package - -echo "--- Distribution list" -dir="${BEATS_PROJECT_NAME}/build/distributions" -buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512" - -echo "--- Docker image list" -docker images - -define_tags - -targetSuffix="" -if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then - targetSuffix="-arm64" -fi - -for variant in "${VARIANTS[@]}"; do - source="beats/${BEATS_PROJECT_NAME}${variant}" - - for tag in "${tags[@]}"; do - targetTag=$tag${targetSuffix} - - sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}" - targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}" - #TODO Remove following line once beats fully migrated to Buildkite and Jenkins builds will be disabled - #Avoid clashing with the Jenkins produced images - targetName="${targetName}-buildkite" - - if docker image inspect "${sourceName}" &>/dev/null; then - echo "--- Tag & Push with target: $targetName" - echo "Source name: $sourceName" - docker tag "$sourceName" "$targetName" - docker push "$targetName" - else - echo "Docker image ${sourceName} does not exist" - fi - done -done diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh deleted file mode 100755 index 56f8d7257d6..00000000000 --- a/.buildkite/scripts/setenv.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -WORKSPACE=${WORKSPACE:-"$(pwd)"} -GO_VERSION=$(cat .go-version) - -export REPO="beats" -export DOCKER_REGISTRY="docker.elastic.co" -export SETUP_GVM_VERSION="v0.5.1" -export DOCKER_COMPOSE_VERSION="1.21.0" -export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" - -export ASDF_NODEJS_VERSION="18.17.1" -export AWS_REGION="eu-central-1" - -export WORKSPACE -export GO_VERSION - -exportVars() { - local platform_type="$(uname)" - local arch_type="$(uname -m)" - if [ "${arch_type}" == "x86_64" ]; then - case "${platform_type}" in - Linux|Darwin) - export GOX_FLAGS="-arch amd64" - export testResults="**/build/TEST*.xml" - export artifacts="**/build/TEST*.out" - ;; - MINGW*) - export GOX_FLAGS="-arch 386" - export testResults="**\\build\\TEST*.xml" - export artifacts="**\\build\\TEST*.out" - ;; - esac - elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then - export GOX_FLAGS="-arch arm" - export testResults="**/build/TEST*.xml" - export artifacts="**/build/TEST*.out" - else - echo "Unsupported OS" - fi -} - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then - exportVars - export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" -fi - -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then - source .buildkite/scripts/common.sh - # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" -fi diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index b3122c23168..2e79a4bcf28 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -194,6 +194,8 @@ steps: steps: - label: ":ubuntu: Winlogbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd winlogbeat mage package @@ -205,8 +207,6 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "winlogbeat: Packaging Linux" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 1c7e675a57e..3e3baa4b138 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -290,6 +290,8 @@ steps: steps: - label: ":ubuntu: x-pack/auditbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd x-pack/auditbeat mage package @@ -303,14 +305,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/auditbeat: Packaging Linux" - label: ":ubuntu: x-pack/auditbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd x-pack/auditbeat mage package @@ -322,9 +325,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/auditbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 526785e61ac..30264774b63 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -108,7 +108,6 @@ steps: key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true command: | cd x-pack/dockerlogbeat mage package @@ -129,7 +128,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true command: | cd x-pack/dockerlogbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index a641390727c..df7b39f3f03 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -295,18 +295,18 @@ steps: skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467" # Related issue: https://github.com/elastic/ingest-dev/issues/3467 if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - command: | - set -euo pipefail - source .buildkite/scripts/initCloudEnv.sh - echo "~~~ Running tests" - cd x-pack/filebeat - mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" MODULE: "aws" # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" + command: | + set -euo pipefail + source .buildkite/scripts/initCloudEnv.sh + echo "~~~ Running tests" + cd x-pack/filebeat + mage build test goIntegTest agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -332,6 +332,8 @@ steps: steps: - label: ":linux: x-pack/filebeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd x-pack/filebeat mage package @@ -345,14 +347,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/filebeat: Packaging Linux" - label: ":linux: x-pack/filebeat: Packaging arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd x-pack/filebeat mage package @@ -364,9 +367,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/filebeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 1227b8e7570..53dbab37716 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -286,6 +286,8 @@ steps: steps: - label: ":ubuntu: x-pack/heartbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd x-pack/heartbeat mage package @@ -299,14 +301,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/heartbeat: Packaging Linux" - label: ":ubuntu: x-pack/heartbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd x-pack/heartbeat mage package @@ -318,9 +321,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/heartbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 693b10c8683..64ca511350d 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -78,6 +78,8 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Go (MODULE) Integration Tests" key: "mandatory-int-test" + env: + TEST_TAGS: "oracle" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -101,6 +103,8 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Python (MODULE) Integration Tests" key: "mandatory-python-int-test" + env: + TEST_TAGS: "oracle" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -280,18 +284,18 @@ steps: - label: ":ubuntu: x-pack/metricbeat: AWS Tests" key: "x-pack-metricbeat-extended-cloud-test" if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - command: | - set -euo pipefail - source .buildkite/scripts/initCloudEnv.sh - echo "~~~ Running tests" - cd x-pack/metricbeat - mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/metricbeat/module/aws" MODULE: "aws" # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" + command: | + set -euo pipefail + source .buildkite/scripts/initCloudEnv.sh + echo "~~~ Running tests" + cd x-pack/metricbeat + mage build test goIntegTest agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -317,6 +321,8 @@ steps: steps: - label: ":ubuntu: x-pack/metricbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd x-pack/metricbeat mage package @@ -330,14 +336,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/metricbeat: Packaging Linux" - label: ":ubuntu: x-pack/metricbeat: Packaging linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd x-pack/metricbeat mage package @@ -349,9 +356,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/metricbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 18ac6cac9a7..3072c7b5bda 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -257,6 +257,8 @@ steps: steps: - label: ":ubuntu: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd x-pack/osquerybeat mage package @@ -270,8 +272,6 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/osquerybeat: Packaging Linux" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 55f944352aa..6b96973d122 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -347,6 +347,8 @@ steps: steps: - label: ":ubuntu: x-pack/packetbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd x-pack/packetbeat mage package @@ -360,14 +362,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/packetbeat: Packaging Linux" - label: ":ubuntu: x-pack/packetbeat: Packaging arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd x-pack/packetbeat mage package @@ -379,9 +382,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/packetbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index fc561ef2fae..7853f651670 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -198,7 +198,11 @@ steps: steps: - label: ":ubuntu: x-pack/winlogbeat: Packaging Linux" key: "packaging-linux" - command: "cd x-pack/winlogbeat && mage package" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: | + cd x-pack/winlogbeat + mage package retry: automatic: - limit: 3 @@ -209,8 +213,6 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/winlogbeat: Packaging Linux"