From 6d85349865137a6d19f422c6fbf472a038a4c764 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 15 Jul 2024 10:45:18 +0300 Subject: [PATCH] Postmigration setenv.sh && common.sh cleanup 7.17 (#40146) Removed unused/unnecessary scripts and updated its' usage in other corresponding scripts --- .buildkite/auditbeat/auditbeat-pipeline.yml | 4 - .../kubernetes/scripts/setup-k8s-env.sh | 2 +- .buildkite/filebeat/filebeat-pipeline.yml | 10 +- .buildkite/heartbeat/heartbeat-pipeline.yml | 3 +- .buildkite/hooks/pre-command | 42 +- .buildkite/metricbeat/pipeline.yml | 12 +- .buildkite/packetbeat/pipeline.packetbeat.yml | 10 +- .buildkite/scripts/common.sh | 477 ------------------ .buildkite/scripts/initCloudEnv.sh | 2 +- .buildkite/scripts/install_macos_tools.sh | 18 +- .buildkite/scripts/setenv.sh | 54 -- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 4 +- .../elastic-agent/scripts/steps/setenv.sh | 2 +- 13 files changed, 45 insertions(+), 595 deletions(-) delete mode 100755 .buildkite/scripts/common.sh delete mode 100755 .buildkite/scripts/setenv.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index e5f950c98d5..f49326990a2 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -124,8 +124,6 @@ steps: - label: ":ubuntu: Auditbeat: Crosscompile" command: | make -C auditbeat crosscompile - env: - GOX_FLAGS: "-arch amd64" retry: automatic: - limit: 3 @@ -244,7 +242,6 @@ steps: key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" - SNAPSHOT: true command: | cd auditbeat mage package @@ -265,7 +262,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true command: | cd auditbeat mage package diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh index 7d2bbfb2256..012bf92f918 100755 --- a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh +++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh @@ -12,7 +12,7 @@ if [[ ! -d "${BIN}" ]]; then fi export PATH="${PATH}:${BIN}" -echo "--- Installing kind & kubectl" +echo "~~~ Installing kind & kubectl" asdf plugin add kind asdf install kind "$ASDF_KIND_VERSION" diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 2715a2f2caa..22b47b4ce21 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -285,6 +285,8 @@ steps: steps: - label: ":ubuntu: Filebeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" command: | cd filebeat mage package @@ -298,14 +300,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: context: "filebeat: Packaging Linux" - label: ":ubuntu: Filebeat: Packaging arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd filebeat mage package @@ -317,9 +320,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "filebeat: Packaging arm64" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 7591cb8b93a..c4f8ef7f6fe 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -20,6 +20,7 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 + ASDF_NODEJS_VERSION: 18.17.1 # Unit tests RACE_DETECTOR: "true" @@ -269,7 +270,6 @@ steps: key: "heartbeat-package-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" - SNAPSHOT: true command: | cd heartbeat mage package @@ -290,7 +290,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true command: | cd heartbeat mage package diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ac3245831a9..6468046facc 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,28 +6,28 @@ set -euo pipefail # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" -ENABLED_BEATS_PIPELINES_SLUGS=( - "auditbeat" - "heartbeat" - "filebeat" - "beats-metricbeat" - "beats-winlogbeat" - "beats-xpack-winlogbeat" - "beats-xpack-dockerlogbeat" - "beats-xpack-auditbeat" - "beats-xpack-metricbeat" - "beats-xpack-heartbeat" - "deploy-k8s" -) - -for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do - if [[ "$BUILDKITE_PIPELINE_SLUG" == "$slug" ]]; then - source .buildkite/scripts/setenv.sh - break - fi -done - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET fi + +CPU_ARCH=$(uname -m) +PLATFORM_TYPE=$(uname) + +if [[ "${CPU_ARCH}" == "x86_64" ]]; then + case "${PLATFORM_TYPE}" in + Linux|Darwin) + export GOX_FLAGS="-arch amd64" + export GO_ARCH_TYPE="amd64" + ;; + MINGW*) + export GOX_FLAGS="-arch 386" + ;; + esac +elif [[ "${CPU_ARCH}" == "aarch64" || "${CPU_ARCH}" == "arm64" ]]; then + export GOX_FLAGS="-arch arm" + export GO_ARCH_TYPE="arm64" +else + echo "Unsupported OS" + exit 1 +fi diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index a4d8bfcb350..f8e0895b562 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -58,6 +58,7 @@ steps: - group: "Mandatory Tests" key: "metricbeat-mandatory-tests" + steps: - label: ":ubuntu: Metricbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" @@ -248,9 +249,12 @@ steps: - group: "Packaging" key: "metricbeat-packaging" + steps: - label: ":ubuntu: Metricbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" command: | cd metricbeat mage package @@ -264,14 +268,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: context: "metricbeat: Packaging Linux" - label: ":ubuntu: Metricbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd metricbeat mage package @@ -283,9 +288,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "metricbeat: Packaging Linux arm64" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index d2d2c3b9b54..0098c163db9 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -196,6 +196,8 @@ steps: steps: - label: ":ubuntu: Packetbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" command: | cd packetbeat mage package @@ -209,14 +211,15 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: context: "packetbeat: Packaging Linux" - label: ":ubuntu: Packetbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd packetbeat mage package @@ -228,9 +231,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "packetbeat: Packaging Linux arm64" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh deleted file mode 100755 index dc7a61feea1..00000000000 --- a/.buildkite/scripts/common.sh +++ /dev/null @@ -1,477 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -WORKSPACE=${WORKSPACE:-"$(pwd)"} -BIN="${WORKSPACE}/bin" -platform_type="$(uname)" -platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') -arch_type="$(uname -m)" -GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} -GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} -ONLY_DOCS=${ONLY_DOCS:-"true"} -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -# define if needed run the whole pipeline for the particular beat -[ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" -[ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" - -# define if needed run ARM platform-specific tests for the particular beat -[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" - -# define if needed run MacOS platform-specific tests for the particular beat -[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" - -# define if needed run cloud-specific tests for the particular beat -[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" - -ci_changeset=( - "^.buildkite/.*" - ) - -go_mod_changeset=( - "^go.mod" - ) - -oss_changeset=( - "^go.mod" - "^pytest.ini" - "^dev-tools/.*" - "^libbeat/.*" - "^testing/.*" -) - -xpack_changeset=( - "${oss_changeset[@]}" -) - -docs_changeset=( - ".*\\.(asciidoc|md)" - "deploy/kubernetes/.*-kubernetes\\.yaml" - ) - -packaging_changeset=( - "^dev-tools/packaging/.*" - ".go-version" - ) - -case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-xpack-metricbeat") - BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} - ;; - *) - echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." - ;; -esac - -check_and_set_beat_vars() { - local BEATS_PROJECT_NAME=${BEATS_PROJECT_NAME:=""} - if [[ "${BEATS_PROJECT_NAME:=""}" == *"x-pack/"* ]]; then - BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - - BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ - BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels - BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} - TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" - TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" - TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" - TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" - TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_win_tests" - echo "--- Beats project name is $BEATS_XPACK_PROJECT_NAME" - mandatory_changeset=( - "${BEAT_CHANGESET_REFERENCE[@]}" - "${xpack_changeset[@]}" - "${ci_changeset[@]}" - ) - else - BEATS_GH_LABEL=${BEATS_PROJECT_NAME} - TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" - TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" - TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" - TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" - TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_PROJECT_NAME}_win_tests" - echo "--- Beats project name is $BEATS_PROJECT_NAME" - mandatory_changeset=( - "${BEAT_CHANGESET_REFERENCE[@]}" - "${oss_changeset[@]}" - "${ci_changeset[@]}" - ) - fi - BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" - BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" - BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" - BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" - BEATS_GH_WIN_COMMENT="${BEATS_GH_COMMENT} for windows" - BEATS_GH_MACOS_LABEL="macOS" - BEATS_GH_ARM_LABEL="arm" - BEATS_GH_AWS_LABEL="aws" - BEATS_GH_WIN_LABEL="windows" -} - -with_docker_compose() { - local version=$1 - echo "Setting up the Docker-compose environment..." - create_workspace - retry 3 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" - chmod +x ${BIN}/docker-compose - export PATH="${BIN}:${PATH}" - docker-compose version -} - -create_workspace() { - if [[ ! -d "${BIN}" ]]; then - mkdir -p "${BIN}" - fi -} - -add_bin_path() { - echo "Adding PATH to the environment variables..." - create_workspace - export PATH="${BIN}:${PATH}" -} - -check_platform_architeture() { - case "${arch_type}" in - "x86_64") - go_arch_type="amd64" - ;; - "aarch64") - go_arch_type="arm64" - ;; - "arm64") - go_arch_type="arm64" - ;; - *) - echo "The current platform or OS type is unsupported yet" - ;; - esac -} - -with_mage() { - local install_packages=( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "gotest.tools/gotestsum" - ) - create_workspace - for pkg in "${install_packages[@]}"; do - go install "${pkg}@latest" - done - echo "Download modules to local cache" - retry 3 go mod download -} - -with_go() { - echo "Setting up the Go environment..." - create_workspace - check_platform_architeture - retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${go_arch_type}" - chmod +x "${BIN}/gvm" - eval "$(gvm $GO_VERSION)" - go version - which go - local go_path="$(go env GOPATH):$(go env GOPATH)/bin" - export PATH="${go_path}:${PATH}" -} - -checkLinuxType() { - if [ "${platform_type}" == "Linux" ]; then - if grep -q 'ubuntu' /etc/os-release; then - echo "ubuntu" - elif grep -q 'rhel' /etc/os-release; then - echo "rhel" - else - echo "Unsupported Linux" - fi - else - echo "This is not a Linux" - fi -} - -with_python() { - local linuxType="$(checkLinuxType)" - echo "${linuxType}" - if [ "${platform_type}" == "Linux" ]; then - if [ "${linuxType}" = "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y python3-pip python3-venv - elif [ "${linuxType}" = "rhel" ]; then - sudo dnf update -y - sudo dnf install -y python3 python3-pip - pip3 install virtualenv - fi - elif [ "${platform_type}" == "Darwin" ]; then - brew update - pip3 install virtualenv - ulimit -Sn 10000 - fi -} - -with_dependencies() { - local linuxType="$(checkLinuxType)" - echo "${linuxType}" - if [ "${platform_type}" == "Linux" ]; then - if [ "${linuxType}" = "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y libsystemd-dev libpcap-dev librpm-dev - elif [ "${linuxType}" = "rhel" ]; then - # sudo dnf update -y - sudo dnf install -y systemd-devel rpm-devel - wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image - sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image - fi - elif [ "${platform_type}" == "Darwin" ]; then - pip3 install libpcap - fi -} - -config_git() { - if [ -z "$(git config --get user.email)" ]; then - git config --global user.email "beatsmachine@users.noreply.github.com" - git config --global user.name "beatsmachine" - fi -} - -retry() { - local retries=$1 - shift - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - >&2 echo "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - return 0 -} - -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -are_changed_only_paths() { - local patterns=("${@}") - local changed_files=($(git diff --name-only HEAD@{1} HEAD)) - local matched_files=() - for pattern in "${patterns[@]}"; do - local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}")) - if [ "${#matched[@]}" -gt 0 ]; then - matched_files+=("${matched[@]}") - fi - done - if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then - return 0 - fi - return 1 -} - -are_conditions_met_mandatory_tests() { - if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then - return 0 - fi - return 1 -} - -are_conditions_met_macos_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi - fi - return 1 -} - -are_conditions_met_aws_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi - fi - return 1 -} - -are_conditions_met_packaging() { - if are_conditions_met_mandatory_tests; then - if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - return 0 - fi - fi - return 1 -} - -defineModuleFromTheChangeSet() { - # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. - # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. - # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module - local project_path=$1 - local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g') - local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$" - local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - - if [[ "$project_path" == *"x-pack/"* ]]; then - local pattern=("$XPACK_MODULE_PATTERN") - else - local pattern=("$OSS_MODULE_PATTERN") - fi - local changed_modules="" - local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d) - for module_dir in $module_dirs; do - if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then - if [[ -z "$changed_modules" ]]; then - changed_modules=$(basename "$module_dir") - else - changed_modules+=",$(basename "$module_dir")" - fi - fi - done - if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - export MODULE="aws" - else - export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - fi -} - -terraformInit() { - local dir=$1 - echo "Terraform Init on $dir" - pushd "${dir}" > /dev/null - terraform init - popd > /dev/null -} - -withAWS() { - # This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names. - export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY - export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY - export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" -} - -startCloudTestEnv() { - local dir=$1 - withAWS - echo "--- Run docker-compose services for emulated cloud env" - docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d - with_Terraform - terraformInit "$dir" - export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') - export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" - export TF_VAR_CREATED_DATE=$(date +%s) - export TF_VAR_ENVIRONMENT="ci" - export TF_VAR_REPO="${REPO}" - pushd "${dir}" > /dev/null - terraform apply -auto-approve - popd > /dev/null -} - -withNodeJSEnv() { - # HOME="${WORKSPACE}" - local version=$1 - # local nvmPath="${HOME}/.nvm/versions/node/${version}/bin" - echo "Installing nvm" - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - echo "Installing the NodeJs version $version" - nvm install "$version" - # export PATH="${nvmPath}:${PATH}" - nvm use "$version" - node --version -} - -installNodeJsDependencies() { - # Install dependencies to run browsers - if [ "${platform_type}" == "Linux" ]; then - sudo apt-get install -y \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libcups2 \ - libxkbcommon0 \ - libatspi2.0-0 \ - libxcomposite1 \ - libxdamage1 \ - libxfixes3 \ - libxrandr2 \ - libgbm1 \ - libpango-1.0-0 \ - libcairo2 \ - libasound2 - if [ $? -ne 0 ]; then - echo "Error: Failed to install dependencies." - exit 1 - else - echo "Dependencies installed successfully." - fi - elif [ "${platform_type}" == "Darwin" ]; then - echo "TBD" - else - echo "Unsupported platform type." - exit 1 - fi -} - -teardown() { - # Teardown resources after using them - echo "---Terraform Cleanup" - .buildkite/scripts/terraform-cleanup.sh "${MODULE_DIR}" - - echo "---Docker Compose Cleanup" - docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v -} - -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then - unset "$var" - fi - done -} - -if ! are_changed_only_paths "${docs_changeset[@]}" ; then - export ONLY_DOCS="false" - echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." -else - echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." -fi - -if are_paths_changed "${go_mod_changeset[@]}" ; then - export GO_MOD_CHANGES="true" -fi - -if are_paths_changed "${packaging_changeset[@]}" ; then - export PACKAGING_CHANGES="true" -fi - -if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then - # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" -fi - -check_and_set_beat_vars diff --git a/.buildkite/scripts/initCloudEnv.sh b/.buildkite/scripts/initCloudEnv.sh index 0f16d2cb001..ff1f68803ac 100755 --- a/.buildkite/scripts/initCloudEnv.sh +++ b/.buildkite/scripts/initCloudEnv.sh @@ -25,7 +25,7 @@ terraformApply() { TF_VAR_CREATED_DATE=$(date +%s) export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" export TF_VAR_ENVIRONMENT="ci" - export TF_VAR_REPO="${REPO}" + export TF_VAR_REPO="beats" export TF_VAR_BRANCH export TF_VAR_CREATED_DATE diff --git a/.buildkite/scripts/install_macos_tools.sh b/.buildkite/scripts/install_macos_tools.sh index 4572afb4cdc..a29606a922d 100755 --- a/.buildkite/scripts/install_macos_tools.sh +++ b/.buildkite/scripts/install_macos_tools.sh @@ -27,21 +27,6 @@ retry() { return 0 } -define_arch() { - local platform_type="$(uname)" - local arch_type="$(uname -m)" - if [ "${arch_type}" == "x86_64" ]; then - export GOX_FLAGS="-arch amd64" - go_arch_type="amd64" - elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then - export GOX_FLAGS="-arch arm" - go_arch_type="arm64" - else - echo "+++ Unsupported OS archictecture; uname: $platform_type and uname -m: $arch_type" - exit 1 - fi -} - create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -83,8 +68,7 @@ with_mage() { with_go() { echo "Setting up the Go environment..." create_workspace - define_arch - retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE_LOWERCASE}-${go_arch_type}" + retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE_LOWERCASE}-${GO_ARCH_TYPE}" chmod +x "${BIN}/gvm" eval "$(gvm $GO_VERSION)" go version diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh deleted file mode 100755 index 56f8d7257d6..00000000000 --- a/.buildkite/scripts/setenv.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -WORKSPACE=${WORKSPACE:-"$(pwd)"} -GO_VERSION=$(cat .go-version) - -export REPO="beats" -export DOCKER_REGISTRY="docker.elastic.co" -export SETUP_GVM_VERSION="v0.5.1" -export DOCKER_COMPOSE_VERSION="1.21.0" -export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" - -export ASDF_NODEJS_VERSION="18.17.1" -export AWS_REGION="eu-central-1" - -export WORKSPACE -export GO_VERSION - -exportVars() { - local platform_type="$(uname)" - local arch_type="$(uname -m)" - if [ "${arch_type}" == "x86_64" ]; then - case "${platform_type}" in - Linux|Darwin) - export GOX_FLAGS="-arch amd64" - export testResults="**/build/TEST*.xml" - export artifacts="**/build/TEST*.out" - ;; - MINGW*) - export GOX_FLAGS="-arch 386" - export testResults="**\\build\\TEST*.xml" - export artifacts="**\\build\\TEST*.out" - ;; - esac - elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then - export GOX_FLAGS="-arch arm" - export testResults="**/build/TEST*.xml" - export artifacts="**/build/TEST*.out" - else - echo "Unsupported OS" - fi -} - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then - exportVars - export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" -fi - -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then - source .buildkite/scripts/common.sh - # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" -fi diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 2e4fcb0bd18..adde781a5b3 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -173,6 +173,8 @@ steps: steps: - label: ":ubuntu: Winlogbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" command: | cd winlogbeat mage package @@ -184,8 +186,6 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: context: "winlogbeat: Packaging Linux" diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh index 1af666be983..7bc0cfd1694 100644 --- a/.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh +++ b/.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh @@ -53,4 +53,4 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then export RACE_DETECTOR="true" export TEST_COVERAGE="true" export DOCKER_PULL="0" -fi \ No newline at end of file +fi