diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh index 3a4418a18f93..d5d8c690891a 100755 --- a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh +++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh @@ -2,15 +2,18 @@ set -euo pipefail -source .buildkite/env-scripts/util.sh - export KUBECONFIG="${WORKSPACE}/kubecfg" export BIN="${WORKSPACE}/bin" -add_bin_path -echo "--- Installing kind & kubectl" +echo "--- Add ${BIN} to PATH" +if [[ ! -d "${BIN}" ]]; then + mkdir -p "${BIN}" +fi +export PATH="${PATH}:${BIN}" + +echo "~~~ Installing kind & kubectl" asdf plugin add kind -asdf install kind $ASDF_KIND_VERSION +asdf install kind "$ASDF_KIND_VERSION" echo "~~~ Setting up kind" max_retries=3 diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh deleted file mode 100644 index 58624e300e69..000000000000 --- a/.buildkite/env-scripts/env.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/env-scripts/util.sh - -DOCS_CHANGESET="^.*\.(asciidoc|md)$ -deploy/kubernetes/.*-kubernetes.yaml" -PACKAGING_CHANGESET="^dev-tools/packaging/ -^.go-version" - -REPO="beats" -WORKSPACE="$(pwd)" -BIN="${WORKSPACE}/bin" -HW_TYPE="$(uname -m)" -PLATFORM_TYPE="$(uname)" -SNAPSHOT="true" -PYTEST_ADDOPTS="" -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" - -SETUP_GVM_VERSION="v0.5.1" -ASDF_MAGE_VERSION="1.14.0" -SETUP_WIN_PYTHON_VERSION="3.11.0" - -# Docker & DockerHub -DOCKER_COMPOSE_VERSION="1.21.0" -DOCKER_REGISTRY="docker.elastic.co" - -ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET") -PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET") -GO_MOD_CHANGES=$(changeset_applies "^go.mod") - -KIND_VERSION="v0.20.0" -KUBECONFIG="${WORKSPACE}/kubecfg" - -export REPO -export WORKSPACE -export BIN -export HW_TYPE -export PLATFORM_TYPE -export SNAPSHOT -export PYTEST_ADDOPTS -export OSS_MODULE_PATTERN -export XPACK_MODULE_PATTERN - -export SETUP_GVM_VERSION -export ASDF_MAGE_VERSION -export SETUP_WIN_PYTHON_VERSION - -export DOCKER_COMPOSE_VERSION -export DOCKER_REGISTRY - -export ONLY_DOCS -export PACKAGING_CHANGES -export GO_MOD_CHANGES - -export KIND_VERSION -export KUBECONFIG - -add_bin_path diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh deleted file mode 100755 index 68fd08a75dfd..000000000000 --- a/.buildkite/env-scripts/util.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -add_bin_path() { - echo "Adding PATH to the environment variables..." - create_bin - export PATH="${PATH}:${BIN}" -} - -with_go() { - local go_version="${GO_VERSION}" - echo "Setting up the Go environment..." - create_bin - check_platform_architecture - retry_with_count 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}" - export PATH="${PATH}:${BIN}" - chmod +x ${BIN}/gvm - eval "$(gvm "$go_version")" - go version - which go - export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin" -} - -with_mage() { - local install_packages=( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "gotest.tools/gotestsum" - ) - create_bin - for pkg in "${install_packages[@]}"; do - go install "${pkg}@latest" - done -} - -create_bin() { - if [[ ! -d "${BIN}" ]]; then - mkdir -p ${BIN} - fi -} - -check_platform_architecture() { -# for downloading the GVM and Terraform packages - case "${HW_TYPE}" in - "x86_64") - arch_type="amd64" - ;; - "aarch64") - arch_type="arm64" - ;; - "arm64") - arch_type="arm64" - ;; - *) - echo "The current platform/OS type is unsupported yet" - ;; - esac -} - -retry_with_count() { - local retries=$1 - shift - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - >&2 echo "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - return 0 -} - -are_files_changed() { - local changeset=$1 - - if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then - return 0; - else - return 1; - fi -} - -changeset_applies() { - local changeset=$1 - if are_files_changed "$changeset"; then - echo true - else - echo false - fi -} - -set_git_config() { - git config user.name "${GITHUB_USERNAME_SECRET}" - git config user.email "${GITHUB_EMAIL_SECRET}" -} diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh deleted file mode 100644 index 931051d550af..000000000000 --- a/.buildkite/env-scripts/win-env.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -if [[ ${PLATFORM_TYPE} = MINGW* ]]; then - echo "--- Installing Python on Win" - choco install mingw -y - choco install python --version=3.11.0 -y -fi diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b22cee81ec91..f860e5fcff75 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,56 +2,9 @@ set -euo pipefail -source .buildkite/env-scripts/util.sh - # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" -DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" -GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then - source .buildkite/env-scripts/env.sh - if [[ -z "${GO_VERSION-""}" ]]; then - export GO_VERSION=$(cat "${WORKSPACE}/.go-version") - fi - - # Images with prefix "platform-ingest-beats-*" has Go pre-setup. - # Image itself takes care of Go version download/installation based on .go-version file - if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then - if [[ -z "${GO_VERSION-""}" ]]; then - GO_VERSION=$(cat "${WORKSPACE}/.go-version") - export GO_VERSION - fi - - # To prevent "OSError: [Errno 24] Too many open files" - ulimit -Sn 150000 - echo "--- Ulimit: $(ulimit)" - - echo "--- Setting up environment" - add_bin_path - with_go - with_mage - fi - - if [[ "$BUILDKITE_STEP_KEY" == *packaging* ]]; then - DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") - DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") - GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) - - export DOCKER_USERNAME_SECRET - export DOCKER_PASSWORD_SECRET - export GITHUB_TOKEN_SECRET - - docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null - - github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH}) - github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH}) - - git config user.name "$github_username" - git config user.email "$github_email" - fi -fi ENABLED_BEATS_PIPELINES_SLUGS=( "auditbeat" @@ -74,10 +27,8 @@ for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do fi done -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then - if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then - PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry_with_count 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then + PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET - fi fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index d1ff6e0ac1c3..0d6d60d4b7a3 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -2,14 +2,24 @@ set -euo pipefail -source .buildkite/hooks/scripts/util.sh +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi + done +} -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then - if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then - docker logout "${DOCKER_REGISTRY}" +cleanup() { + if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then + echo "Deleting temporary files..." + if [[ -n "${BIN:-}" ]]; then + rm -rf "${BIN}" + fi + echo "Done." fi +} + +unset_secrets +cleanup - # Ensure that any temporal files created during any step are removed - cleanup - unset_secrets -fi diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh deleted file mode 100755 index b441991e6feb..000000000000 --- a/.buildkite/hooks/scripts/util.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then - unset "$var" - fi - done -} - -google_cloud_logout_active_account() { - local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null) - if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then - echo "Logging out from GCP for active account" - gcloud auth revoke $active_account > /dev/null 2>&1 - else - echo "No active GCP accounts found." - fi - if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then - unset GOOGLE_APPLICATION_CREDENTIALS - cleanup - fi -} - -cleanup() { - if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then - echo "Deleting temporary files..." - if [[ -n "${BIN:-}" ]]; then - rm -rf "${BIN}" - fi - echo "Done." - fi -} diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh deleted file mode 100755 index c5627aaf0073..000000000000 --- a/.buildkite/scripts/generate_pipeline.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "~~~ Install dependencies" -python3 -mpip install --quiet "ruamel.yaml<0.18.0" -# temporary solution until we have this into a base container -curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -chmod a+x /usr/bin/yq - -.buildkite/scripts/run_dynamic_pipeline_tests.sh - -echo "+++ Run pipeline generator in dry-run mode" -python3 .buildkite/pipeline.py | yq . - -# Temporary commenting this, until we restart this work-stream -#echo "~~~ Upload pipeline" -#python3 .buildkite/pipeline.py | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/initCloudEnv.sh b/.buildkite/scripts/initCloudEnv.sh index d8feea0d49ee..ac6cb1857883 100755 --- a/.buildkite/scripts/initCloudEnv.sh +++ b/.buildkite/scripts/initCloudEnv.sh @@ -8,8 +8,8 @@ exportAwsSecrets() { local awsSecretKey local awsAccessKey - awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) - awsAccessKey=$(retry -t 5 -- vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}") + awsAccessKey=$(retry -t 5 -- vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}") echo "~~~ Exporting AWS secrets" export AWS_ACCESS_KEY_ID=$awsAccessKey diff --git a/.buildkite/scripts/install-msitools.sh b/.buildkite/scripts/install-msitools.sh deleted file mode 100755 index 682a7da68f93..000000000000 --- a/.buildkite/scripts/install-msitools.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -sudo apt-get update -y -DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends --yes msitools \ No newline at end of file diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh deleted file mode 100755 index 5f04ed3849bc..000000000000 --- a/.buildkite/scripts/packaging/package-step.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/util.sh - -changeset="^${BEATS_PROJECT_NAME}/ -^go.mod -^pytest.ini -^dev-tools/ -^libbeat/ -^testing/ -^\.buildkite/${BEATS_PROJECT_NAME}/" - -if are_files_changed "$changeset"; then - bk_pipeline=$(cat <<-YAML - steps: - - label: ":ubuntu: ${BEATS_PROJECT_NAME}/Packaging Linux X86" - key: "package-linux-x86" - env: - PLATFORMS: $PACKAGING_PLATFORMS - SNAPSHOT: true - command: ".buildkite/scripts/packaging/package.sh" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}/Packaging: Linux X86" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - - - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM" - key: "package-linux-arm" - env: - PLATFORMS: $PACKAGING_ARM_PLATFORMS - PACKAGES: "docker" - SNAPSHOT: true - command: ".buildkite/scripts/packaging/package.sh" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}/Packaging: ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" -YAML -) - echo "${bk_pipeline}" | buildkite-agent pipeline upload -else - buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' - exit 0 -fi diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index f1a0d2a90f84..18ac6cac9a75 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -258,7 +258,6 @@ steps: - label: ":ubuntu: Packaging Linux" key: "packaging-linux" command: | - .buildkite/scripts/install-msitools.sh cd x-pack/osquerybeat mage package retry: