Ess terraform + run integration tests on BK agents

.buildkite/bk.integration.pipeline.yml

@@ -0,0 +1,163 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+  ASDF_MAGE_VERSION: 1.14.0
+  ASDF_GOLANG_VERSION: 1.22.6
+  ASDF_TERRAFORM_VERSION: 1.9.3
+
+steps:
+  # - label: "Integration tests: packaging"
+  #   key: "package-it"
+  #   command: ".buildkite/scripts/steps/integration-package.sh"
+  #   artifact_paths:
+  #     - build/distributions/**
+  #   agents:
+  #     provider: "gcp"
+  #     machineType: "n1-standard-8"
+  - label: Start ESS stack for integration tests
+    key: integration-ess
+    # depends_on:
+    #   - package-it
+    command: |
+      #!/usr/bin/env bash
+      set -euo pipefail
+      source .buildkite/scripts/steps/ess_start.sh
+    artifact_paths:
+      - test_infra/ess/*.tfstate
+      - test_infra/ess/*.lock.hcl
+    agents:
+      provider: "gcp"
+      imageProject: elastic-images-qa
+      machineType: "n1-standard-2"
+      image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+  - group: "Stateful IT (Sudo): Ubuntu"
+    key: integration-tests
+    depends_on:
+      # - package-it
+      - integration-ess
+    steps:
+      - label: "Default"
+        key: stateful-ubuntu-default-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build 01924359-c420-41d6-ad55-4b4be6ea551e
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "default" "^(TestAPMConfig|TestDiagnosticsOptionalValues|TestIsolatedUnitsDiagnosticsOptionalValues|TestDiagnosticsCommand|TestIsolatedUnitsDiagnosticsCommand|TestEventLogFile|TestFakeComponent|TestFakeIsolatedUnitsComponent|TestOtelFileProcessing|TestOtelLogsIngestion|TestOtelAPMIngestion|TestPackageVersion)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Upgrade"
+        key: stateful-ubuntu-upgrade-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "upgrade" "^(TestUpgradeBrokenPackageVersion|TestStandaloneUpgradeWithGPGFallback|TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing|TestStandaloneUpgradeRollback|TestStandaloneUpgradeRollbackOnRestarts|TestStandaloneUpgradeFailsWhenUpgradeIsInProgress|TestStandaloneUpgradeRetryDownload|TestStandaloneUpgradeSameCommit|TestStandaloneUpgrade|TestStandaloneUpgradeUninstallKillWatcher)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet"
+        key: stateful-ubuntu-fleet-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet" "^(TestLongRunningAgentForLeaks|TestDelayEnroll|TestDelayEnrollUnprivileged|TestInstallAndCLIUninstallWithEndpointSecurity|TestInstallAndUnenrollWithEndpointSecurity|TestInstallWithEndpointSecurityAndRemoveEndpointIntegration|TestEndpointSecurityNonDefaultBasePath|TestEndpointSecurityUnprivileged|TestEndpointSecurityCannotSwitchToUnprivileged|TestEndpointLogsAreCollectedInDiagnostics|TestForceInstallOverProtectedPolicy|TestSetLogLevelFleetManaged|TestLogIngestionFleetManaged|TestMetricsMonitoringCorrectBinaries|TestEndpointAgentServiceMonitoring|TestMonitoringPreserveTextConfig|TestMonitoringLivenessReloadable|TestComponentBuildHashInDiagnostics|TestProxyURL|TestFleetManagedUpgradeUnprivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "FQDN"
+        key: stateful-ubuntu-fqdn-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fqdn" "^(TestFQDN)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Deb"
+        key: stateful-ubuntu-deb-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "deb" "^(TestDebLogIngestFleetManaged|TestDebFleetUpgrade)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Airgapped"
+        key: stateful-ubuntu-fleet-airgapped-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped" "^(TestFleetAirGappedUpgradeUnprivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Privileged"
+        key: stateful-ubuntu-fleet-privileged-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-privileged" "^(TestInstallFleetServerBootstrap|TestFleetManagedUpgradePrivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Airgapped Privileged"
+        key: stateful-ubuntu-fleet-airgapped-privileged-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped-privileged" "^(TestFleetAirGappedUpgradePrivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+  - label: ESS stack cleanup
+    depends_on: integration-tests
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
+      ls -lah test_infra/ess
+      .buildkite/scripts/steps/ess_down.sh
+    agents:
+      provider: "gcp"
+      imageProject: elastic-images-qa
+      machineType: "n1-standard-2"
+      image: "family/platform-ingest-elastic-agent-ubuntu-2204"

.buildkite/scripts/common2.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/retry.sh
+
+if [[ -z "${WORKSPACE-""}" ]]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+    export WORKSPACE
+fi
+if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then
+    SETUP_MAGE_VERSION="1.14.0"
+fi
+if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then
+    SETUP_GVM_VERSION="v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+fi
+BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go")
+export BEAT_VERSION
+
+getOSOptions() {
+  case $(uname | tr '[:upper:]' '[:lower:]') in
+    linux*)
+      export AGENT_OS_NAME=linux
+      ;;
+    darwin*)
+      export AGENT_OS_NAME=darwin
+      ;;
+    msys*)
+      export AGENT_OS_NAME=windows
+      ;;
+    *)
+      export AGENT_OS_NAME=notset
+      ;;
+  esac
+  case $(uname -m | tr '[:upper:]' '[:lower:]') in
+    aarch64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    arm64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    amd64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    x86_64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    *)
+      export AGENT_OS_ARCH=notset
+      ;;
+  esac
+}
+
+# Wrapper function for executing mage
+install_mage() {
+    go version
+    if ! [ -x "$(type -P mage | sed 's/mage is //g')" ];
+    then
+        echo "installing mage ${SETUP_MAGE_VERSION}"
+        make mage
+    fi
+    pushd "$WORKSPACE"
+}
+
+install_go() {
+    # Search for the go in the Path
+    if ! [ -x "$(type -P go | sed 's/go is //g')" ];
+    then
+        getOSOptions
+        echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" "
+        local _bin="${WORKSPACE}/bin"
+        mkdir -p "${_bin}"
+        retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${AGENT_OS_NAME}-${AGENT_OS_ARCH}"
+  chmod +x "${_bin}/gvm"
+        eval "$(command "${_bin}/gvm" "${GO_VERSION}" )"
+        export GOPATH=$(command go env GOPATH)
+        export PATH="${PATH}:${GOPATH}/bin"
+    fi
+    pushd "$WORKSPACE"
+}
+
+google_cloud_auth() {
+    local keyFile=$1
+
+    gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null
+
+    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}

.buildkite/scripts/retry.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euo pipefail
+
+retry() {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}

.buildkite/scripts/steps/ess.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/retry.sh
+
+function ess_up() {
+  install_terraform
+  echo "~~~ Staring ESS Stack"  
+  local WORKSPACE=$(git rev-parse --show-toplevel)
+  local TF_DIR="${WORKSPACE}/test_infra/ess/"
+  local STACK_VERSION=$1
+  local ESS_REGION=${2:-"gcp-us-west2"}
+
+  if [ -z "$STACK_VERSION" ]; then
+    echo "Error: Specify stack version: ess_up [stack_version]" >&2
+    return 1
+  fi
+
+  export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
+
+  if [[ -z "${EC_API_KEY}" ]]; then
+    echo "Error: Failed to get EC API key from vault" >&2
+    exit 1
+  fi
+
+  BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
+  BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
+  BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
+
+  pushd "${TF_DIR}"    
+  terraform init
+  terraform apply \
+    -auto-approve \
+    -var="stack_version=${STACK_VERSION}" \
+    -var="ess_region=${ESS_REGION}" \
+    -var="creator=${BUILDKITE_BUILD_CREATOR}" \
+    -var="buildkite_id=${BUILDKITE_BUILD_NUMBER}" \
+    -var="pipeline=${BUILDKITE_PIPELINE_SLUG}"
+
+  export ELASTICSEARCH_HOST=$(terraform output -raw es_host)
+  export ELASTICSEARCH_USERNAME=$(terraform output -raw es_username)
+  export ELASTICSEARCH_PASSWORD=$(terraform output -raw es_password)
+  export KIBANA_HOST=$(terraform output -raw kibana_endpoint)
+  export KIBANA_USERNAME=$ELASTICSEARCH_USERNAME
+  export KIBANA_PASSWORD=$ELASTICSEARCH_PASSWORD
+  popd
+}
+
+function ess_down() {
+  install_terraform
+  echo "~~~ Tearing down the ESS Stack"  
+  local WORKSPACE=$(git rev-parse --show-toplevel)
+  local TF_DIR="${WORKSPACE}/test_infra/ess/"
+  if [ -z "${EC_API_KEY:-}" ]; then
+    export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
+  fi
+
+  pushd "${TF_DIR}"
+  terraform init
+  terraform destroy -auto-approve
+  popd
+}
+
+function get_git_user_email() {
+  if ! git rev-parse --is-inside-work-tree &>/dev/null; then
+    echo "unknown"  
+    return
+  fi
+
+  local email
+  email=$(git config --get user.email)
+
+  if [ -z "$email" ]; then
+    echo "unknown"  
+  else
+    echo "$email"
+  fi
+}
+
+# remove when use custom images
+install_terraform() {
+  if command -v terraform &> /dev/null; then    
+    return 0
+  fi
+  TERRAFORM_VERSION="1.9.1"
+  echo "~~~ Installing Terraform ${TERRAFORM_VERSION}"
+  DOWNLOAD_URL="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
+  curl -o terraform_${TERRAFORM_VERSION}_linux_amd64.zip $DOWNLOAD_URL
+  unzip -o terraform_${TERRAFORM_VERSION}_linux_amd64.zip
+  sudo mv terraform /usr/local/bin/
+  rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip
+  echo "Terraform version $(terraform -v) installed successfully."
+}

.buildkite/scripts/steps/ess_down.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common2.sh
+
+source .buildkite/scripts/steps/ess.sh
+
+OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
+OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"
+
+ess_down || echo "Failed to stop ESS stack" >&2