Ess terraform + run integration tests on BK agents (#5113)

* ESS by terraforms

* Use bk runners

* added state to artifacts

* Add common.sh to scripts

* Add depends_on

* Inlined paclage variables

* tmp disbale unit tests

* tmp disbale unit tests

* install tf

* install tf

* install tf

* install tf

* fix ec creds

* Added auto-approve

* Test run

* package again

* package again

* package again

* package again

* package

* package

* added mage build:testBinaries

* Install mage

* Install mage

* Install mage

* Install mage

* Install mage

* fix go

* Merged

* Merged

* fix

* fix

* Fix tests

* Debug

* Use custom image

* New suod tests

* Test run

* Test run

* Test run

* Debug

* Debug

* Debug

* Debug

* Debug

* Cleanup

* Using subshell in newgrp

* Using subshell in newgrp

* Test new custom VM image

* Test new custom VM image

* Set terraform version

* Cleanup

* terraform 1.9.3

* terraform 1.9.3

* Debug

* Explicit image version

* Explicit image version

* Explicit image version

* Sudo tests

* +x

* Try mage integration:local

* Use gotestsum

* Use gotestsum

* Run sudo tests

* Run sudo tests

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Increased test timeout

* Groupped sudo tests

* Quoted regexps

* BK steps by groups

* BK steps by groups

* BK steps by groups

* BK steps by groups

* fix param propagation

* Debug

* Big run

* reuse EC_API_KEY

* Debug test

* Debug a test

* Revert debug

* Debug

* Html report

* groupping, auto-retry, packaging

* removed reruns

* Reuse the initial ESS stack. Start a new stack for retries

* Fix ess start path

* Fix ess start path

* Fix ess start path

* Fix ess start path

* Debug

* Debug

* Debug

* Fix EC_KEY recovery

* Added lock to artifacts

* Fix artifacts download path

* Fix artifacts download path

* Fix artifacts download path

* Removed unnecessaru build dependencies

* Added build id

* Separated BK integration tests to a dedicated pipeline

* CLeanup

* Applied proposed changes

* Applied proposed changes

* Applied proposed changes

(cherry picked from commit 7aa8bb2)

# Conflicts:
#	.buildkite/bk.integration.pipeline.yml
#	.buildkite/scripts/steps/ess.sh
#	.buildkite/scripts/sudo-integration-tests.sh

.buildkite/bk.integration.pipeline.yml

@@ -5,7 +5,11 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   ASDF_MAGE_VERSION: 1.14.0
   ASDF_GOLANG_VERSION: 1.22.6
+<<<<<<< HEAD
   ASDF_TERRAFORM_VERSION: 1.9.2
+=======
+  ASDF_TERRAFORM_VERSION: 1.9.3
+>>>>>>> 7aa8bb222f (Ess terraform + run integration tests on BK agents (#5113))
 
 steps:
   - label: "Integration tests: packaging"

.buildkite/scripts/steps/ess.sh

@@ -13,7 +13,11 @@ function ess_up() {
     return 1
   fi
 
+<<<<<<< HEAD
   export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
+=======
+  export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
+>>>>>>> 7aa8bb222f (Ess terraform + run integration tests on BK agents (#5113))
 
   if [[ -z "${EC_API_KEY}" ]]; then
     echo "Error: Failed to get EC API key from vault" >&2
@@ -48,7 +52,11 @@ function ess_down() {
   local WORKSPACE=$(git rev-parse --show-toplevel)
   local TF_DIR="${WORKSPACE}/test_infra/ess/"
   if [ -z "${EC_API_KEY:-}" ]; then
+<<<<<<< HEAD
     export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
+=======
+    export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
+>>>>>>> 7aa8bb222f (Ess terraform + run integration tests on BK agents (#5113))
   fi
 
   pushd "${TF_DIR}"

.buildkite/scripts/sudo-integration-tests.sh

@@ -9,13 +9,20 @@ TESTS_TO_RUN=$2
 
 echo "~~~ Running integration tests as $USER"
 echo "~~~ Integration tests: ${GROUP_NAME}"
+<<<<<<< HEAD
 gotestsum --version
 PACKAGE_VERSION="$(cat .package-version)"
 if [[ -n "$PACKAGE_VERSION" ]]; then
     PACKAGE_VERSION=${PACKAGE_VERSION}"-SNAPSHOT"
 fi
 set +e
 AGENT_VERSION="${PACKAGE_VERSION}" SNAPSHOT=true TEST_DEFINE_PREFIX="sudo_${GROUP_NAME}_ubuntu" gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s -test.run "${TESTS_TO_RUN}" github.com/elastic/elastic-agent/testing/integration
+=======
+# TODO: Pass the actual version of the agen
+gotestsum --version
+set +e
+SNAPSHOT=true TEST_DEFINE_PREFIX="sudo_${GROUP_NAME}_ubuntu" gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s -test.run "${TESTS_TO_RUN}" github.com/elastic/elastic-agent/testing/integration
+>>>>>>> 7aa8bb222f (Ess terraform + run integration tests on BK agents (#5113))
 TESTS_EXIT_STATUS=$?
 set -e