updated cloud env setup script

.buildkite/scripts/initCloudEnv.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exportAwsSecrets() {
+  echo "~~~ Exporting AWS secrets"
+  export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
+  export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY
+  export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws"
+
+  # AWS_REGION is not set here, since AWS region is taken from *.tf file:
+  # - x-pack/metricbeat/module/aws/terraform.tf
+  # - x-pack/filebeat/input/awscloudwatch/_meta/terraform/variables.tf
+}
+
+terraformApply() {
+  echo "~~~ Terraform Init on $MODULE_DIR"
+  terraform -chdir="$MODULE_DIR" init
+
+  TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
+  TF_VAR_CREATED_DATE=$(date +%s)
+  export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}"
+  export TF_VAR_ENVIRONMENT="ci"
+  export TF_VAR_REPO="${REPO}"
+  export TF_VAR_BRANCH
+  export TF_VAR_CREATED_DATE
+
+  echo "~~~ Terraform Apply on $MODULE_DIR"
+  terraform -chdir="$MODULE_DIR" apply -auto-approve
+}
+
+terraformDestroy() {
+  find "$MODULE_DIR" -name terraform.tfstate -print0 | while IFS= read -r -d '' tfstate; do
+    cd "$(dirname "$tfstate")"
+    if ! terraform destroy -auto-approve; then
+      return 1
+    fi
+    cd -
+  done
+  return 0
+}
+
+trap 'terraformDestroy' EXIT
+exportAwsSecrets
+
+max_retries=2
+timeout=5
+retries=0
+
+while true; do
+  echo "--- Setting up Terraform"
+  out=$(terraformApply 2>&1)
+  exit_code=$?
+
+  if [ $exit_code -eq 0 ]; then
+    break
+  else
+    retries=$((retries + 1))
+
+    if [ $retries -gt $max_retries ]; then
+      terraformDestroy
+      echo "+++ Terraform init & apply failed: $out"
+      exit 1
+    fi
+
+    terraformDestroy
+
+    sleep_time=$((timeout * retries))
+    echo "~~~~ Retry #$retries failed. Retrying after ${sleep_time}s..."
+    sleep $sleep_time
+  fi
+done

.buildkite/x-pack/pipeline.xpack.metricbeat.yml

@@ -195,15 +195,15 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
+          source .buildkite/scripts/initCloudEnv.sh
           echo "~~~ Running tests"
-          source .buildkite/scripts/setup_cloud_env.sh
           cd x-pack/metricbeat
           mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
-          AWS_REGION: "eu-central-1"
-          MODULE_DIR: x-pack/metricbeat/module/aws
-          REPO: beats
+#          AWS_REGION: "eu-central-1"
+          MODULE_DIR: "x-pack/metricbeat/module/aws"
+#          REPO: beats
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -217,8 +217,8 @@ steps:
 
       - label: ":ubuntu: x-pack/metricbeat Cloud AWS (MODULE) Tests"
         key: "x-pack-metricbeat-extended-cloud-test-aws"
-#        skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3170"
-        # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3170
+        skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3399"
+        # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3399
         if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
           set -euo pipefail
@@ -229,6 +229,7 @@ steps:
           echo "~~~ Running tests"
           source .buildkite/scripts/setup_cloud_env.sh
           cd x-pack/metricbeat
+
           mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2