Switch back to Ubuntu-based Docker images in the cloud

.buildkite/bk.integration.pipeline.yml

@@ -0,0 +1,193 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+  ASDF_MAGE_VERSION: 1.14.0
+
+steps:
+  - label: Start ESS stack for integration tests
+    key: integration-ess
+    env:
+      ASDF_TERRAFORM_VERSION: 1.9.2
+    command: |
+      #!/usr/bin/env bash
+      set -euo pipefail
+      source .buildkite/scripts/steps/ess_start.sh
+    artifact_paths:
+      - test_infra/ess/*.tfstate
+      - test_infra/ess/*.lock.hcl
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - group: "Stateful: Windows"
+    key: integration-tests-win
+    depends_on:
+      - integration-ess
+    steps:
+      - label: "Win2022:sudo:{{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-windows-2022"
+        matrix:
+          - default
+          - fleet
+          - fleet-privileged
+          - upgrade
+
+      - label: "Win2022:non-sudo:{{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} false
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-windows-2022"
+        matrix:
+          - default
+
+  - group: "Stateful:Ubuntu"
+    key: integration-tests-ubuntu
+    depends_on:
+      - integration-ess
+    steps:
+      - label: "x86_64:non-sudo: {{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        matrix:
+          - default
+
+      - label: "x86_64:sudo: {{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        matrix:
+          - default
+          - container
+          - fleet-upgrade-to-pr-build
+          - upgrade
+          - fleet
+          - fqdn
+          - deb
+          - fleet-airgapped
+          - fleet-privileged
+          - fleet-airgapped-privileged
+
+      - label: "arm:sudo: {{matrix}}"
+        skip: true
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "aws"
+          imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
+          instanceType: "m6g.2xlarge"
+        matrix:
+          - default
+          - container
+          - fleet-upgrade-to-pr-build
+          - upgrade
+          - fleet
+          - fqdn
+          - deb
+          - fleet-airgapped
+          - fleet-privileged
+          - fleet-airgapped-privileged
+
+      - label: "arm:non-sudo: {{matrix}}"
+        skip: true
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "aws"
+          imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
+          instanceType: "m6g.xlarge"
+        matrix:
+          - default
+
+  - group: "Stateful(Sudo):RHEL8"
+    key: integration-tests-rhel8
+    depends_on:
+      - integration-ess
+    steps:
+      - label: "x86_64:sudo:rpm"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh rpm true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-rhel-8"
+
+  - label: ESS stack cleanup
+    depends_on:
+      - integration-tests-ubuntu
+      - integration-tests-win
+      - integration-tests-rhel8
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
+      ls -lah test_infra/ess
+      .buildkite/scripts/steps/ess_down.sh
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - label: Aggregate test reports
+    # Warning: The key has a hook in pre-command
+    key: aggregate-reports
+    depends_on:
+      - integration-tests-ubuntu
+      - integration-tests-win
+      - integration-tests-rhel8
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "build/*.xml" .
+      buildkite-agent artifact download "build\*.xml" .
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+    soft_fail:
+      - exit_status: "*"
+    plugins:
+      - test-collector#v1.10.1:
+          files: "build/*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true

.buildkite/hooks/pre-command

@@ -58,6 +58,18 @@ if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   export API_KEY_TOKEN=$(vault kv get -field apiKey ${CI_ESS_PATH})
   echo ${API_KEY_TOKEN} > ./apiKey
   export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
+
+  # BK analytics
+  echo "--- Prepare BK test analytics token :vault:"
+  BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
+  export BUILDKITE_ANALYTICS_TOKEN
+fi
+
+if [[ "$BUILDKITE_STEP_KEY" == *"aggregate-reports"* ]]; then
+  # BK analytics
+  echo "--- Prepare BK test analytics token :vault:"
+  BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
+  export BUILDKITE_ANALYTICS_TOKEN
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then

.buildkite/hooks/pre-command.ps1

@@ -1,29 +1,2 @@
-# Install gcc TODO: Move to the VM image
-choco install mingw
-Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
-refreshenv
-
-# Install gvm and go
-# TODO: Move GVM download to the base VM image
-$env:GvmVersion = "0.5.2"
-[Net.ServicePointManager]::SecurityProtocol = "tls12"
-$env:GoVersion = Get-Content -Path .go-version
-Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
-gvm --format=powershell $env:GoVersion | Invoke-Expression
-go version
-
-$GOPATH = $(go env GOPATH)
-$env:Path = "$GOPATH\bin;" + $env:Path
-$env:GOTMPDIR = "$GOPATH\tmp"
-New-Item -ItemType Directory -Force -Path $env:GOTMPDIR
-[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
-[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine)
-[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
-go env
-
-# Install tools
-go install github.com/magefile/mage
-go install github.com/elastic/go-licenser
-go install golang.org/x/tools/cmd/goimports
-go install github.com/jstemmer/go-junit-report
-go install gotest.tools/gotestsum
+# Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer
+$env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length))

.buildkite/hooks/pre-exit

@@ -26,8 +26,6 @@ if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then
   fi
 fi
 
-source .buildkite/scripts/unset-secrets.sh
-
 if command -v docker &>/dev/null; then
   DOCKER_REGISTRY="docker.elastic.co"
   docker logout $DOCKER_REGISTRY

.buildkite/integration.pipeline.yml

@@ -57,26 +57,13 @@ steps:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
 
-  - label: "Integration tests"
-    key: "integration-tests"
+  - label: "Triggering Integration tests"
     depends_on:
       - package-it
-    concurrency_group: elastic-agent-extended-testing/integration
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh stateful
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Integration tests"
+    trigger: "elastic-agent-extended-testing-bk"
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
 
   - label: "Serverless Beats Tests"
     depends_on:
@@ -89,10 +76,29 @@ steps:
     command: |
       buildkite-agent artifact download build/distributions/** . --step 'package-it'
       .buildkite/scripts/steps/beats_tests.sh
-    # if: "build.env('CRON') == 'yes'"
     agents:
       provider: "gcp"
       machineType: "n1-standard-8"
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
+
+  - label: "Kubernetes Integration tests"
+    key: "k8s-integration-tests"
+    env:
+      K8S_VERSION: "v1.31.0"
+      KIND_VERSION: "v0.24.0"
+    command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
+    artifact_paths:
+      - "build/k8s-logs*/*"
+      - "build/k8s-logs*/**/*"
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "c2-standard-16"
+      image: "family/core-ubuntu-2204"
+      diskSizeGb: 400
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"

.buildkite/pipeline.elastic-agent-binary-dra.yml

@@ -10,7 +10,7 @@ env:
 steps:
   - group: ":beats: DRA Elastic-Agent Core Snapshot :beats:"
     key: "dra-core-snapshot"
-    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_SNAPSHOT") == "true"
     steps:
     - label: ":package: Build Elastic-Agent Core Snapshot"
       commands:
@@ -43,7 +43,7 @@ steps:
 
   - group: ":beats: DRA Elastic-Agent Core Staging :beats:"
     key: "dra-core-staging"
-    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_STAGING") == "true"
     steps:
     - label: ":package: Build Elastic-Agent Core staging"
       commands: