[8.14](backport #5129) [CI] Separated packaging step (#5143)

* [CI] Separated packaging step (#5129) * [CI] Separated packaging step * [CI] Separated packaging step * Fix mistype * Fix mistype * Cleanup * Update .buildkite/scripts/steps/integration_tests.sh Co-authored-by: Paolo Chilà <[email protected]> * Update .buildkite/scripts/steps/integration-package.sh Co-authored-by: Paolo Chilà <[email protected]> * Cleanup --------- Co-authored-by: Paolo Chilà <[email protected]> (cherry picked from commit 9c49b37) # Conflicts: # .buildkite/scripts/steps/integration_tests.sh * Merged --------- Co-authored-by: Pavel Zorin <[email protected]>
elastic · Jul 25, 2024 · 15b68e7 · 15b68e7
1 parent a952dd0
commit 15b68e7
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 14 deletions.
diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -5,14 +5,27 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
 steps:
+  - label: "Integration tests: packaging"
+    key: "package-it"
+    command: ".buildkite/scripts/steps/integration-package.sh"
+    artifact_paths:
+      - build/distributions/**
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/serverless-integration
     concurrency: 8
     env:
       # we run each step in a different data center to spread the load
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -25,11 +38,15 @@ steps:
 
   - label: "Extended runtime leak tests"
     key: "extended-integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/leak-tests
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -42,11 +59,15 @@ steps:
 
   - label: "Integration tests"
     key: "integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh stateful
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -58,12 +79,16 @@ steps:
           context: "buildkite/elastic-agent-extended-testing - Integration tests"
 
   - label: "Serverless Beats Tests"
+    depends_on:
+      - package-it
     key: "serverless-beats-integration-tests"
     concurrency_group: elastic-agent-extended-testing/beats-integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/beats_tests.sh"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/beats_tests.sh
     # if: "build.env('CRON') == 'yes'"
     agents:
       provider: "gcp"

diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64  SNAPSHOT=true EXTERNAL=true  DEV=true mage package
diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh
@@ -7,21 +7,19 @@ STACK_PROVISIONER="${1:-"stateful"}"
 MAGE_TARGET="${2:-"integration:test"}"
 MAGE_SUBTARGET="${3:-""}"
 
-# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
+
+# Override the stack version from `.package-version` contents
+# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically
 # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
-OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
 
-if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
-  OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"
-else
- OVERRIDE_TEST_AGENT_VERSION=""
+STACK_VERSION="$(cat .package-version)"
+if [[ -n "$STACK_VERSION" ]]; then
+    STACK_VERSION=${STACK_VERSION}"-SNAPSHOT"
 fi
-# PACKAGE
-AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip,rpm,deb mage package
 
 # Run integration tests
 set +e
-AGENT_VERSION="${OVERRIDE_TEST_AGENT_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
+AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
 TESTS_EXIT_STATUS=$?
 set -e
 
@@ -35,4 +33,4 @@ else
     echo "Cannot generate HTML test report: $outputXML not found"
 fi
 
-exit $TESTS_EXIT_STATUS
+exit $TESTS_EXIT_STATUS