[CI] Separated packaging step (#5129)

* [CI] Separated packaging step * [CI] Separated packaging step * Fix mistype * Fix mistype * Cleanup * Update .buildkite/scripts/steps/integration_tests.sh Co-authored-by: Paolo Chilà <[email protected]> * Update .buildkite/scripts/steps/integration-package.sh Co-authored-by: Paolo Chilà <[email protected]> * Cleanup --------- Co-authored-by: Paolo Chilà <[email protected]>
elastic · Jul 16, 2024 · 9c49b37 · 9c49b37
1 parent 27dacc5
commit 9c49b37
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 7 deletions.
diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -5,14 +5,27 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
 steps:
+  - label: "Integration tests: packaging"
+    key: "package-it"
+    command: ".buildkite/scripts/steps/integration-package.sh"
+    artifact_paths:
+      - build/distributions/**
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/serverless-integration
     concurrency: 8
     env:
       # we run each step in a different data center to spread the load
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -25,11 +38,15 @@ steps:
 
   - label: "Extended runtime leak tests"
     key: "extended-integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/leak-tests
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -42,11 +59,15 @@ steps:
 
   - label: "Integration tests"
     key: "integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh stateful
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -58,12 +79,16 @@ steps:
           context: "buildkite/elastic-agent-extended-testing - Integration tests"
 
   - label: "Serverless Beats Tests"
+    depends_on:
+      - package-it
     key: "serverless-beats-integration-tests"
     concurrency_group: elastic-agent-extended-testing/beats-integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/beats_tests.sh"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/beats_tests.sh
     # if: "build.env('CRON') == 'yes'"
     agents:
       provider: "gcp"

diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64  SNAPSHOT=true EXTERNAL=true  DEV=true mage package
diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh
@@ -8,9 +8,9 @@ MAGE_TARGET="${2:-"integration:test"}"
 MAGE_SUBTARGET="${3:-""}"
 
 
-
-# PACKAGE
-DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip,rpm,deb mage package
+# Override the stack version from `.package-version` contents
+# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically
+# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
 
 STACK_VERSION="$(cat .package-version)"
 if [[ -n "$STACK_VERSION" ]]; then