From 9c49b374080227fbf3983797bf2875282f543d47 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 16 Jul 2024 21:06:04 +0200 Subject: [PATCH] [CI] Separated packaging step (#5129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [CI] Separated packaging step * [CI] Separated packaging step * Fix mistype * Fix mistype * Cleanup * Update .buildkite/scripts/steps/integration_tests.sh Co-authored-by: Paolo Chilà * Update .buildkite/scripts/steps/integration-package.sh Co-authored-by: Paolo Chilà * Cleanup --------- Co-authored-by: Paolo Chilà --- .buildkite/integration.pipeline.yml | 33 ++++++++++++++++--- .../scripts/steps/integration-package.sh | 6 ++++ .buildkite/scripts/steps/integration_tests.sh | 6 ++-- 3 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 .buildkite/scripts/steps/integration-package.sh diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 37b4c2d1d25..ec14a58ee2f 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -5,14 +5,27 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" steps: + - label: "Integration tests: packaging" + key: "package-it" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n1-standard-8" + - label: "Serverless integration test" key: "serverless-integration-tests" + depends_on: + - package-it concurrency_group: elastic-agent-extended-testing/serverless-integration concurrency: 8 env: # we run each step in a different data center to spread the load TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" - command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite + command: | + buildkite-agent artifact download build/distributions/** . --step 'package-it' + .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite artifact_paths: - "build/TEST-**" - "build/diagnostics/*" @@ -25,11 +38,15 @@ steps: - label: "Extended runtime leak tests" key: "extended-integration-tests" + depends_on: + - package-it concurrency_group: elastic-agent-extended-testing/leak-tests concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" - command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks" + command: | + buildkite-agent artifact download build/distributions/** . --step 'package-it' + .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks artifact_paths: - "build/TEST-**" - "build/diagnostics/*" @@ -42,11 +59,15 @@ steps: - label: "Integration tests" key: "integration-tests" + depends_on: + - package-it concurrency_group: elastic-agent-extended-testing/integration concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f" - command: ".buildkite/scripts/steps/integration_tests.sh stateful" + command: | + buildkite-agent artifact download build/distributions/** . --step 'package-it' + .buildkite/scripts/steps/integration_tests.sh stateful artifact_paths: - "build/TEST-**" - "build/diagnostics/*" @@ -58,12 +79,16 @@ steps: context: "buildkite/elastic-agent-extended-testing - Integration tests" - label: "Serverless Beats Tests" + depends_on: + - package-it key: "serverless-beats-integration-tests" concurrency_group: elastic-agent-extended-testing/beats-integration concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" - command: ".buildkite/scripts/steps/beats_tests.sh" + command: | + buildkite-agent artifact download build/distributions/** . --step 'package-it' + .buildkite/scripts/steps/beats_tests.sh # if: "build.env('CRON') == 'yes'" agents: provider: "gcp" diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh new file mode 100644 index 00000000000..8b15722066d --- /dev/null +++ b/.buildkite/scripts/steps/integration-package.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +source .buildkite/scripts/common.sh + +PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true DEV=true mage package \ No newline at end of file diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index a94826b0a15..834da1cd4c6 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -8,9 +8,9 @@ MAGE_TARGET="${2:-"integration:test"}" MAGE_SUBTARGET="${3:-""}" - -# PACKAGE -DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip,rpm,deb mage package +# Override the stack version from `.package-version` contents +# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically +# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. STACK_VERSION="$(cat .package-version)" if [[ -n "$STACK_VERSION" ]]; then