From 54582e28fcddae74c623377be9c155fcdd7787be Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Thu, 19 Dec 2024 12:31:16 +0200 Subject: [PATCH] updated script and pipelines --- .buildkite/bk.integration.pipeline.yml | 14 +- .buildkite/integration.pipeline.yml | 239 +++++++++++------- .../scripts/steps/integration-package.sh | 2 + 3 files changed, 160 insertions(+), 95 deletions(-) diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index 6d61b8743fb..51aaa012ecc 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -28,7 +28,7 @@ steps: steps: - label: "Win2022:sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} true artifact_paths: - build/** @@ -45,7 +45,7 @@ steps: - label: "Win2022:non-sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} false artifact_paths: - build/** @@ -64,7 +64,7 @@ steps: steps: - label: "x86_64:non-sudo: {{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -78,7 +78,7 @@ steps: - label: "x86_64:sudo: {{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -102,7 +102,7 @@ steps: - label: "arm:sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -126,7 +126,7 @@ steps: - label: "arm:non-sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -145,7 +145,7 @@ steps: steps: - label: "x86_64:sudo:rpm" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-containers' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh rpm true artifact_paths: - build/** diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 3095024feff..6ff80f8254a 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -4,6 +4,11 @@ env: DOCKER_REGISTRY: "docker.elastic.co" VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" + # Packaging + SNAPSHOT: true + EXTERNAL: true + DEV: true + steps: - label: "Integration tests: packaging" key: "package-it" @@ -13,92 +18,150 @@ steps: agents: provider: "gcp" machineType: "n1-standard-8" +# - group: "Integration tests: packaging" +# key: "package-it" +# steps: +# - label: "Packaging: Ubuntu x86_64" +# key: "packaging-ubuntu-x86-64" +# env: +# PACKAGES: "tar.gz" +# PLATFORMS: "linux/amd64" +# command: ".buildkite/scripts/steps/integration-package.sh" +# artifact_paths: +# - build/distributions/** +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# +# - label: "Packaging: Ubuntu arm64" +# key: "packaging-ubuntu-arm64" +# env: +# PACKAGES: "tar.gz" +# PLATFORMS: "linux/amd64,linux/arm64" +# command: ".buildkite/scripts/steps/integration-package.sh" +# artifact_paths: +# - build/distributions/** +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# +# - label: "Packaging: Windows" +# key: "packaging-windows" +# env: +# PACKAGES: "zip" +# PLATFORMS: "windows/amd64" +# command: ".buildkite/scripts/steps/integration-package.sh" +# artifact_paths: +# - build/distributions/** +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# +# - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}" +# key: "packaging-containers" +# env: +# PACKAGES: "{{matrix.ext}}" +# PLATFORMS: "{{matrix.arch}}" +# command: ".buildkite/scripts/steps/integration-package.sh" +# artifact_paths: +# - build/distributions/** +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# matrix: +# setup: +# arch: +# - linux/amd64 +# - linux/arm64 +# ext: +# - rpm +# - deb - - label: "Serverless integration test" - key: "serverless-integration-tests" - depends_on: - - package-it - concurrency_group: elastic-agent-extended-testing/serverless-integration - concurrency: 8 - env: - # we run each step in a different data center to spread the load - TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" - command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' - .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - agents: - provider: "gcp" - machineType: "n1-standard-8" - notify: - - github_commit_status: - context: "buildkite/elastic-agent-extended-testing - Serverless integration test" - - - label: "Extended runtime leak tests" - key: "extended-integration-tests" - depends_on: - - package-it - concurrency_group: elastic-agent-extended-testing/leak-tests - concurrency: 8 - env: - TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" - command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' - .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - agents: - provider: "gcp" - machineType: "n1-standard-8" - notify: - - github_commit_status: - context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests" - - - label: "Triggering Integration tests" - depends_on: - - package-it - trigger: "elastic-agent-extended-testing-bk" - build: - commit: "${BUILDKITE_COMMIT}" - branch: "${BUILDKITE_BRANCH}" - - - label: "Serverless Beats Tests" - depends_on: - - package-it - key: "serverless-beats-integration-tests" - concurrency_group: elastic-agent-extended-testing/beats-integration - concurrency: 8 - env: - TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" - command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' - .buildkite/scripts/steps/beats_tests.sh - agents: - provider: "gcp" - machineType: "n1-standard-8" - notify: - - github_commit_status: - context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests" - - - label: "Kubernetes Integration tests" - key: "k8s-integration-tests" - env: - K8S_VERSION: "v1.31.0" - KIND_VERSION: "v0.24.0" - command: ".buildkite/scripts/steps/k8s-extended-tests.sh" - artifact_paths: - - "build/k8s-logs*/*" - - "build/k8s-logs*/**/*" - - "build/TEST-**" - - "build/diagnostics/*" - agents: - provider: "gcp" - machineType: "c2-standard-16" - image: "family/core-ubuntu-2204" - diskSizeGb: 400 - notify: - - github_commit_status: - context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests" +# - label: "Serverless integration test" +# key: "serverless-integration-tests" +# depends_on: +# - package-it +# concurrency_group: elastic-agent-extended-testing/serverless-integration +# concurrency: 8 +# env: +# # we run each step in a different data center to spread the load +# TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" +# command: | +# buildkite-agent artifact download build/distributions/** . --step 'package-it' +# .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite +# artifact_paths: +# - "build/TEST-**" +# - "build/diagnostics/*" +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# notify: +# - github_commit_status: +# context: "buildkite/elastic-agent-extended-testing - Serverless integration test" +# +# - label: "Extended runtime leak tests" +# key: "extended-integration-tests" +# depends_on: +# - package-it +# concurrency_group: elastic-agent-extended-testing/leak-tests +# concurrency: 8 +# env: +# TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" +# command: | +# buildkite-agent artifact download build/distributions/** . --step 'package-it' +# .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks +# artifact_paths: +# - "build/TEST-**" +# - "build/diagnostics/*" +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# notify: +# - github_commit_status: +# context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests" +# +# - label: "Triggering Integration tests" +# depends_on: +# - package-it +# trigger: "elastic-agent-extended-testing-bk" +# build: +# commit: "${BUILDKITE_COMMIT}" +# branch: "${BUILDKITE_BRANCH}" +# +# - label: "Serverless Beats Tests" +# depends_on: +# - package-it +# key: "serverless-beats-integration-tests" +# concurrency_group: elastic-agent-extended-testing/beats-integration +# concurrency: 8 +# env: +# TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" +# command: | +# buildkite-agent artifact download build/distributions/** . --step 'package-it' +# .buildkite/scripts/steps/beats_tests.sh +# agents: +# provider: "gcp" +# machineType: "n1-standard-8" +# notify: +# - github_commit_status: +# context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests" +# +# - label: "Kubernetes Integration tests" +# key: "k8s-integration-tests" +# env: +# K8S_VERSION: "v1.31.0" +# KIND_VERSION: "v0.24.0" +# command: ".buildkite/scripts/steps/k8s-extended-tests.sh" +# artifact_paths: +# - "build/k8s-logs*/*" +# - "build/k8s-logs*/**/*" +# - "build/TEST-**" +# - "build/diagnostics/*" +# agents: +# provider: "gcp" +# machineType: "c2-standard-16" +# image: "family/core-ubuntu-2204" +# diskSizeGb: 400 +# notify: +# - github_commit_status: +# context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests" diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index 623a67573bc..d59755cc5ec 100644 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -4,3 +4,5 @@ set -euo pipefail source .buildkite/scripts/common.sh PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package + +#PACKAGES="${PACKAGES}" PLATFORMS="${PLATFORMS}" SNAPSHOT="${SNAPSHOT}" EXTERNAL="${EXTERNAL}" DEV="${DEV}" mage package