diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 6ff80f8254..69866d6d4f 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -7,161 +7,158 @@ env: # Packaging SNAPSHOT: true EXTERNAL: true - DEV: true steps: - - label: "Integration tests: packaging" - key: "package-it" - command: ".buildkite/scripts/steps/integration-package.sh" + - group: "Integration tests: packaging" + steps: + - label: "Packaging: Ubuntu x86_64" + key: "packaging-ubuntu-x86-64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n1-standard-8" + + - label: "Packaging: Ubuntu arm64" + key: "packaging-ubuntu-arm64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/arm64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n1-standard-8" + + - label: "Packaging: Windows" + key: "packaging-windows" + env: + PACKAGES: "zip" + PLATFORMS: "windows/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n1-standard-8" + + - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}" + key: "packaging-containers" + env: + PACKAGES: "{{matrix.ext}}" + PLATFORMS: "{{matrix.arch}}" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n1-standard-8" + matrix: + setup: + arch: + - linux/amd64 + - linux/arm64 + ext: + - rpm + - deb + + - label: "Serverless integration test" + key: "serverless-integration-tests" + depends_on: + - packaging-ubuntu-x86-64 + - packaging-windows + concurrency_group: elastic-agent-extended-testing/serverless-integration + concurrency: 8 + env: + # we run each step in a different data center to spread the load + TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" + commands: + - buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' + - buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' + - .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + machineType: "n1-standard-8" + notify: + - github_commit_status: + context: "buildkite/elastic-agent-extended-testing - Serverless integration test" + + - label: "Extended runtime leak tests" + key: "extended-integration-tests" + depends_on: + - packaging-ubuntu-x86-64 + - packaging-windows + concurrency_group: elastic-agent-extended-testing/leak-tests + concurrency: 8 + env: + TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" + command: | + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' + .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks artifact_paths: - - build/distributions/** + - "build/TEST-**" + - "build/diagnostics/*" agents: provider: "gcp" machineType: "n1-standard-8" -# - group: "Integration tests: packaging" -# key: "package-it" -# steps: -# - label: "Packaging: Ubuntu x86_64" -# key: "packaging-ubuntu-x86-64" -# env: -# PACKAGES: "tar.gz" -# PLATFORMS: "linux/amd64" -# command: ".buildkite/scripts/steps/integration-package.sh" -# artifact_paths: -# - build/distributions/** -# agents: -# provider: "gcp" -# machineType: "n1-standard-8" -# -# - label: "Packaging: Ubuntu arm64" -# key: "packaging-ubuntu-arm64" -# env: -# PACKAGES: "tar.gz" -# PLATFORMS: "linux/amd64,linux/arm64" -# command: ".buildkite/scripts/steps/integration-package.sh" -# artifact_paths: -# - build/distributions/** -# agents: -# provider: "gcp" -# machineType: "n1-standard-8" -# -# - label: "Packaging: Windows" -# key: "packaging-windows" -# env: -# PACKAGES: "zip" -# PLATFORMS: "windows/amd64" -# command: ".buildkite/scripts/steps/integration-package.sh" -# artifact_paths: -# - build/distributions/** -# agents: -# provider: "gcp" -# machineType: "n1-standard-8" -# -# - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}" -# key: "packaging-containers" -# env: -# PACKAGES: "{{matrix.ext}}" -# PLATFORMS: "{{matrix.arch}}" -# command: ".buildkite/scripts/steps/integration-package.sh" -# artifact_paths: -# - build/distributions/** -# agents: -# provider: "gcp" -# machineType: "n1-standard-8" -# matrix: -# setup: -# arch: -# - linux/amd64 -# - linux/arm64 -# ext: -# - rpm -# - deb + notify: + - github_commit_status: + context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests" -# - label: "Serverless integration test" -# key: "serverless-integration-tests" -# depends_on: -# - package-it -# concurrency_group: elastic-agent-extended-testing/serverless-integration -# concurrency: 8 -# env: -# # we run each step in a different data center to spread the load -# TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" -# command: | -# buildkite-agent artifact download build/distributions/** . --step 'package-it' -# .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite -# artifact_paths: -# - "build/TEST-**" -# - "build/diagnostics/*" -# agents: -# provider: "gcp" -# machineType: "n1-standard-8" -# notify: -# - github_commit_status: -# context: "buildkite/elastic-agent-extended-testing - Serverless integration test" -# -# - label: "Extended runtime leak tests" -# key: "extended-integration-tests" -# depends_on: -# - package-it -# concurrency_group: elastic-agent-extended-testing/leak-tests -# concurrency: 8 -# env: -# TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" -# command: | -# buildkite-agent artifact download build/distributions/** . --step 'package-it' -# .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks -# artifact_paths: -# - "build/TEST-**" -# - "build/diagnostics/*" -# agents: -# provider: "gcp" -# machineType: "n1-standard-8" -# notify: -# - github_commit_status: -# context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests" -# -# - label: "Triggering Integration tests" -# depends_on: -# - package-it -# trigger: "elastic-agent-extended-testing-bk" -# build: -# commit: "${BUILDKITE_COMMIT}" -# branch: "${BUILDKITE_BRANCH}" -# -# - label: "Serverless Beats Tests" -# depends_on: -# - package-it -# key: "serverless-beats-integration-tests" -# concurrency_group: elastic-agent-extended-testing/beats-integration -# concurrency: 8 -# env: -# TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" -# command: | -# buildkite-agent artifact download build/distributions/** . --step 'package-it' -# .buildkite/scripts/steps/beats_tests.sh -# agents: -# provider: "gcp" -# machineType: "n1-standard-8" -# notify: -# - github_commit_status: -# context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests" -# -# - label: "Kubernetes Integration tests" -# key: "k8s-integration-tests" -# env: -# K8S_VERSION: "v1.31.0" -# KIND_VERSION: "v0.24.0" -# command: ".buildkite/scripts/steps/k8s-extended-tests.sh" -# artifact_paths: -# - "build/k8s-logs*/*" -# - "build/k8s-logs*/**/*" -# - "build/TEST-**" -# - "build/diagnostics/*" -# agents: -# provider: "gcp" -# machineType: "c2-standard-16" -# image: "family/core-ubuntu-2204" -# diskSizeGb: 400 -# notify: -# - github_commit_status: -# context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests" + - label: "Triggering Integration tests" + depends_on: + - packaging-ubuntu-x86-64 + - packaging-ubuntu-arm64 + - packaging-windows + - packaging-containers + trigger: "elastic-agent-extended-testing-bk" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + + - label: "Serverless Beats Tests" + depends_on: + - packaging-ubuntu-x86-64 + key: "serverless-beats-integration-tests" + concurrency_group: elastic-agent-extended-testing/beats-integration + concurrency: 8 + env: + TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" + command: | + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' + .buildkite/scripts/steps/beats_tests.sh + agents: + provider: "gcp" + machineType: "n1-standard-8" + notify: + - github_commit_status: + context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests" + + - label: "Kubernetes Integration tests" + key: "k8s-integration-tests" + env: + K8S_VERSION: "v1.31.0" + KIND_VERSION: "v0.24.0" + command: ".buildkite/scripts/steps/k8s-extended-tests.sh" + artifact_paths: + - "build/k8s-logs*/*" + - "build/k8s-logs*/**/*" + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + image: "family/core-ubuntu-2204" + diskSizeGb: 400 + notify: + - github_commit_status: + context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests" diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index d59755cc5e..42baf7e95a 100644 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# Set required env vars source .buildkite/scripts/common.sh -PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package - -#PACKAGES="${PACKAGES}" PLATFORMS="${PLATFORMS}" SNAPSHOT="${SNAPSHOT}" EXTERNAL="${EXTERNAL}" DEV="${DEV}" mage package +mage package