Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split packaging into separate steps #6401

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+92 −29
Open

Split packaging into separate steps #6401

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0dd05e7
updated script and pipelines
oakrizan Dec 19, 2024
86dd799
test
oakrizan Dec 19, 2024
3a313df
updated artifact download
oakrizan Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .buildkite/bk.integration.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ steps:
steps:
- label: "Win2022:sudo:{{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
artifact_paths:
- build/**
Expand All @@ -45,7 +45,7 @@ steps:

- label: "Win2022:non-sudo:{{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
artifact_paths:
- build/**
Expand All @@ -64,7 +64,7 @@ steps:
steps:
- label: "x86_64:non-sudo: {{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
artifact_paths:
- build/**
Expand All @@ -78,7 +78,7 @@ steps:

- label: "x86_64:sudo: {{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
artifact_paths:
- build/**
Expand All @@ -102,7 +102,7 @@ steps:
- label: "arm:sudo: {{matrix}}"
skip: true
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
artifact_paths:
- build/**
Expand All @@ -126,7 +126,7 @@ steps:
- label: "arm:non-sudo: {{matrix}}"
skip: true
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
artifact_paths:
- build/**
Expand All @@ -145,7 +145,7 @@ steps:
steps:
- label: "x86_64:sudo:rpm"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh rpm true
artifact_paths:
- build/**
Expand Down
104 changes: 83 additions & 21 deletions .buildkite/integration.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,84 @@ env:
DOCKER_REGISTRY: "docker.elastic.co"
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"

# Packaging
SNAPSHOT: true
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd recommend returning those variables to the packaging script to reduce the scope

EXTERNAL: true

steps:
- label: "Integration tests: packaging"
key: "package-it"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n1-standard-8"
- group: "Integration tests: packaging"
steps:
- label: "Packaging: Ubuntu x86_64"
key: "packaging-ubuntu-x86-64"
env:
PACKAGES: "tar.gz"
PLATFORMS: "linux/amd64"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n1-standard-8"

- label: "Packaging: Ubuntu arm64"
key: "packaging-ubuntu-arm64"
env:
PACKAGES: "tar.gz"
PLATFORMS: "linux/arm64"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n1-standard-8"

- label: "Packaging: Windows"
key: "packaging-windows"
env:
PACKAGES: "zip"
PLATFORMS: "windows/amd64"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n1-standard-8"

- label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
key: "packaging-containers"
env:
PACKAGES: "{{matrix.ext}}"
PLATFORMS: "{{matrix.arch}}"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n1-standard-8"
matrix:
setup:
arch:
- linux/amd64
- linux/arm64
ext:
- rpm
- deb

- label: "Serverless integration test"
key: "serverless-integration-tests"
depends_on:
- package-it
- packaging-ubuntu-x86-64
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: Can we use the group key instead of mentioning all the steps?

- packaging-ubuntu-arm64
- packaging-windows
- packaging-containers
concurrency_group: elastic-agent-extended-testing/serverless-integration
concurrency: 8
env:
# we run each step in a different data center to spread the load
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
commands:
- buildkite-agent artifact download "build/distributions/**" . --build $BUILDKITE_BUILD_ID
- .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
Expand All @@ -39,14 +95,17 @@ steps:
- label: "Extended runtime leak tests"
key: "extended-integration-tests"
depends_on:
- package-it
- packaging-ubuntu-x86-64
- packaging-ubuntu-arm64
- packaging-windows
- packaging-containers
concurrency_group: elastic-agent-extended-testing/leak-tests
concurrency: 8
env:
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
commands:
- buildkite-agent artifact download "build/distributions/**" . --build $BUILDKITE_BUILD_ID
- .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
Expand All @@ -59,23 +118,26 @@ steps:

- label: "Triggering Integration tests"
depends_on:
- package-it
- packaging-ubuntu-x86-64
- packaging-ubuntu-arm64
- packaging-windows
- packaging-containers
trigger: "elastic-agent-extended-testing-bk"
build:
commit: "${BUILDKITE_COMMIT}"
branch: "${BUILDKITE_BRANCH}"

- label: "Serverless Beats Tests"
depends_on:
- package-it
- packaging-ubuntu-x86-64
key: "serverless-beats-integration-tests"
concurrency_group: elastic-agent-extended-testing/beats-integration
concurrency: 8
env:
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/beats_tests.sh
commands:
- buildkite-agent artifact download "build/distributions/**" . --build $BUILDKITE_BUILD_ID
- .buildkite/scripts/steps/beats_tests.sh
agents:
provider: "gcp"
machineType: "n1-standard-8"
Expand Down
3 changes: 2 additions & 1 deletion .buildkite/scripts/steps/integration-package.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail

# Set required env vars
source .buildkite/scripts/common.sh

PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package
mage package
Loading