updated script and pipelines

elastic · Dec 19, 2024 · 54582e2 · 54582e2
1 parent 52ad5f0
commit 54582e2
Show file tree

Hide file tree

Showing 3 changed files with 160 additions and 95 deletions.
diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml
@@ -28,7 +28,7 @@ steps:
     steps:
       - label: "Win2022:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
@@ -45,7 +45,7 @@ steps:
 
       - label: "Win2022:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
@@ -64,7 +64,7 @@ steps:
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -78,7 +78,7 @@ steps:
 
       - label: "x86_64:sudo: {{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -102,7 +102,7 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -126,7 +126,7 @@ steps:
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -145,7 +145,7 @@ steps:
     steps:
       - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-containers' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -4,6 +4,11 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
+  # Packaging
+  SNAPSHOT: true
+  EXTERNAL: true
+  DEV: true
+
 steps:
   - label: "Integration tests: packaging"
     key: "package-it"
@@ -13,92 +18,150 @@ steps:
     agents:
       provider: "gcp"
       machineType: "n1-standard-8"
+#  - group: "Integration tests: packaging"
+#    key: "package-it"
+#    steps:
+#      - label: "Packaging: Ubuntu x86_64"
+#        key: "packaging-ubuntu-x86-64"
+#        env:
+#          PACKAGES: "tar.gz"
+#          PLATFORMS: "linux/amd64"
+#        command: ".buildkite/scripts/steps/integration-package.sh"
+#        artifact_paths:
+#        - build/distributions/**
+#        agents:
+#          provider: "gcp"
+#          machineType: "n1-standard-8"
+#
+#      - label: "Packaging: Ubuntu arm64"
+#        key: "packaging-ubuntu-arm64"
+#        env:
+#          PACKAGES: "tar.gz"
+#          PLATFORMS: "linux/amd64,linux/arm64"
+#        command: ".buildkite/scripts/steps/integration-package.sh"
+#        artifact_paths:
+#          - build/distributions/**
+#        agents:
+#          provider: "gcp"
+#          machineType: "n1-standard-8"
+#
+#      - label: "Packaging: Windows"
+#        key: "packaging-windows"
+#        env:
+#          PACKAGES: "zip"
+#          PLATFORMS: "windows/amd64"
+#        command: ".buildkite/scripts/steps/integration-package.sh"
+#        artifact_paths:
+#          - build/distributions/**
+#        agents:
+#          provider: "gcp"
+#          machineType: "n1-standard-8"
+#
+#      - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
+#        key: "packaging-containers"
+#        env:
+#          PACKAGES: "{{matrix.ext}}"
+#          PLATFORMS: "{{matrix.arch}}"
+#        command: ".buildkite/scripts/steps/integration-package.sh"
+#        artifact_paths:
+#          - build/distributions/**
+#        agents:
+#          provider: "gcp"
+#          machineType: "n1-standard-8"
+#        matrix:
+#          setup:
+#            arch:
+#              - linux/amd64
+#              - linux/arm64
+#            ext:
+#              - rpm
+#              - deb
 
-  - label: "Serverless integration test"
-    key: "serverless-integration-tests"
-    depends_on:
-      - package-it
-    concurrency_group: elastic-agent-extended-testing/serverless-integration
-    concurrency: 8
-    env:
-      # we run each step in a different data center to spread the load
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
-
-  - label: "Extended runtime leak tests"
-    key: "extended-integration-tests"
-    depends_on:
-      - package-it
-    concurrency_group: elastic-agent-extended-testing/leak-tests
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
-
-  - label: "Triggering Integration tests"
-    depends_on:
-      - package-it
-    trigger: "elastic-agent-extended-testing-bk"
-    build:
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
-
-  - label: "Serverless Beats Tests"
-    depends_on:
-      - package-it
-    key: "serverless-beats-integration-tests"
-    concurrency_group: elastic-agent-extended-testing/beats-integration
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/beats_tests.sh
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
-
-  - label: "Kubernetes Integration tests"
-    key: "k8s-integration-tests"
-    env:
-      K8S_VERSION: "v1.31.0"
-      KIND_VERSION: "v0.24.0"
-    command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
-    artifact_paths:
-      - "build/k8s-logs*/*"
-      - "build/k8s-logs*/**/*"
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "c2-standard-16"
-      image: "family/core-ubuntu-2204"
-      diskSizeGb: 400
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"
+#  - label: "Serverless integration test"
+#    key: "serverless-integration-tests"
+#    depends_on:
+#      - package-it
+#    concurrency_group: elastic-agent-extended-testing/serverless-integration
+#    concurrency: 8
+#    env:
+#      # we run each step in a different data center to spread the load
+#      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+#    command: |
+#      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+#      .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+#    artifact_paths:
+#      - "build/TEST-**"
+#      - "build/diagnostics/*"
+#    agents:
+#      provider: "gcp"
+#      machineType: "n1-standard-8"
+#    notify:
+#      - github_commit_status:
+#          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
+#
+#  - label: "Extended runtime leak tests"
+#    key: "extended-integration-tests"
+#    depends_on:
+#      - package-it
+#    concurrency_group: elastic-agent-extended-testing/leak-tests
+#    concurrency: 8
+#    env:
+#      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
+#    command: |
+#      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+#      .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
+#    artifact_paths:
+#      - "build/TEST-**"
+#      - "build/diagnostics/*"
+#    agents:
+#      provider: "gcp"
+#      machineType: "n1-standard-8"
+#    notify:
+#      - github_commit_status:
+#          context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
+#
+#  - label: "Triggering Integration tests"
+#    depends_on:
+#      - package-it
+#    trigger: "elastic-agent-extended-testing-bk"
+#    build:
+#      commit: "${BUILDKITE_COMMIT}"
+#      branch: "${BUILDKITE_BRANCH}"
+#
+#  - label: "Serverless Beats Tests"
+#    depends_on:
+#      - package-it
+#    key: "serverless-beats-integration-tests"
+#    concurrency_group: elastic-agent-extended-testing/beats-integration
+#    concurrency: 8
+#    env:
+#      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+#    command: |
+#      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+#      .buildkite/scripts/steps/beats_tests.sh
+#    agents:
+#      provider: "gcp"
+#      machineType: "n1-standard-8"
+#    notify:
+#      - github_commit_status:
+#          context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
+#
+#  - label: "Kubernetes Integration tests"
+#    key: "k8s-integration-tests"
+#    env:
+#      K8S_VERSION: "v1.31.0"
+#      KIND_VERSION: "v0.24.0"
+#    command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
+#    artifact_paths:
+#      - "build/k8s-logs*/*"
+#      - "build/k8s-logs*/**/*"
+#      - "build/TEST-**"
+#      - "build/diagnostics/*"
+#    agents:
+#      provider: "gcp"
+#      machineType: "c2-standard-16"
+#      image: "family/core-ubuntu-2204"
+#      diskSizeGb: 400
+#    notify:
+#      - github_commit_status:
+#          context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"
diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh
@@ -4,3 +4,5 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64  SNAPSHOT=true EXTERNAL=true  mage package
+
+#PACKAGES="${PACKAGES}" PLATFORMS="${PLATFORMS}" SNAPSHOT="${SNAPSHOT}" EXTERNAL="${EXTERNAL}" DEV="${DEV}" mage package
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,3 +4,5 @@ set -euo pipefail
		source .buildkite/scripts/common.sh

		PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package

		#PACKAGES="${PACKAGES}" PLATFORMS="${PLATFORMS}" SNAPSHOT="${SNAPSHOT}" EXTERNAL="${EXTERNAL}" DEV="${DEV}" mage package