From 2348f2b73e63bc06e941a56f93942d2ea30b30fb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Sep 2024 15:48:41 +0200 Subject: [PATCH 01/23] Add support for 9.0 stack --- .buildkite/pipeline.schedule-daily.yml | 12 +++ .buildkite/pipeline.schedule-weekly.yml | 13 +++ .buildkite/pipeline.yml | 3 +- .buildkite/scripts/common.sh | 11 ++- ...ml => bump-latest-8x-snapshot-version.yml} | 0 .../bump-latest-9x-snapshot-version.yml | 84 +++++++++++++++++++ 6 files changed, 120 insertions(+), 3 deletions(-) rename .github/workflows/updatecli/updatecli.d/{bump-latest-snapshot-version.yml => bump-latest-8x-snapshot-version.yml} (100%) create mode 100644 .github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 26d50b1ffc8..8d7822136b4 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -51,6 +51,18 @@ steps: - step: "check" allow_failure: false + - label: "Check integrations local stacks - Stack Version v9.0" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 9.0.0-SNAPSHOT + PUBLISH_COVERAGE_REPORTS: "false" + depends_on: + - step: "check" + allow_failure: false + - label: "Check integrations in serverless - project: Observability" key: "trigger-integrations-serverless-obs" trigger: "integrations-serverless" diff --git a/.buildkite/pipeline.schedule-weekly.yml b/.buildkite/pipeline.schedule-weekly.yml index f936b0ff848..1f32de03f67 100644 --- a/.buildkite/pipeline.schedule-weekly.yml +++ b/.buildkite/pipeline.schedule-weekly.yml @@ -27,3 +27,16 @@ steps: depends_on: - step: "check" allow_failure: false + + - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v9.0" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 9.0.0-SNAPSHOT + PUBLISH_COVERAGE_REPORTS: "false" + ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true" + depends_on: + - step: "check" + allow_failure: false diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 96c1a8f6ded..fabaa7de2c3 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -102,5 +102,6 @@ steps: # not fail build if this step fails soft_fail: true # run this step when if it is triggered by the daily job + # skipped for now notification for tests related to STACK_VERSION: 9 if: > - build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" + build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && build.env('STACK_VERSION') !~ /^9/ diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 1aa68d3c582..49f73d3a2c5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -464,12 +464,19 @@ is_supported_stack() { if [ "${kibana_version}" == "null" ]; then return 0 fi - if [[ ! "${kibana_version}" =~ \^7\. && "${STACK_VERSION}" =~ ^7\. ]]; then + if [[ ( ! "${kibana_version}" =~ \^7\. ) && "${STACK_VERSION}" =~ ^7\. ]]; then return 1 fi - if [[ ! "${kibana_version}" =~ \^8\. && "${STACK_VERSION}" =~ ^8\. ]]; then + if [[ ( ! "${kibana_version}" =~ \^8\. ) && "${STACK_VERSION}" =~ ^8\. ]]; then return 1 fi + + // TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too. + // This workaround should be removed once packages have been updated their constraints for 9.0 stack. + if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then + return 1 + fi + return 0 } diff --git a/.github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml similarity index 100% rename from .github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml rename to .github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml diff --git a/.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml new file mode 100644 index 00000000000..5616f94a593 --- /dev/null +++ b/.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml @@ -0,0 +1,84 @@ +--- +name: Bump latest elastic-stack test version +pipelineid: 'bump-latest-elastic-stack-version' + +actions: + default: + title: '[updatecli] Update latest snapshot to {{ source "latestSnapshot" }}' + kind: github/pullrequest + scmid: default + spec: + labels: + - automation + - dependency + +scms: + default: + kind: github + spec: + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + commitusingapi: true + branch: main + +sources: + latestSnapshot: + name: Get latest snapshot + kind: json + spec: + file: https://storage.googleapis.com/artifacts-api/snapshots/main.json + key: .version + latestSnapshotMajorMinor: + name: Get latest snapshort major and minor + kind: json + spec: + file: https://storage.googleapis.com/artifacts-api/snapshots/main.json + key: .version + transformers: + - findsubmatch: + pattern: '(^\d+\.\d+)\.[^\s]+$' + captureindex: 1 + +targets: + update-snapshot-daily: + name: '[updatecli] [daily] Update latest snapshot to {{ source "latestSnapshot" }}' + kind: file + sourceid: latestSnapshot + scmid: default + spec: + file: '.buildkite/pipeline.schedule-daily.yml' + matchpattern: '(STACK_VERSION:) 9\.[^\s]*\.[^\s]*' + replacepattern: '$1 {{ source "latestSnapshot" }}' + + update-snapshot-label-daily: + name: '[updatecli] [daily] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}' + kind: file + sourceid: latestSnapshotMajorMinor + scmid: default + spec: + file: '.buildkite/pipeline.schedule-daily.yml' + matchpattern: '(Stack Version) v9\.\d+"' + replacepattern: 'Stack Version v{{ source "latestSnapshotMajorMinor" }}"' + + update-snapshot-weekly: + name: '[updatecli] [weekly] Update latest snapshot to {{ source "latestSnapshot" }}' + kind: file + sourceid: latestSnapshot + scmid: default + spec: + file: '.buildkite/pipeline.schedule-weekly.yml' + matchpattern: '(STACK_VERSION:) 9\.[^\s]*\.[^\s]*' + replacepattern: '$1 {{ source "latestSnapshot" }}' + + update-snapshot-label-weekly: + name: '[updatecli] [weekly] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}' + kind: file + sourceid: latestSnapshotMajorMinor + scmid: default + spec: + file: '.buildkite/pipeline.schedule-weekly.yml' + matchpattern: '(Stack Version) v9\.\d+"' + replacepattern: 'Stack Version v{{ source "latestSnapshotMajorMinor" }}"' From 90042ff4f58ec3474861040bd8b617efdc34ae32 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Sep 2024 15:49:35 +0200 Subject: [PATCH 02/23] Test STACK_VERSION 9.0.0-SNAPSHOT --- .buildkite/pipeline.yml | 1 + .buildkite/scripts/common.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fabaa7de2c3..d423e1c2557 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -8,6 +8,7 @@ env: YQ_VERSION: 'v4.35.2' JQ_VERSION: '1.7' GH_CLI_VERSION: "2.29.0" + STACK_VERSION: 9.0.0-SNAPSHOT # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 49f73d3a2c5..a5cbd8cc63d 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -473,7 +473,7 @@ is_supported_stack() { // TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too. // This workaround should be removed once packages have been updated their constraints for 9.0 stack. - if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then + if [[ ( ! ( "${kibana_version}" =~ \^9\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then return 1 fi From 40f864d1acb069422cb23cc8b0205d715f50860a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Sep 2024 16:11:15 +0200 Subject: [PATCH 03/23] Allow to test packages with constraint ^8.x for STACK_VERSION 9.0.0 --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a5cbd8cc63d..49f73d3a2c5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -473,7 +473,7 @@ is_supported_stack() { // TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too. // This workaround should be removed once packages have been updated their constraints for 9.0 stack. - if [[ ( ! ( "${kibana_version}" =~ \^9\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then + if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then return 1 fi From ea11883c946b48289f11bb5172d20ff7c0d3d8a1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Sep 2024 17:12:04 +0200 Subject: [PATCH 04/23] Fix comments --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 49f73d3a2c5..2a9354f30e5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -471,8 +471,8 @@ is_supported_stack() { return 1 fi - // TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too. - // This workaround should be removed once packages have been updated their constraints for 9.0 stack. + # TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too. + # This workaround should be removed once packages have been updated their constraints for 9.0 stack. if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then return 1 fi From 4d3ba1e0e0a1b4553aa7747b11eb8684f2a51589 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Sep 2024 17:12:32 +0200 Subject: [PATCH 05/23] Test again just 9.0.0 stack --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2a9354f30e5..24c45ce87d2 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -473,7 +473,7 @@ is_supported_stack() { # TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too. # This workaround should be removed once packages have been updated their constraints for 9.0 stack. - if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then + if [[ ( ! ( "${kibana_version}" =~ \^9\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then return 1 fi From be020b3e1df954d7a319537506b0391759d4d841 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Sep 2024 18:10:45 +0200 Subject: [PATCH 06/23] Restore allowing constraint 8.0.0 when testing with 9.0.0 stack --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 24c45ce87d2..2a9354f30e5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -473,7 +473,7 @@ is_supported_stack() { # TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too. # This workaround should be removed once packages have been updated their constraints for 9.0 stack. - if [[ ( ! ( "${kibana_version}" =~ \^9\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then + if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then return 1 fi From 5538d14e34abc1ebc11e6a9a81b24cb63d6abdd7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Sep 2024 18:26:47 +0200 Subject: [PATCH 07/23] Update pipelines --- .../updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml | 4 ++-- .../updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml index 9af2a074fe8..4b54a0d503d 100644 --- a/.github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml +++ b/.github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml @@ -1,6 +1,6 @@ --- -name: Bump latest elastic-stack test version -pipelineid: 'bump-latest-elastic-stack-version' +name: Bump latest 8.x-SNAPSHOT elastic-stack test version +pipelineid: 'bump-latest-8x-elastic-stack-version' actions: default: diff --git a/.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml index 5616f94a593..54e9afc68b1 100644 --- a/.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml +++ b/.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml @@ -1,6 +1,6 @@ --- -name: Bump latest elastic-stack test version -pipelineid: 'bump-latest-elastic-stack-version' +name: Bump latest 9.x-SNAPSHOT elastic-stack test version +pipelineid: 'bump-latest-9x-elastic-stack-version' actions: default: From 0dcfdccf616c2f0419ed12baf59e952642f0a24a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 17 Sep 2024 10:15:23 +0200 Subject: [PATCH 08/23] Update label for weekly step --- .buildkite/pipeline.schedule-weekly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.schedule-weekly.yml b/.buildkite/pipeline.schedule-weekly.yml index 1f32de03f67..7fd4f0b63b5 100644 --- a/.buildkite/pipeline.schedule-weekly.yml +++ b/.buildkite/pipeline.schedule-weekly.yml @@ -28,7 +28,7 @@ steps: - step: "check" allow_failure: false - - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v9.0" + - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.0" trigger: "integrations" build: env: From c4f13fdf9791a0a95c9fb650c2f5794090b570df Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 17 Sep 2024 10:15:31 +0200 Subject: [PATCH 09/23] Remove STACK_VERSION env var --- .buildkite/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d423e1c2557..fabaa7de2c3 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -8,7 +8,6 @@ env: YQ_VERSION: 'v4.35.2' JQ_VERSION: '1.7' GH_CLI_VERSION: "2.29.0" - STACK_VERSION: 9.0.0-SNAPSHOT # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" From a0cbd5be816039ca3e184dc8bb2555d976f7a9f2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 17 Sep 2024 10:21:49 +0200 Subject: [PATCH 10/23] Update CI docs --- docs/ci_pipelines.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/ci_pipelines.md b/docs/ci_pipelines.md index 2ab2f873d33..f9d696d9509 100644 --- a/docs/ci_pipelines.md +++ b/docs/ci_pipelines.md @@ -7,6 +7,7 @@ Currently, there are five different pipelines: - https://buildkite.com/elastic/integrations-serverless: pipeline in charge of testing all packages using a Elastic Serverless project. More info at [section](#serverless-pipeline). - https://buildkite.com/elastic/integrations-publish: pipeline to publish the new versions of packages. More info at [section](#publish-packages). - https://buildkite.com/elastic/integrations-schedule-daily/: pipeline running every night to test packages in different scenarios. More info at [section](#daily-job). +- https://buildkite.com/elastic/integrations-schedule-weekly/: pipeline running once per week to test packages in different scenarios. More info at [section](#weekly-job). - https://buildkite.com/elastic/integrations-backport/: pipeline to create backport branches (just from UI). More info at [section](#backport-branches-pipeline). ## Pull Requests and pushes to specific branches @@ -22,6 +23,7 @@ Special comments that can be added in the Pull Request (by Elastic employees): There are some environment variables that can be added into this pipeline: - **FORCE_CHECK_ALL**: If `true`, this forces the CI to check all packages even if those packages have no file updated/added/deleted. Default: `false`. - **STACK_VERSION**: Force the CI steps to spin up a specific Elastic stack version to run the tests. Default: unset. +- **STACK_LOGSDB_ENABLED**: Force to enable LogsDB in Elasticsearch service. Default: false. - **PUBLISH_COVERAGE_REPORTS**: If `true`, it enables reporting coverage reports. Currently, it is just set for the build triggered with the current major Elastic stack from the daily job. Default: `false`. @@ -120,13 +122,17 @@ This environment variable can be defined at: Every night it is configured to run a daily job that will be in charge of testing all packages with different scenarios: https://buildkite.com/elastic/integrations-schedule-daily/ -The schedule of this job can be checked [here](https://github.com/elastic/integrations/blob/5714f5665bbe3bc29b9e2444c6a94dbc2d5eebe9/catalog-info.yaml#L93). +The schedule of this job can be checked [here](https://github.com/elastic/integrations/blob/2e72e8524728daca2d47c814d8042031b8f5804f/catalog-info.yaml#L145). The scenarios that are tested in this daily job are: - Test packages with a local Elastic stack running the previous major version of the stack (7.17.X SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations -- Test packages with a local Elastic stack running the latest major version of the stack (currently 8.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest 8.x major version of the stack (currently 8.X.Y-SNAPSHOT). + - Triggered pipeline: https://buildkite.com/elastic/integrations +- Test packages with a local Elastic stack running the latest 8.x major version of the stack with LogsDB enabled (currently 8.X.Y-SNAPSHOT). + - Triggered pipeline: https://buildkite.com/elastic/integrations +- Test packages with a local Elastic stack running the latest major version of the stack (currently 9.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations - Test packages with an Elastic Serverless Observability project. - Triggered pipeline: https://buildkite.com/elastic/integration-serverless @@ -141,6 +147,24 @@ have been published by triggering the pipeline https://buildkite.com/elastic/int Each step triggering a new pipeline can be customized through environment variables. Environment variables that can be used in each pipeline are detailed in the corresponding sections of each pipeline. +## Weekly job + +**Note**: Just available for Employees at Elastic. + +Every week it is configured to run a job that will be in charge of testing all packages with non-Wolfi Elastic Agent docker images: https://buildkite.com/elastic/integrations-schedule-weekly/ + +The schedule of this job can be checked [here](https://github.com/elastic/integrations/blob/5714f5665bbe3bc29b9e2444c6a94dbc2d5eebe9/catalog-info.yaml#L93). + +The scenarios that are tested in this weekly job are: + +- Test packages with a local Elastic stack running the latest 8.x major version of the stack with Elastic Agent images based on Ubuntu iamges (currently 8.X.Y-SNAPSHOT). + - Triggered pipeline: https://buildkite.com/elastic/integrations +- Test packages with a local Elastic stack running the latest 9.x major version of the stack with Elastic Agent images based on non-wolfi images (currently 9.X.Y-SNAPSHOT). + - Triggered pipeline: https://buildkite.com/elastic/integrations + +Each step triggering a new pipeline can be customized through environment variables. Environment variables that can +be used in each pipeline are detailed in the corresponding sections of each pipeline. + ## Backport branches pipeline From 72f4e8d5783fb093b79389484b914422d6af5052 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 17 Sep 2024 10:39:52 +0200 Subject: [PATCH 11/23] Update cron links to daily and weekly pipelines --- docs/ci_pipelines.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ci_pipelines.md b/docs/ci_pipelines.md index f9d696d9509..0130fc6ff10 100644 --- a/docs/ci_pipelines.md +++ b/docs/ci_pipelines.md @@ -122,7 +122,7 @@ This environment variable can be defined at: Every night it is configured to run a daily job that will be in charge of testing all packages with different scenarios: https://buildkite.com/elastic/integrations-schedule-daily/ -The schedule of this job can be checked [here](https://github.com/elastic/integrations/blob/2e72e8524728daca2d47c814d8042031b8f5804f/catalog-info.yaml#L145). +The schedule of this job can be checked [here](https://github.com/elastic/integrations/blob/5714f5665bbe3bc29b9e2444c6a94dbc2d5eebe9/catalog-info.yaml#L93). The scenarios that are tested in this daily job are: @@ -153,7 +153,7 @@ be used in each pipeline are detailed in the corresponding sections of each pipe Every week it is configured to run a job that will be in charge of testing all packages with non-Wolfi Elastic Agent docker images: https://buildkite.com/elastic/integrations-schedule-weekly/ -The schedule of this job can be checked [here](https://github.com/elastic/integrations/blob/5714f5665bbe3bc29b9e2444c6a94dbc2d5eebe9/catalog-info.yaml#L93). +The schedule of this job can be checked [here](https://github.com/elastic/integrations/blob/2e72e8524728daca2d47c814d8042031b8f5804f/catalog-info.yaml#L145). The scenarios that are tested in this weekly job are: From 34555089c3bb6c45b4f771a8ac8f1bd52cba9309 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 17 Sep 2024 10:42:18 +0200 Subject: [PATCH 12/23] Update stack version references --- docs/ci_pipelines.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/ci_pipelines.md b/docs/ci_pipelines.md index 0130fc6ff10..72243a26bd9 100644 --- a/docs/ci_pipelines.md +++ b/docs/ci_pipelines.md @@ -126,11 +126,11 @@ The schedule of this job can be checked [here](https://github.com/elastic/integr The scenarios that are tested in this daily job are: -- Test packages with a local Elastic stack running the previous major version of the stack (7.17.X SNAPSHOT). +- Test packages with a local Elastic stack running the latest 7.x version of the stack (7.17.X SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations -- Test packages with a local Elastic stack running the latest 8.x major version of the stack (currently 8.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest 8.x version of the stack (8.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations -- Test packages with a local Elastic stack running the latest 8.x major version of the stack with LogsDB enabled (currently 8.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest 8.x version of the stack with LogsDB enabled (8.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations - Test packages with a local Elastic stack running the latest major version of the stack (currently 9.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations @@ -157,9 +157,9 @@ The schedule of this job can be checked [here](https://github.com/elastic/integr The scenarios that are tested in this weekly job are: -- Test packages with a local Elastic stack running the latest 8.x major version of the stack with Elastic Agent images based on Ubuntu iamges (currently 8.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest 8.x version of the stack with Elastic Agent images based on Ubuntu iamges (8.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations -- Test packages with a local Elastic stack running the latest 9.x major version of the stack with Elastic Agent images based on non-wolfi images (currently 9.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest major version of the stack with Elastic Agent images based on non-wolfi images (currently 9.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations Each step triggering a new pipeline can be customized through environment variables. Environment variables that can From e6c4f010ce2ee70a978169da32b0b3b516d789e5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 17 Sep 2024 10:49:03 +0200 Subject: [PATCH 13/23] Update docs ci --- docs/ci_pipelines.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/ci_pipelines.md b/docs/ci_pipelines.md index 72243a26bd9..b7a450cbfb3 100644 --- a/docs/ci_pipelines.md +++ b/docs/ci_pipelines.md @@ -23,15 +23,15 @@ Special comments that can be added in the Pull Request (by Elastic employees): There are some environment variables that can be added into this pipeline: - **FORCE_CHECK_ALL**: If `true`, this forces the CI to check all packages even if those packages have no file updated/added/deleted. Default: `false`. - **STACK_VERSION**: Force the CI steps to spin up a specific Elastic stack version to run the tests. Default: unset. -- **STACK_LOGSDB_ENABLED**: Force to enable LogsDB in Elasticsearch service. Default: false. +- **STACK_LOGSDB_ENABLED**: Enable LogsDB setting in Elasticsearch service. Default: false. - **PUBLISH_COVERAGE_REPORTS**: If `true`, it enables reporting coverage reports. Currently, it is just set for the build triggered with the current major Elastic stack from the daily job. Default: `false`. These environment variables can be defined in different locations: - In the [global `env` section](https://github.com/elastic/integrations/blob/5276ef63712f8f2311818770881688870e8422fe/.buildkite/pipeline.yml#L2). -- In the case of the scheduled daily job here: - - [Running tests with previous major Elastic stack](https://github.com/elastic/integrations/blob/5276ef63712f8f2311818770881688870e8422fe/.buildkite/pipeline.schedule-daily.yml#L21). - - [Running tests with current major Elastic stack](https://github.com/elastic/integrations/blob/d6d99792b90838d18844f6df9343bc5f16130666/.buildkite/pipeline.schedule-daily.yml#L32). +- In the case of the scheduled daily job in each step. Some examples: + - [Running tests with 7.x major Elastic stack](https://github.com/elastic/integrations/blob/5276ef63712f8f2311818770881688870e8422fe/.buildkite/pipeline.schedule-daily.yml#L21). + - [Running tests with 8.x major Elastic stack](https://github.com/elastic/integrations/blob/d6d99792b90838d18844f6df9343bc5f16130666/.buildkite/pipeline.schedule-daily.yml#L32). More details about this CI pipeline: @@ -130,7 +130,7 @@ The scenarios that are tested in this daily job are: - Triggered pipeline: https://buildkite.com/elastic/integrations - Test packages with a local Elastic stack running the latest 8.x version of the stack (8.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations -- Test packages with a local Elastic stack running the latest 8.x version of the stack with LogsDB enabled (8.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest 8.x version of the stack with LogsDB setting enabled (8.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations - Test packages with a local Elastic stack running the latest major version of the stack (currently 9.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations @@ -157,15 +157,14 @@ The schedule of this job can be checked [here](https://github.com/elastic/integr The scenarios that are tested in this weekly job are: -- Test packages with a local Elastic stack running the latest 8.x version of the stack with Elastic Agent images based on Ubuntu iamges (8.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest 8.x version of the stack with Elastic Agent images based on Ubuntu images (8.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations -- Test packages with a local Elastic stack running the latest major version of the stack with Elastic Agent images based on non-wolfi images (currently 9.X.Y-SNAPSHOT). +- Test packages with a local Elastic stack running the latest major version of the stack with Elastic Agent images based on non-Wolfi images (currently 9.X.Y-SNAPSHOT). - Triggered pipeline: https://buildkite.com/elastic/integrations Each step triggering a new pipeline can be customized through environment variables. Environment variables that can be used in each pipeline are detailed in the corresponding sections of each pipeline. - ## Backport branches pipeline **Note**: Just available for Employees at Elastic. From 5163e39e318b4d5f64cf215ccf62c16d2580878d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 10:45:04 +0200 Subject: [PATCH 14/23] Update depends relationships in daily job --- .buildkite/pipeline.schedule-daily.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 8d7822136b4..41e5f8b1fed 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -15,7 +15,9 @@ steps: cpu: "8" memory: "4G" + # TODO: Once removing this step, review depends_on configuration in 9.x step - label: "Check integrations local stacks - Stack Version v7.17" + key: "local-7-stack" trigger: "integrations" build: env: @@ -27,6 +29,7 @@ steps: allow_failure: false - label: "Check integrations local stacks - Stack Version v8.16" + key: "local-8-stack" trigger: "integrations" build: env: @@ -39,6 +42,7 @@ steps: allow_failure: false - label: "Check integrations local stacks - Stack Version v8.16 - LogsDB" + key: "local-8-stack-logsdb" trigger: "integrations" build: env: @@ -52,6 +56,7 @@ steps: allow_failure: false - label: "Check integrations local stacks - Stack Version v9.0" + key: "local-9-stack-logsdb" trigger: "integrations" build: env: @@ -59,9 +64,9 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 9.0.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" - depends_on: - - step: "check" - allow_failure: false + depends_on: # TODO: once removed 7.x step , update depends_on.step value to "check" and allow_failure to false + - step: "local-8-stack" + allow_failure: true - label: "Check integrations in serverless - project: Observability" key: "trigger-integrations-serverless-obs" From c36dd3176711cff8623f16cfb8a32ff61f3f4984 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 10:50:48 +0200 Subject: [PATCH 15/23] Update cron for weekly job --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index b7d693f6a46..eaf9faed9e9 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -142,7 +142,7 @@ spec: schedules: main_weekly: branch: "main" - cronline: "30 3 * * 1" + cronline: "00 5 * * 1" message: "Run the weekly jobs" provider_settings: trigger_mode: none # don't trigger jobs from github activity From 9954b20e805ac31243da7afe5b4f93ecf3f0f3db Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 15:58:01 +0200 Subject: [PATCH 16/23] Revert depends_on changes --- .buildkite/pipeline.schedule-daily.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 41e5f8b1fed..8d7822136b4 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -15,9 +15,7 @@ steps: cpu: "8" memory: "4G" - # TODO: Once removing this step, review depends_on configuration in 9.x step - label: "Check integrations local stacks - Stack Version v7.17" - key: "local-7-stack" trigger: "integrations" build: env: @@ -29,7 +27,6 @@ steps: allow_failure: false - label: "Check integrations local stacks - Stack Version v8.16" - key: "local-8-stack" trigger: "integrations" build: env: @@ -42,7 +39,6 @@ steps: allow_failure: false - label: "Check integrations local stacks - Stack Version v8.16 - LogsDB" - key: "local-8-stack-logsdb" trigger: "integrations" build: env: @@ -56,7 +52,6 @@ steps: allow_failure: false - label: "Check integrations local stacks - Stack Version v9.0" - key: "local-9-stack-logsdb" trigger: "integrations" build: env: @@ -64,9 +59,9 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 9.0.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" - depends_on: # TODO: once removed 7.x step , update depends_on.step value to "check" and allow_failure to false - - step: "local-8-stack" - allow_failure: true + depends_on: + - step: "check" + allow_failure: false - label: "Check integrations in serverless - project: Observability" key: "trigger-integrations-serverless-obs" From 441f7e85d0e5927b5bd3b1233a964d7b2c04e1b6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 18:25:27 +0200 Subject: [PATCH 17/23] Test skipping reporting in 9.0 --- .buildkite/pipeline.schedule-daily.yml | 131 ++++++++++++------------- .buildkite/pipeline.yml | 2 +- .buildkite/scripts/common.sh | 2 +- 3 files changed, 66 insertions(+), 69 deletions(-) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 8d7822136b4..d79dd05aebc 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -7,49 +7,49 @@ env: # The pipeline is triggered by the scheduler every day steps: - - label: ":white_check_mark: Check go sources" - key: "check" - command: ".buildkite/scripts/check_sources.sh" - agents: - image: "${LINUX_AGENT_IMAGE}" - cpu: "8" - memory: "4G" + # - label: ":white_check_mark: Check go sources" + # key: "check" + # command: ".buildkite/scripts/check_sources.sh" + # agents: + # image: "${LINUX_AGENT_IMAGE}" + # cpu: "8" + # memory: "4G" - - label: "Check integrations local stacks - Stack Version v7.17" - trigger: "integrations" - build: - env: - SERVERLESS: "false" - FORCE_CHECK_ALL: "true" - STACK_VERSION: 7.17.25-SNAPSHOT - depends_on: - - step: "check" - allow_failure: false + # - label: "Check integrations local stacks - Stack Version v7.17" + # trigger: "integrations" + # build: + # env: + # SERVERLESS: "false" + # FORCE_CHECK_ALL: "true" + # STACK_VERSION: 7.17.25-SNAPSHOT + # depends_on: + # - step: "check" + # allow_failure: false - - label: "Check integrations local stacks - Stack Version v8.16" - trigger: "integrations" - build: - env: - SERVERLESS: "false" - FORCE_CHECK_ALL: "true" - STACK_VERSION: 8.16.0-SNAPSHOT - PUBLISH_COVERAGE_REPORTS: "true" - depends_on: - - step: "check" - allow_failure: false + # - label: "Check integrations local stacks - Stack Version v8.16" + # trigger: "integrations" + # build: + # env: + # SERVERLESS: "false" + # FORCE_CHECK_ALL: "true" + # STACK_VERSION: 8.16.0-SNAPSHOT + # PUBLISH_COVERAGE_REPORTS: "true" + # depends_on: + # - step: "check" + # allow_failure: false - - label: "Check integrations local stacks - Stack Version v8.16 - LogsDB" - trigger: "integrations" - build: - env: - SERVERLESS: "false" - FORCE_CHECK_ALL: "true" - STACK_VERSION: 8.16.0-SNAPSHOT - STACK_LOGSDB_ENABLED: "true" - PUBLISH_COVERAGE_REPORTS: "false" - depends_on: - - step: "check" - allow_failure: false + # - label: "Check integrations local stacks - Stack Version v8.16 - LogsDB" + # trigger: "integrations" + # build: + # env: + # SERVERLESS: "false" + # FORCE_CHECK_ALL: "true" + # STACK_VERSION: 8.16.0-SNAPSHOT + # STACK_LOGSDB_ENABLED: "true" + # PUBLISH_COVERAGE_REPORTS: "false" + # depends_on: + # - step: "check" + # allow_failure: false - label: "Check integrations local stacks - Stack Version v9.0" trigger: "integrations" @@ -59,33 +59,30 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 9.0.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" - depends_on: - - step: "check" - allow_failure: false - - label: "Check integrations in serverless - project: Observability" - key: "trigger-integrations-serverless-obs" - trigger: "integrations-serverless" - build: - env: - SERVERLESS_PROJECT: observability - depends_on: - - step: "check" - allow_failure: false + # - label: "Check integrations in serverless - project: Observability" + # key: "trigger-integrations-serverless-obs" + # trigger: "integrations-serverless" + # build: + # env: + # SERVERLESS_PROJECT: observability + # depends_on: + # - step: "check" + # allow_failure: false - - label: "Check integrations in serverless - project: Security" - key: "trigger-integrations-serverless-security" - trigger: "integrations-serverless" - build: - env: - SERVERLESS_PROJECT: security - depends_on: - - step: "check" - allow_failure: false + # - label: "Check integrations in serverless - project: Security" + # key: "trigger-integrations-serverless-security" + # trigger: "integrations-serverless" + # build: + # env: + # SERVERLESS_PROJECT: security + # depends_on: + # - step: "check" + # allow_failure: false - - label: ":package: Publish missing packages" - key: "trigger-integrations-publish" - trigger: "integrations-publish" - depends_on: - - step: "check" - allow_failure: false + # - label: ":package: Publish missing packages" + # key: "trigger-integrations-publish" + # trigger: "integrations-publish" + # depends_on: + # - step: "check" + # allow_failure: false diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fabaa7de2c3..8b38528c4d6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -103,5 +103,5 @@ steps: soft_fail: true # run this step when if it is triggered by the daily job # skipped for now notification for tests related to STACK_VERSION: 9 - if: > + if: | build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && build.env('STACK_VERSION') !~ /^9/ diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2a9354f30e5..ca0befdb9aa 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -753,7 +753,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep elastic_package_registry } check_package() { From 846b407267bcf55c413222288e3cd4028896ff66 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 18:37:12 +0200 Subject: [PATCH 18/23] Update condition --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8b38528c4d6..ae22588688b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -104,4 +104,4 @@ steps: # run this step when if it is triggered by the daily job # skipped for now notification for tests related to STACK_VERSION: 9 if: | - build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && build.env('STACK_VERSION') !~ /^9/ + build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && ( build.env('STACK_VERSION') !~ /^9\./ ) From 19ea0efa21affa51195f75ec5683977a7c151091 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 18:42:36 +0200 Subject: [PATCH 19/23] Update condition --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ae22588688b..09016719eb6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -104,4 +104,4 @@ steps: # run this step when if it is triggered by the daily job # skipped for now notification for tests related to STACK_VERSION: 9 if: | - build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && ( build.env('STACK_VERSION') !~ /^9\./ ) + build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && build.env('STACK_VERSION') != "9.0.0-SNAPSHOT" From db0bfed4387744585a2a97415f47d04e5362a6d0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 18:56:36 +0200 Subject: [PATCH 20/23] Test without refspec --- .buildkite/pipeline.schedule-daily.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index d79dd05aebc..38dd5b10731 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -59,6 +59,8 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 9.0.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" + BUILDKITE_REFSPEC: "refs/${BUILDKITE_BRANCH}" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" # - label: "Check integrations in serverless - project: Observability" # key: "trigger-integrations-serverless-obs" From c8ed84be23b963e6dbab18829972a0105df31c00 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 18:57:13 +0200 Subject: [PATCH 21/23] restore regex --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 09016719eb6..e332f8c8722 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -104,4 +104,4 @@ steps: # run this step when if it is triggered by the daily job # skipped for now notification for tests related to STACK_VERSION: 9 if: | - build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && build.env('STACK_VERSION') != "9.0.0-SNAPSHOT" + build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && build.env('STACK_VERSION') !~ /^9/ From bbeef796ad9533ecd46f96a5bcc9e41a5f80ab2d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 19:02:10 +0200 Subject: [PATCH 22/23] Test with other pipeline --- .buildkite/pipeline.schedule-daily.yml | 19 +++++++++---------- .buildkite/pipeline.yml | 4 +++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 38dd5b10731..4ced65f6263 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -15,16 +15,15 @@ steps: # cpu: "8" # memory: "4G" - # - label: "Check integrations local stacks - Stack Version v7.17" - # trigger: "integrations" - # build: - # env: - # SERVERLESS: "false" - # FORCE_CHECK_ALL: "true" - # STACK_VERSION: 7.17.25-SNAPSHOT - # depends_on: - # - step: "check" - # allow_failure: false + - label: "Check integrations local stacks - Stack Version v7.17" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 7.17.25-SNAPSHOT + BUILDKITE_REFSPEC: "refs/${BUILDKITE_BRANCH}" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" # - label: "Check integrations local stacks - Stack Version v8.16" # trigger: "integrations" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e332f8c8722..a3dda8cc9da 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -104,4 +104,6 @@ steps: # run this step when if it is triggered by the daily job # skipped for now notification for tests related to STACK_VERSION: 9 if: | - build.source == "trigger_job" && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && build.env('STACK_VERSION') !~ /^9/ + build.source == "trigger_job" && + build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" && + build.env('STACK_VERSION') !~ /^9/ From 26db499ee2cab00a31a6195d8590533c439854bb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Sep 2024 19:14:34 +0200 Subject: [PATCH 23/23] Remove testing --- .buildkite/pipeline.schedule-daily.yml | 118 +++++++++++++------------ .buildkite/scripts/common.sh | 2 +- 2 files changed, 61 insertions(+), 59 deletions(-) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 4ced65f6263..8d7822136b4 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -7,13 +7,13 @@ env: # The pipeline is triggered by the scheduler every day steps: - # - label: ":white_check_mark: Check go sources" - # key: "check" - # command: ".buildkite/scripts/check_sources.sh" - # agents: - # image: "${LINUX_AGENT_IMAGE}" - # cpu: "8" - # memory: "4G" + - label: ":white_check_mark: Check go sources" + key: "check" + command: ".buildkite/scripts/check_sources.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "4G" - label: "Check integrations local stacks - Stack Version v7.17" trigger: "integrations" @@ -22,33 +22,34 @@ steps: SERVERLESS: "false" FORCE_CHECK_ALL: "true" STACK_VERSION: 7.17.25-SNAPSHOT - BUILDKITE_REFSPEC: "refs/${BUILDKITE_BRANCH}" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + depends_on: + - step: "check" + allow_failure: false - # - label: "Check integrations local stacks - Stack Version v8.16" - # trigger: "integrations" - # build: - # env: - # SERVERLESS: "false" - # FORCE_CHECK_ALL: "true" - # STACK_VERSION: 8.16.0-SNAPSHOT - # PUBLISH_COVERAGE_REPORTS: "true" - # depends_on: - # - step: "check" - # allow_failure: false + - label: "Check integrations local stacks - Stack Version v8.16" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 8.16.0-SNAPSHOT + PUBLISH_COVERAGE_REPORTS: "true" + depends_on: + - step: "check" + allow_failure: false - # - label: "Check integrations local stacks - Stack Version v8.16 - LogsDB" - # trigger: "integrations" - # build: - # env: - # SERVERLESS: "false" - # FORCE_CHECK_ALL: "true" - # STACK_VERSION: 8.16.0-SNAPSHOT - # STACK_LOGSDB_ENABLED: "true" - # PUBLISH_COVERAGE_REPORTS: "false" - # depends_on: - # - step: "check" - # allow_failure: false + - label: "Check integrations local stacks - Stack Version v8.16 - LogsDB" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + STACK_VERSION: 8.16.0-SNAPSHOT + STACK_LOGSDB_ENABLED: "true" + PUBLISH_COVERAGE_REPORTS: "false" + depends_on: + - step: "check" + allow_failure: false - label: "Check integrations local stacks - Stack Version v9.0" trigger: "integrations" @@ -58,32 +59,33 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 9.0.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" - BUILDKITE_REFSPEC: "refs/${BUILDKITE_BRANCH}" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + depends_on: + - step: "check" + allow_failure: false - # - label: "Check integrations in serverless - project: Observability" - # key: "trigger-integrations-serverless-obs" - # trigger: "integrations-serverless" - # build: - # env: - # SERVERLESS_PROJECT: observability - # depends_on: - # - step: "check" - # allow_failure: false + - label: "Check integrations in serverless - project: Observability" + key: "trigger-integrations-serverless-obs" + trigger: "integrations-serverless" + build: + env: + SERVERLESS_PROJECT: observability + depends_on: + - step: "check" + allow_failure: false - # - label: "Check integrations in serverless - project: Security" - # key: "trigger-integrations-serverless-security" - # trigger: "integrations-serverless" - # build: - # env: - # SERVERLESS_PROJECT: security - # depends_on: - # - step: "check" - # allow_failure: false + - label: "Check integrations in serverless - project: Security" + key: "trigger-integrations-serverless-security" + trigger: "integrations-serverless" + build: + env: + SERVERLESS_PROJECT: security + depends_on: + - step: "check" + allow_failure: false - # - label: ":package: Publish missing packages" - # key: "trigger-integrations-publish" - # trigger: "integrations-publish" - # depends_on: - # - step: "check" - # allow_failure: false + - label: ":package: Publish missing packages" + key: "trigger-integrations-publish" + trigger: "integrations-publish" + depends_on: + - step: "check" + allow_failure: false diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ca0befdb9aa..2a9354f30e5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -753,7 +753,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep elastic_package_registry + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort } check_package() {