From f803d486128bec754078ece49ca572c39df1b0de Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 12:44:31 +0200 Subject: [PATCH 01/14] Debug DRA --- .buildkite/pipeline.elastic-agent-package.yml | 101 +++++++++--------- .buildkite/scripts/steps/dra-publish.sh | 2 +- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 25c538f25f9..0d0432ecaa3 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -1,8 +1,8 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: DOCKER_REGISTRY: "docker.elastic.co" - SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 - SETUP_MAGE_VERSION: '1.14.0' + SETUP_GVM_VERSION: "v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 + SETUP_MAGE_VERSION: "1.14.0" # this is required in order to allow the build process to override the default PWD of the BEAT_NAME. BEAT_NAME: "elastic-agent" # after moving elastic-agent out of beats, we should update the URL of the packaging. @@ -53,57 +53,57 @@ steps: - wait: ~ if: build.env("MANIFEST_URL") == null - - group: ":Packaging Artefacts" - key: "package" - steps: - - label: ":package: Cross Building and package elastic-agent" - key: package_elastic-agent - agents: - provider: "gcp" - machineType: "c2-standard-16" - diskSizeGb: 400 - command: | - if [[ -z "${MANIFEST_URL}" ]]; then - export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") - if [[ -z "${MANIFEST_URL}" ]]; then - echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" - exit 1 - fi - fi - if [[ -z "${MAGEFILE_VERBOSE}" ]]; then - export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") - fi - .buildkite/scripts/steps/package.sh - artifact_paths: - - "build/distributions/**/*" - - label: ":package: Package ARM elastic-agent" - key: package_elastic-agent-arm - agents: - provider: "aws" - instanceType: "t4g.2xlarge" - imagePrefix: "core-ubuntu-2004-aarch64" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - command: | - if [[ -z "${MANIFEST_URL}" ]]; then - export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") - if [[ -z "${MANIFEST_URL}" ]]; then - echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" - exit 1 - fi - fi - if [[ -z "${MAGEFILE_VERBOSE}" ]]; then - export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") - fi - .buildkite/scripts/steps/package.sh - artifact_paths: - - "build/distributions/**/*" + # - group: ":Packaging Artefacts" + # key: "package" + # steps: + # - label: ":package: Cross Building and package elastic-agent" + # key: package_elastic-agent + # agents: + # provider: "gcp" + # machineType: "c2-standard-16" + # diskSizeGb: 400 + # command: | + # if [[ -z "${MANIFEST_URL}" ]]; then + # export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") + # if [[ -z "${MANIFEST_URL}" ]]; then + # echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" + # exit 1 + # fi + # fi + # if [[ -z "${MAGEFILE_VERBOSE}" ]]; then + # export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") + # fi + # .buildkite/scripts/steps/package.sh + # artifact_paths: + # - "build/distributions/**/*" + # - label: ":package: Package ARM elastic-agent" + # key: package_elastic-agent-arm + # agents: + # provider: "aws" + # instanceType: "t4g.2xlarge" + # imagePrefix: "core-ubuntu-2004-aarch64" + # env: + # PLATFORMS: "linux/arm64" + # PACKAGES: "docker" + # command: | + # if [[ -z "${MANIFEST_URL}" ]]; then + # export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") + # if [[ -z "${MANIFEST_URL}" ]]; then + # echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" + # exit 1 + # fi + # fi + # if [[ -z "${MAGEFILE_VERBOSE}" ]]; then + # export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") + # fi + # .buildkite/scripts/steps/package.sh + # artifact_paths: + # - "build/distributions/**/*" - label: ":elastic-stack: Publishing to DRA" if: build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == null || build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") != "independent-agent-release-staging" key: dra-publish - depends_on: package + # depends_on: package agents: provider: "gcp" env: @@ -111,9 +111,10 @@ steps: DRA_PROJECT_ARTIFACT_ID: "agent-package" command: | echo "+++ Restoring Artifacts" - buildkite-agent artifact download "build/**/*" . + buildkite-agent artifact download "build/**/*" . --build 0190c8dd-5fa9-4894-ab74-61c817b930ab echo "+++ Changing permissions for the release manager" sudo chown -R :1000 build/distributions/ + ls -lahR build/ echo "+++ Running DRA publish step" if [[ -z "${MAGEFILE_VERBOSE}" ]]; then export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") diff --git a/.buildkite/scripts/steps/dra-publish.sh b/.buildkite/scripts/steps/dra-publish.sh index bda5dd5eabd..6cbea281a99 100755 --- a/.buildkite/scripts/steps/dra-publish.sh +++ b/.buildkite/scripts/steps/dra-publish.sh @@ -10,7 +10,7 @@ PACKAGE_VERSION="${DRA_VERSION:="${BEAT_VERSION:=""}"}" # force main branch on PR's or it won't execute # because the PR branch does not have a project folder in release-manager -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "debug_dra" ]]; then BRANCH=main DRY_RUN="--dry-run" echo "+++ Running in PR and setting branch main and --dry-run" From 3ce6955c9c51b07134f469c05dee1b59bca53b15 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 12:58:06 +0200 Subject: [PATCH 02/14] Debug DRA --- .buildkite/pipeline.elastic-agent-package.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 0d0432ecaa3..44d030c4768 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -111,9 +111,13 @@ steps: DRA_PROJECT_ARTIFACT_ID: "agent-package" command: | echo "+++ Restoring Artifacts" + echo "UMASK: $(umask)" + umask 0022 + buildkite-agent artifact download "build/**/*" . --build 0190c8dd-5fa9-4894-ab74-61c817b930ab echo "+++ Changing permissions for the release manager" - sudo chown -R :1000 build/distributions/ + + sudo chown -R 1000:1000 build/distributions/ ls -lahR build/ echo "+++ Running DRA publish step" if [[ -z "${MAGEFILE_VERBOSE}" ]]; then From 3b7bac1d6ec9e1fe35bb0dacaa74f8dbfd8f702b Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 13:04:00 +0200 Subject: [PATCH 03/14] Debug DRA --- .buildkite/pipeline.elastic-agent-package.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 44d030c4768..11dfe603a19 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -112,12 +112,15 @@ steps: command: | echo "+++ Restoring Artifacts" echo "UMASK: $(umask)" - umask 0022 + # umask 0022 buildkite-agent artifact download "build/**/*" . --build 0190c8dd-5fa9-4894-ab74-61c817b930ab echo "+++ Changing permissions for the release manager" - sudo chown -R 1000:1000 build/distributions/ + ls -lahR build/ + sudo chmod -R g+r build/distributions/ + sudo chown -R :1000 build/distributions/ + ls -lahR build/ echo "+++ Running DRA publish step" if [[ -z "${MAGEFILE_VERBOSE}" ]]; then From 5f6356a01290748e7c65ea32719159a7fc0e8501 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 13:16:21 +0200 Subject: [PATCH 04/14] [CI] Fix packaging permissions --- .buildkite/pipeline.elastic-agent-package.yml | 104 +++++++++--------- .buildkite/scripts/steps/dra-publish.sh | 2 +- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 11dfe603a19..3922afce1ad 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -53,57 +53,57 @@ steps: - wait: ~ if: build.env("MANIFEST_URL") == null - # - group: ":Packaging Artefacts" - # key: "package" - # steps: - # - label: ":package: Cross Building and package elastic-agent" - # key: package_elastic-agent - # agents: - # provider: "gcp" - # machineType: "c2-standard-16" - # diskSizeGb: 400 - # command: | - # if [[ -z "${MANIFEST_URL}" ]]; then - # export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") - # if [[ -z "${MANIFEST_URL}" ]]; then - # echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" - # exit 1 - # fi - # fi - # if [[ -z "${MAGEFILE_VERBOSE}" ]]; then - # export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") - # fi - # .buildkite/scripts/steps/package.sh - # artifact_paths: - # - "build/distributions/**/*" - # - label: ":package: Package ARM elastic-agent" - # key: package_elastic-agent-arm - # agents: - # provider: "aws" - # instanceType: "t4g.2xlarge" - # imagePrefix: "core-ubuntu-2004-aarch64" - # env: - # PLATFORMS: "linux/arm64" - # PACKAGES: "docker" - # command: | - # if [[ -z "${MANIFEST_URL}" ]]; then - # export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") - # if [[ -z "${MANIFEST_URL}" ]]; then - # echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" - # exit 1 - # fi - # fi - # if [[ -z "${MAGEFILE_VERBOSE}" ]]; then - # export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") - # fi - # .buildkite/scripts/steps/package.sh - # artifact_paths: - # - "build/distributions/**/*" + - group: ":Packaging Artefacts" + key: "package" + steps: + - label: ":package: Cross Building and package elastic-agent" + key: package_elastic-agent + agents: + provider: "gcp" + machineType: "c2-standard-16" + diskSizeGb: 400 + command: | + if [[ -z "${MANIFEST_URL}" ]]; then + export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") + if [[ -z "${MANIFEST_URL}" ]]; then + echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" + exit 1 + fi + fi + if [[ -z "${MAGEFILE_VERBOSE}" ]]; then + export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") + fi + .buildkite/scripts/steps/package.sh + artifact_paths: + - "build/distributions/**/*" + - label: ":package: Package ARM elastic-agent" + key: package_elastic-agent-arm + agents: + provider: "aws" + instanceType: "t4g.2xlarge" + imagePrefix: "core-ubuntu-2004-aarch64" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: | + if [[ -z "${MANIFEST_URL}" ]]; then + export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") + if [[ -z "${MANIFEST_URL}" ]]; then + echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" + exit 1 + fi + fi + if [[ -z "${MAGEFILE_VERBOSE}" ]]; then + export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") + fi + .buildkite/scripts/steps/package.sh + artifact_paths: + - "build/distributions/**/*" - label: ":elastic-stack: Publishing to DRA" if: build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == null || build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") != "independent-agent-release-staging" key: dra-publish - # depends_on: package + depends_on: package agents: provider: "gcp" env: @@ -111,17 +111,13 @@ steps: DRA_PROJECT_ARTIFACT_ID: "agent-package" command: | echo "+++ Restoring Artifacts" - echo "UMASK: $(umask)" - # umask 0022 - - buildkite-agent artifact download "build/**/*" . --build 0190c8dd-5fa9-4894-ab74-61c817b930ab - echo "+++ Changing permissions for the release manager" + buildkite-agent artifact download "build/**/*" . - ls -lahR build/ + echo "+++ Changing permissions for the release manager" sudo chmod -R g+r build/distributions/ sudo chown -R :1000 build/distributions/ - ls -lahR build/ + echo "+++ Running DRA publish step" if [[ -z "${MAGEFILE_VERBOSE}" ]]; then export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") diff --git a/.buildkite/scripts/steps/dra-publish.sh b/.buildkite/scripts/steps/dra-publish.sh index 6cbea281a99..bda5dd5eabd 100755 --- a/.buildkite/scripts/steps/dra-publish.sh +++ b/.buildkite/scripts/steps/dra-publish.sh @@ -10,7 +10,7 @@ PACKAGE_VERSION="${DRA_VERSION:="${BEAT_VERSION:=""}"}" # force main branch on PR's or it won't execute # because the PR branch does not have a project folder in release-manager -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "debug_dra" ]]; then +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then BRANCH=main DRY_RUN="--dry-run" echo "+++ Running in PR and setting branch main and --dry-run" From 24990dd40fa845e2fa419adbff06a6737c3a97ee Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 13:33:27 +0200 Subject: [PATCH 05/14] [CI] Fix packaging permissions --- .buildkite/pipeline.elastic-agent-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 3922afce1ad..2d1491b80de 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -1,8 +1,8 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: DOCKER_REGISTRY: "docker.elastic.co" - SETUP_GVM_VERSION: "v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 - SETUP_MAGE_VERSION: "1.14.0" + SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 + SETUP_MAGE_VERSION: '1.14.0' # this is required in order to allow the build process to override the default PWD of the BEAT_NAME. BEAT_NAME: "elastic-agent" # after moving elastic-agent out of beats, we should update the URL of the packaging. From 2e38e394c8b16476bbea83af2357888650d21746 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 13:45:20 +0200 Subject: [PATCH 06/14] CHanged added read permissions --- .buildkite/pipeline.elastic-agent-package.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 2d1491b80de..4671bec213d 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -1,8 +1,8 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: DOCKER_REGISTRY: "docker.elastic.co" - SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 - SETUP_MAGE_VERSION: '1.14.0' + SETUP_GVM_VERSION: "v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 + SETUP_MAGE_VERSION: "1.14.0" # this is required in order to allow the build process to override the default PWD of the BEAT_NAME. BEAT_NAME: "elastic-agent" # after moving elastic-agent out of beats, we should update the URL of the packaging. @@ -97,6 +97,7 @@ steps: export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") fi .buildkite/scripts/steps/package.sh + ls -lahR build/ artifact_paths: - "build/distributions/**/*" @@ -114,7 +115,7 @@ steps: buildkite-agent artifact download "build/**/*" . echo "+++ Changing permissions for the release manager" - sudo chmod -R g+r build/distributions/ + sudo chmod -R +r build/distributions/ sudo chown -R :1000 build/distributions/ ls -lahR build/ From ee3e09b3528cb9afd168a86f5cdaeeab022585b8 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 14:33:27 +0200 Subject: [PATCH 07/14] CHanged added read permissions --- .buildkite/pipeline.elastic-agent-package.yml | 2 +- .buildkite/scripts/common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 4671bec213d..129b8e12a18 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -115,7 +115,7 @@ steps: buildkite-agent artifact download "build/**/*" . echo "+++ Changing permissions for the release manager" - sudo chmod -R +r build/distributions/ + sudo chmod -R a+r build/distributions/ sudo chown -R :1000 build/distributions/ ls -lahR build/ diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 86c370ae360..ff4b890ff39 100644 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -69,7 +69,7 @@ mage() { go(){ # Search for the go in the Path if ! [ -x "$(type -P go | sed 's/go is //g')" ]; - then + then getOSOptions echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" " local _bin="${WORKSPACE}/bin" From f591e83b5dc0d523f322dffb0b5e4a4f1e3e2b1f Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 15:30:30 +0200 Subject: [PATCH 08/14] Fix veriables interpolation --- .buildkite/pipeline.elastic-agent-package.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 129b8e12a18..14d0793b14f 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -63,14 +63,14 @@ steps: machineType: "c2-standard-16" diskSizeGb: 400 command: | - if [[ -z "${MANIFEST_URL}" ]]; then + if [[ -z "$${MANIFEST_URL}" ]]; then export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") - if [[ -z "${MANIFEST_URL}" ]]; then + if [[ -z "$${MANIFEST_URL}" ]]; then echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" exit 1 fi fi - if [[ -z "${MAGEFILE_VERBOSE}" ]]; then + if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") fi .buildkite/scripts/steps/package.sh @@ -86,14 +86,14 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - if [[ -z "${MANIFEST_URL}" ]]; then + if [[ -z "$${MANIFEST_URL}" ]]; then export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "") - if [[ -z "${MANIFEST_URL}" ]]; then + if [[ -z "$${MANIFEST_URL}" ]]; then echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided" exit 1 fi fi - if [[ -z "${MAGEFILE_VERBOSE}" ]]; then + if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") fi .buildkite/scripts/steps/package.sh @@ -120,14 +120,14 @@ steps: ls -lahR build/ echo "+++ Running DRA publish step" - if [[ -z "${MAGEFILE_VERBOSE}" ]]; then + if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0") fi - if [[ -z "${DRA_DRY_RUN}" ]]; then + if [[ -z "$${DRA_DRY_RUN}" ]]; then DRA_DRY_RUN=$(buildkite-agent meta-data get DRA_DRY_RUN --default "") export DRA_DRY_RUN fi - if [[ -z "${DRA_VERSION}" ]]; then + if [[ -z "$${DRA_VERSION}" ]]; then DRA_VERSION=$(buildkite-agent meta-data get DRA_VERSION --default "") export DRA_VERSION fi @@ -163,7 +163,7 @@ steps: .buildkite/scripts/steps/dra-publish.sh # Artifacts will be uploaded via the artifact_paths entry above echo "+++ Set job metadata if TRIGGER_JOB_ID is properly set" - if [[ -z "${TRIGGER_JOB_ID}" ]]; then + if [[ -z "$${TRIGGER_JOB_ID}" ]]; then echo "TRIGGER_JOB_ID is not set, so not setting metadata" else # If a pipeline that triggered this build passes in a "TRIGGER_JOB_ID" env var, that From fa0e113c9b27e589ec3e8d72ffc86b5812ae70c3 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 16:02:33 +0200 Subject: [PATCH 09/14] Debug --- .buildkite/pipeline.yml | 534 +++++++++++++++++------------------ .buildkite/scripts/common.sh | 4 +- 2 files changed, 270 insertions(+), 268 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d92d10db541..494fcffaa2f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,285 +13,285 @@ steps: retry: manual: allowed: true - - group: "Unit tests" - key: "unit-tests" - steps: - - label: "Unit tests - Ubuntu 22.04" - key: "unit-tests-2204" - command: ".buildkite/scripts/steps/unit-tests.sh" - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - retry: - manual: - allowed: true + # - group: "Unit tests" + # key: "unit-tests" + # steps: + # - label: "Unit tests - Ubuntu 22.04" + # key: "unit-tests-2204" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # image: "family/core-ubuntu-2204" + # retry: + # manual: + # allowed: true - - label: "Unit tests - Ubuntu 22.04 ARM64" - key: "unit-tests-2204-arm64" - command: ".buildkite/scripts/steps/unit-tests.sh" - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "aws" - imagePrefix: "core-ubuntu-2204-aarch64" - diskSizeGb: 200 - instanceType: "m6g.xlarge" - retry: - manual: - allowed: true + # - label: "Unit tests - Ubuntu 22.04 ARM64" + # key: "unit-tests-2204-arm64" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "aws" + # imagePrefix: "core-ubuntu-2204-aarch64" + # diskSizeGb: 200 + # instanceType: "m6g.xlarge" + # retry: + # manual: + # allowed: true - - label: "Unit tests - Windows 2022" - key: "unit-tests-win2022" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - image: "family/core-windows-2022" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - retry: - manual: - allowed: true + # - label: "Unit tests - Windows 2022" + # key: "unit-tests-win2022" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # image: "family/core-windows-2022" + # machine_type: "n2-standard-8" + # disk_size: 200 + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true - - label: "Unit tests - Windows 2016" - key: "unit-tests-win2016" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - image: "family/core-windows-2016" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - retry: - manual: - allowed: true + # - label: "Unit tests - Windows 2016" + # key: "unit-tests-win2016" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # image: "family/core-windows-2016" + # machine_type: "n2-standard-8" + # disk_size: 200 + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true - # Runs inly on the main branch - - label: "Unit tests - MacOS 13 ARM" - key: "unit-tests-macos-13-arm" - command: ".buildkite/scripts/steps/unit-tests.sh" - branches: main - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: orka - imagePrefix: generic-13-ventura-arm - retry: - manual: - allowed: true + # # Runs inly on the main branch + # - label: "Unit tests - MacOS 13 ARM" + # key: "unit-tests-macos-13-arm" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # branches: main + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: orka + # imagePrefix: generic-13-ventura-arm + # retry: + # manual: + # allowed: true - - label: "Unit tests - MacOS 13" - key: "unit-tests-macos-13" - command: ".buildkite/scripts/steps/unit-tests.sh" - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: orka - imagePrefix: generic-13-ventura-x64 - retry: - manual: - allowed: true + # - label: "Unit tests - MacOS 13" + # key: "unit-tests-macos-13" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: orka + # imagePrefix: generic-13-ventura-x64 + # retry: + # manual: + # allowed: true - - group: "Desktop Windows tests" - key: "extended-windows" - steps: - - label: "Unit tests - Windows 10" - key: "unit-tests-win10" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - # TODO create own image - image: "family/general-windows-10" - machine_type: "n2-standard-8" - disk_type: "pd-ssd" - retry: - manual: - allowed: true - - label: "Unit tests - Windows 11" - key: "unit-tests-win11" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-go-unit.html" - - "build/TEST-go-unit.xml" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - # TODO create own image - image: "family/general-windows-11" - machine_type: "n2-standard-8" - disk_type: "pd-ssd" - retry: - manual: - allowed: true + # - group: "Desktop Windows tests" + # key: "extended-windows" + # steps: + # - label: "Unit tests - Windows 10" + # key: "unit-tests-win10" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # # TODO create own image + # image: "family/general-windows-10" + # machine_type: "n2-standard-8" + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true + # - label: "Unit tests - Windows 11" + # key: "unit-tests-win11" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-go-unit.html" + # - "build/TEST-go-unit.xml" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # # TODO create own image + # image: "family/general-windows-11" + # machine_type: "n2-standard-8" + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true - - label: "Merge coverage reports" - key: "merge-coverage" - env: - BUILDKITE_REPO: "" - command: " - .buildkite/scripts/steps/merge.sh - unit-tests-2204 - unit-tests-2204-arm64 - unit-tests-win2016 - unit-tests-win2022 - unit-tests-macos-13 - unit-tests-win10 - unit-tests-win11 - " - artifact_paths: - - "build/TEST-go-unit.cov" - agents: - image: "golang:1.20.10" - depends_on: - - unit-tests - - extended-windows - allow_dependency_failure: true + # - label: "Merge coverage reports" + # key: "merge-coverage" + # env: + # BUILDKITE_REPO: "" + # command: " + # .buildkite/scripts/steps/merge.sh + # unit-tests-2204 + # unit-tests-2204-arm64 + # unit-tests-win2016 + # unit-tests-win2022 + # unit-tests-macos-13 + # unit-tests-win10 + # unit-tests-win11 + # " + # artifact_paths: + # - "build/TEST-go-unit.cov" + # agents: + # image: "golang:1.20.10" + # depends_on: + # - unit-tests + # - extended-windows + # allow_dependency_failure: true - - group: "K8s tests" - key: "k8s-tests" - steps: - - label: "K8s tests: {{matrix.k8s_version}}" - env: - K8S_VERSION: "v{{matrix.k8s_version}}" - KIND_VERSION: "v0.20.0" - command: ".buildkite/scripts/steps/k8s-tests.sh" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - matrix: - setup: - k8s_version: - - "1.30.0" - - "1.29.4" - - "1.28.9" - - "1.27.13" - retry: - manual: - allowed: true - - label: "K8s integration tests" - key: "k8s-integration-tests" - env: - K8S_VERSION: "v1.30.2,v1.29.4,v1.28.9" - KIND_VERSION: "v0.20.0" - command: ".buildkite/scripts/steps/k8s-extended-tests.sh" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - retry: - manual: - allowed: true + # - group: "K8s tests" + # key: "k8s-tests" + # steps: + # - label: "K8s tests: {{matrix.k8s_version}}" + # env: + # K8S_VERSION: "v{{matrix.k8s_version}}" + # KIND_VERSION: "v0.20.0" + # command: ".buildkite/scripts/steps/k8s-tests.sh" + # agents: + # provider: "gcp" + # image: "family/core-ubuntu-2204" + # matrix: + # setup: + # k8s_version: + # - "1.30.0" + # - "1.29.4" + # - "1.28.9" + # - "1.27.13" + # retry: + # manual: + # allowed: true + # - label: "K8s integration tests" + # key: "k8s-integration-tests" + # env: + # K8S_VERSION: "v1.30.2,v1.29.4,v1.28.9" + # KIND_VERSION: "v0.20.0" + # command: ".buildkite/scripts/steps/k8s-extended-tests.sh" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # agents: + # provider: "gcp" + # image: "family/core-ubuntu-2204" + # retry: + # manual: + # allowed: true - - label: ":sonarqube: Continuous Code Inspection" - env: - VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" - agents: - image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" - command: - - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." - - "/scan-source-code.sh" - depends_on: - - "merge-coverage" - retry: - manual: - allowed: true + # - label: ":sonarqube: Continuous Code Inspection" + # env: + # VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" + # agents: + # image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" + # command: + # - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." + # - "/scan-source-code.sh" + # depends_on: + # - "merge-coverage" + # retry: + # manual: + # allowed: true - # Triggers a dynamic step: Sync K8s - # Runs only on main and if k8s files are changed - - label: "Trigger k8s sync" - branches: main - plugins: - - monebag/monorepo-diff#v2.5.9: - diff: "git diff --name-only HEAD~1" - watch: - - path: - - deploy/kubernetes/* - - version/docs/version.asciidoc - config: - label: "Sync K8s" - command: ".buildkite/scripts/steps/sync-k8s.sh" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - env: - - GH_VERSION=2.4.0 + # # Triggers a dynamic step: Sync K8s + # # Runs only on main and if k8s files are changed + # - label: "Trigger k8s sync" + # branches: main + # plugins: + # - monebag/monorepo-diff#v2.5.9: + # diff: "git diff --name-only HEAD~1" + # watch: + # - path: + # - deploy/kubernetes/* + # - version/docs/version.asciidoc + # config: + # label: "Sync K8s" + # command: ".buildkite/scripts/steps/sync-k8s.sh" + # agents: + # provider: "gcp" + # image: "family/core-ubuntu-2204" + # env: + # - GH_VERSION=2.4.0 - # Trigger for pull requests - - label: "Trigger Integration tests for Pull request" - if: | - (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) || - build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/ + # # Trigger for pull requests + # - label: "Trigger Integration tests for Pull request" + # if: | + # (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) || + # build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/ - plugins: - - monorepo-diff#v1.0.1: - diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" - watch: - - path: - - internal/ - - dev-tools/ - - pkg/ - - testing/ - - version/ - - specs/ - - .agent-versions.json - - .go-version - - .package-version - - go.mod - - go.sum - - magefile.go - - main.go + # plugins: + # - monorepo-diff#v1.0.1: + # diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + # watch: + # - path: + # - internal/ + # - dev-tools/ + # - pkg/ + # - testing/ + # - version/ + # - specs/ + # - .agent-versions.json + # - .go-version + # - .package-version + # - go.mod + # - go.sum + # - magefile.go + # - main.go - - .buildkite/integration.pipeline.yml - - .buildkite/pipeline.yml - - .buildkite/scripts/ - - .buildkite/hooks/ + # - .buildkite/integration.pipeline.yml + # - .buildkite/pipeline.yml + # - .buildkite/scripts/ + # - .buildkite/hooks/ - config: - trigger: "elastic-agent-extended-testing" - build: - commit: "${BUILDKITE_COMMIT}" - branch: "${BUILDKITE_BRANCH}" - env: - - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + # config: + # trigger: "elastic-agent-extended-testing" + # build: + # commit: "${BUILDKITE_COMMIT}" + # branch: "${BUILDKITE_BRANCH}" + # env: + # - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + # - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + # - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} - # Trigger for branches - - label: "Triggering Integration tests for branches" - if: build.pull_request.id == null - trigger: "elastic-agent-extended-testing" - build: - commit: "${BUILDKITE_COMMIT}" - branch: "${BUILDKITE_BRANCH}" + # # Trigger for branches + # - label: "Triggering Integration tests for branches" + # if: build.pull_request.id == null + # trigger: "elastic-agent-extended-testing" + # build: + # commit: "${BUILDKITE_COMMIT}" + # branch: "${BUILDKITE_BRANCH}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ff4b890ff39..9fddd505b67 100644 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -69,7 +69,8 @@ mage() { go(){ # Search for the go in the Path if ! [ -x "$(type -P go | sed 's/go is //g')" ]; - then + then + set -x getOSOptions echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" " local _bin="${WORKSPACE}/bin" @@ -79,6 +80,7 @@ go(){ eval "$(command "${_bin}/gvm" "${GO_VERSION}" )" export GOPATH=$(command go env GOPATH) export PATH="${PATH}:${GOPATH}/bin" + set +x fi pushd "$WORKSPACE" command go "$@" From 416485fea9e97ba5bce8ee6e8b24476efbe71e2b Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 16:19:03 +0200 Subject: [PATCH 10/14] Debug --- .buildkite/pipeline.elastic-agent-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 14d0793b14f..815ba7f5994 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: DOCKER_REGISTRY: "docker.elastic.co" - SETUP_GVM_VERSION: "v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 + SETUP_GVM_VERSION: v0.5.0 # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 SETUP_MAGE_VERSION: "1.14.0" # this is required in order to allow the build process to override the default PWD of the BEAT_NAME. BEAT_NAME: "elastic-agent" From e07a3b43f22dc6c2b640caaec0912532374e498a Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 16:21:19 +0200 Subject: [PATCH 11/14] Debug --- .buildkite/pipeline.elastic-agent-package.yml | 2 +- .buildkite/scripts/common.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index 815ba7f5994..b866bd1ac79 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: DOCKER_REGISTRY: "docker.elastic.co" - SETUP_GVM_VERSION: v0.5.0 # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 + # SETUP_GVM_VERSION: "v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 SETUP_MAGE_VERSION: "1.14.0" # this is required in order to allow the build process to override the default PWD of the BEAT_NAME. BEAT_NAME: "elastic-agent" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9fddd505b67..0aa05f2b83f 100644 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -8,10 +8,10 @@ if [[ -z "${WORKSPACE-""}" ]]; then fi PIPELINE="${WORKSPACE}/.buildkite/pipeline.elastic-agent-package.yml" if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then - SETUP_MAGE_VERSION=$(grep -oe "SETUP_MAGE_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" ) + SETUP_MAGE_VERSION="1.14.0" fi if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then - SETUP_GVM_VERSION=$(grep -oe "SETUP_GVM_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" ) + SETUP_GVM_VERSION="v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 fi BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go") export BEAT_VERSION From f3b4e0549c186c262c229053d8333dea505fdad8 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 16:23:14 +0200 Subject: [PATCH 12/14] Debug --- .buildkite/pipeline.elastic-agent-package.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/pipeline.elastic-agent-package.yml b/.buildkite/pipeline.elastic-agent-package.yml index b866bd1ac79..a1fe16d696a 100644 --- a/.buildkite/pipeline.elastic-agent-package.yml +++ b/.buildkite/pipeline.elastic-agent-package.yml @@ -1,8 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: DOCKER_REGISTRY: "docker.elastic.co" - # SETUP_GVM_VERSION: "v0.5.0" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 - SETUP_MAGE_VERSION: "1.14.0" # this is required in order to allow the build process to override the default PWD of the BEAT_NAME. BEAT_NAME: "elastic-agent" # after moving elastic-agent out of beats, we should update the URL of the packaging. From 73a607889a54614d966a7d09a6fb6acb58bb6cfb Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 16:29:12 +0200 Subject: [PATCH 13/14] Cleanup --- .buildkite/pipeline.yml | 534 +++++++++++++++++------------------ .buildkite/scripts/common.sh | 2 - 2 files changed, 267 insertions(+), 269 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 494fcffaa2f..d92d10db541 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,285 +13,285 @@ steps: retry: manual: allowed: true - # - group: "Unit tests" - # key: "unit-tests" - # steps: - # - label: "Unit tests - Ubuntu 22.04" - # key: "unit-tests-2204" - # command: ".buildkite/scripts/steps/unit-tests.sh" - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # image: "family/core-ubuntu-2204" - # retry: - # manual: - # allowed: true + - group: "Unit tests" + key: "unit-tests" + steps: + - label: "Unit tests - Ubuntu 22.04" + key: "unit-tests-2204" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + retry: + manual: + allowed: true - # - label: "Unit tests - Ubuntu 22.04 ARM64" - # key: "unit-tests-2204-arm64" - # command: ".buildkite/scripts/steps/unit-tests.sh" - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "aws" - # imagePrefix: "core-ubuntu-2204-aarch64" - # diskSizeGb: 200 - # instanceType: "m6g.xlarge" - # retry: - # manual: - # allowed: true + - label: "Unit tests - Ubuntu 22.04 ARM64" + key: "unit-tests-2204-arm64" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "aws" + imagePrefix: "core-ubuntu-2204-aarch64" + diskSizeGb: 200 + instanceType: "m6g.xlarge" + retry: + manual: + allowed: true - # - label: "Unit tests - Windows 2022" - # key: "unit-tests-win2022" - # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # image: "family/core-windows-2022" - # machine_type: "n2-standard-8" - # disk_size: 200 - # disk_type: "pd-ssd" - # retry: - # manual: - # allowed: true + - label: "Unit tests - Windows 2022" + key: "unit-tests-win2022" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2022" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true - # - label: "Unit tests - Windows 2016" - # key: "unit-tests-win2016" - # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # image: "family/core-windows-2016" - # machine_type: "n2-standard-8" - # disk_size: 200 - # disk_type: "pd-ssd" - # retry: - # manual: - # allowed: true + - label: "Unit tests - Windows 2016" + key: "unit-tests-win2016" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2016" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true - # # Runs inly on the main branch - # - label: "Unit tests - MacOS 13 ARM" - # key: "unit-tests-macos-13-arm" - # command: ".buildkite/scripts/steps/unit-tests.sh" - # branches: main - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: orka - # imagePrefix: generic-13-ventura-arm - # retry: - # manual: - # allowed: true + # Runs inly on the main branch + - label: "Unit tests - MacOS 13 ARM" + key: "unit-tests-macos-13-arm" + command: ".buildkite/scripts/steps/unit-tests.sh" + branches: main + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-arm + retry: + manual: + allowed: true - # - label: "Unit tests - MacOS 13" - # key: "unit-tests-macos-13" - # command: ".buildkite/scripts/steps/unit-tests.sh" - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: orka - # imagePrefix: generic-13-ventura-x64 - # retry: - # manual: - # allowed: true + - label: "Unit tests - MacOS 13" + key: "unit-tests-macos-13" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-x64 + retry: + manual: + allowed: true - # - group: "Desktop Windows tests" - # key: "extended-windows" - # steps: - # - label: "Unit tests - Windows 10" - # key: "unit-tests-win10" - # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # # TODO create own image - # image: "family/general-windows-10" - # machine_type: "n2-standard-8" - # disk_type: "pd-ssd" - # retry: - # manual: - # allowed: true - # - label: "Unit tests - Windows 11" - # key: "unit-tests-win11" - # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - # artifact_paths: - # - "build/TEST-go-unit.html" - # - "build/TEST-go-unit.xml" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # # TODO create own image - # image: "family/general-windows-11" - # machine_type: "n2-standard-8" - # disk_type: "pd-ssd" - # retry: - # manual: - # allowed: true + - group: "Desktop Windows tests" + key: "extended-windows" + steps: + - label: "Unit tests - Windows 10" + key: "unit-tests-win10" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + # TODO create own image + image: "family/general-windows-10" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + retry: + manual: + allowed: true + - label: "Unit tests - Windows 11" + key: "unit-tests-win11" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-go-unit.html" + - "build/TEST-go-unit.xml" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + # TODO create own image + image: "family/general-windows-11" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + retry: + manual: + allowed: true - # - label: "Merge coverage reports" - # key: "merge-coverage" - # env: - # BUILDKITE_REPO: "" - # command: " - # .buildkite/scripts/steps/merge.sh - # unit-tests-2204 - # unit-tests-2204-arm64 - # unit-tests-win2016 - # unit-tests-win2022 - # unit-tests-macos-13 - # unit-tests-win10 - # unit-tests-win11 - # " - # artifact_paths: - # - "build/TEST-go-unit.cov" - # agents: - # image: "golang:1.20.10" - # depends_on: - # - unit-tests - # - extended-windows - # allow_dependency_failure: true + - label: "Merge coverage reports" + key: "merge-coverage" + env: + BUILDKITE_REPO: "" + command: " + .buildkite/scripts/steps/merge.sh + unit-tests-2204 + unit-tests-2204-arm64 + unit-tests-win2016 + unit-tests-win2022 + unit-tests-macos-13 + unit-tests-win10 + unit-tests-win11 + " + artifact_paths: + - "build/TEST-go-unit.cov" + agents: + image: "golang:1.20.10" + depends_on: + - unit-tests + - extended-windows + allow_dependency_failure: true - # - group: "K8s tests" - # key: "k8s-tests" - # steps: - # - label: "K8s tests: {{matrix.k8s_version}}" - # env: - # K8S_VERSION: "v{{matrix.k8s_version}}" - # KIND_VERSION: "v0.20.0" - # command: ".buildkite/scripts/steps/k8s-tests.sh" - # agents: - # provider: "gcp" - # image: "family/core-ubuntu-2204" - # matrix: - # setup: - # k8s_version: - # - "1.30.0" - # - "1.29.4" - # - "1.28.9" - # - "1.27.13" - # retry: - # manual: - # allowed: true - # - label: "K8s integration tests" - # key: "k8s-integration-tests" - # env: - # K8S_VERSION: "v1.30.2,v1.29.4,v1.28.9" - # KIND_VERSION: "v0.20.0" - # command: ".buildkite/scripts/steps/k8s-extended-tests.sh" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # agents: - # provider: "gcp" - # image: "family/core-ubuntu-2204" - # retry: - # manual: - # allowed: true + - group: "K8s tests" + key: "k8s-tests" + steps: + - label: "K8s tests: {{matrix.k8s_version}}" + env: + K8S_VERSION: "v{{matrix.k8s_version}}" + KIND_VERSION: "v0.20.0" + command: ".buildkite/scripts/steps/k8s-tests.sh" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + matrix: + setup: + k8s_version: + - "1.30.0" + - "1.29.4" + - "1.28.9" + - "1.27.13" + retry: + manual: + allowed: true + - label: "K8s integration tests" + key: "k8s-integration-tests" + env: + K8S_VERSION: "v1.30.2,v1.29.4,v1.28.9" + KIND_VERSION: "v0.20.0" + command: ".buildkite/scripts/steps/k8s-extended-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + retry: + manual: + allowed: true - # - label: ":sonarqube: Continuous Code Inspection" - # env: - # VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" - # agents: - # image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" - # command: - # - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." - # - "/scan-source-code.sh" - # depends_on: - # - "merge-coverage" - # retry: - # manual: - # allowed: true + - label: ":sonarqube: Continuous Code Inspection" + env: + VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" + agents: + image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" + command: + - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." + - "/scan-source-code.sh" + depends_on: + - "merge-coverage" + retry: + manual: + allowed: true - # # Triggers a dynamic step: Sync K8s - # # Runs only on main and if k8s files are changed - # - label: "Trigger k8s sync" - # branches: main - # plugins: - # - monebag/monorepo-diff#v2.5.9: - # diff: "git diff --name-only HEAD~1" - # watch: - # - path: - # - deploy/kubernetes/* - # - version/docs/version.asciidoc - # config: - # label: "Sync K8s" - # command: ".buildkite/scripts/steps/sync-k8s.sh" - # agents: - # provider: "gcp" - # image: "family/core-ubuntu-2204" - # env: - # - GH_VERSION=2.4.0 + # Triggers a dynamic step: Sync K8s + # Runs only on main and if k8s files are changed + - label: "Trigger k8s sync" + branches: main + plugins: + - monebag/monorepo-diff#v2.5.9: + diff: "git diff --name-only HEAD~1" + watch: + - path: + - deploy/kubernetes/* + - version/docs/version.asciidoc + config: + label: "Sync K8s" + command: ".buildkite/scripts/steps/sync-k8s.sh" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + env: + - GH_VERSION=2.4.0 - # # Trigger for pull requests - # - label: "Trigger Integration tests for Pull request" - # if: | - # (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) || - # build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/ + # Trigger for pull requests + - label: "Trigger Integration tests for Pull request" + if: | + (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) || + build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/ - # plugins: - # - monorepo-diff#v1.0.1: - # diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" - # watch: - # - path: - # - internal/ - # - dev-tools/ - # - pkg/ - # - testing/ - # - version/ - # - specs/ - # - .agent-versions.json - # - .go-version - # - .package-version - # - go.mod - # - go.sum - # - magefile.go - # - main.go + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - internal/ + - dev-tools/ + - pkg/ + - testing/ + - version/ + - specs/ + - .agent-versions.json + - .go-version + - .package-version + - go.mod + - go.sum + - magefile.go + - main.go - # - .buildkite/integration.pipeline.yml - # - .buildkite/pipeline.yml - # - .buildkite/scripts/ - # - .buildkite/hooks/ + - .buildkite/integration.pipeline.yml + - .buildkite/pipeline.yml + - .buildkite/scripts/ + - .buildkite/hooks/ - # config: - # trigger: "elastic-agent-extended-testing" - # build: - # commit: "${BUILDKITE_COMMIT}" - # branch: "${BUILDKITE_BRANCH}" - # env: - # - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - # - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - # - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + config: + trigger: "elastic-agent-extended-testing" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} - # # Trigger for branches - # - label: "Triggering Integration tests for branches" - # if: build.pull_request.id == null - # trigger: "elastic-agent-extended-testing" - # build: - # commit: "${BUILDKITE_COMMIT}" - # branch: "${BUILDKITE_BRANCH}" + # Trigger for branches + - label: "Triggering Integration tests for branches" + if: build.pull_request.id == null + trigger: "elastic-agent-extended-testing" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 0aa05f2b83f..b732a4c5634 100644 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -70,7 +70,6 @@ go(){ # Search for the go in the Path if ! [ -x "$(type -P go | sed 's/go is //g')" ]; then - set -x getOSOptions echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" " local _bin="${WORKSPACE}/bin" @@ -80,7 +79,6 @@ go(){ eval "$(command "${_bin}/gvm" "${GO_VERSION}" )" export GOPATH=$(command go env GOPATH) export PATH="${PATH}:${GOPATH}/bin" - set +x fi pushd "$WORKSPACE" command go "$@" From 33eb785159a1020736ff84def4fe56f65d3613e2 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Jul 2024 16:35:08 +0200 Subject: [PATCH 14/14] Cleanup --- .buildkite/scripts/common.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b732a4c5634..43a18a2dbc1 100644 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -6,7 +6,6 @@ if [[ -z "${WORKSPACE-""}" ]]; then WORKSPACE=$(git rev-parse --show-toplevel) export WORKSPACE fi -PIPELINE="${WORKSPACE}/.buildkite/pipeline.elastic-agent-package.yml" if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then SETUP_MAGE_VERSION="1.14.0" fi