From 099587f6e60720bfa8dc44eb84bce7f9da8f5e25 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 30 Oct 2023 15:44:05 +0100 Subject: [PATCH] [CI] Pull requests migration to buildkite (#3573) * buildkite test on Ubuntu * Fixed step ids * fix image family * Use aws from arm64 * try windows post-checkout hook * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * set GOPATH * set GOPATH * set GOPATH * set GOPATH * set GOPATH * Installed GCC * Installed GCC * Installed GCC * Windows 2016 * Propagate exit code on windows * Added GOTMPDIR * Added GOTMPDIR * Merge coverage reports step * Merge coverage reports step * Merge coverage reports step * changed installation path * changed installation path * changed installation path * changed installation path * changed installation path * changed installation path * test * test * merge.sh * Added coverage.out to simplify artifacts download * Fixed artifacts copy * buildite orka integration * fix os name * Add err message to upgrade test * Fix autocrlf for windows * Craig's patch * Complete test pipeline * Complete test pipeline * Remove macos GHA workflow * Fix Processing test results step * fix k8s jenkins step * Removed m1 stage * Enabled race detector for unit tests * Enables race detector for windows (cherry picked from commit bf78cc1e1498d19651857da3e69930f6db185903) --- .buildkite/hooks/post-checkout.ps1 | 60 +++++++++ .buildkite/hooks/pre-command.ps1 | 25 ++++ .buildkite/pipeline.yml | 117 +++++++++++++++- .buildkite/scripts/common.sh | 2 +- .buildkite/scripts/steps/merge.sh | 23 ++++ .buildkite/scripts/steps/unit-tests.ps1 | 28 ++++ .buildkite/scripts/steps/unit-tests.sh | 4 +- .ci/Jenkinsfile | 126 +----------------- .github/workflows/macos.yml | 23 ---- .../agent/application/upgrade/upgrade_test.go | 2 +- 10 files changed, 256 insertions(+), 154 deletions(-) create mode 100644 .buildkite/hooks/post-checkout.ps1 create mode 100644 .buildkite/hooks/pre-command.ps1 create mode 100755 .buildkite/scripts/steps/merge.sh create mode 100644 .buildkite/scripts/steps/unit-tests.ps1 delete mode 100644 .github/workflows/macos.yml diff --git a/.buildkite/hooks/post-checkout.ps1 b/.buildkite/hooks/post-checkout.ps1 new file mode 100644 index 00000000000..f2c590378fa --- /dev/null +++ b/.buildkite/hooks/post-checkout.ps1 @@ -0,0 +1,60 @@ +# Set error handling +$ErrorActionPreference = "Stop" + +# Define a function to checkout and merge +function Checkout-Merge { + param ( + [string]$targetBranch, + [string]$prCommit, + [string]$mergeBranch + ) + + if (-not $targetBranch) { + Write-Host "No pull request target branch" + exit 1 + } + + git fetch -v origin $targetBranch + git checkout FETCH_HEAD + Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # Create a temporary branch to merge the PR with the target branch + git checkout -b $mergeBranch + Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # Set author identity so it can be used for git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit $prCommit + + if ($LASTEXITCODE -ne 0) { + $mergeResult = $LASTEXITCODE + Write-Host "Merge failed: $mergeResult" + git merge --abort + exit $mergeResult + } +} + +$pullRequest = $env:BUILDKITE_PULL_REQUEST + +if ($pullRequest -eq "false") { + Write-Host "Not a pull request, skipping" + exit 0 +} + +$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH +$prCommit = $env:BUILDKITE_COMMIT +$prId = $env:BUILDKITE_PULL_REQUEST +$mergeBranch = "pr_merge_$prId" + +Checkout-Merge $targetBranch $prCommit $mergeBranch + +Write-Host "Commit information" +git --no-pager log --format=%B -n 1 + +Write-Host "Fixing CRLF in git checkout --" +git config core.autocrlf true + +# Ensure Buildkite groups are rendered +Write-Host "" diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1 new file mode 100644 index 00000000000..ca5b34bc920 --- /dev/null +++ b/.buildkite/hooks/pre-command.ps1 @@ -0,0 +1,25 @@ +# Install gcc TODO: Move to the VM image +choco install mingw +Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 +refreshenv + +# Install gvm and go +# TODO: Move GVM download to the base VM image +$env:GvmVersion = "0.5.2" +[Net.ServicePointManager]::SecurityProtocol = "tls12" +$env:GoVersion = Get-Content -Path .go-version +Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe +gvm --format=powershell $env:GoVersion | Invoke-Expression +go version + +$GOPATH = $(go env GOPATH) +$env:Path = "$GOPATH\bin;" + $env:Path +[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine) + +# Install tools +go install github.com/magefile/mage +go install github.com/elastic/go-licenser +go install golang.org/x/tools/cmd/goimports +go install github.com/jstemmer/go-junit-report +go install gotest.tools/gotestsum diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8ff198708fd..3936aa8b30f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,29 +4,134 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" DOCKER_REGISTRY: "docker.elastic.co" steps: - - label: "Unit tests" - key: "unit-tests" + - label: "Unit tests - Ubuntu 22.04" + key: "unit-tests-2204" command: ".buildkite/scripts/steps/unit-tests.sh" artifact_paths: - "build/TEST-**" - "build/diagnostics/*" + - "coverage.out" agents: provider: "gcp" + image: "family/core-ubuntu-2204" retry: manual: allowed: true + - label: "Unit tests - Ubuntu 22.04 ARM64" + key: "unit-tests-2204-arm64" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "aws" + imagePrefix: "core-ubuntu-2204-aarch64" + diskSizeGb: 200 + instanceType: "m6g.4xlarge" + retry: + manual: + allowed: true + + - label: "Unit tests - Windows 2022" + key: "unit-tests-win2022" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2022" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true + + - label: "Unit tests - Windows 2016" + key: "unit-tests-win2016" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2016" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true + + - label: "Unit tests - MacOS 13 ARM" + key: "unit-tests-macos-13-arm" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-arm + retry: + manual: + allowed: true + + - label: "Unit tests - MacOS 13" + key: "unit-tests-macos-13" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-x64 + retry: + manual: + allowed: true + + - label: "Merge coverage reports" + key: "merge-coverage" + env: + BUILDKITE_REPO: "" + command: " + .buildkite/scripts/steps/merge.sh + unit-tests-2204 + unit-tests-2204-arm64 + unit-tests-win2016 + unit-tests-win2022 + unit-tests-macos-13 + unit-tests-macos-13-arm + " + artifact_paths: + - "build/TEST-**" + agents: + image: "golang:1.20.10" + depends_on: + - unit-tests-2204 + - unit-tests-2204-arm64 + - unit-tests-win2022 + - unit-tests-win2016 + - unit-tests-macos-13 + - unit-tests-macos-13-arm + allow_dependency_failure: true + - label: ":sonarqube: Continuous Code Inspection" env: VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" agents: image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" command: - - "buildkite-agent artifact download build/TEST-go-unit.cov ." - - "buildkite-agent artifact download build/TEST-go-unit.out.json ." + - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." - "/scan-source-code.sh" depends_on: - - "unit-tests" + - "merge-coverage" retry: manual: allowed: true @@ -48,4 +153,4 @@ steps: provider: "gcp" plugins: - junit-annotate#v2.4.1: - artifacts: build/TEST-go-*.xml + artifacts: build/TEST-go-integration*.xml diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b1ec9608143..16d32fb77ab 100644 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -22,7 +22,7 @@ getOSOptions() { export AGENT_OS_NAME=linux ;; darwin*) - export AGENT_OS_NAME=osx + export AGENT_OS_NAME=darwin ;; msys*) export AGENT_OS_NAME=windows diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh new file mode 100755 index 00000000000..4eda3e2fd3b --- /dev/null +++ b/.buildkite/scripts/steps/merge.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov). +# Usage: merge.sh ... Where is the id of the step that contains the coverage artifact.# + +set -exuo pipefail + +COV_ARTIFACT="coverage.out" +MERGED_COV_FILE="build/TEST-go-unit.cov" +# Space separated list of paths to coverage files +COV_PATHS="" + +go install github.com/wadey/gocovmerge@latest +mkdir -p build + +for STEP_ID in "$@"; do + mkdir -p $STEP_ID + buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID + COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT" +done + +gocovmerge $COV_PATHS > $MERGED_COV_FILE +echo "Merged coverage file: $MERGED_COV_FILE. See artifacts" \ No newline at end of file diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1 new file mode 100644 index 00000000000..e1645554532 --- /dev/null +++ b/.buildkite/scripts/steps/unit-tests.ps1 @@ -0,0 +1,28 @@ +$ErrorActionPreference = "Stop" + +Write-Host "-- Fixing CRLF in git checkout --" +git config core.autocrlf input +git rm --quiet --cached -r . +git reset --quiet --hard + +$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH" + +Write-Host "--- Build" +mage build + +if ($LASTEXITCODE -ne 0) { + exit 1 +} + +Write-Host "--- Unit tests" +$env:TEST_COVERAGE = $true +$env:RACE_DETECTOR = $true +mage unitTest +# Copy coverage file to build directory so it can be downloaded as an artifact +cp .\build\TEST-go-unit.cov coverage.out + +if ($LASTEXITCODE -ne 0) { + exit 1 +} + + diff --git a/.buildkite/scripts/steps/unit-tests.sh b/.buildkite/scripts/steps/unit-tests.sh index b4d399801bf..6df75a26a08 100644 --- a/.buildkite/scripts/steps/unit-tests.sh +++ b/.buildkite/scripts/steps/unit-tests.sh @@ -4,4 +4,6 @@ set -euxo pipefail source .buildkite/scripts/common.sh echo "--- Unit tests" -TEST_COVERAGE=true mage unitTest \ No newline at end of file +RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest +# Copy coverage file to build directory so it can be downloaded as an artifact +cp build/TEST-go-unit.cov coverage.out \ No newline at end of file diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 237e62ab2fc..5cde7f706ed 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -95,57 +95,7 @@ pipeline { values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64' } } - stages { - stage('build'){ - steps { - withGithubNotify(context: "Build-${PLATFORM}") { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - withMageEnv(){ - dir("${BASE_DIR}"){ - cmd(label: 'Go build', script: 'mage build') - } - } - } - } - } - stage('Test') { - steps { - withGithubNotify(context: "Test-${PLATFORM}") { - withMageEnv(){ - dir("${BASE_DIR}"){ - withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) { - cmd(label: 'Go unitTest', script: 'mage unitTest') - } - } - } - } - } - post { - always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml") - whenTrue(isCodeCoverageEnabled()) { - coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml') - } - } - } - } - stage('Integration tests') { - when { - // Always when running builds on branches/tags - // Enable if integration test related changes. - beforeAgent true - anyOf { - not { changeRequest() } - expression { return params.integration_tests_ci } - expression { return env.GITHUB_COMMENT?.contains('integration tests') } - expression { matchesPrLabel(label: 'ci:integration') } - } - } - steps { - echo 'TBD' - } - } + stages { stage('K8s') { when { beforeAgent true @@ -160,6 +110,8 @@ pipeline { } } steps { + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") runK8s(k8sVersion: 'v1.27.3', kindVersion: 'v0.20.0', context: "K8s-${PLATFORM}") } } @@ -348,77 +300,7 @@ pipeline { } } } - } - stage('m1') { - agent { label 'orka && darwin && aarch64' } - options { skipDefaultCheckout() } - when { - // Always when running builds on branches/tags - // Enable if extended M1 support related changes. - beforeAgent true - anyOf { - not { changeRequest() } - expression { return isExtendedM1Enabled() && env.ONLY_DOCS == "false"} - } - } - stages { - stage('build'){ - steps { - withGithubNotify(context: "Build-darwin-aarch64") { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - withMageEnv(){ - dir("${BASE_DIR}"){ - cmd(label: 'Go build', script: 'mage build') - } - } - } - } - } - stage('Test') { - steps { - withGithubNotify(context: "Test-darwin-aarch64") { - withMageEnv(){ - dir("${BASE_DIR}"){ - withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) { - cmd(label: 'Go unitTest', script: 'mage unitTest') - } - } - } - } - } - post { - always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml") - whenTrue(isCodeCoverageEnabled()) { - coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml') - } - } - } - } - } - } - stage('e2e tests') { - when { - // Disable until we fix the e2e test. - // This should be running when there is a comment or the e2e label - beforeAgent true - anyOf { - // package artifacts are not generated if ONLY_DOCS, therefore e2e should not run if ONLY_DOCS - expression { return isE2eEnabled() && env.ONLY_DOCS == "false"} - } - } - steps { - runE2E(testMatrixFile: '.ci/.e2e-tests-for-elastic-agent.yaml', - beatVersion: "${env.BEAT_VERSION}-SNAPSHOT", - elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT", - gitHubCheckName: "e2e-tests", - gitHubCheckRepo: env.REPO, - gitHubCheckSha1: env.GIT_BASE_COMMIT, - propagate: true, - wait: true) - } - } + } } post { cleanup { diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index 23a7c6a4da8..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: macos - -on: - pull_request: - push: - branches: - - main - - 8.* - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: mage build - - name: Run test - run: mage unitTest diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go index edbf1dd9b6a..d8de6923cfe 100644 --- a/internal/pkg/agent/application/upgrade/upgrade_test.go +++ b/internal/pkg/agent/application/upgrade/upgrade_test.go @@ -99,7 +99,7 @@ func Test_CopyFile(t *testing.T) { } err := copyDir(l, tc.From, tc.To, tc.IgnoreErr) - require.Equal(t, tc.ExpectedErr, err != nil) + require.Equal(t, tc.ExpectedErr, err != nil, err) }) } }