-
Notifications
You must be signed in to change notification settings - Fork 148
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CI] Pull requests migration to buildkite (#3573)
* buildkite test on Ubuntu * Fixed step ids * fix image family * Use aws from arm64 * try windows post-checkout hook * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * set GOPATH * set GOPATH * set GOPATH * set GOPATH * set GOPATH * Installed GCC * Installed GCC * Installed GCC * Windows 2016 * Propagate exit code on windows * Added GOTMPDIR * Added GOTMPDIR * Merge coverage reports step * Merge coverage reports step * Merge coverage reports step * changed installation path * changed installation path * changed installation path * changed installation path * changed installation path * changed installation path * test * test * merge.sh * Added coverage.out to simplify artifacts download * Fixed artifacts copy * buildite orka integration * fix os name * Add err message to upgrade test * Fix autocrlf for windows * Craig's patch * Complete test pipeline * Complete test pipeline * Remove macos GHA workflow * Fix Processing test results step * fix k8s jenkins step * Removed m1 stage * Enabled race detector for unit tests * Enables race detector for windows (cherry picked from commit bf78cc1) # Conflicts: # .buildkite/pipeline.yml # .buildkite/scripts/common.sh # .buildkite/scripts/steps/unit-tests.sh # .ci/Jenkinsfile
- Loading branch information
1 parent
e902065
commit 1067dc3
Showing
10 changed files
with
427 additions
and
146 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| # Set error handling | ||
| $ErrorActionPreference = "Stop" | ||
|
|
||
| # Define a function to checkout and merge | ||
| function Checkout-Merge { | ||
| param ( | ||
| [string]$targetBranch, | ||
| [string]$prCommit, | ||
| [string]$mergeBranch | ||
| ) | ||
|
|
||
| if (-not $targetBranch) { | ||
| Write-Host "No pull request target branch" | ||
| exit 1 | ||
| } | ||
|
|
||
| git fetch -v origin $targetBranch | ||
| git checkout FETCH_HEAD | ||
| Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)" | ||
|
|
||
| # Create a temporary branch to merge the PR with the target branch | ||
| git checkout -b $mergeBranch | ||
| Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)" | ||
|
|
||
| # Set author identity so it can be used for git merge | ||
| git config user.name "github-merged-pr-post-checkout" | ||
| git config user.email "auto-merge@buildkite" | ||
|
|
||
| git merge --no-edit $prCommit | ||
|
|
||
| if ($LASTEXITCODE -ne 0) { | ||
| $mergeResult = $LASTEXITCODE | ||
| Write-Host "Merge failed: $mergeResult" | ||
| git merge --abort | ||
| exit $mergeResult | ||
| } | ||
| } | ||
|
|
||
| $pullRequest = $env:BUILDKITE_PULL_REQUEST | ||
|
|
||
| if ($pullRequest -eq "false") { | ||
| Write-Host "Not a pull request, skipping" | ||
| exit 0 | ||
| } | ||
|
|
||
| $targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH | ||
| $prCommit = $env:BUILDKITE_COMMIT | ||
| $prId = $env:BUILDKITE_PULL_REQUEST | ||
| $mergeBranch = "pr_merge_$prId" | ||
|
|
||
| Checkout-Merge $targetBranch $prCommit $mergeBranch | ||
|
|
||
| Write-Host "Commit information" | ||
| git --no-pager log --format=%B -n 1 | ||
|
|
||
| Write-Host "Fixing CRLF in git checkout --" | ||
| git config core.autocrlf true | ||
|
|
||
| # Ensure Buildkite groups are rendered | ||
| Write-Host "" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # Install gcc TODO: Move to the VM image | ||
| choco install mingw | ||
| Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 | ||
| refreshenv | ||
|
|
||
| # Install gvm and go | ||
| # TODO: Move GVM download to the base VM image | ||
| $env:GvmVersion = "0.5.2" | ||
| [Net.ServicePointManager]::SecurityProtocol = "tls12" | ||
| $env:GoVersion = Get-Content -Path .go-version | ||
| Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe | ||
| gvm --format=powershell $env:GoVersion | Invoke-Expression | ||
| go version | ||
|
|
||
| $GOPATH = $(go env GOPATH) | ||
| $env:Path = "$GOPATH\bin;" + $env:Path | ||
| [Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine) | ||
| [Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine) | ||
|
|
||
| # Install tools | ||
| go install github.com/magefile/mage | ||
| go install github.com/elastic/go-licenser | ||
| go install golang.org/x/tools/cmd/goimports | ||
| go install github.com/jstemmer/go-junit-report | ||
| go install gotest.tools/gotestsum |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| if [[ -z "${WORKSPACE-""}" ]]; then | ||
| WORKSPACE=$(git rev-parse --show-toplevel) | ||
| export WORKSPACE | ||
| fi | ||
| PIPELINE="${WORKSPACE}/.buildkite/pipeline.elastic-agent-package.yml" | ||
| if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then | ||
| SETUP_MAGE_VERSION=$(grep -oe "SETUP_MAGE_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" ) | ||
| fi | ||
| if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then | ||
| SETUP_GVM_VERSION=$(grep -oe "SETUP_GVM_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" ) | ||
| fi | ||
| BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go") | ||
| export BEAT_VERSION | ||
|
|
||
| getOSOptions() { | ||
| case $(uname | tr '[:upper:]' '[:lower:]') in | ||
| linux*) | ||
| export AGENT_OS_NAME=linux | ||
| ;; | ||
| darwin*) | ||
| export AGENT_OS_NAME=darwin | ||
| ;; | ||
| msys*) | ||
| export AGENT_OS_NAME=windows | ||
| ;; | ||
| *) | ||
| export AGENT_OS_NAME=notset | ||
| ;; | ||
| esac | ||
| case $(uname -m | tr '[:upper:]' '[:lower:]') in | ||
| aarch64*) | ||
| export AGENT_OS_ARCH=arm64 | ||
| ;; | ||
| arm64*) | ||
| export AGENT_OS_ARCH=arm64 | ||
| ;; | ||
| amd64*) | ||
| export AGENT_OS_ARCH=amd64 | ||
| ;; | ||
| x86_64*) | ||
| export AGENT_OS_ARCH=amd64 | ||
| ;; | ||
| *) | ||
| export AGENT_OS_ARCH=notset | ||
| ;; | ||
| esac | ||
| } | ||
|
|
||
| # Wrapper function for executing mage | ||
| mage() { | ||
| go version | ||
| if ! [ -x "$(type -p mage | sed 's/mage is //g')" ]; | ||
| then | ||
| echo "installing mage ${SETUP_MAGE_VERSION}" | ||
| make mage | ||
| fi | ||
| pushd "$WORKSPACE" | ||
| command "mage" "$@" | ||
| ACTUAL_EXIT_CODE=$? | ||
| popd | ||
| return $ACTUAL_EXIT_CODE | ||
| } | ||
|
|
||
| # Wrapper function for executing go | ||
| go(){ | ||
| # Search for the go in the Path | ||
| if ! [ -x "$(type -p go | sed 's/go is //g')" ]; | ||
| then | ||
| getOSOptions | ||
| echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" " | ||
| local _bin="${WORKSPACE}/bin" | ||
| mkdir -p "${_bin}" | ||
| retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${AGENT_OS_NAME}-${AGENT_OS_ARCH}" | ||
| chmod +x "${_bin}/gvm" | ||
| eval "$(command "${_bin}/gvm" "${GO_VERSION}" )" | ||
| export GOPATH=$(command go env GOPATH) | ||
| export PATH="${PATH}:${GOPATH}/bin" | ||
| fi | ||
| pushd "$WORKSPACE" | ||
| command go "$@" | ||
| ACTUAL_EXIT_CODE=$? | ||
| popd | ||
| return $ACTUAL_EXIT_CODE | ||
| } | ||
|
|
||
| google_cloud_auth() { | ||
| local keyFile=$1 | ||
|
|
||
| gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null | ||
|
|
||
| export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation} | ||
| } | ||
|
|
||
| retry() { | ||
| local retries=$1 | ||
| shift | ||
|
|
||
| local count=0 | ||
| until "$@"; do | ||
| exit=$? | ||
| wait=$((2 ** count)) | ||
| count=$((count + 1)) | ||
| if [ $count -lt "$retries" ]; then | ||
| >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." | ||
| sleep $wait | ||
| else | ||
| >&2 echo "Retry $count/$retries exited $exit, no more retries left." | ||
| return $exit | ||
| fi | ||
| done | ||
| return 0 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov). | ||
| # Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.# | ||
|
|
||
| set -exuo pipefail | ||
|
|
||
| COV_ARTIFACT="coverage.out" | ||
| MERGED_COV_FILE="build/TEST-go-unit.cov" | ||
| # Space separated list of paths to coverage files | ||
| COV_PATHS="" | ||
|
|
||
| go install github.com/wadey/gocovmerge@latest | ||
| mkdir -p build | ||
|
|
||
| for STEP_ID in "$@"; do | ||
| mkdir -p $STEP_ID | ||
| buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID | ||
| COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT" | ||
| done | ||
|
|
||
| gocovmerge $COV_PATHS > $MERGED_COV_FILE | ||
| echo "Merged coverage file: $MERGED_COV_FILE. See artifacts" |
Oops, something went wrong.