Skip to content

Commit

Permalink
Buildkite integration tests: Windows 2022 Sudo tests (#6010)
Browse files Browse the repository at this point in the history
* [CI]Run Ubuntu x86 and ARM integration tests on buildkite

* Updated the new pipeline

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* Test Windown 2022 in buildkite

* test pre-command hook

* Test windows flags

* Fix pre-command

* Newer win image

* newer image

* Newer win image

* quotes

* quotes

* Debug gotestsum

* Debug gotestsum

* Debug gotestsum

* Test PS5.1

* go test, no gotestsum

* raw go test

* pwsh

* pwsh

* Merged with main

* gotestsum

* gotestsum

* gotestsum

* Run with stack

* Test all OS

* Test all OS

* Test all OS

* Added gotestsum for testing purposes

* added asdf install, newer artifacts

* Cleanup

* Cleanup

* Removed build_id
  • Loading branch information
pazone authored Nov 13, 2024
1 parent b7bbe52 commit be5debd
Show file tree
Hide file tree
Showing 8 changed files with 133 additions and 43 deletions.
35 changes: 29 additions & 6 deletions .buildkite/bk.integration.pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ env:
DOCKER_REGISTRY: "docker.elastic.co"
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
ASDF_MAGE_VERSION: 1.14.0
ASDF_GOLANG_VERSION: 1.22.6
ASDF_TERRAFORM_VERSION: 1.9.2

steps:
- label: "Integration tests: packaging"
Expand All @@ -19,6 +17,8 @@ steps:

- label: Start ESS stack for integration tests
key: integration-ess
env:
ASDF_TERRAFORM_VERSION: 1.9.2
depends_on:
- package-it
command: |
Expand All @@ -32,8 +32,29 @@ steps:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
useCustomGlobalHooks: true

- group: "Stateful IT (Sudo): Ubuntu"
key: integration-tests
- group: "Stateful(Sudo): Windows"
key: integration-tests-win
depends_on:
- integration-ess
steps:
- label: "Win2022:{{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/integration-tests.ps1 {{matrix}}
artifact_paths:
- build/**
agents:
provider: "gcp"
machineType: "n1-standard-8"
image: "platform-ingest-elastic-agent-windows-2022"
matrix:
- default
- fleet
- fleet-privileged
- upgrade

- group: "Stateful(Sudo): Ubuntu"
key: integration-tests-ubuntu
depends_on:
- package-it
- integration-ess
Expand All @@ -58,7 +79,7 @@ steps:
- fleet-privileged
- fleet-airgapped-privileged

- label: "arm:Group:{{matrix}}"
- label: "arm:Group: {{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests_tf_sudo.sh {{matrix}}
Expand All @@ -79,7 +100,9 @@ steps:
- fleet-airgapped-privileged

- label: ESS stack cleanup
depends_on: integration-tests
depends_on:
- integration-tests-ubuntu
- integration-tests-win
allow_dependency_failure: true
command: |
buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
Expand Down
30 changes: 0 additions & 30 deletions .buildkite/hooks/pre-command.ps1
Original file line number Diff line number Diff line change
@@ -1,32 +1,2 @@
# Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer
$env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length))

# Install gcc TODO: Move to the VM image
choco install mingw
Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
refreshenv

# Install gvm and go
# TODO: Move GVM download to the base VM image
$env:GvmVersion = "0.5.2"
[Net.ServicePointManager]::SecurityProtocol = "tls12"
$env:GoVersion = Get-Content -Path .go-version
Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
gvm --format=powershell $env:GoVersion | Invoke-Expression
go version

$GOPATH = $(go env GOPATH)
$env:Path = "$GOPATH\bin;" + $env:Path
$env:GOTMPDIR = "$GOPATH\tmp"
New-Item -ItemType Directory -Force -Path $env:GOTMPDIR
[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine)
[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
go env

# Install tools
go install github.com/magefile/mage
go install github.com/elastic/go-licenser
go install golang.org/x/tools/cmd/goimports
go install github.com/jstemmer/go-junit-report
go install gotest.tools/gotestsum
18 changes: 12 additions & 6 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ steps:

- label: "Unit tests - Windows 2022"
key: "unit-tests-win2022"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
command: |
.\\.buildkite\\scripts\\install-go.ps1
.\\.buildkite\\scripts\\steps\\unit-tests.ps1
artifact_paths:
- "build/TEST-*.html"
- "build/TEST-*.xml"
Expand All @@ -68,7 +70,9 @@ steps:

- label: "Unit tests - Windows 2016"
key: "unit-tests-win2016"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
command: |
.\\.buildkite\\scripts\\install-go.ps1
.\\.buildkite\\scripts\\steps\\unit-tests.ps1
artifact_paths:
- "build/TEST-*.html"
- "build/TEST-*.xml"
Expand Down Expand Up @@ -121,7 +125,9 @@ steps:
steps:
- label: "Unit tests - Windows 10"
key: "unit-tests-win10"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
command: |
.\\.buildkite\\scripts\\install-go.ps1
.\\.buildkite\\scripts\\steps\\unit-tests.ps1
artifact_paths:
- "build/TEST-*.html"
- "build/TEST-*.xml"
Expand All @@ -138,7 +144,9 @@ steps:
allowed: true
- label: "Unit tests - Windows 11"
key: "unit-tests-win11"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
command: |
.\\.buildkite\\scripts\\install-go.ps1
.\\.buildkite\\scripts\\steps\\unit-tests.ps1
artifact_paths:
- "build/TEST-*.html"
- "build/TEST-*.xml"
Expand All @@ -154,7 +162,6 @@ steps:
manual:
allowed: true


- label: ":junit: Junit annotate"
plugins:
- junit-annotate#v2.5.0:
Expand All @@ -180,7 +187,6 @@ steps:
- step: "unit-tests-win11"
allow_failure: true


- label: "Merge coverage reports"
key: "merge-coverage"
env:
Expand Down
29 changes: 29 additions & 0 deletions .buildkite/scripts/install-go.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# # Install gcc TODO: Move to the VM image
choco install mingw
Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
refreshenv

# Install gvm and go
# TODO: Move GVM download to the base VM image
$env:GvmVersion = "0.5.2"
[Net.ServicePointManager]::SecurityProtocol = "tls12"
$env:GoVersion = Get-Content -Path .go-version
Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
gvm --format=powershell $env:GoVersion | Invoke-Expression
go version

$GOPATH = $(go env GOPATH)
$env:Path = "$GOPATH\bin;" + $env:Path
$env:GOTMPDIR = "$GOPATH\tmp"
New-Item -ItemType Directory -Force -Path $env:GOTMPDIR
[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine)
[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
go env

Install tools
go install github.com/magefile/mage
go install github.com/elastic/go-licenser
go install golang.org/x/tools/cmd/goimports
go install github.com/jstemmer/go-junit-report
go install gotest.tools/gotestsum
55 changes: 55 additions & 0 deletions .buildkite/scripts/integration-tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
param (
[string]$GROUP_NAME
)

Write-Output "~~~ Switching to PowerShell 7"
pwsh
$PSVersionTable.PSVersion

# TODO: dedicated ESS strack for retries
Write-Output "~~~ Receiving ESS stack metadata"
$env:ELASTICSEARCH_HOST = (buildkite-agent meta-data get "es.host")
$env:ELASTICSEARCH_USERNAME = (buildkite-agent meta-data get "es.username")
$env:ELASTICSEARCH_PASSWORD = (buildkite-agent meta-data get "es.pwd")
$env:KIBANA_HOST = (buildkite-agent meta-data get "kibana.host")
$env:KIBANA_USERNAME = (buildkite-agent meta-data get "kibana.username")
$env:KIBANA_PASSWORD = (buildkite-agent meta-data get "kibana.pwd")

Write-Output "~~~ Running integration tests as $env:USERNAME"
Write-Output "~~~ Integration tests: $GROUP_NAME"

go install gotest.tools/gotestsum
gotestsum --version

# Read package version from .package-version file
$PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue
if ($PACKAGE_VERSION) {
$PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT"
}

echo "~~~ Building test binaries"
mage build:testBinaries

# Run integration tests with gotestsum
echo "~~~ Running tests"
$env:TEST_BINARY_NAME = "elastic-agent"
$env:AGENT_VERSION = $PACKAGE_VERSION
$env:SNAPSHOT = $true

$ErrorActionPreference = 'Continue'

gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=true"
$TESTS_EXIT_STATUS = $LASTEXITCODE
$ErrorActionPreference = 'Stop'

# Generate HTML report if XML output exists
$outputXML = "build/${GROUP_NAME}.integration.xml"
if (Test-Path $outputXML) {
# Install junit2html if not installed
go install github.com/alexec/junit2html@latest
Get-Content $outputXML | junit2html > "build/TEST-report.html"
} else {
Write-Output "Cannot generate HTML test report: $outputXML not found"
}

exit $TESTS_EXIT_STATUS
3 changes: 3 additions & 0 deletions .buildkite/scripts/steps/integration_tests_tf_sudo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ source .buildkite/scripts/common2.sh

source .buildkite/scripts/steps/ess.sh

# Make sure that all tools are installed
asdf install

GROUP_NAME=$1
if [ -z "$GROUP_NAME" ]; then
echo "Error: Specify the group name: sudo-integration-tests.sh [group_name]" >&2
Expand Down
4 changes: 4 additions & 0 deletions .buildkite/scripts/sudo-integration-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@
source /opt/buildkite-agent/hooks/pre-command
source .buildkite/hooks/pre-command || echo "No pre-command hook found"

# Make sure that all tools are installed
asdf install

GROUP_NAME=$1

echo "~~~ Running integration tests as $USER"
echo "~~~ Integration tests: ${GROUP_NAME}"
go install gotest.tools/gotestsum
gotestsum --version
PACKAGE_VERSION="$(cat .package-version)"
if [[ -n "$PACKAGE_VERSION" ]]; then
Expand Down
2 changes: 1 addition & 1 deletion .tool-versions
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
mage 1.14.0
golang 1.22.8
golang 1.22.9
terraform 1.9.3

0 comments on commit be5debd

Please sign in to comment.