From be5debd626ca8c615a61928b217f10d9b7ad6852 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 13 Nov 2024 16:42:07 +0100 Subject: [PATCH] Buildkite integration tests: Windows 2022 Sudo tests (#6010) * [CI]Run Ubuntu x86 and ARM integration tests on buildkite * Updated the new pipeline * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * Test Windown 2022 in buildkite * test pre-command hook * Test windows flags * Fix pre-command * Newer win image * newer image * Newer win image * quotes * quotes * Debug gotestsum * Debug gotestsum * Debug gotestsum * Test PS5.1 * go test, no gotestsum * raw go test * pwsh * pwsh * Merged with main * gotestsum * gotestsum * gotestsum * Run with stack * Test all OS * Test all OS * Test all OS * Added gotestsum for testing purposes * added asdf install, newer artifacts * Cleanup * Cleanup * Removed build_id --- .buildkite/bk.integration.pipeline.yml | 35 ++++++++++-- .buildkite/hooks/pre-command.ps1 | 30 ---------- .buildkite/pipeline.yml | 18 ++++-- .buildkite/scripts/install-go.ps1 | 29 ++++++++++ .buildkite/scripts/integration-tests.ps1 | 55 +++++++++++++++++++ .../steps/integration_tests_tf_sudo.sh | 3 + .buildkite/scripts/sudo-integration-tests.sh | 4 ++ .tool-versions | 2 +- 8 files changed, 133 insertions(+), 43 deletions(-) create mode 100644 .buildkite/scripts/install-go.ps1 create mode 100755 .buildkite/scripts/integration-tests.ps1 diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index 0a6611cc086..0e64e5f2065 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -4,8 +4,6 @@ env: DOCKER_REGISTRY: "docker.elastic.co" VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" ASDF_MAGE_VERSION: 1.14.0 - ASDF_GOLANG_VERSION: 1.22.6 - ASDF_TERRAFORM_VERSION: 1.9.2 steps: - label: "Integration tests: packaging" @@ -19,6 +17,8 @@ steps: - label: Start ESS stack for integration tests key: integration-ess + env: + ASDF_TERRAFORM_VERSION: 1.9.2 depends_on: - package-it command: | @@ -32,8 +32,29 @@ steps: image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" useCustomGlobalHooks: true - - group: "Stateful IT (Sudo): Ubuntu" - key: integration-tests + - group: "Stateful(Sudo): Windows" + key: integration-tests-win + depends_on: + - integration-ess + steps: + - label: "Win2022:{{matrix}}" + command: | + buildkite-agent artifact download build/distributions/** . --step 'package-it' + .buildkite/scripts/integration-tests.ps1 {{matrix}} + artifact_paths: + - build/** + agents: + provider: "gcp" + machineType: "n1-standard-8" + image: "platform-ingest-elastic-agent-windows-2022" + matrix: + - default + - fleet + - fleet-privileged + - upgrade + + - group: "Stateful(Sudo): Ubuntu" + key: integration-tests-ubuntu depends_on: - package-it - integration-ess @@ -58,7 +79,7 @@ steps: - fleet-privileged - fleet-airgapped-privileged - - label: "arm:Group:{{matrix}}" + - label: "arm:Group: {{matrix}}" command: | buildkite-agent artifact download build/distributions/** . --step 'package-it' .buildkite/scripts/steps/integration_tests_tf_sudo.sh {{matrix}} @@ -79,7 +100,9 @@ steps: - fleet-airgapped-privileged - label: ESS stack cleanup - depends_on: integration-tests + depends_on: + - integration-tests-ubuntu + - integration-tests-win allow_dependency_failure: true command: | buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess" diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1 index e3a6257b2f1..13eecf8c14b 100644 --- a/.buildkite/hooks/pre-command.ps1 +++ b/.buildkite/hooks/pre-command.ps1 @@ -1,32 +1,2 @@ # Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer $env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length)) - -# Install gcc TODO: Move to the VM image -choco install mingw -Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 -refreshenv - -# Install gvm and go -# TODO: Move GVM download to the base VM image -$env:GvmVersion = "0.5.2" -[Net.ServicePointManager]::SecurityProtocol = "tls12" -$env:GoVersion = Get-Content -Path .go-version -Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe -gvm --format=powershell $env:GoVersion | Invoke-Expression -go version - -$GOPATH = $(go env GOPATH) -$env:Path = "$GOPATH\bin;" + $env:Path -$env:GOTMPDIR = "$GOPATH\tmp" -New-Item -ItemType Directory -Force -Path $env:GOTMPDIR -[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine) -[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine) -[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine) -go env - -# Install tools -go install github.com/magefile/mage -go install github.com/elastic/go-licenser -go install golang.org/x/tools/cmd/goimports -go install github.com/jstemmer/go-junit-report -go install gotest.tools/gotestsum diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d7de45d2bc8..59668a515dd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -50,7 +50,9 @@ steps: - label: "Unit tests - Windows 2022" key: "unit-tests-win2022" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + command: | + .\\.buildkite\\scripts\\install-go.ps1 + .\\.buildkite\\scripts\\steps\\unit-tests.ps1 artifact_paths: - "build/TEST-*.html" - "build/TEST-*.xml" @@ -68,7 +70,9 @@ steps: - label: "Unit tests - Windows 2016" key: "unit-tests-win2016" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + command: | + .\\.buildkite\\scripts\\install-go.ps1 + .\\.buildkite\\scripts\\steps\\unit-tests.ps1 artifact_paths: - "build/TEST-*.html" - "build/TEST-*.xml" @@ -121,7 +125,9 @@ steps: steps: - label: "Unit tests - Windows 10" key: "unit-tests-win10" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + command: | + .\\.buildkite\\scripts\\install-go.ps1 + .\\.buildkite\\scripts\\steps\\unit-tests.ps1 artifact_paths: - "build/TEST-*.html" - "build/TEST-*.xml" @@ -138,7 +144,9 @@ steps: allowed: true - label: "Unit tests - Windows 11" key: "unit-tests-win11" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + command: | + .\\.buildkite\\scripts\\install-go.ps1 + .\\.buildkite\\scripts\\steps\\unit-tests.ps1 artifact_paths: - "build/TEST-*.html" - "build/TEST-*.xml" @@ -154,7 +162,6 @@ steps: manual: allowed: true - - label: ":junit: Junit annotate" plugins: - junit-annotate#v2.5.0: @@ -180,7 +187,6 @@ steps: - step: "unit-tests-win11" allow_failure: true - - label: "Merge coverage reports" key: "merge-coverage" env: diff --git a/.buildkite/scripts/install-go.ps1 b/.buildkite/scripts/install-go.ps1 new file mode 100644 index 00000000000..b9bbcf926b2 --- /dev/null +++ b/.buildkite/scripts/install-go.ps1 @@ -0,0 +1,29 @@ +# # Install gcc TODO: Move to the VM image +choco install mingw +Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 +refreshenv + +# Install gvm and go +# TODO: Move GVM download to the base VM image +$env:GvmVersion = "0.5.2" +[Net.ServicePointManager]::SecurityProtocol = "tls12" +$env:GoVersion = Get-Content -Path .go-version +Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe +gvm --format=powershell $env:GoVersion | Invoke-Expression +go version + +$GOPATH = $(go env GOPATH) +$env:Path = "$GOPATH\bin;" + $env:Path +$env:GOTMPDIR = "$GOPATH\tmp" +New-Item -ItemType Directory -Force -Path $env:GOTMPDIR +[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine) +go env + +Install tools +go install github.com/magefile/mage +go install github.com/elastic/go-licenser +go install golang.org/x/tools/cmd/goimports +go install github.com/jstemmer/go-junit-report +go install gotest.tools/gotestsum diff --git a/.buildkite/scripts/integration-tests.ps1 b/.buildkite/scripts/integration-tests.ps1 new file mode 100755 index 00000000000..e62dcf024d7 --- /dev/null +++ b/.buildkite/scripts/integration-tests.ps1 @@ -0,0 +1,55 @@ +param ( + [string]$GROUP_NAME +) + +Write-Output "~~~ Switching to PowerShell 7" +pwsh +$PSVersionTable.PSVersion + +# TODO: dedicated ESS strack for retries +Write-Output "~~~ Receiving ESS stack metadata" +$env:ELASTICSEARCH_HOST = (buildkite-agent meta-data get "es.host") +$env:ELASTICSEARCH_USERNAME = (buildkite-agent meta-data get "es.username") +$env:ELASTICSEARCH_PASSWORD = (buildkite-agent meta-data get "es.pwd") +$env:KIBANA_HOST = (buildkite-agent meta-data get "kibana.host") +$env:KIBANA_USERNAME = (buildkite-agent meta-data get "kibana.username") +$env:KIBANA_PASSWORD = (buildkite-agent meta-data get "kibana.pwd") + +Write-Output "~~~ Running integration tests as $env:USERNAME" +Write-Output "~~~ Integration tests: $GROUP_NAME" + +go install gotest.tools/gotestsum +gotestsum --version + +# Read package version from .package-version file +$PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue +if ($PACKAGE_VERSION) { + $PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT" +} + +echo "~~~ Building test binaries" +mage build:testBinaries + +# Run integration tests with gotestsum +echo "~~~ Running tests" +$env:TEST_BINARY_NAME = "elastic-agent" +$env:AGENT_VERSION = $PACKAGE_VERSION +$env:SNAPSHOT = $true + +$ErrorActionPreference = 'Continue' + +gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=true" +$TESTS_EXIT_STATUS = $LASTEXITCODE +$ErrorActionPreference = 'Stop' + +# Generate HTML report if XML output exists +$outputXML = "build/${GROUP_NAME}.integration.xml" +if (Test-Path $outputXML) { + # Install junit2html if not installed + go install github.com/alexec/junit2html@latest + Get-Content $outputXML | junit2html > "build/TEST-report.html" +} else { + Write-Output "Cannot generate HTML test report: $outputXML not found" +} + +exit $TESTS_EXIT_STATUS \ No newline at end of file diff --git a/.buildkite/scripts/steps/integration_tests_tf_sudo.sh b/.buildkite/scripts/steps/integration_tests_tf_sudo.sh index 5a832748ca6..0ce1bc03b7e 100755 --- a/.buildkite/scripts/steps/integration_tests_tf_sudo.sh +++ b/.buildkite/scripts/steps/integration_tests_tf_sudo.sh @@ -5,6 +5,9 @@ source .buildkite/scripts/common2.sh source .buildkite/scripts/steps/ess.sh +# Make sure that all tools are installed +asdf install + GROUP_NAME=$1 if [ -z "$GROUP_NAME" ]; then echo "Error: Specify the group name: sudo-integration-tests.sh [group_name]" >&2 diff --git a/.buildkite/scripts/sudo-integration-tests.sh b/.buildkite/scripts/sudo-integration-tests.sh index 8ebc5f2ff90..95eda22c72b 100755 --- a/.buildkite/scripts/sudo-integration-tests.sh +++ b/.buildkite/scripts/sudo-integration-tests.sh @@ -4,10 +4,14 @@ source /opt/buildkite-agent/hooks/pre-command source .buildkite/hooks/pre-command || echo "No pre-command hook found" +# Make sure that all tools are installed +asdf install + GROUP_NAME=$1 echo "~~~ Running integration tests as $USER" echo "~~~ Integration tests: ${GROUP_NAME}" +go install gotest.tools/gotestsum gotestsum --version PACKAGE_VERSION="$(cat .package-version)" if [[ -n "$PACKAGE_VERSION" ]]; then diff --git a/.tool-versions b/.tool-versions index 2d608f9b5e2..1d08234864f 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ mage 1.14.0 -golang 1.22.8 +golang 1.22.9 terraform 1.9.3 \ No newline at end of file