Skip to content

Commit

Permalink
[CI] BK Integration tests: Run non-sudo tests (#6045)
Browse files Browse the repository at this point in the history
* Sudo test script refactoring

* sudo param

* Test non-sudo

* Added apm to TF deployment

* Test build non-sudo

* removed apm

* Fix Kibana config

* Fix Kibana config

* Fix Kibana config

* Non-sudo test run

* Non-sudo test run

* Cleanup

* Cleanup

* stop reading system log files for OtelTest

* Renamed sudo-integration-tests.sh to buildkite-integration-tests.sh

---------

Co-authored-by: Lee E. Hinman <[email protected]>
  • Loading branch information
pazone and leehinman authored Nov 19, 2024
1 parent 825a15d commit f64785c
Show file tree
Hide file tree
Showing 6 changed files with 128 additions and 73 deletions.
63 changes: 52 additions & 11 deletions .buildkite/bk.integration.pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,16 @@ steps:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
useCustomGlobalHooks: true

- group: "Stateful(Sudo): Windows"
- group: "Stateful: Windows"
key: integration-tests-win
depends_on:
- integration-ess
- package-it
steps:
- label: "Win2022:{{matrix}}"
- label: "Win2022:sudo:{{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/integration-tests.ps1 {{matrix}}
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
artifact_paths:
- build/**
agents:
Expand All @@ -53,15 +54,42 @@ steps:
- fleet-privileged
- upgrade

- group: "Stateful(Sudo): Ubuntu"
- label: "Win2022:non-sudo:{{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
artifact_paths:
- build/**
agents:
provider: "gcp"
machineType: "n1-standard-8"
image: "family/platform-ingest-elastic-agent-windows-2022"
matrix:
- default

- group: "Stateful:Ubuntu"
key: integration-tests-ubuntu
depends_on:
- integration-ess
- package-it
steps:
- label: "x86_64:Group: {{matrix}}"
- label: "x86_64:non-sudo: {{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
artifact_paths:
- build/**
agents:
provider: "gcp"
machineType: "n1-standard-8"
image: "family/platform-ingest-elastic-agent-ubuntu-2404"
matrix:
- default

- label: "x86_64:sudo: {{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests_tf_sudo.sh {{matrix}}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
artifact_paths:
- build/**
agents:
Expand All @@ -80,10 +108,10 @@ steps:
- fleet-privileged
- fleet-airgapped-privileged

- label: "arm:Group: {{matrix}}"
- label: "arm:sudo: {{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests_tf_sudo.sh {{matrix}}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
artifact_paths:
- build/**
agents:
Expand All @@ -102,16 +130,29 @@ steps:
- fleet-privileged
- fleet-airgapped-privileged

- group: "Stateful(Sudo): RHEL8"
- label: "arm:non-sudo: {{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
artifact_paths:
- build/**
agents:
provider: "aws"
imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
instanceType: "m6g.xlarge"
matrix:
- default

- group: "Stateful(Sudo):RHEL8"
key: integration-tests-rhel8
depends_on:
- package-it
- integration-ess
steps:
- label: "x86_64:Group: rpm"
- label: "x86_64:sudo:rpm"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
.buildkite/scripts/steps/integration_tests_tf_sudo.sh rpm
.buildkite/scripts/steps/integration_tests_tf.sh rpm true
artifact_paths:
- build/**
agents:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,27 +1,42 @@
#!/usr/bin/env bash

# Fixes source asdf.sh, otherwise it relies on unexisting /root/.asdf directory
export HOME=/opt/buildkite-agent

# The script is used to run integration tests with sudo
source /opt/buildkite-agent/hooks/pre-command
source .buildkite/hooks/pre-command || echo "No pre-command hook found"
GROUP_NAME=$1
TEST_SUDO=$2

if [ -z "$GROUP_NAME" ]; then
echo "Error: Specify the group name: sudo-integration-tests.sh [group_name]" >&2
exit 1
fi

if [ -z "$TEST_SUDO" ]; then
echo "Error: Specify the test sudo: sudo-integration-tests.sh [group_name] [test_sudo]" >&2
exit 1
fi

if [ "$TEST_SUDO" == "true" ]; then
echo "Re-initializing ASDF. The user is changed to root..."
export HOME=/opt/buildkite-agent
source /opt/buildkite-agent/hooks/pre-command
source .buildkite/hooks/pre-command || echo "No pre-command hook found"
fi

# Make sure that all tools are installed
asdf install

GROUP_NAME=$1

echo "~~~ Running integration tests as $USER"
echo "~~~ Integration tests: ${GROUP_NAME}"

go install gotest.tools/gotestsum
gotestsum --version

PACKAGE_VERSION="$(cat .package-version)"
if [[ -n "$PACKAGE_VERSION" ]]; then
PACKAGE_VERSION=${PACKAGE_VERSION}"-SNAPSHOT"
fi

set +e
TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${PACKAGE_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo=true
TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${PACKAGE_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo="${TEST_SUDO}"
TESTS_EXIT_STATUS=$?
set -e

Expand Down
5 changes: 3 additions & 2 deletions .buildkite/scripts/integration-tests.ps1
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
param (
[string]$GROUP_NAME
[string]$GROUP_NAME,
[string]$TEST_SUDO
)

echo "~~~ Preparing environment"
Expand All @@ -26,7 +27,7 @@ mage build:testBinaries
try {
Get-Ess-Stack -StackVersion $PACKAGE_VERSION
Write-Output "~~~ Running integration test group: $GROUP_NAME as user: $env:USERNAME"
gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=true"
gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=$TEST_SUDO"
} finally {
ess_down
# Generate HTML report if XML output exists
Expand Down
49 changes: 43 additions & 6 deletions .buildkite/scripts/steps/integration_tests_tf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,57 @@
set -euo pipefail

source .buildkite/scripts/common2.sh

source .buildkite/scripts/steps/ess.sh

# Make sure that all tools are installed
asdf install

GROUP_NAME=$1
TEST_SUDO=$2
if [ -z "$GROUP_NAME" ]; then
echo "Error: Specify the group name: integration_tests_tf.sh [group_name]" >&2
exit 1
fi

if [ -z "$TEST_SUDO" ]; then
echo "Error: Specify the test sudo: integration_tests_tf.sh [group_name] [test_sudo]" >&2
exit 1
fi

# Override the agent package version using a string with format <major>.<minor>.<patch>
# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"
OVERRIDE_STACK_VERSION="$(cat .package-version)"
OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT"

echo "~~~ Bulding test binaries"
echo "~~~ Building test binaries"
mage build:testBinaries

ess_up $OVERRIDE_TEST_AGENT_VERSION || echo "Failed to start ESS stack" >&2
trap 'ess_down' EXIT
# If the step is retried, we start the stack again.
# BUILDKITE_RETRY_COUNT == "0" for the first run
# BUILDKITE_RETRY_COUNT > 0 for the retries
if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then
echo "~~~ The steps is retried, starting the ESS stack again"
trap 'ess_down' EXIT
ess_up $OVERRIDE_STACK_VERSION || echo "Failed to start ESS stack" >&2
else
# For the first run, we start the stack in the start_ess.sh step and it sets the meta-data
echo "~~~ Receiving ESS stack metadata"
export ELASTICSEARCH_HOST=$(buildkite-agent meta-data get "es.host")
export ELASTICSEARCH_USERNAME=$(buildkite-agent meta-data get "es.username")
export ELASTICSEARCH_PASSWORD=$(buildkite-agent meta-data get "es.pwd")
export KIBANA_HOST=$(buildkite-agent meta-data get "kibana.host")
export KIBANA_USERNAME=$(buildkite-agent meta-data get "kibana.username")
export KIBANA_PASSWORD=$(buildkite-agent meta-data get "kibana.pwd")
fi

# Run integration tests
echo "~~~ Running integration tests"
AGENT_VERSION="8.16.0-SNAPSHOT" SNAPSHOT=true TEST_DEFINE_PREFIX=non_sudo_linux gotestsum --no-color -f standard-verbose --junitfile build/TEST-go-integration.xml --jsonfile build/TEST-go-integration.out.json -- -tags integration github.com/elastic/elastic-agent/testing/integration

if [ "$TEST_SUDO" == "true" ]; then
sudo -E .buildkite/scripts/buildkite-integration-tests.sh $@
else
.buildkite/scripts/buildkite-integration-tests.sh $@
fi

46 changes: 0 additions & 46 deletions .buildkite/scripts/steps/integration_tests_tf_sudo.sh

This file was deleted.

7 changes: 7 additions & 0 deletions test_infra/ess/deployment.tf
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,13 @@ resource "ec_deployment" "integration-testing" {
kibana = {
size = "1g"
zone_count = 1
config = {
user_settings_json = jsonencode({
"xpack.fleet.enableExperimental" = ["agentTamperProtectionEnabled"]
"xpack.fleet.internal.registry.kibanaVersionCheckEnabled" = false
"server.restrictInternalApis" = false
})
}
}

integrations_server = {
Expand Down

0 comments on commit f64785c

Please sign in to comment.