Skip to content

Commit

Permalink
[CI]Buildkite k8s tests migration (#3680)
Browse files Browse the repository at this point in the history
* Buildkite k8s tests

* Buildkite k8s tests

* Added kind setup script

* Added kind setup script

* Added kind setup script

* Added kind setup script

* Branch configuration for k8s tests

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* Removed k8s tests from Jenkinsfile

* Run k8s tests on every PR

* Fix review comments

* install kind: moved path update to the top

* Moved PATH declaration to the top leve script

* Removed junit-annotate from k8s tests

* Moved k8s tests to upper level in the pipeline

(cherry picked from commit 72653ac)

# Conflicts:
#	.buildkite/pipeline.yml
#	.buildkite/scripts/steps/unit-tests.sh
#	.ci/Jenkinsfile
  • Loading branch information
pazone authored and mergify[bot] committed Nov 8, 2023
1 parent 3b4a759 commit 36d13c6
Show file tree
Hide file tree
Showing 6 changed files with 311 additions and 0 deletions.
172 changes: 172 additions & 0 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,178 @@ env:
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
DOCKER_REGISTRY: "docker.elastic.co"
steps:
<<<<<<< HEAD
=======
- label: "Unit tests - Ubuntu 22.04"
key: "unit-tests-2204"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-ubuntu-2204"
retry:
manual:
allowed: true

- label: "Unit tests - Ubuntu 22.04 ARM64"
key: "unit-tests-2204-arm64"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "aws"
imagePrefix: "core-ubuntu-2204-aarch64"
diskSizeGb: 200
instanceType: "m6g.4xlarge"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2022"
key: "unit-tests-win2022"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2022"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2016"
key: "unit-tests-win2016"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2016"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13 ARM"
key: "unit-tests-macos-13-arm"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-arm
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13"
key: "unit-tests-macos-13"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-x64
retry:
manual:
allowed: true

- label: "Merge coverage reports"
key: "merge-coverage"
env:
BUILDKITE_REPO: ""
command: "
.buildkite/scripts/steps/merge.sh
unit-tests-2204
unit-tests-2204-arm64
unit-tests-win2016
unit-tests-win2022
unit-tests-macos-13
unit-tests-macos-13-arm
"
artifact_paths:
- "build/TEST-**"
agents:
image: "golang:1.20.10"
depends_on:
- unit-tests-2204
- unit-tests-2204-arm64
- unit-tests-win2022
- unit-tests-win2016
- unit-tests-macos-13
- unit-tests-macos-13-arm
allow_dependency_failure: true

- group: "K8s tests"
key: "k8s-tests"
steps:
- label: "K8s tests: {{matrix.k8s_version}}"
env:
K8S_VERSION: "v{{matrix.k8s_version}}"
KIND_VERSION: "v0.20.0"
command: ".buildkite/scripts/steps/k8s-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-ubuntu-2204"
matrix:
setup:
k8s_version:
- "1.28.0"
- "1.27.3"
- "1.26.6"
retry:
manual:
allowed: true

- label: ":sonarqube: Continuous Code Inspection"
env:
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
agents:
image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
command:
- "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
- "/scan-source-code.sh"
depends_on:
- "merge-coverage"
retry:
manual:
allowed: true

- label: "Serverless integration test"
key: "serverless-integration-tests"
env:
TEST_INTEG_AUTH_ESS_REGION: us-east-1
command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
agents:
provider: "gcp"
machineType: "n1-standard-8"

>>>>>>> 72653ac2b0 ([CI]Buildkite k8s tests migration (#3680))
- label: "Integration tests"
key: "integration-tests"
command: ".buildkite/scripts/steps/integration_tests.sh"
Expand Down
45 changes: 45 additions & 0 deletions .buildkite/scripts/install-kind.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -exuo pipefail

echo "--- Install Kind"

MSG="environment variable missing."
DEFAULT_HOME="/usr/local"
KIND_VERSION=${KIND_VERSION:?$MSG}
HOME=${HOME:?$DEFAULT_HOME}
KIND_CMD="${HOME}/bin/kind"

if command -v kind
then
set +e
echo "Found Kind. Checking version.."
FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
then
echo "Versions match. No need to install Kind. Exiting."
exit 0
fi
set -e
fi

echo "Installing Kind"

OS=$(uname -s| tr '[:upper:]' '[:lower:]')
ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
if [ "${ARCH}" == "aarch64" ] ; then
ARCH_SUFFIX=arm64
else
ARCH_SUFFIX=amd64
fi

mkdir -p "${HOME}/bin"

if curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
chmod +x "${KIND_CMD}"
else
echo "Something bad with the download, let's delete the corrupted binary"
if [ -e "${KIND_CMD}" ] ; then
rm "${KIND_CMD}"
fi
exit 1
fi
45 changes: 45 additions & 0 deletions .buildkite/scripts/install-kubectl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -euo pipefail

echo "--- Install kubectl"

MSG="parameter missing."
DEFAULT_HOME="/usr/local"
K8S_VERSION=${K8S_VERSION:?$MSG}
HOME=${HOME:?$DEFAULT_HOME}
KUBECTL_CMD="${HOME}/bin/kubectl"

if command -v kubectl
then
set +e
echo "Found kubectl. Checking version.."
FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}')
if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ]
then
echo "Versions match. No need to install kubectl. Exiting."
exit 0
fi
set -e
fi

echo "Installing kubectl"

mkdir -p "${HOME}/bin"

OS=$(uname -s| tr '[:upper:]' '[:lower:]')
ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
if [ "${ARCH}" == "aarch64" ] ; then
ARCH_SUFFIX=arm64
else
ARCH_SUFFIX=amd64
fi

if curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then
chmod +x "${KUBECTL_CMD}"
else
echo "Something bad with the download, let's delete the corrupted binary"
if [ -e "${KUBECTL_CMD}" ] ; then
rm "${KUBECTL_CMD}"
fi
exit 1
fi
30 changes: 30 additions & 0 deletions .buildkite/scripts/steps/k8s-tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash
set -euo pipefail

export PATH=$HOME/bin:${PATH}
source .buildkite/scripts/install-kubectl.sh
source .buildkite/scripts/install-kind.sh

kind create cluster --image "kindest/node:${K8S_VERSION}" --config - <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
scheduler:
extraArgs:
bind-address: "0.0.0.0"
port: "10251"
secure-port: "10259"
controllerManager:
extraArgs:
bind-address: "0.0.0.0"
port: "10252"
secure-port: "10257"
EOF
kubectl cluster-info


make -C deploy/kubernetes test
11 changes: 11 additions & 0 deletions .buildkite/scripts/steps/unit-tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -uo pipefail

source .buildkite/scripts/common.sh

echo "--- Unit tests"
RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
TESTS_EXIT_STATUS=$?
# Copy coverage file to build directory so it can be downloaded as an artifact
cp build/TEST-go-unit.cov coverage.out
exit $TESTS_EXIT_STATUS
8 changes: 8 additions & 0 deletions .ci/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ pipeline {
values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64'
}
}
<<<<<<< HEAD
stages {
stage('build'){
steps {
Expand Down Expand Up @@ -163,6 +164,9 @@ pipeline {
runK8s(k8sVersion: 'v1.26.0', kindVersion: 'v0.17.0', context: "K8s-${PLATFORM}")
}
}
=======
stages {
>>>>>>> 72653ac2b0 ([CI]Buildkite k8s tests migration (#3680))
stage('Package') {
when {
beforeAgent true
Expand Down Expand Up @@ -215,6 +219,7 @@ pipeline {
}
}
}
<<<<<<< HEAD
}
stage('Full K8s') {
when {
Expand Down Expand Up @@ -251,6 +256,9 @@ pipeline {
}
}
}
=======
}
>>>>>>> 72653ac2b0 ([CI]Buildkite k8s tests migration (#3680))
stage('Sync K8s') { //This stage opens a PR to kibana Repository in order to sync k8s manifests
when {
// Only on main branch
Expand Down

0 comments on commit 36d13c6

Please sign in to comment.