From 72653ac2b0b57deedd276c3abc8d19f9085b3154 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 8 Nov 2023 19:25:12 +0100
Subject: [PATCH] [CI]Buildkite k8s tests migration (#3680)

* Buildkite k8s tests

* Buildkite k8s tests

* Added kind setup script

* Added kind setup script

* Added kind setup script

* Added kind setup script

* Branch configuration for k8s tests

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* transformed to matrix build

* Removed k8s tests from Jenkinsfile

* Run k8s tests on every PR

* Fix review comments

* install kind: moved path update to the top

* Moved PATH declaration to the top leve script

* Removed junit-annotate from k8s tests

* Moved k8s tests to upper level in the pipeline
---
 .buildkite/pipeline.yml                | 25 +++++++++++
 .buildkite/scripts/install-kind.sh     | 45 ++++++++++++++++++++
 .buildkite/scripts/install-kubectl.sh  | 45 ++++++++++++++++++++
 .buildkite/scripts/steps/k8s-tests.sh  | 30 +++++++++++++
 .buildkite/scripts/steps/unit-tests.sh |  6 ++-
 .ci/Jenkinsfile                        | 58 +-------------------------
 6 files changed, 151 insertions(+), 58 deletions(-)
 create mode 100644 .buildkite/scripts/install-kind.sh
 create mode 100644 .buildkite/scripts/install-kubectl.sh
 create mode 100644 .buildkite/scripts/steps/k8s-tests.sh

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index cbd0ae35884..fc1ff11871a 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -122,6 +122,31 @@ steps:
       - unit-tests-macos-13-arm
     allow_dependency_failure: true
 
+  - group: "K8s tests"
+    key: "k8s-tests"
+    steps:
+      - label: "K8s tests: {{matrix.k8s_version}}"
+        env:
+          K8S_VERSION: "v{{matrix.k8s_version}}"
+          KIND_VERSION: "v0.20.0"
+        command: ".buildkite/scripts/steps/k8s-tests.sh"
+        artifact_paths:
+          - "build/TEST-**"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "gcp"
+          image: "family/core-ubuntu-2204"
+        matrix:
+          setup:
+            k8s_version:
+              - "1.28.0"
+              - "1.27.3"
+              - "1.26.6"
+        retry:
+          manual:
+            allowed: true
+
   - label: ":sonarqube: Continuous Code Inspection"
     env:
       VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
diff --git a/.buildkite/scripts/install-kind.sh b/.buildkite/scripts/install-kind.sh
new file mode 100644
index 00000000000..171480d7685
--- /dev/null
+++ b/.buildkite/scripts/install-kind.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+echo "--- Install Kind"
+
+MSG="environment variable missing."
+DEFAULT_HOME="/usr/local"
+KIND_VERSION=${KIND_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KIND_CMD="${HOME}/bin/kind"
+
+if command -v kind
+then
+    set +e
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
+    then
+        echo "Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "Installing Kind"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+mkdir -p "${HOME}/bin"
+
+if curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
+    chmod +x "${KIND_CMD}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KIND_CMD}" ] ; then
+        rm "${KIND_CMD}"
+    fi
+    exit 1
+fi
diff --git a/.buildkite/scripts/install-kubectl.sh b/.buildkite/scripts/install-kubectl.sh
new file mode 100644
index 00000000000..fc558448c6c
--- /dev/null
+++ b/.buildkite/scripts/install-kubectl.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "--- Install kubectl"
+
+MSG="parameter missing."
+DEFAULT_HOME="/usr/local"
+K8S_VERSION=${K8S_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KUBECTL_CMD="${HOME}/bin/kubectl"
+
+if command -v kubectl
+then
+    set +e
+    echo "Found kubectl. Checking version.."
+    FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}')
+    if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ]
+    then
+        echo "Versions match. No need to install kubectl. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "Installing kubectl"
+
+mkdir -p "${HOME}/bin"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+if curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then
+    chmod +x "${KUBECTL_CMD}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KUBECTL_CMD}" ] ; then
+        rm "${KUBECTL_CMD}"
+    fi
+    exit 1
+fi
diff --git a/.buildkite/scripts/steps/k8s-tests.sh b/.buildkite/scripts/steps/k8s-tests.sh
new file mode 100644
index 00000000000..bb9affe2edc
--- /dev/null
+++ b/.buildkite/scripts/steps/k8s-tests.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -euo pipefail
+
+export PATH=$HOME/bin:${PATH}
+source .buildkite/scripts/install-kubectl.sh
+source .buildkite/scripts/install-kind.sh
+
+kind create cluster --image "kindest/node:${K8S_VERSION}" --config - <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    scheduler:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10251"
+        secure-port: "10259"
+    controllerManager:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10252"
+        secure-port: "10257"
+EOF
+kubectl cluster-info
+
+
+make -C deploy/kubernetes test
\ No newline at end of file
diff --git a/.buildkite/scripts/steps/unit-tests.sh b/.buildkite/scripts/steps/unit-tests.sh
index 6df75a26a08..a7203202105 100644
--- a/.buildkite/scripts/steps/unit-tests.sh
+++ b/.buildkite/scripts/steps/unit-tests.sh
@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
-set -euxo pipefail
+set -uo pipefail
 
 source .buildkite/scripts/common.sh
 
 echo "--- Unit tests"
 RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
+TESTS_EXIT_STATUS=$?
 # Copy coverage file to build directory so it can be downloaded as an artifact
-cp build/TEST-go-unit.cov coverage.out
\ No newline at end of file
+cp build/TEST-go-unit.cov coverage.out
+exit $TESTS_EXIT_STATUS
\ No newline at end of file
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 81e33aea0c5..870e9684742 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -95,26 +95,7 @@ pipeline {
             values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64'
           }
         }
-        stages {
-          stage('K8s') {
-            when {
-              beforeAgent true
-              // Always when running builds on branches/tags
-              // Enable if k8s related changes.
-              allOf {
-                expression { return env.PLATFORM == 'ubuntu-22 && immutable' }
-                anyOf {
-                  not { changeRequest() }                           // If no PR
-                  expression { return env.K8S_CHANGES == "true" }
-                }
-              }
-            }
-            steps {
-              deleteDir()
-              unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-              runK8s(k8sVersion: 'v1.28.0', kindVersion: 'v0.20.0', context: "K8s-${PLATFORM}")
-            }
-          }
+        stages {                  
           stage('Package') {
             when {
               beforeAgent true
@@ -167,42 +148,7 @@ pipeline {
           }
         }
       }
-    }
-    stage('Full K8s') {
-      when {
-        // Always when running builds on branches/tags
-        // Enable if k8s related changes.
-        anyOf {
-          not { changeRequest() }                           // If no PR
-          expression { return env.K8S_CHANGES == "true" }   // If k8s changes
-        }
-      }
-      failFast false
-      matrix {
-        agent {label 'ubuntu-22 && immutable'}
-        options { skipDefaultCheckout() }
-        axes {
-          axis {
-            name 'K8S_VERSION'
-            values "v1.28.0","v1.27.3","v1.26.6"
-          }
-        }
-        stages {
-          stage('K8s') {
-            steps {
-              deleteDir()
-              unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-              runK8s(k8sVersion: K8S_VERSION, kindVersion: 'v0.20.0', context: "K8s-${K8S_VERSION}")
-            }
-            post {
-              always {
-                junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
-              }
-            }
-          }
-        }
-      }
-    }
+    }    
     stage('Sync K8s') { //This stage opens a PR to kibana Repository in order to sync k8s manifests
       when {
         // Only on main branch