Add testing for buildkite dynamic pipeline generator (elastic#38492)

This PR changes the main dynamic pipeline to detect if there are changes in the Python or yaml scripts and runs some Python tests to make sure that everything works before moving into triggering the pipeline.

This will ensure that we don't accidentally break the dynamic pipeline generator.

Signed-off-by: Alexandros Sapranidis <[email protected]>
Co-authored-by: Dimitrios Liappis <[email protected]>

.buildkite/pipeline.yml

@@ -5,6 +5,3 @@ env:
 steps:
   - label: "Generate dynamic pipeline"
     command: ".buildkite/scripts/generate_pipeline.sh"
-    agents:
-      provider: "gcp"
-      image: "family/platform-ingest-beats-ubuntu-2204"

.buildkite/pytest.ini

@@ -0,0 +1,11 @@
+[pytest]
+junit_family=xunit1
+
+addopts = --strict-markers
+markers =
+    load: Load tests
+    tag(name): Tag tests with Go-like semantics
+
+# Ignore setup and teardown for the timeout
+#timeout_func_only = True
+

.buildkite/scripts/generate_pipeline.sh

@@ -2,7 +2,12 @@
 set -euo pipefail
 
 echo "~~~ Install dependencies"
-pip3 install --quiet "ruamel.yaml<0.18.0"
+python3 -mpip install --quiet "ruamel.yaml<0.18.0"
+# temporary solution until we have this into a base container
+curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+chmod a+x /usr/bin/yq
+
+.buildkite/scripts/run_dynamic_pipeline_tests.sh
 
 echo "+++ Run pipeline generator in dry-run mode"
 python3 .buildkite/pipeline.py | yq .

.buildkite/scripts/run_dynamic_pipeline_tests.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed
+# this will allow us to fail fast, if e.g. a PR has broken the generator
+
+set -euo pipefail
+
+are_paths_changed() {
+  local patterns=("${@}")
+  local changelist=()
+  for pattern in "${patterns[@]}"; do
+    changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern"))
+    if [ "${#changed_files[@]}" -gt 0 ]; then
+      changelist+=("${changed_files[@]}")
+    fi
+  done
+
+  if [ "${#changelist[@]}" -gt 0 ]; then
+    echo "Files changed:"
+    echo "${changelist[*]}"
+    return 0
+  else
+    echo "No files changed within specified changeset:"
+    echo "${patterns[*]}"
+    return 1
+  fi
+}
+
+pipeline_generator_changeset=(
+  "^.buildkite/pipeline.py"
+  "^*/buildkite.yml"
+)
+
+if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
+  echo "~~~ Skipping pipeline generator tests"
+  exit
+fi
+
+echo "~~~ Running pipeline generator tests"
+
+python3 -mpip install --quiet "pytest"
+pushd .buildkite
+pytest .
+popd

.buildkite/test_pipeline.py

@@ -0,0 +1,71 @@
+import os
+
+import pytest
+import pipeline
+
+
+@pytest.fixture
+def ubuntu2204_aws_agent():
+    return {
+        "command": "fake-cmd",
+        "platform": "platform-ingest-beats-ubuntu-2204-aarch64",
+        "provider": "aws"
+    }
+
+
+@pytest.fixture()
+def fake_simple_group():
+    return {
+        "unitTest": {
+            "command": "fake-cmd",
+            "platform": "family/platform-ingest-beats-ubuntu-2204",
+        },
+        "integrationTest": {
+            "command": "fake-integration",
+            "platform": "family/platform-ingest-beats-ubuntu-2204",
+            "env": {
+                "FOO": "BAR",
+            },
+        },
+    }
+
+
+def test_fetch_stage(ubuntu2204_aws_agent):
+    step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category")
+    assert step.create_entity() == {
+        "label": "fake test",
+        "command": ["cd fake", "fake-cmd"],
+        "notify": [
+            {
+                "github_commit_status": {
+                    "context": "Fake: test",
+                }
+            }
+        ],
+        "agents": {
+            "provider": "aws",
+            "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64",
+            "instanceType": "t4g.large",
+        },
+        "artifact_paths": [
+            "fake/build/*.xml",
+            "fake/build/*.json",
+        ],
+    }
+
+
+def test_fetch_group(fake_simple_group):
+    group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing")
+    assert len(group.steps) == 2
+    for step in group.steps:
+        assert "testing" == step.category
+        assert "gcp" == step.agent.provider
+
+    assert group.steps[1].env.get("FOO") == "BAR"
+
+
+def test_is_pr():
+    os.environ["BUILDKITE_PULL_REQUEST"] = "1234"
+    assert pipeline.is_pr() is True
+    os.environ["BUILDKITE_PULL_REQUEST"] = "false"
+    assert pipeline.is_pr() is False