From 62235efd7aa2ea74031981962580cac1dbf48977 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Fri, 22 Mar 2024 11:38:03 +0200 Subject: [PATCH] Add testing for buildkite dynamic pipeline generator (#38492) This PR changes the main dynamic pipeline to detect if there are changes in the Python or yaml scripts and runs some Python tests to make sure that everything works before moving into triggering the pipeline. This will ensure that we don't accidentally break the dynamic pipeline generator. Signed-off-by: Alexandros Sapranidis Co-authored-by: Dimitrios Liappis --- .buildkite/pipeline.yml | 3 - .buildkite/pytest.ini | 11 +++ .buildkite/scripts/generate_pipeline.sh | 7 +- .../scripts/run_dynamic_pipeline_tests.sh | 43 +++++++++++ .buildkite/test_pipeline.py | 71 +++++++++++++++++++ 5 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 .buildkite/pytest.ini create mode 100755 .buildkite/scripts/run_dynamic_pipeline_tests.sh create mode 100644 .buildkite/test_pipeline.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8f124fea4ae..84f725981fe 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -5,6 +5,3 @@ env: steps: - label: "Generate dynamic pipeline" command: ".buildkite/scripts/generate_pipeline.sh" - agents: - provider: "gcp" - image: "family/platform-ingest-beats-ubuntu-2204" diff --git a/.buildkite/pytest.ini b/.buildkite/pytest.ini new file mode 100644 index 00000000000..3eff7473d9f --- /dev/null +++ b/.buildkite/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +junit_family=xunit1 + +addopts = --strict-markers +markers = + load: Load tests + tag(name): Tag tests with Go-like semantics + +# Ignore setup and teardown for the timeout +#timeout_func_only = True + diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index 342dc8bf7ac..877ea018f06 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -2,7 +2,12 @@ set -euo pipefail echo "~~~ Install dependencies" -pip3 install --quiet "ruamel.yaml<0.18.0" +python3 -mpip install --quiet "ruamel.yaml<0.18.0" +# temporary solution until we have this into a base container +curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 +chmod a+x /usr/bin/yq + +.buildkite/scripts/run_dynamic_pipeline_tests.sh echo "+++ Run pipeline generator in dry-run mode" python3 .buildkite/pipeline.py | yq . diff --git a/.buildkite/scripts/run_dynamic_pipeline_tests.sh b/.buildkite/scripts/run_dynamic_pipeline_tests.sh new file mode 100755 index 00000000000..8eb72d7a96b --- /dev/null +++ b/.buildkite/scripts/run_dynamic_pipeline_tests.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed +# this will allow us to fail fast, if e.g. a PR has broken the generator + +set -euo pipefail + +are_paths_changed() { + local patterns=("${@}") + local changelist=() + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +pipeline_generator_changeset=( + "^.buildkite/pipeline.py" + "^*/buildkite.yml" +) + +if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then + echo "~~~ Skipping pipeline generator tests" + exit +fi + +echo "~~~ Running pipeline generator tests" + +python3 -mpip install --quiet "pytest" +pushd .buildkite +pytest . +popd diff --git a/.buildkite/test_pipeline.py b/.buildkite/test_pipeline.py new file mode 100644 index 00000000000..75fd949ccc8 --- /dev/null +++ b/.buildkite/test_pipeline.py @@ -0,0 +1,71 @@ +import os + +import pytest +import pipeline + + +@pytest.fixture +def ubuntu2204_aws_agent(): + return { + "command": "fake-cmd", + "platform": "platform-ingest-beats-ubuntu-2204-aarch64", + "provider": "aws" + } + + +@pytest.fixture() +def fake_simple_group(): + return { + "unitTest": { + "command": "fake-cmd", + "platform": "family/platform-ingest-beats-ubuntu-2204", + }, + "integrationTest": { + "command": "fake-integration", + "platform": "family/platform-ingest-beats-ubuntu-2204", + "env": { + "FOO": "BAR", + }, + }, + } + + +def test_fetch_stage(ubuntu2204_aws_agent): + step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category") + assert step.create_entity() == { + "label": "fake test", + "command": ["cd fake", "fake-cmd"], + "notify": [ + { + "github_commit_status": { + "context": "Fake: test", + } + } + ], + "agents": { + "provider": "aws", + "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64", + "instanceType": "t4g.large", + }, + "artifact_paths": [ + "fake/build/*.xml", + "fake/build/*.json", + ], + } + + +def test_fetch_group(fake_simple_group): + group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing") + assert len(group.steps) == 2 + for step in group.steps: + assert "testing" == step.category + assert "gcp" == step.agent.provider + + assert group.steps[1].env.get("FOO") == "BAR" + + +def test_is_pr(): + os.environ["BUILDKITE_PULL_REQUEST"] = "1234" + assert pipeline.is_pr() is True + os.environ["BUILDKITE_PULL_REQUEST"] = "false" + assert pipeline.is_pr() is False