From 0f8c60155ea6098fb5b249bdc542aa3f211bf23b Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Thu, 1 Aug 2024 17:19:31 -0400 Subject: [PATCH] Adding CI tests Signed-off-by: Vishnu Challa --- .github/workflows/builders.yaml | 20 +++++ .github/workflows/ci-tests.yaml | 23 ++++++ .../{build-push.yaml => image-push.yaml} | 10 ++- .github/workflows/{pylint.yml => pylint.yaml} | 11 +-- .github/workflows/tests.yaml | 45 +++++++++++ ...y.yml => small-scale-cluster-density.yaml} | 0 ....yml => small-scale-node-density-cni.yaml} | 0 ...y.yml => trt-payload-cluster-density.yaml} | 0 .../label-small-scale-cluster-density.yaml | 4 +- examples/payload-scale-415.yaml | 8 +- examples/payload-scale-416.yaml | 8 +- examples/readout-control-plane-cdv2.yaml | 28 +++---- .../readout-control-plane-node-density.yaml | 28 +++---- examples/readout-netperf-tcp.yaml | 2 + examples/small-scale-cluster-density.yaml | 4 +- examples/small-scale-node-density-cni.yaml | 4 +- .../trt-external-payload-cluster-density.yaml | 4 +- examples/trt-external-payload-crd-scale.yaml | 4 +- .../trt-external-payload-node-density.yaml | 4 +- examples/trt-payload-cluster-density.yaml | 4 +- pkg/daemon.py | 7 +- pkg/utils.py | 2 +- test.bats | 76 +++++++++++++++++++ 23 files changed, 234 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/builders.yaml create mode 100644 .github/workflows/ci-tests.yaml rename .github/workflows/{build-push.yaml => image-push.yaml} (89%) rename .github/workflows/{pylint.yml => pylint.yaml} (91%) create mode 100644 .github/workflows/tests.yaml rename configs/{small-scale-cluster-density.yml => small-scale-cluster-density.yaml} (100%) rename configs/{small-scale-node-density-cni.yml => small-scale-node-density-cni.yaml} (100%) rename configs/{trt-payload-cluster-density.yml => trt-payload-cluster-density.yaml} (100%) create mode 100644 test.bats diff --git a/.github/workflows/builders.yaml b/.github/workflows/builders.yaml new file mode 100644 index 0000000..2bae5a1 --- /dev/null +++ b/.github/workflows/builders.yaml @@ -0,0 +1,20 @@ +name: Builders +on: + workflow_call: +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: build and install + run: | + pip install . \ No newline at end of file diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml new file mode 100644 index 0000000..47b0d16 --- /dev/null +++ b/.github/workflows/ci-tests.yaml @@ -0,0 +1,23 @@ +name: CI tests +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + +jobs: + lint: + uses: ./.github/workflows/pylint.yaml + + build: + uses: ./.github/workflows/builders.yaml + + tests: + needs: build + uses: ./.github/workflows/tests.yaml + secrets: inherit + diff --git a/.github/workflows/build-push.yaml b/.github/workflows/image-push.yaml similarity index 89% rename from .github/workflows/build-push.yaml rename to .github/workflows/image-push.yaml index 2670210..f6873ae 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/image-push.yaml @@ -1,9 +1,13 @@ -name: Build and Push Image -on: [ push ] +name: Image Push +on: + push: + branches: + - master + - main jobs: build: - name: Build and push image + name: Image Push runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yaml similarity index 91% rename from .github/workflows/pylint.yml rename to .github/workflows/pylint.yaml index 25699ee..075ad9c 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yaml @@ -1,25 +1,26 @@ name: Pylint - -on: [push,pull_request] - +on: + workflow_call: jobs: - build: + lint: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.11"] steps: - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip pip install pylint pip install -r requirements.txt - pip install . + - name: Analysing the code with pylint run: | pylint -d C0103 -d R0912 $(git ls-files '*/*.py' '*.py') \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..50cdfb8 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,45 @@ +name: Execute tests +on: + workflow_call: +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + + - name: Install bats + uses: bats-core/bats-action@1.5.4 + with: + bats-version: 1.10.0 + support-install: false + assert-install: false + detik-install: false + file-install: false + + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + pip install -r requirements.txt + pip install . + + - name: Execute Tests + run: | + bats -F pretty -T --print-output-on-failure test.bats + env: + TERM: linux + QE_ES_SERVER: ${{ secrets.QE_ES_SERVER }} \ No newline at end of file diff --git a/configs/small-scale-cluster-density.yml b/configs/small-scale-cluster-density.yaml similarity index 100% rename from configs/small-scale-cluster-density.yml rename to configs/small-scale-cluster-density.yaml diff --git a/configs/small-scale-node-density-cni.yml b/configs/small-scale-node-density-cni.yaml similarity index 100% rename from configs/small-scale-node-density-cni.yml rename to configs/small-scale-node-density-cni.yaml diff --git a/configs/trt-payload-cluster-density.yml b/configs/trt-payload-cluster-density.yaml similarity index 100% rename from configs/trt-payload-cluster-density.yml rename to configs/trt-payload-cluster-density.yaml diff --git a/examples/label-small-scale-cluster-density.yaml b/examples/label-small-scale-cluster-density.yaml index 924e0b1..13b69fd 100644 --- a/examples/label-small-scale-cluster-density.yaml +++ b/examples/label-small-scale-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS masterNodesType: m6a.xlarge diff --git a/examples/payload-scale-415.yaml b/examples/payload-scale-415.yaml index 475a98b..7d65d3e 100644 --- a/examples/payload-scale-415.yaml +++ b/examples/payload-scale-415.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed @@ -52,8 +52,8 @@ tests : value: duration agg_type: avg - name : aws-payload-scale-node-density - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/payload-scale-416.yaml b/examples/payload-scale-416.yaml index 6232235..5adb728 100644 --- a/examples/payload-scale-416.yaml +++ b/examples/payload-scale-416.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed @@ -52,8 +52,8 @@ tests : value: duration agg_type: avg - name : aws-payload-scale-node-density - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/readout-control-plane-cdv2.yaml b/examples/readout-control-plane-cdv2.yaml index 1fd0614..8d52bee 100644 --- a/examples/readout-control-plane-cdv2.yaml +++ b/examples/readout-control-plane-cdv2.yaml @@ -1,7 +1,7 @@ tests : - name : cluster-density-v2-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -37,8 +37,8 @@ tests : agg_type: avg - name : cluster-density-v2-120nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -74,8 +74,8 @@ tests : agg_type: avg - name : cluster-density-v2-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -111,8 +111,8 @@ tests : agg_type: avg - name : cdv2-kube-apiserver-etcd-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -160,8 +160,8 @@ tests : agg_type: avg - name : cd-v2-controller-manager-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -204,8 +204,8 @@ tests : agg_type: avg - name : cd-v2-api-request-latency-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -249,8 +249,8 @@ tests : agg_type: avg - name : cd-v2-etcd-latency-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS diff --git a/examples/readout-control-plane-node-density.yaml b/examples/readout-control-plane-node-density.yaml index c46c347..e7a73b3 100644 --- a/examples/readout-control-plane-node-density.yaml +++ b/examples/readout-control-plane-node-density.yaml @@ -1,7 +1,7 @@ tests : - name : node-density-heavy-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -49,8 +49,8 @@ tests : agg_type: avg - name : node-density-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -81,8 +81,8 @@ tests : agg_type: avg - name : node-density-120nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -113,8 +113,8 @@ tests : agg_type: avg - name : node-density-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -145,8 +145,8 @@ tests : agg_type: avg - name : node-density-cni-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -177,8 +177,8 @@ tests : agg_type: avg - name : node-density-cni-120nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -209,8 +209,8 @@ tests : agg_type: avg - name : node-density-cni-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS diff --git a/examples/readout-netperf-tcp.yaml b/examples/readout-netperf-tcp.yaml index 3394371..db70044 100644 --- a/examples/readout-netperf-tcp.yaml +++ b/examples/readout-netperf-tcp.yaml @@ -1,5 +1,7 @@ tests : - name : k8s-netperf-tcp + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS diff --git a/examples/small-scale-cluster-density.yaml b/examples/small-scale-cluster-density.yaml index c2b13b3..2cb7d4a 100644 --- a/examples/small-scale-cluster-density.yaml +++ b/examples/small-scale-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS masterNodesType: m6a.xlarge diff --git a/examples/small-scale-node-density-cni.yaml b/examples/small-scale-node-density-cni.yaml index 64b99f4..e45119d 100644 --- a/examples/small-scale-node-density-cni.yaml +++ b/examples/small-scale-node-density-cni.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-node-density-cni - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS masterNodesType: m6a.xlarge diff --git a/examples/trt-external-payload-cluster-density.yaml b/examples/trt-external-payload-cluster-density.yaml index e672294..192a7f0 100644 --- a/examples/trt-external-payload-cluster-density.yaml +++ b/examples/trt-external-payload-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : payload-cluster-density-v2 - index: perf_scale_ci-* - benchmarkIndex: ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/trt-external-payload-crd-scale.yaml b/examples/trt-external-payload-crd-scale.yaml index 2f6371b..f91e3c2 100644 --- a/examples/trt-external-payload-crd-scale.yaml +++ b/examples/trt-external-payload-crd-scale.yaml @@ -1,7 +1,7 @@ tests : - name : payload-crd-scale - index: perf_scale_ci-* - benchmarkIndex: ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/trt-external-payload-node-density.yaml b/examples/trt-external-payload-node-density.yaml index c7eb9c6..2613edf 100644 --- a/examples/trt-external-payload-node-density.yaml +++ b/examples/trt-external-payload-node-density.yaml @@ -1,7 +1,7 @@ tests : - name : payload-node-density - index: perf_scale_ci-* - benchmarkIndex: ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/trt-payload-cluster-density.yaml b/examples/trt-payload-cluster-density.yaml index 713d3f5..192a7f0 100644 --- a/examples/trt-payload-cluster-density.yaml +++ b/examples/trt-payload-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : payload-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/pkg/daemon.py b/pkg/daemon.py index cf9366f..63f8bd3 100644 --- a/pkg/daemon.py +++ b/pkg/daemon.py @@ -36,7 +36,7 @@ async def daemon_changepoint( # pylint: disable = R0913 json: json object of the changepoints and metrics """ parameters = {"version": version} - config_file_name=f"{test_name}.yml" + config_file_name=f"{test_name}.yaml" config_path = pkg_resources.resource_filename("configs", config_file_name) option_arguments = { "config": config_file_name, @@ -54,9 +54,9 @@ async def daemon_changepoint( # pylint: disable = R0913 True if filter_changepoints == "true" else False # pylint: disable = R1719 ) result = runTest.run(**option_arguments) - result = {k:json.loads(v) for k,v in result.items()} if result is None: return {"Error":"No UUID with given metadata"} + result = {k:json.loads(v) for k,v in result.items()} if filter_changepoints: for key, value in result.items(): result[key] = list(filter(lambda x: x.get("is_changepoint", False), value)) @@ -130,9 +130,10 @@ async def daemon_anomaly( # pylint: disable = R0913, R0914 True if filter_points == "true" else False # pylint: disable = R1719 ) result = runTest.run(**option_arguments) - result = {k:json.loads(v) for k,v in result.items()} if result is None: return {"Error":"No UUID with given metadata"} + result = {k:json.loads(v) for k,v in result.items()} + if filter_points: for key, value in result.items(): result[key] = list(filter(lambda x: x.get("is_changepoint", False), value)) diff --git a/pkg/utils.py b/pkg/utils.py index b6b8710..289016b 100644 --- a/pkg/utils.py +++ b/pkg/utils.py @@ -229,7 +229,7 @@ def process_test( uuids.append(options["uuid"]) buildUrls = get_build_urls(fingerprint_index, uuids, match) elif not uuids: - logger.error("No UUID present for given metadata") + logger.info("No UUID present for given metadata") return None, None benchmark_index = test["benchmarkIndex"] diff --git a/test.bats b/test.bats new file mode 100644 index 0000000..303ee25 --- /dev/null +++ b/test.bats @@ -0,0 +1,76 @@ +#!/usr/bin/env bats +# vi: ft=bash +# shellcheck disable=SC2086,SC2030,SC2031,SC2164 + + +run_cmd(){ + echo "$@" + ${@} +} + +setup() { + orion daemon --port 8000 & + export ES_SERVER="$QE_ES_SERVER" + export es_metadata_index="perf_scale_ci*" + export es_benchmark_index="ripsaw-kube-burner*" +} + +@test "orion cmd label small scale cluster density with hunter-analyze " { + run_cmd orion cmd --config "examples/label-small-scale-cluster-density.yaml" --lookback 5d --hunter-analyze +} + +@test "orion cmd payload scale 4.15 " { + run_cmd orion cmd --config "examples/payload-scale-415.yaml" --lookback 5d +} + +@test "orion cmd payload scale 4.16 without lookback period " { + run_cmd orion cmd --config "examples/payload-scale-416.yaml" +} + +@test "orion cmd readout control plane cdv2 with text output " { + run_cmd orion cmd --config "examples/readout-control-plane-cdv2.yaml" --hunter-analyze --output-format text --save-output-path=output.txt +} + +@test "orion cmd readout control plane node-density with json output " { + run_cmd orion cmd --config "examples/readout-control-plane-node-density.yaml" --hunter-analyze --output-format json --save-output-path=output.json +} + +@test "orion cmd readout netperf tcp with junit output " { + run_cmd orion cmd --config "examples/readout-netperf-tcp.yaml" --output-format junit --hunter-analyze --save-output-path=output.xml +} + +@test "orion cmd small scale cluster density with anomaly detection " { + run_cmd orion cmd --config "examples/small-scale-cluster-density.yaml" --lookback 5d --anomaly-detection +} + +@test "orion cmd small scale node density cni anomaly detection with a window " { + run_cmd orion cmd --config "examples/small-scale-node-density-cni.yaml" --anomaly-detection --anomaly-window 3 +} + +@test "orion cmd trt external payload cluster density anomaly detection with a minimum percentage " { + run_cmd orion cmd --config "examples/trt-external-payload-cluster-density.yaml" --anomaly-detection --anomaly-window 3 --min-anomaly-percent 5 +} + +@test "orion cmd trt external payload crd scale with default anomaly detection " { + run_cmd orion cmd --config "examples/trt-external-payload-crd-scale.yaml" --anomaly-detection +} + +@test "orion cmd trt external payload node density " { + run_cmd orion cmd --config "examples/trt-external-payload-node-density.yaml" --hunter-analyze +} + +@test "orion cmd trt external payload cluster density " { + run_cmd orion cmd --config "examples/trt-payload-cluster-density.yaml" --hunter-analyze +} + +@test "orion daemon small scale cluster density with anomaly detection " { + run_cmd curl http://127.0.0.1:8000/daemon/anomaly?convert_tinyurl=True&test_name=small-scale-cluster-density +} + +@test "orion daemon small scale node density cni with changepoint detection " { + run_cmd curl http://127.0.0.1:8000/daemon/changepoint?filter_changepoints=true&test_name=small-scale-node-density-cni +} + +@test "orion daemon trt payload cluster density with version parameter " { + run_cmd curl http://127.0.0.1:8000/daemon/changepoint?version=4.17&filter_changepoints=false&test_name=trt-payload-cluster-density +} \ No newline at end of file