From cc5bf2e91039debde97ff368260a09b2c71dfac0 Mon Sep 17 00:00:00 2001
From: Shashank Reddy Boyapally <sboyapal@redhat.com>
Date: Wed, 17 Jan 2024 16:40:10 -0500
Subject: [PATCH 1/2] added unit tests to fmatch

---
 .github/workflows/unit_test.yml |  25 ++++
 fmatch/matcher.py               |   5 +-
 fmatch/requirements.txt         |  42 ++++++
 fmatch/tests/__init__.py        |   3 +
 fmatch/tests/test_matcher.py    | 248 ++++++++++++++++++++++++++++++++
 5 files changed, 322 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/unit_test.yml
 create mode 100644 fmatch/tests/__init__.py
 create mode 100644 fmatch/tests/test_matcher.py

diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml
new file mode 100644
index 0000000..7734155
--- /dev/null
+++ b/.github/workflows/unit_test.yml
@@ -0,0 +1,25 @@
+name: Run pytest on Pull Request
+
+on: [push,pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt  # Replace with your requirements file if applicable
+
+    - name: Run pytest
+      run: |
+        python -m pytest pytest fmatch/tests --cov --random-order
diff --git a/fmatch/matcher.py b/fmatch/matcher.py
index 551d6f0..420262d 100644
--- a/fmatch/matcher.py
+++ b/fmatch/matcher.py
@@ -173,7 +173,7 @@ def burner_results(self, uuid, uuids, index):
         Returns:
             _type_: _description_
         """
-        if len(uuids) > 1:
+        if len(uuids) >= 1:
             if len(uuid) > 0:
                 uuids.remove(uuid)
         if len(uuids) < 1:
@@ -261,7 +261,9 @@ def parse_burner_cpu_results(self, data: dict):
         """
         res = []
         stamps = data['aggregations']['time']['buckets']
+        print("stamp",stamps)
         cpu = data['aggregations']['uuid']['buckets']
+        print("cpu",cpu)
         for stamp in stamps:
             dat = {}
             dat['uuid'] = stamp['key']
@@ -269,6 +271,7 @@ def parse_burner_cpu_results(self, data: dict):
             acpu = next(item for item in cpu if item["key"] == stamp['key'])
             dat['cpu_avg'] = acpu['cpu']['value']
             res.append(dat)
+        print("result",res)
         return res
 
     def convert_to_df(self, data, columns=None):
diff --git a/fmatch/requirements.txt b/fmatch/requirements.txt
index 75bae53..a800fe3 100644
--- a/fmatch/requirements.txt
+++ b/fmatch/requirements.txt
@@ -1,11 +1,53 @@
+astroid==3.0.2
+asttokens==2.4.1
+cbor2==5.5.1
 certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+coverage==7.4.0
+dill==0.3.7
+docutils==0.20.1
 elastic-transport==8.11.0
 elasticsearch==8.11.1
 elasticsearch7==7.13.0
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+isort==5.13.2
+jaraco.classes==3.3.0
+keyring==24.3.0
+markdown-it-py==3.0.0
+mccabe==0.7.0
+mdurl==0.1.2
+more-itertools==10.2.0
+nh3==0.2.15
 numpy==1.26.3
+packaging==23.2
 pandas==2.1.4
+pip-name==1.0.2
+pkginfo==1.9.6
+platformdirs==4.1.0
+pluggy==1.3.0
+pycodestyle==2.11.1
+pycryptodome==3.20.0
+pyflakes==3.2.0
+Pygments==2.17.2
+pylint==3.0.3
+pytest==7.4.4
+pytest-cov==4.1.0
+pytest-random-order==1.1.0
 python-dateutil==2.8.2
 pytz==2023.3.post1
+PyYAML==6.0.1
+readme-renderer==42.0
+requests==2.31.0
+requests-toolbelt==1.0.0
+rfc3986==2.0.0
+rich==13.7.0
 six==1.16.0
+tomlkit==0.12.3
+twine==4.0.2
 tzdata==2023.4
 urllib3==1.26.18
+vyper==0.3.10
+zipp==3.17.0
diff --git a/fmatch/tests/__init__.py b/fmatch/tests/__init__.py
new file mode 100644
index 0000000..ca5288b
--- /dev/null
+++ b/fmatch/tests/__init__.py
@@ -0,0 +1,3 @@
+"""
+This is __init__.py file for tests of fmatch
+"""
diff --git a/fmatch/tests/test_matcher.py b/fmatch/tests/test_matcher.py
new file mode 100644
index 0000000..439d007
--- /dev/null
+++ b/fmatch/tests/test_matcher.py
@@ -0,0 +1,248 @@
+"""
+Unit Test file for fmatch.py
+"""
+#pylint: disable = redefined-outer-name
+#pylint: disable = missing-function-docstring
+import os
+
+from elasticsearch.exceptions import NotFoundError
+import pytest
+import pandas as pd
+
+#pylint: disable = import-error
+from fmatch.matcher import Matcher
+
+
+@pytest.fixture
+def matcher_instance():
+    sample_output = {
+        "hits": {
+            "hits": [
+                {"_source": {"uuid": "uuid1", "field1": "value1"}},
+                {"_source": {"uuid": "uuid2", "field1": "value2"}},
+            ]
+        }
+    }
+    match = Matcher(index="perf-scale-ci")
+    match.es.search = lambda *args, **kwargs: sample_output
+    return match
+
+
+def test_get_metadata_by_uuid_found(matcher_instance):
+    uuid = "test_uuid"
+    result = matcher_instance.get_metadata_by_uuid(uuid)
+    expected = {"uuid": "uuid1", "field1": "value1"}
+    assert result == expected
+
+
+def test_get_metadata_by_uuid_not_found(matcher_instance):
+    def raise_exception():
+        raise NotFoundError(404, "index_not_found_exception", "no such index [sample]")
+
+    # Mock Elasticsearch response for testing NotFoundError
+    matcher_instance.es.search = lambda *args, **kwargs: raise_exception()
+    uuid = "nonexistent_uuid"
+    result = matcher_instance.get_metadata_by_uuid(uuid=uuid, index="sample index")
+    expected = {}
+    assert result == expected
+
+
+def test_query_index(matcher_instance):
+    index = "test_uuid"
+    query = "test_query"
+    result = matcher_instance.query_index(index, query)
+    expected = {
+        "hits": {
+            "hits": [
+                {"_source": {"uuid": "uuid1", "field1": "value1"}},
+                {"_source": {"uuid": "uuid2", "field1": "value2"}},
+            ]
+        }
+    }
+    assert result == expected
+
+
+def test_get_uuid_by_metadata(matcher_instance):
+    matcher_instance.es.search = lambda *args, **kwargs: {
+        "hits": {
+            "hits": [{"_source": {"uuid": "uuid1"}}, {"_source": {"uuid": "uuid2"}}]
+        }
+    }
+    meta = {"field1": "value1", "ocpVersion": "4.15"}
+    result = matcher_instance.get_uuid_by_metadata(meta)
+    expected = ["uuid1", "uuid2"]
+    assert result == expected
+
+
+def test_match_k8s_netperf(matcher_instance):
+    result = matcher_instance.match_k8s_netperf(["uuid1"])
+    expected = [
+        {"uuid": "uuid1", "field1": "value1"},
+        {"uuid": "uuid2", "field1": "value2"},
+    ]
+    assert result == expected
+
+
+def test_match_kube_bruner(matcher_instance):
+    result = matcher_instance.match_kube_burner(["uuid1"])
+    expected = [
+        {"uuid": "uuid1", "field1": "value1"},
+        {"uuid": "uuid2", "field1": "value2"},
+    ]
+    assert result == expected
+
+
+def test_burner_results(matcher_instance):
+    result = matcher_instance.burner_results(
+        "uuid1", ["uuid1", "uuid1"], "sample_index"
+    )
+    expected = [
+        {"uuid": "uuid1", "field1": "value1"},
+        {"uuid": "uuid2", "field1": "value2"},
+    ]
+    assert result == expected
+
+
+def test_burner_results_single_element(matcher_instance):
+    result = matcher_instance.burner_results("uuid1", ["uuid1"], "sample_index")
+    expected = []
+    assert result == expected
+
+
+def test_burner_cpu_results(matcher_instance):
+    matcher_instance.parse_burner_cpu_results = lambda *args, **kwargs: {}
+    result = matcher_instance.burner_cpu_results(
+        ["uuid1", "uuid2"], "sample_namespace", "sample_index"
+    )
+    expected = {}
+    assert result == expected
+
+
+def test_filter_runs(matcher_instance):
+    mock_data = [
+        {
+            "timestamp": "2024-01-15T20:00:46.307453873Z",
+            "endTimestamp": "2024-01-15T20:30:57.853708171Z",
+            "elapsedTime": 1812,
+            "uuid": "90189fbf-7181-4129-8ca5-3cc8d656b595",
+            "metricName": "jobSummary",
+            "jobConfig": {
+                "jobIterations": 216,
+                "name": "cluster-density-v2",
+                "jobType": "create",
+                "qps": 20,
+                "burst": 20,
+                "namespace": "cluster-density-v2",
+                "maxWaitTimeout": 14400000000000,
+                "waitForDeletion": True,
+                "waitWhenFinished": True,
+                "cleanup": True,
+                "namespacedIterations": True,
+                "iterationsPerNamespace": 1,
+                "verifyObjects": True,
+                "errorOnVerify": True,
+                "preLoadImages": True,
+                "preLoadPeriod": 15000000000,
+                "churn": True,
+                "churnPercent": 10,
+                "churnDuration": 1200000000000,
+                "churnDelay": 120000000000,
+            },
+            "metadata": {
+                "k8sVersion": "v1.28.5+c84a6b8",
+                "ocpMajorVersion": "4.15",
+                "ocpVersion": "4.15.0-0.nightly-2024-01-15-022811",
+                "platform": "AWS",
+                "sdnType": "OVNKubernetes",
+                "totalNodes": 30,
+            },
+            "version": "1.7.12@f0b89ccdbeb2a7d65512f5970d5a25a82ec386b2",
+        },
+        {
+            "timestamp": "2024-01-15T20:32:11.681417765Z",
+            "endTimestamp": "2024-01-15T20:37:24.591361376Z",
+            "elapsedTime": 313,
+            "uuid": "90189fbf-7181-4129-8ca5-3cc8d656b595",
+            "metricName": "jobSummary",
+            "jobConfig": {"name": "garbage-collection"},
+            "metadata": {
+                "k8sVersion": "v1.28.5+c84a6b8",
+                "ocpMajorVersion": "4.15",
+                "ocpVersion": "4.15.0-0.nightly-2024-01-15-022811",
+                "platform": "AWS",
+                "sdnType": "OVNKubernetes",
+                "totalNodes": 30,
+            },
+            "version": "1.7.12@f0b89ccdbeb2a7d65512f5970d5a25a82ec386b2",
+        },
+    ]
+    result = matcher_instance.filter_runs(mock_data, mock_data)
+    expected = ["90189fbf-7181-4129-8ca5-3cc8d656b595"]
+    assert result == expected
+
+
+def test_parse_burner_cpu_results(matcher_instance):
+    mock_data = {"aggregations": {"time": {"buckets": []}, "uuid": {"buckets": []}}}
+    mock_data["aggregations"]["time"]["buckets"] = [
+        {
+            "key": "90189fbf-7181-4129-8ca5-3cc8d656b595",
+            "doc_count": 1110,
+            "time": {
+                "value": 1705349944941.3918,
+                "value_as_string": "2024-01-15T20:19:04.941Z",
+            },
+        }
+    ]
+    mock_data["aggregations"]["uuid"]["buckets"] = [
+        {
+            "key": "90189fbf-7181-4129-8ca5-3cc8d656b595",
+            "doc_count": 1110,
+            "cpu": {"value": 10.818089329872935},
+        }
+    ]
+    expected = [
+        {
+            "uuid": "90189fbf-7181-4129-8ca5-3cc8d656b595",
+            "timestamp": "2024-01-15T20:19:04.941Z",
+            "cpu_avg": 10.818089329872935,
+        }
+    ]
+    result = matcher_instance.parse_burner_cpu_results(mock_data)
+    assert result == expected
+
+
+def test_convert_to_df(matcher_instance):
+    mock_data = [
+        {
+            "uuid": "90189fbf-7181-4129-8ca5-3cc8d656b595",
+            "timestamp": "2024-01-15T20:19:04.941Z",
+            "cpu_avg": 10.818089329872935,
+        }
+    ]
+    odf = pd.json_normalize(mock_data)
+    expected = odf.sort_values(by=["timestamp"])
+    result = matcher_instance.convert_to_df(
+        mock_data, columns=["uuid", "timestamp", "cpu_avg"]
+    )
+    assert result.equals(expected)
+
+
+def test_save_results(matcher_instance):
+    mock_data = [
+        {
+            "uuid": "90189fbf-7181-4129-8ca5-3cc8d656b595",
+            "timestamp": "2024-01-15T20:19:04.941Z",
+            "cpu_avg": 10.818089329872935,
+        }
+    ]
+    mock_file_name = "test_output.csv"
+    mock_df = pd.json_normalize(mock_data)
+    matcher_instance.save_results(
+        mock_df, csv_file_path=mock_file_name, columns=["uuid", "timestamp", "cpu_avg"]
+    )
+    assert os.path.isfile(mock_file_name)
+    os.remove(mock_file_name)
+
+
+if __name__ == "__main__":
+    pytest.main()

From 0bfdf8e19166b59890f54b75aa0cb48b5a65e8d9 Mon Sep 17 00:00:00 2001
From: Shashank Reddy Boyapally <sboyapal@redhat.com>
Date: Wed, 17 Jan 2024 16:41:05 -0500
Subject: [PATCH 2/2] added github action for unit tests

---
 .github/workflows/unit_test.yml | 9 ++++++---
 fmatch/matcher.py               | 3 ---
 fmatch/requirements.txt         | 2 +-
 fmatch/tests/test_matcher.py    | 1 +
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml
index 7734155..574dddf 100644
--- a/.github/workflows/unit_test.yml
+++ b/.github/workflows/unit_test.yml
@@ -6,6 +6,9 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
+    env:
+      ES_SERVER: ${{ secrets.ES_SERVER }}
+
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
@@ -13,13 +16,13 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.11
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt  # Replace with your requirements file if applicable
+        pip install -r fmatch/requirements.txt 
 
     - name: Run pytest
       run: |
-        python -m pytest pytest fmatch/tests --cov --random-order
+        python -m pytest fmatch/tests --cov --random-order
diff --git a/fmatch/matcher.py b/fmatch/matcher.py
index 420262d..69814a5 100644
--- a/fmatch/matcher.py
+++ b/fmatch/matcher.py
@@ -261,9 +261,7 @@ def parse_burner_cpu_results(self, data: dict):
         """
         res = []
         stamps = data['aggregations']['time']['buckets']
-        print("stamp",stamps)
         cpu = data['aggregations']['uuid']['buckets']
-        print("cpu",cpu)
         for stamp in stamps:
             dat = {}
             dat['uuid'] = stamp['key']
@@ -271,7 +269,6 @@ def parse_burner_cpu_results(self, data: dict):
             acpu = next(item for item in cpu if item["key"] == stamp['key'])
             dat['cpu_avg'] = acpu['cpu']['value']
             res.append(dat)
-        print("result",res)
         return res
 
     def convert_to_df(self, data, columns=None):
diff --git a/fmatch/requirements.txt b/fmatch/requirements.txt
index a800fe3..2777deb 100644
--- a/fmatch/requirements.txt
+++ b/fmatch/requirements.txt
@@ -21,7 +21,7 @@ mccabe==0.7.0
 mdurl==0.1.2
 more-itertools==10.2.0
 nh3==0.2.15
-numpy==1.26.3
+numpy==1.24.0
 packaging==23.2
 pandas==2.1.4
 pip-name==1.0.2
diff --git a/fmatch/tests/test_matcher.py b/fmatch/tests/test_matcher.py
index 439d007..8689b5e 100644
--- a/fmatch/tests/test_matcher.py
+++ b/fmatch/tests/test_matcher.py
@@ -3,6 +3,7 @@
 """
 #pylint: disable = redefined-outer-name
 #pylint: disable = missing-function-docstring
+#pylint: disable = import-error
 import os
 
 from elasticsearch.exceptions import NotFoundError