intermittents commenter: use table for failures by platform and variant

tests/conftest.py

@@ -6,6 +6,7 @@
 import time
 from os.path import join, dirname
 from unittest.mock import MagicMock
+import yaml
 
 import kombu
 import pytest
@@ -892,6 +893,27 @@ def _fetch_intermittent_bugs(additional_params, limit, duplicate_chain_length):
     )
 
 
+@pytest.fixture
+def mock_test_variants_firefoxci_request(monkeypatch):
+    """
+    Mock fetch_test_variants() used to retreive test variants from firefox-ci
+    """
+
+    def _fetch_test_variants(self):
+        tests_folder = os.path.dirname(__file__)
+        file_name = "variants.yml"
+        data_path = os.path.join(tests_folder, "sample_data", file_name)
+        with open(data_path) as f:
+            data = yaml.safe_load(f)
+        return data
+
+    monkeypatch.setattr(
+        treeherder.intermittents_commenter.commenter.Commenter,
+        "fetch_test_variants",
+        _fetch_test_variants,
+    )
+
+
 class MockResponse:
     def __init__(self):
         self.status_code = 200
@@ -1134,6 +1156,22 @@ def bug_data(eleven_jobs_stored, test_repository, test_push, bugs):
     }
 
 
+@pytest.fixture
+def bug_data_with_5_failures(eleven_jobs_stored, test_repository, test_push, bugs):
+    jobs = th_models.Job.objects.all().order_by("id")
+    bug_id = bugs[0].id
+    for index, job in enumerate(jobs[:5]):
+        th_models.BugJobMap.create(job_id=job.id, bug_id=bug_id)
+        if index > 2:
+            signature = th_models.BugJobMap.failures.last().job.signature
+            signature.job_type_name = "mochitest-plain-headless-1"
+            signature.save()
+
+    return {
+        "bug_id": bug_id,
+    }
+
+
 @pytest.fixture
 def test_run_data(bug_data):
     pushes = th_models.Push.objects.all()

tests/intermittents_commenter/expected_comment.text

@@ -1,13 +1,19 @@
+## Summary
 1 failures in 10 pushes (0.1 failures/push) were associated with this bug in the last 7 days.
 
 This is the #1 most frequent failure this week.
 
-Repository breakdown:
+## Repository breakdown:
 * mozilla-central: 1
 
-Platform and build breakdown:
+## Platform and build breakdown:
 * linux1804: 1
   * debug: 1
 
-For more details, see:
+## Table
+| |**no_variant**|
+|---|:-:|
+|**linux1804/debug**|1|
+
+## For more details, see:
 https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all

tests/intermittents_commenter/expected_comment_with_5_failures.text

@@ -0,0 +1,30 @@
+## Summary
+5 failures in 10 pushes (0.5 failures/push) were associated with this bug in the last 7 days.
+
+This is the #1 most frequent failure this week.
+
+## Repository breakdown:
+* mozilla-central: 5
+
+## Platform and build breakdown:
+* linux1804: 2
+  * debug: 1
+  * opt: 1
+* mac1015: 1
+  * debug: 1
+* mac1120: 1
+  * debug: 1
+* windows7-32: 1
+  * debug: 1
+
+## Table
+| |**headless**|**no_variant**|
+|---|:-:|:-:|
+|**linux1804/debug**| |1|
+|**linux1804/opt**| |1|
+|**mac1015/debug**| |1|
+|**mac1120/debug**|1| |
+|**windows7-32/debug**|1| |
+
+## For more details, see:
+https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all

tests/intermittents_commenter/test_commenter.py

@@ -5,7 +5,7 @@
 
 
 @responses.activate
-def test_intermittents_commenter(bug_data):
+def test_intermittents_commenter(bug_data, mock_test_variants_firefoxci_request):
     startday = "2012-05-09"
     endday = "2018-05-10"
     alt_startday = startday
@@ -39,6 +39,44 @@ def test_intermittents_commenter(bug_data):
 
     with open("tests/intermittents_commenter/expected_comment.text") as comment:
         expected_comment = comment.read()
-    print(len(expected_comment))
-    print(len(comment_params[0]["changes"]["comment"]["body"]))
+    assert comment_params[0]["changes"]["comment"]["body"] == expected_comment
+
+
+@responses.activate
+def test_intermittents_commenter_with_failures(
+    bug_data_with_5_failures, mock_test_variants_firefoxci_request
+):
+    startday = "2012-05-09"
+    endday = "2018-05-10"
+    alt_startday = startday
+    alt_endday = endday
+
+    process = Commenter(weekly_mode=True, dry_run=True)
+    params = {"include_fields": "product%2C+component%2C+priority%2C+whiteboard%2C+id"}
+    url = "{}/rest/bug?id={}&include_fields={}".format(
+        settings.BZ_API_URL, bug_data_with_5_failures["bug_id"], params["include_fields"]
+    )
+
+    content = {
+        "bugs": [
+            {
+                "component": "General",
+                "priority": "P3",
+                "product": "Testing",
+                "whiteboard": "[stockwell infra] [see summary at comment 92]",
+                "id": bug_data_with_5_failures["bug_id"],
+            }
+        ],
+        "faults": [],
+    }
+
+    responses.add(responses.Response(method="GET", url=url, json=content, status=200))
+
+    resp = process.fetch_bug_details(bug_data_with_5_failures["bug_id"])
+    assert resp == content["bugs"]
+
+    comment_params = process.generate_bug_changes(startday, endday, alt_startday, alt_endday)
+
+    with open("tests/intermittents_commenter/expected_comment_with_5_failures.text") as comment:
+        expected_comment = comment.read()
     assert comment_params[0]["changes"]["comment"]["body"] == expected_comment

tests/sample_data/job_data2.txt

@@ -0,0 +1,6 @@
+{"project": "mozilla-central", "job": {"build_platform": {"platform": "linux1804", "os_name": "linux", "architecture": "x86"}, "submit_timestamp": 1384353546, "start_timestamp": 1384353553, "job_guid": "f1c75261017c7c5ce3000931dce4c442fe0a1297", "name": "linux-x64-tsan-build", "reference_data_name": "mozilla-release_linux-debug_dep", "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/tinderbox-builds/mozilla-release-emulator-jb-debug/1384353546/mozilla-release_linux-debug_dep-bm62-build1-build17.txt.gz", "name": "live_backing_log"}], "option_collection": {"debug": true}, "who": "mozilla-release", "group_symbol": "?", "state": "completed", "artifact": {"log_urls": [], "type": "", "name": "", "blob": ""}, "machine_platform": {"platform": "linux1804", "os_name": "linux", "architecture": "x86"}, "machine": "bld-linux64-ec2-132", "reason": "scheduler", "result": "success", "job_symbol": "B", "group_name": null, "product_name": "firefox", "end_timestamp": 1384364989}, "resultset_id": 1, "revision": "45f8637cb9f78f19cb8463ff174e81756805d8cf"}
+{"project": "mozilla-central", "job": {"build_platform": {"platform": "linux1804", "os_name": "linux", "architecture": "x86"}, "submit_timestamp": 1384353546, "start_timestamp": 1384353555, "job_guid": "9abb6f7d54a49d763c584926377f09835c5e1a32", "name": "linux-x64-tsan-build", "reference_data_name": "mozilla-release_emulator-jb_dep", "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/tinderbox-builds/mozilla-release-emulator-jb/1384353546/mozilla-release_emulator-jb_dep-bm58-build1-build29.txt.gz", "name": "live_backing_log"}], "option_collection": {"opt": true}, "who": "mozilla-release", "group_symbol": "?", "state": "completed", "artifact": {"log_urls": [], "type": "", "name": "", "blob": ""}, "machine_platform": {"platform": "linux1804", "os_name": "linux", "architecture": "x86"}, "machine": "bld-linux64-ec2-462", "reason": "scheduler", "result": "success", "job_symbol": "B", "group_name": null, "product_name": "firefox", "end_timestamp": 1384364662}, "resultset_id": 1, "revision": "45f8637cb9f78f19cb8463ff174e81756805d8cf"}
+{"project": "mozilla-central", "job": {"build_platform": {"platform": "mac1015", "os_name": "mac", "architecture": "x86_64"}, "submit_timestamp": 1384356854, "start_timestamp": 1384356876, "job_guid": "d9cb3668f75f8f1b0b4ef503778ad0b90595a693", "name": "Mochitest-headless-plain-1", "reference_data_name": "Rev4 MacOSX Snow Leopard 10.6 mozilla-release debug test mochitest-browser-chrome", "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-release-macosx64-debug/1384353545/mozilla-release_snowleopard-debug_test-mochitest-browser-chrome-bm76-tests1-macosx-build14.txt.gz", "name": "live_backing_log"}], "option_collection": {"debug": true}, "who": "tests-mozilla-release-snowleopard-debug-unittest", "group_symbol": "M", "state": "completed", "artifact": {"log_urls": [], "type": "", "name": "", "blob": ""}, "machine_platform": {"platform": "mac1015", "os_name": "mac", "architecture": "x86_64"}, "machine": "talos-r4-snow-007", "reason": "scheduler", "result": "success", "job_symbol": "bc", "group_name": "Mochitest", "product_name": "firefox", "end_timestamp": 1384363994}, "resultset_id": 1, "revision": "45f8637cb9f78f19cb8463ff174e81756805d8cf"}
+{"project": "mozilla-central", "job": {"build_platform": {"platform": "mac1120", "os_name": "mac", "architecture": "x86_64"}, "submit_timestamp": 1384356854, "start_timestamp": 1384356880, "job_guid": "ab952a4bbbc74f1d9fb3cf536073b371029dbd02", "name": "Mochitest-headless-plain-1", "reference_data_name": "Rev4 MacOSX Lion 10.7 mozilla-release debug test mochitest-browser-chrome", "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-release-macosx64-debug/1384353545/mozilla-release_lion-debug_test-mochitest-browser-chrome-bm76-tests1-macosx-build12.txt.gz", "name": "live_backing_log"}], "option_collection": {"debug": true}, "who": "tests-mozilla-release-lion-debug-unittest", "group_symbol": "M", "state": "completed", "artifact": {"log_urls": [], "type": "", "name": "", "blob": ""}, "machine_platform": {"platform": "mac1120", "os_name": "mac", "architecture": "x86_64"}, "machine": "talos-r4-lion-011", "reason": "scheduler", "result": "success", "job_symbol": "bc", "group_name": "Mochitest", "product_name": "firefox", "end_timestamp": 1384364849}, "resultset_id": 1, "revision": "45f8637cb9f78f19cb8463ff174e81756805d8cf"}
+{"resultset_id": 1, "superseded": ["f1c75261017c7c5ce3000931dce4c442fe0a1297", "9abb6f7d54a49d763c584926377f09835c5e1a32", "ab952a4bbbc74f1d9fb3cf536073b371029dbd02"], "project": "mozilla-central", "job": {"build_platform": {"platform": "windows7-32", "os_name": "win", "architecture": "x86"}, "submit_timestamp": 1384358654, "start_timestamp": 1384358660, "job_guid": "5da36fb825bc52d13fed5b805d44015b0f2f2f16", "name": "Mochitest-headless-plain-1", "reference_data_name": "Windows 7 32-bit mozilla-release debug test mochitest-browser-chrome", "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-release-win32-debug/1384353545/mozilla-release_win7-ix-debug_test-mochitest-browser-chrome-bm74-tests1-windows-build2.txt.gz", "name": "live_backing_log"}], "option_collection": {"debug": true}, "who": "tests-mozilla-release-win7-ix-debug-unittest", "group_symbol": "M", "state": "completed", "artifact": {"log_urls": [], "type": "", "name": "", "blob": ""}, "machine_platform": {"platform": "windows7-32", "os_name": "win", "architecture": "x86"}, "machine": "t-w732-ix-117", "reason": "scheduler", "result": "success", "job_symbol": "bc", "group_name": "Mochitest", "product_name": "firefox", "end_timestamp": 1384363533}, "revision": "45f8637cb9f78f19cb8463ff174e81756805d8cf"}
+