JUnit reports, Allure report, and test comparison reports are availab…

…le as artifacts.

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI Job to Generate JUnit Reports with Diff
+name: CI Job to Generate JUnit Reports with Diff and Allure Reports
 
 on:
   push:
@@ -29,36 +29,41 @@ jobs:
       run: chmod +x scripts/run_commit_tests.sh
 
     - name: Run Shell Script to Generate Input File
+      continue-on-error: true  # extra: Continue even if this step fails
       run: |
         ./scripts/run_commit_tests.sh
 
     - name: Run JUnit Report Generation Script
+      continue-on-error: true  # extra: Continue even if this step fails
       run: |
         python scripts/into_junit.py /tmp/SHARED.UNITS > junit.xml
 
     - name: Convert JUnit XML to Standard HTML Report
-      continue-on-error: true  # Continue to the next step even if this one fails
+      continue-on-error: true  # extra: Continue even if this step fails
       run: |
         junit2html junit.xml junit-standard-report.html
 
     - name: Convert JUnit XML to Matrix HTML Report
-      continue-on-error: true  # Continue to the next step even if this one fails
+      continue-on-error: true  # extra: Continue even if this step fails
       run: |
         junit2html --report-matrix junit-matrix-report.html junit.xml
 
     - name: Upload JUnit XML Report
+      continue-on-error: true  # extra: Continue even if this step fails
       uses: actions/upload-artifact@v3
       with:
         name: junit-report
         path: junit.xml
 
     - name: Upload Standard HTML Report
+      continue-on-error: true  # extra: Continue even if this step fails
       uses: actions/upload-artifact@v3
       with:
         name: junit-standard-html-report
         path: junit-standard-report.html
 
     - name: Upload Matrix HTML Report
+      continue-on-error: true  # extra: Continue even if this step fails
       uses: actions/upload-artifact@v3
       with:
         name: junit-matrix-html-report
@@ -73,10 +78,11 @@ jobs:
         fail-on-error: false  # Do not fail the job if tests fail
 
     - name: Download Previous JUnit Results
+      continue-on-error: true  # extra: Continue even if this step fails
       uses: actions/download-artifact@v3
       with:
         name: junit-report
-        path: previous-junit.xml  # Save as previous-junit.xml
+        path: previous-junit.xml
 
     - name: Install ReportGenerator
       run: |
@@ -90,11 +96,38 @@ jobs:
           -reporttypes:"HtmlSummary;HtmlChart"
 
     - name: Upload JUnit Comparison Report
+      continue-on-error: true  # extra: Continue even if this step fails
       uses: actions/upload-artifact@v3
       with:
         name: junit-comparison-html-report
         path: ./comparison-report
 
-    - name: Display HTML Report Information
+    - name: Install Allure
       run: |
-        echo "Standard, matrix, and comparison HTML reports are available as artifacts."
+        curl -sLo allure-2.17.2.tgz https://github.com/allure-framework/allure2/releases/download/2.17.2/allure-2.17.2.tgz
+        tar -zxvf allure-2.17.2.tgz
+        sudo mv allure-2.17.2 /opt/allure
+        sudo ln -s /opt/allure/bin/allure /usr/bin/allure
+
+    - name: Prepare Allure Results Directory
+      run: |
+        mkdir -p ./allure-results
+        cp junit.xml ./allure-results/
+        if [ -f "previous-junit.xml" ]; then
+          cp previous-junit.xml ./allure-results/
+        fi
+
+    - name: Generate Allure Report
+      run: |
+        allure generate --clean --output ./allure-report ./allure-results
+
+    - name: Upload Allure Report as Artifact
+      continue-on-error: true  # extra: Continue even if this step fails
+      uses: actions/upload-artifact@v3
+      with:
+        name: allure-html-report
+        path: ./allure-report
+
+    - name: Provide Report Links
+      run: |
+        echo "JUnit reports, Allure report, and test comparison reports are available as artifacts."

scripts/into_junit.py

@@ -1,71 +1,80 @@
 import xml.etree.ElementTree as ET
 import sys
 import re
+from collections import defaultdict
 
 def create_testcase_element(testclass, testname, stdout, identifier, got, expected, status, url):
+    # Create the testcase XML element with the class and test name attributes
     testcase = ET.Element("testcase", classname=testclass, name=testname)
-
-    description = f"Test {identifier} with URL: {url}"
-
+
     if status == "PASS":
+        # If the test passed, add system-out with a clickable link and details
         system_out = ET.SubElement(testcase, "system-out")
         system_out.text = f"<![CDATA[\n<a href=\"{url}\">Test Report</a>\n\nAssertion: {stdout}\nExpected: {expected}\nActual: {got}\n]]>"
     else:  # status == "FAIL"
+        # If the test failed, add a failure element with details and a clickable link
         failure_message = f"Test failed: Expected '{expected}' but got '{got}'"
         failure = ET.SubElement(testcase, "failure", message=failure_message, type="AssertionError")
         failure.text = f"<![CDATA[\nAssertionError: {failure_message}\n]]>"
         system_out = ET.SubElement(testcase, "system-out")
         system_out.text = f"<![CDATA[\n<a href=\"{url}\">Test Report</a>\n\nAssertion: {stdout}\nExpected: {expected}\nActual: {got}\n]]>"
-    
+
     return testcase
 
 def parse_test_line(line):
+    # Split the line into parts based on the table format
     parts = re.split(r'\s*\|\s*(?![^()]*\))', line.strip())
     if len(parts) < 7:
         raise ValueError(f"Line does not have the expected number of parts: {len(parts)} found")
 
-    full_identifier = parts[1].strip()  # The second field contains the test class and name
+    full_identifier = parts[1].strip()  # The second field contains the test package/class and name
     status = parts[2].strip()  # The third field contains the pass/fail status
     url = re.search(r'\((.*?)\)', parts[3]).group(1).strip()  # Extract the URL inside the parentheses
     stdout = parts[4].strip()  # The fifth field contains the assertion
-    got = parts[5].strip()
-    expected = parts[6].strip()
+    got = parts[5].strip()  # The sixth field contains the actual result
+    expected = parts[6].strip()  # The seventh field contains the expected result
 
     try:
-        testclass, testname = full_identifier.rsplit('.', 1)
-        if '.' in testclass:
-            testname = f"{testclass.split('.')[-1]}.{testname}"  # Combine to form the correct testname
-        if not testclass or not testname:
-            raise ValueError("Test class or test name is empty after splitting.")
+        # Split the identifier into the package, class, and test names
+        testpackage, testname = full_identifier.split('.', 1)
+        if not testpackage or not testname:
+            raise ValueError("Test package or test name is empty after splitting.")
     except ValueError as e:
         raise ValueError(f"Identifier does not contain the expected format: {full_identifier}. Error: {str(e)}")
 
-    return testclass, testname, stdout, full_identifier, got, expected, status, url
+    return testpackage, testname, stdout, full_identifier, got, expected, status, url
 
 def generate_junit_xml(input_file):
     testsuites = ET.Element("testsuites")
-    testsuite = ET.Element("testsuite", name="Metta Tests")
-    testsuites.append(testsuite)
+    packages_dict = defaultdict(list)  # Dictionary to group test cases by their testpackage
 
     with open(input_file, 'r') as file:
         for line in file:
             parts = None
             if line.startswith("|"):
                 try:
                     parts = re.split(r'\s*\|\s*(?![^()]*\))', line.strip())
-                    testclass, testname, stdout, full_identifier, got, expected, status, url = parse_test_line(line)
-                    testcase = create_testcase_element(testclass, testname, stdout, full_identifier, got, expected, status, url)
-                    testsuite.append(testcase)
-                    print(f"Processing {testclass}.{testname}: {status}", file=sys.stderr)
+                    testpackage, testname, stdout, full_identifier, got, expected, status, url = parse_test_line(line)
+                    testcase = create_testcase_element(testpackage, testname, stdout, full_identifier, got, expected, status, url)
+                    packages_dict[testpackage].append(testcase)
+                    print(f"Processing {testpackage}.{testname}: {status}", file=sys.stderr)
                 except ValueError as e:
                     print(f"Skipping line due to error: {e}\nLine: {line}\nParts: {parts}", file=sys.stderr)
 
+    # Create a testsuite for each testpackage group
+    for testpackage, testcases in packages_dict.items():
+        testsuite = ET.Element("testsuite", name=testpackage)
+        for testcase in testcases:
+            testsuite.append(testcase)
+        testsuites.append(testsuite)
+
+    # Generate the XML tree and return it as a string
     tree = ET.ElementTree(testsuites)
     return ET.tostring(testsuites, encoding="utf-8", xml_declaration=True).decode("utf-8")
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:
-        print("Usage: python generate_junit.py <input_file>")
+        print("Usage: python scripts/into_junit.py <input_file>")
         sys.exit(1)
 
     input_file = sys.argv[1]