From 4ea431da65f17a93679215b9aa651b5b9a94c75f Mon Sep 17 00:00:00 2001 From: TeamSPoon Date: Wed, 14 Aug 2024 06:09:12 -0700 Subject: [PATCH 1/2] JUnit reports, Allure report, and test comparison reports are available as artifacts. --- .github/workflows/ci.yml | 45 +++++++++++++++++++++++++++++++----- scripts/into_junit.py | 49 ++++++++++++++++++++++++---------------- 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9590a8cf1b..38068ceb192 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI Job to Generate JUnit Reports with Diff +name: CI Job to Generate JUnit Reports with Diff and Allure Reports on: push: @@ -29,36 +29,41 @@ jobs: run: chmod +x scripts/run_commit_tests.sh - name: Run Shell Script to Generate Input File + continue-on-error: true # extra: Continue even if this step fails run: | ./scripts/run_commit_tests.sh - name: Run JUnit Report Generation Script + continue-on-error: true # extra: Continue even if this step fails run: | python scripts/into_junit.py /tmp/SHARED.UNITS > junit.xml - name: Convert JUnit XML to Standard HTML Report - continue-on-error: true # Continue to the next step even if this one fails + continue-on-error: true # extra: Continue even if this step fails run: | junit2html junit.xml junit-standard-report.html - name: Convert JUnit XML to Matrix HTML Report - continue-on-error: true # Continue to the next step even if this one fails + continue-on-error: true # extra: Continue even if this step fails run: | junit2html --report-matrix junit-matrix-report.html junit.xml - name: Upload JUnit XML Report + continue-on-error: true # extra: Continue even if this step fails uses: actions/upload-artifact@v3 with: name: junit-report path: junit.xml - name: Upload Standard HTML Report + continue-on-error: true # extra: Continue even if this step fails uses: actions/upload-artifact@v3 with: name: junit-standard-html-report path: junit-standard-report.html - name: Upload Matrix HTML Report + continue-on-error: true # extra: Continue even if this step fails uses: actions/upload-artifact@v3 with: name: junit-matrix-html-report @@ -73,10 +78,11 @@ jobs: fail-on-error: false # Do not fail the job if tests fail - name: Download Previous JUnit Results + continue-on-error: true # extra: Continue even if this step fails uses: actions/download-artifact@v3 with: name: junit-report - path: previous-junit.xml # Save as previous-junit.xml + path: previous-junit.xml - name: Install ReportGenerator run: | @@ -90,11 +96,38 @@ jobs: -reporttypes:"HtmlSummary;HtmlChart" - name: Upload JUnit Comparison Report + continue-on-error: true # extra: Continue even if this step fails uses: actions/upload-artifact@v3 with: name: junit-comparison-html-report path: ./comparison-report - - name: Display HTML Report Information + - name: Install Allure run: | - echo "Standard, matrix, and comparison HTML reports are available as artifacts." + curl -sLo allure-2.17.2.tgz https://github.com/allure-framework/allure2/releases/download/2.17.2/allure-2.17.2.tgz + tar -zxvf allure-2.17.2.tgz + sudo mv allure-2.17.2 /opt/allure + sudo ln -s /opt/allure/bin/allure /usr/bin/allure + + - name: Prepare Allure Results Directory + run: | + mkdir -p ./allure-results + cp junit.xml ./allure-results/ + if [ -f "previous-junit.xml" ]; then + cp previous-junit.xml ./allure-results/ + fi + + - name: Generate Allure Report + run: | + allure generate --clean --output ./allure-report ./allure-results + + - name: Upload Allure Report as Artifact + continue-on-error: true # extra: Continue even if this step fails + uses: actions/upload-artifact@v3 + with: + name: allure-html-report + path: ./allure-report + + - name: Provide Report Links + run: | + echo "JUnit reports, Allure report, and test comparison reports are available as artifacts." diff --git a/scripts/into_junit.py b/scripts/into_junit.py index 875e9c15f45..b7d9e3a7ab7 100644 --- a/scripts/into_junit.py +++ b/scripts/into_junit.py @@ -1,51 +1,52 @@ import xml.etree.ElementTree as ET import sys import re +from collections import defaultdict def create_testcase_element(testclass, testname, stdout, identifier, got, expected, status, url): + # Create the testcase XML element with the class and test name attributes testcase = ET.Element("testcase", classname=testclass, name=testname) - - description = f"Test {identifier} with URL: {url}" - + if status == "PASS": + # If the test passed, add system-out with a clickable link and details system_out = ET.SubElement(testcase, "system-out") system_out.text = f"Test Report\n\nAssertion: {stdout}\nExpected: {expected}\nActual: {got}\n]]>" else: # status == "FAIL" + # If the test failed, add a failure element with details and a clickable link failure_message = f"Test failed: Expected '{expected}' but got '{got}'" failure = ET.SubElement(testcase, "failure", message=failure_message, type="AssertionError") failure.text = f"" system_out = ET.SubElement(testcase, "system-out") system_out.text = f"Test Report\n\nAssertion: {stdout}\nExpected: {expected}\nActual: {got}\n]]>" - + return testcase def parse_test_line(line): + # Split the line into parts based on the table format parts = re.split(r'\s*\|\s*(?![^()]*\))', line.strip()) if len(parts) < 7: raise ValueError(f"Line does not have the expected number of parts: {len(parts)} found") - full_identifier = parts[1].strip() # The second field contains the test class and name + full_identifier = parts[1].strip() # The second field contains the test package/class and name status = parts[2].strip() # The third field contains the pass/fail status url = re.search(r'\((.*?)\)', parts[3]).group(1).strip() # Extract the URL inside the parentheses stdout = parts[4].strip() # The fifth field contains the assertion - got = parts[5].strip() - expected = parts[6].strip() + got = parts[5].strip() # The sixth field contains the actual result + expected = parts[6].strip() # The seventh field contains the expected result try: - testclass, testname = full_identifier.rsplit('.', 1) - if '.' in testclass: - testname = f"{testclass.split('.')[-1]}.{testname}" # Combine to form the correct testname - if not testclass or not testname: - raise ValueError("Test class or test name is empty after splitting.") + # Split the identifier into the package, class, and test names + testpackage, testname = full_identifier.split('.', 1) + if not testpackage or not testname: + raise ValueError("Test package or test name is empty after splitting.") except ValueError as e: raise ValueError(f"Identifier does not contain the expected format: {full_identifier}. Error: {str(e)}") - return testclass, testname, stdout, full_identifier, got, expected, status, url + return testpackage, testname, stdout, full_identifier, got, expected, status, url def generate_junit_xml(input_file): testsuites = ET.Element("testsuites") - testsuite = ET.Element("testsuite", name="Metta Tests") - testsuites.append(testsuite) + packages_dict = defaultdict(list) # Dictionary to group test cases by their testpackage with open(input_file, 'r') as file: for line in file: @@ -53,19 +54,27 @@ def generate_junit_xml(input_file): if line.startswith("|"): try: parts = re.split(r'\s*\|\s*(?![^()]*\))', line.strip()) - testclass, testname, stdout, full_identifier, got, expected, status, url = parse_test_line(line) - testcase = create_testcase_element(testclass, testname, stdout, full_identifier, got, expected, status, url) - testsuite.append(testcase) - print(f"Processing {testclass}.{testname}: {status}", file=sys.stderr) + testpackage, testname, stdout, full_identifier, got, expected, status, url = parse_test_line(line) + testcase = create_testcase_element(testpackage, testname, stdout, full_identifier, got, expected, status, url) + packages_dict[testpackage].append(testcase) + print(f"Processing {testpackage}.{testname}: {status}", file=sys.stderr) except ValueError as e: print(f"Skipping line due to error: {e}\nLine: {line}\nParts: {parts}", file=sys.stderr) + # Create a testsuite for each testpackage group + for testpackage, testcases in packages_dict.items(): + testsuite = ET.Element("testsuite", name=testpackage) + for testcase in testcases: + testsuite.append(testcase) + testsuites.append(testsuite) + + # Generate the XML tree and return it as a string tree = ET.ElementTree(testsuites) return ET.tostring(testsuites, encoding="utf-8", xml_declaration=True).decode("utf-8") if __name__ == "__main__": if len(sys.argv) != 2: - print("Usage: python generate_junit.py ") + print("Usage: python scripts/into_junit.py ") sys.exit(1) input_file = sys.argv[1] From 7611d7137a62bd8ed02cc8fd30035592f97843bc Mon Sep 17 00:00:00 2001 From: TeamSPoon Date: Wed, 14 Aug 2024 06:14:22 -0700 Subject: [PATCH 2/2] Add permissions for contents and checks to avoid integration errors --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38068ceb192..fc176d6345f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,10 @@ on: branches: - main +permissions: + contents: write + checks: write + jobs: generate-reports: runs-on: ubuntu-latest