Merge pull request #186 from AsherGlick/reorg_integration_tests

Reorganize integration tests

xml_converter/intigration_tests/.gitignore

@@ -0,0 +1 @@
+outputs/

xml_converter/test_cases/can_fade/inputs/can_fade_is_correct.xml → xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml

@@ -4,5 +4,6 @@
 
 <POIs>
 <POI  CanFade="false" Type="mycategory"/>
+<POI  CanFade="false" Type="mycategory"/>
 </POIs>
 </OverlayData>

xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml

@@ -0,0 +1,13 @@
+<OverlayData>
+<MarkerCategory  Name="mycategory">
+</MarkerCategory>
+
+<POIs>
+<POI CanFade="false" Type="mycategory"/>
+<POI CanFade="0" Type="mycategory"/>
+<!-- TODO: This value should be elided in the output, but it is not currently
+so we are ignoring the test instead of wriitng a bad test
+<POI CanFade="true" Type="mycategory"/>
+<POI CanFade="1" Type="mycategory"/> -->
+</POIs>
+</OverlayData>

xml_converter/intigration_tests/run_tests.py

@@ -0,0 +1,189 @@
+import argparse
+import difflib
+import json
+import subprocess
+import re
+import os
+from typing import List, Optional, Final, Tuple
+from testcases import testcases
+import shutil
+
+# Path to compiled C++ executable
+xml_converter_binary_path: str = "../build/xml_converter"
+
+arg_input_xml: Final[str] = "--input-taco-path"
+arg_output_xml: Final[str] = "--output-taco-path"
+arg_input_proto: Final[str] = "--input-waypoint-path"
+arg_output_proto: Final[str] = "--output-waypoint-path"
+arg_split_proto: Final[str] = "--output-split-waypoint-path"
+
+
+def run_xml_converter(
+    input_xml: Optional[List[str]] = None,
+    output_xml: Optional[List[str]] = None,
+    input_proto: Optional[List[str]] = None,
+    output_proto: Optional[List[str]] = None,
+    split_output_proto: Optional[str] = None,
+) -> Tuple[str, str, int]:
+
+    # Build the command to execute the C++ program with the desired function and arguments
+    cmd: List[str] = [xml_converter_binary_path]
+
+    if input_xml:
+        cmd += [arg_input_xml] + input_xml
+    if output_xml:
+        cmd += [arg_output_xml] + output_xml
+    if input_proto:
+        cmd += [arg_input_proto] + input_proto
+    if output_proto:
+        cmd += [arg_output_proto] + output_proto
+    if split_output_proto:
+        cmd += [arg_split_proto] + [split_output_proto]
+
+    # Run the C++ program and capture its output
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    return (result.stdout, result.stderr, result.returncode)
+
+
+def compare_files(file_path1: str, file_path2: str) -> List[str]:
+    with open(file_path1, 'r') as file1:
+        content1 = file1.readlines()
+    with open(file_path2, 'r') as file2:
+        content2 = file2.readlines()
+
+    diff = list(difflib.Differ().compare(content1, content2))
+
+    return diff
+
+
+def len_diff(lines: List[str]) -> int:
+    diffcount = 0
+
+    for line in lines:
+        if line.startswith("  "):
+            continue
+        diffcount += 1
+    return diffcount
+
+
+
+
+################################################################################
+# remove_ansii_color_escapecodes
+#
+# Remove the ANSII color code escape characters from a string to make it easier
+# to read what is actually contained in the string.
+################################################################################
+pattern_for_color_escape_codes = r"\u001b\[[0-9;]+m"
+
+
+def remove_ansii_color_escapecodes(lines: List[str]) -> List[str]:
+    return [ re.sub(pattern_for_color_escape_codes, '', line) for line in lines ]
+
+
+################################################################################
+# remove_ignored_lines
+#
+# Goes through a list of lines and removes any line that matches a pattern in
+# the `line_patterns_to_ignore` global variable.
+################################################################################
+line_patterns_to_ignore = [
+    r"^Loading taco pack .*$",
+    r"^The taco parse function took [0-9]+ milliseconds to run$",
+    r"^The xml write function took [0-9]+ milliseconds to run$",
+    r"^The protobuf read function took [0-9]+ milliseconds to run$",
+    r"^The protobuf write function took [0-9]+ milliseconds to run$",
+    r"^$"
+]
+
+
+def remove_ignored_lines(lines: List[str]) -> List[str]:
+    filtered_array = []
+    for line in lines:
+        match_found: bool = False
+        for pattern in line_patterns_to_ignore:
+            if re.fullmatch(pattern, line):
+                match_found = True
+                break
+        if not match_found:
+            filtered_array.append(line)
+    return filtered_array
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="A test harness for evaluating the output of the xmlconverter program")
+    parser.add_argument("-v", "--verbose", help="Prints the results from xmlconverter in JSON format", action="store_true")
+    args = parser.parse_args()
+
+    output_parent_dirpath = "./outputs"
+
+    # Ensure that the test output directory is empty
+    if os.path.exists(output_parent_dirpath):
+        shutil.rmtree(output_parent_dirpath)
+
+    for testcase in testcases:
+        xml_output_dir_path = os.path.join(output_parent_dirpath, "xml", testcase.name)
+        proto_output_dir_path = os.path.join(output_parent_dirpath, "proto", testcase.name)
+
+
+        os.makedirs(xml_output_dir_path, exist_ok=True)
+        os.makedirs(proto_output_dir_path, exist_ok=True)
+
+        rawstdout, rawstderr, returncode = run_xml_converter(
+            input_xml=testcase.xml_input_paths,
+            output_xml=[xml_output_dir_path],
+            output_proto=[proto_output_dir_path],
+        )
+
+        # Sanitize and denoise the lines
+        stdout: List[str] = remove_ansii_color_escapecodes(remove_ignored_lines(rawstdout.split("\n")))
+        stderr: List[str] = remove_ansii_color_escapecodes(remove_ignored_lines(rawstderr.split("\n")))
+
+        # Prints the results of xml_converter
+        if args.verbose:
+            print(f"Test {testcase.name}")
+            print("    stdout :  {}".format("\n".join(stdout)))
+            print("    stderr : {}".format("\n".join(stderr)))
+            print("    return_code : {}".format(returncode))
+
+        all_tests_passed: bool = True
+
+        stdout_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stdout, stdout))
+        if len_diff(stdout_diff) != 0:
+            print(f"Standard output did not match for test {testcase.name}")
+            for line in stdout_diff:
+                print(line)
+            all_tests_passed = False
+
+        stderr_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stderr, stderr))
+        if len_diff(stderr_diff) != 0:
+            print(f"Standard error did not match for test {testcase.name}")
+            for line in stderr_diff:
+                print(line)
+            all_tests_passed = False
+
+        if testcase.expected_returncode is not None and testcase.expected_returncode != returncode :
+            print(f"Expected a return code of {testcase.expected_returncode} for {testcase.name} but got {returncode}")
+
+        if testcase.expected_output_xml_path is not None:
+            # TODO: These paths are directories and `xml_file.xml` is just one
+            # possible file in the directories. Eventually we should check all
+            # the files in the directory not just the one.
+            output_xml_filepath = os.path.join(xml_output_dir_path, "xml_file.xml")
+            expected_output_xml_filepath = os.path.join(testcase.expected_output_xml_path, "xml_file.xml")
+
+            xml_diff = compare_files(expected_output_xml_filepath , output_xml_filepath)
+
+            if len_diff(xml_diff) != 0:
+                print(f"XML output was incorrect for test {testcase.name}")
+                for line in xml_diff:
+                    print(line, end="")
+                all_tests_passed = False
+
+        if all_tests_passed:
+            print(f"Success: test {testcase.name}")
+
+
+if __name__ == "__main__":
+    main()

xml_converter/intigration_tests/testcases.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+@dataclass
+class Testcase:
+    name: str
+    xml_input_paths: List[str] = field(default_factory=list)
+    proto_input_paths: List[str] = field(default_factory=list)
+
+    # TODO: Eventually the expected output paths wont be optional
+    expected_output_xml_path: Optional[str] = None
+    expected_output_proto_path: Optional[str] = None
+
+    expected_stdout: List[str] = field(default_factory=list)
+    expected_stderr: List[str] = field(default_factory=list)
+    expected_returncode: int = 0
+
+
+testcases: List[Testcase] = [
+    Testcase(
+        name="canfade_valid",
+        xml_input_paths=["./inputs/xml_can_fade_valid"],
+        expected_output_xml_path="./expected_outputs/xml_can_fade_valid",
+    ),
+    Testcase(
+        name="canfade_invalid",
+        xml_input_paths=["./inputs/xml_can_fade_invalid"],
+        expected_output_xml_path="./expected_outputs/xml_can_fade_invalid",
+        expected_stdout=[
+            "Error: Found a boolean value that was not a '1', '0', 'true', or 'false'",
+            "./inputs/xml_can_fade_invalid/xml_file.xml",
+            '6 |<POI  CanFade="yes" Type="mycategory"/>',
+            "  |               ^^^"
+        ]
+    )
+]

xml_converter/src/xml_converter.cpp

@@ -98,7 +98,7 @@ void read_taco_directory(string directory, map<string, Category>* marker_categor
 
 void write_taco_directory(string directory, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois) {
     // TODO: Exportion of XML Marker Packs File Structure #111
-    string xml_filepath = directory + "xml_file.xml";
+    string xml_filepath = directory + "/xml_file.xml";
     write_xml_file(xml_filepath, marker_categories, parsed_pois);
 }