From 0aac51fa2949df35d1b6a5d7347e288e59bb07b6 Mon Sep 17 00:00:00 2001 From: Asher Glick Date: Thu, 26 Oct 2023 02:19:22 -0500 Subject: [PATCH 1/2] inital moves --- xml_converter/{tests => intigration_tests}/attribute_testing.py | 0 .../can_fade/expected_outputs/can_fade_false.xml | 0 .../can_fade/inputs/can_fade_is_correct.xml | 0 .../can_fade/inputs/can_fade_is_supported_value_not_default.xml | 0 .../can_fade/inputs/can_fade_unsupported_value.xml | 0 .../{tests => intigration_tests}/test_expected_outputs.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename xml_converter/{tests => intigration_tests}/attribute_testing.py (100%) rename xml_converter/{test_cases => intigration_tests}/can_fade/expected_outputs/can_fade_false.xml (100%) rename xml_converter/{test_cases => intigration_tests}/can_fade/inputs/can_fade_is_correct.xml (100%) rename xml_converter/{test_cases => intigration_tests}/can_fade/inputs/can_fade_is_supported_value_not_default.xml (100%) rename xml_converter/{test_cases => intigration_tests}/can_fade/inputs/can_fade_unsupported_value.xml (100%) rename xml_converter/{tests => intigration_tests}/test_expected_outputs.json (100%) diff --git a/xml_converter/tests/attribute_testing.py b/xml_converter/intigration_tests/attribute_testing.py similarity index 100% rename from xml_converter/tests/attribute_testing.py rename to xml_converter/intigration_tests/attribute_testing.py diff --git a/xml_converter/test_cases/can_fade/expected_outputs/can_fade_false.xml b/xml_converter/intigration_tests/can_fade/expected_outputs/can_fade_false.xml similarity index 100% rename from xml_converter/test_cases/can_fade/expected_outputs/can_fade_false.xml rename to xml_converter/intigration_tests/can_fade/expected_outputs/can_fade_false.xml diff --git a/xml_converter/test_cases/can_fade/inputs/can_fade_is_correct.xml b/xml_converter/intigration_tests/can_fade/inputs/can_fade_is_correct.xml similarity index 100% rename from xml_converter/test_cases/can_fade/inputs/can_fade_is_correct.xml rename to xml_converter/intigration_tests/can_fade/inputs/can_fade_is_correct.xml diff --git a/xml_converter/test_cases/can_fade/inputs/can_fade_is_supported_value_not_default.xml b/xml_converter/intigration_tests/can_fade/inputs/can_fade_is_supported_value_not_default.xml similarity index 100% rename from xml_converter/test_cases/can_fade/inputs/can_fade_is_supported_value_not_default.xml rename to xml_converter/intigration_tests/can_fade/inputs/can_fade_is_supported_value_not_default.xml diff --git a/xml_converter/test_cases/can_fade/inputs/can_fade_unsupported_value.xml b/xml_converter/intigration_tests/can_fade/inputs/can_fade_unsupported_value.xml similarity index 100% rename from xml_converter/test_cases/can_fade/inputs/can_fade_unsupported_value.xml rename to xml_converter/intigration_tests/can_fade/inputs/can_fade_unsupported_value.xml diff --git a/xml_converter/tests/test_expected_outputs.json b/xml_converter/intigration_tests/test_expected_outputs.json similarity index 100% rename from xml_converter/tests/test_expected_outputs.json rename to xml_converter/intigration_tests/test_expected_outputs.json From f206e1d183bab080eab49a51949c10e8f630334c Mon Sep 17 00:00:00 2001 From: Asher Glick Date: Thu, 26 Oct 2023 05:32:08 -0500 Subject: [PATCH 2/2] Refactoring Integration Tests When experimenting it turns out the entire test harness was broken, so none of this can really be split out easily because the base system was incorrect and fixing it just to replace it felt like a waste. * All inputs and expected outputs are directories * Valid test cases are combined * A bug with directory concatination was fixed in xml_converter * Diff output is no longer randomly censored * Test cases were moved from JSON to Python to more easily dictate typing. This may change again in the future based on the needs of the test suites. * Test suite runs properly on the current revision of code. --- xml_converter/intigration_tests/.gitignore | 1 + .../intigration_tests/attribute_testing.py | 168 ---------------- ...an_fade_is_supported_value_not_default.xml | 8 - .../xml_can_fade_invalid/xml_file.xml} | 0 .../xml_can_fade_valid/xml_file.xml} | 1 + .../xml_can_fade_invalid/xml_file.xml} | 0 .../inputs/xml_can_fade_valid/xml_file.xml | 13 ++ xml_converter/intigration_tests/run_tests.py | 189 ++++++++++++++++++ .../test_expected_outputs.json | 37 ---- xml_converter/intigration_tests/testcases.py | 36 ++++ xml_converter/src/xml_converter.cpp | 2 +- 11 files changed, 241 insertions(+), 214 deletions(-) create mode 100644 xml_converter/intigration_tests/.gitignore delete mode 100755 xml_converter/intigration_tests/attribute_testing.py delete mode 100644 xml_converter/intigration_tests/can_fade/inputs/can_fade_is_supported_value_not_default.xml rename xml_converter/intigration_tests/{can_fade/expected_outputs/can_fade_false.xml => expected_outputs/xml_can_fade_invalid/xml_file.xml} (100%) rename xml_converter/intigration_tests/{can_fade/inputs/can_fade_is_correct.xml => expected_outputs/xml_can_fade_valid/xml_file.xml} (77%) rename xml_converter/intigration_tests/{can_fade/inputs/can_fade_unsupported_value.xml => inputs/xml_can_fade_invalid/xml_file.xml} (100%) create mode 100644 xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml create mode 100755 xml_converter/intigration_tests/run_tests.py delete mode 100644 xml_converter/intigration_tests/test_expected_outputs.json create mode 100644 xml_converter/intigration_tests/testcases.py diff --git a/xml_converter/intigration_tests/.gitignore b/xml_converter/intigration_tests/.gitignore new file mode 100644 index 00000000..17aa483a --- /dev/null +++ b/xml_converter/intigration_tests/.gitignore @@ -0,0 +1 @@ +outputs/ diff --git a/xml_converter/intigration_tests/attribute_testing.py b/xml_converter/intigration_tests/attribute_testing.py deleted file mode 100755 index 17ee513a..00000000 --- a/xml_converter/intigration_tests/attribute_testing.py +++ /dev/null @@ -1,168 +0,0 @@ -import argparse -import difflib -import json -import subprocess -import re -import os -from typing import List, Optional, Final, Tuple - -# Path to compiled C++ executable -xml_converter_binary_path: str = "../build/xml_converter" -# Paths to data for the tests -json_file_path: str = "./test_expected_outputs.json" -tests_cases_path: str = "../test_cases" - -arg_input_xml: Final[str] = "--input-taco-path" -arg_output_xml: Final[str] = "--output-taco-path" -arg_input_proto: Final[str] = "--input-waypoint-path" -arg_output_proto: Final[str] = "--output-waypoint-path" -arg_split_proto: Final[str] = "--output-split-waypoint-path" - - -def run_xml_converter( - input_xml: Optional[List[str]] = None, - output_xml: Optional[List[str]] = None, - input_proto: Optional[List[str]] = None, - output_proto: Optional[List[str]] = None, - split_output_proto: Optional[str] = None, -) -> Tuple[str, str, int]: - - # Build the command to execute the C++ program with the desired function and arguments - cmd: List[str] = [xml_converter_binary_path] - - if input_xml: - cmd += [arg_input_xml] + input_xml - if output_xml: - cmd += [arg_output_xml] + output_xml - if input_proto: - cmd += [arg_input_proto] + input_proto - if output_proto: - cmd += [arg_output_proto] + output_proto - if split_output_proto: - cmd += [arg_split_proto] + [split_output_proto] - - # Run the C++ program and capture its output - result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - - return (result.stdout, result.stderr, result.returncode) - - -def compare_files(file_path1: str, file_path2: str) -> List[str]: - with open(file_path1, 'r') as file1, open(file_path2, 'r') as file2: - content1 = file1.readlines() - content2 = file2.readlines() - - diff = list(difflib.Differ().compare(content1, content2)) - filtered_diff: List[str] = [] - for line in diff: - if line.startswith("+ ") or line.startswith("- "): - filtered_diff.append(line) - - return filtered_diff - - -patterns_for_noisy_lines = [ - r"^The taco parse function took [0-9]+ milliseconds to run$", - r"^The xml write function took [0-9]+ milliseconds to run$", - r"^The protobuf read function took [0-9]+ milliseconds to run$", - r"^The protobuf write function took [0-9]+ milliseconds to run$", - r"^$" -] - -pattern_for_color_escape_codes = r"\u001b\[[0-9;]+m" - - -def remove_color_tags(string: str) -> str: - string = re.sub(pattern_for_color_escape_codes, '', string) - return string - - -def remove_noisy_lines(lines: List[str]) -> List[str]: - filtered_array = [] - for line in lines: - match_found: bool = False - for pattern in patterns_for_noisy_lines: - if re.fullmatch(pattern, line): - match_found = True - break - if not match_found: - filtered_array.append(remove_color_tags(line)) - return filtered_array - - -def main() -> None: - parser = argparse.ArgumentParser(description="A test harness for evaluating the output of the xmlconverter program") - parser.add_argument("-v", "--verbose", help="Prints the results from xmlconverter in JSON format", action="store_true") - args = parser.parse_args() - - with open(json_file_path, 'r') as json_file: - data = json.load(json_file) - - for attribute_data in data: - attribute_name: str = attribute_data["attribute_name"] - print(attribute_name) - - input_dir_path = os.path.join(tests_cases_path, attribute_name, "inputs") - expected_output_dir_path = os.path.join(tests_cases_path, attribute_name, "expected_outputs") - output_dir_path = os.path.join(tests_cases_path, attribute_name, "outputs") - - # Ensure that the test output directory is created - os.makedirs(output_dir_path, exist_ok=True) - - for test in attribute_data["tests"]: - xml_file_name: str = attribute_name + "_" + test["name"] + ".xml" - input_xml_path = os.path.join(input_dir_path, xml_file_name) - output_xml_path = os.path.join(output_dir_path, xml_file_name) - expected_output_xml_path = os.path.join(expected_output_dir_path, test["expected_output_xml_file_name"]) - - result = run_xml_converter(input_xml=[input_xml_path], output_xml=[output_xml_path]) - - # Remove noisy lines - stdout: List[str] = remove_noisy_lines(result[0].split("\n")) - stderr: List[str] = remove_noisy_lines(result[1].split("\n")) - returncode: int = result[2] - - # Prints the results of xml_converter - if args.verbose: - print(f"Test {attribute_name}_{test['name']}") - print(f"\"stdout\" : {json.dumps(stdout)}") - print(f"\"stderr\" : {json.dumps(stderr)}") - print(f"\"return_code\" : {json.dumps(returncode)}") - - all_tests_passed: bool = True - error_diff: List[str] - - if stdout != test["expected_stdout"]: - print(f"Standard output did not match for test {attribute_name}{test['name']}") - error_diff = list(difflib.Differ().compare(test["expected_stdout"], stdout)) - for line in error_diff: - print(line) - all_tests_passed = False - - if stderr != test["expected_stderr"]: - print(f"Standard error did not match for test {attribute_name}{test['name']}") - error_diff = list(difflib.Differ().compare(test["expected_stderr"], stderr)) - for line in error_diff: - print(line) - all_tests_passed = False - - if returncode != test["expected_returncode"]: - print(f"Return code did not match for test {attribute_name}{test['name']}") - print(f"expected_returncode = {test['expected_returncode']}") - print(f"returncode = {returncode}") - - xml_diff = compare_files(expected_output_xml_path, output_xml_path) - - if xml_diff != []: - print(f"Diff was incorrect for test {attribute_name}{test['name']}") - for line in xml_diff: - if line.startswith("+ "): - print(line) - all_tests_passed = False - - if all_tests_passed: - print(f"Success: test {attribute_name}_{test['name']}") - - -if __name__ == "__main__": - main() diff --git a/xml_converter/intigration_tests/can_fade/inputs/can_fade_is_supported_value_not_default.xml b/xml_converter/intigration_tests/can_fade/inputs/can_fade_is_supported_value_not_default.xml deleted file mode 100644 index fdf72475..00000000 --- a/xml_converter/intigration_tests/can_fade/inputs/can_fade_is_supported_value_not_default.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/xml_converter/intigration_tests/can_fade/expected_outputs/can_fade_false.xml b/xml_converter/intigration_tests/expected_outputs/xml_can_fade_invalid/xml_file.xml similarity index 100% rename from xml_converter/intigration_tests/can_fade/expected_outputs/can_fade_false.xml rename to xml_converter/intigration_tests/expected_outputs/xml_can_fade_invalid/xml_file.xml diff --git a/xml_converter/intigration_tests/can_fade/inputs/can_fade_is_correct.xml b/xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml similarity index 77% rename from xml_converter/intigration_tests/can_fade/inputs/can_fade_is_correct.xml rename to xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml index d5c9dc0b..d0bede6d 100644 --- a/xml_converter/intigration_tests/can_fade/inputs/can_fade_is_correct.xml +++ b/xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml @@ -4,5 +4,6 @@ + diff --git a/xml_converter/intigration_tests/can_fade/inputs/can_fade_unsupported_value.xml b/xml_converter/intigration_tests/inputs/xml_can_fade_invalid/xml_file.xml similarity index 100% rename from xml_converter/intigration_tests/can_fade/inputs/can_fade_unsupported_value.xml rename to xml_converter/intigration_tests/inputs/xml_can_fade_invalid/xml_file.xml diff --git a/xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml b/xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml new file mode 100644 index 00000000..21ecea79 --- /dev/null +++ b/xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml @@ -0,0 +1,13 @@ + + + + + + + + + + diff --git a/xml_converter/intigration_tests/run_tests.py b/xml_converter/intigration_tests/run_tests.py new file mode 100755 index 00000000..155653ad --- /dev/null +++ b/xml_converter/intigration_tests/run_tests.py @@ -0,0 +1,189 @@ +import argparse +import difflib +import json +import subprocess +import re +import os +from typing import List, Optional, Final, Tuple +from testcases import testcases +import shutil + +# Path to compiled C++ executable +xml_converter_binary_path: str = "../build/xml_converter" + +arg_input_xml: Final[str] = "--input-taco-path" +arg_output_xml: Final[str] = "--output-taco-path" +arg_input_proto: Final[str] = "--input-waypoint-path" +arg_output_proto: Final[str] = "--output-waypoint-path" +arg_split_proto: Final[str] = "--output-split-waypoint-path" + + +def run_xml_converter( + input_xml: Optional[List[str]] = None, + output_xml: Optional[List[str]] = None, + input_proto: Optional[List[str]] = None, + output_proto: Optional[List[str]] = None, + split_output_proto: Optional[str] = None, +) -> Tuple[str, str, int]: + + # Build the command to execute the C++ program with the desired function and arguments + cmd: List[str] = [xml_converter_binary_path] + + if input_xml: + cmd += [arg_input_xml] + input_xml + if output_xml: + cmd += [arg_output_xml] + output_xml + if input_proto: + cmd += [arg_input_proto] + input_proto + if output_proto: + cmd += [arg_output_proto] + output_proto + if split_output_proto: + cmd += [arg_split_proto] + [split_output_proto] + + # Run the C++ program and capture its output + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + return (result.stdout, result.stderr, result.returncode) + + +def compare_files(file_path1: str, file_path2: str) -> List[str]: + with open(file_path1, 'r') as file1: + content1 = file1.readlines() + with open(file_path2, 'r') as file2: + content2 = file2.readlines() + + diff = list(difflib.Differ().compare(content1, content2)) + + return diff + + +def len_diff(lines: List[str]) -> int: + diffcount = 0 + + for line in lines: + if line.startswith(" "): + continue + diffcount += 1 + return diffcount + + + + +################################################################################ +# remove_ansii_color_escapecodes +# +# Remove the ANSII color code escape characters from a string to make it easier +# to read what is actually contained in the string. +################################################################################ +pattern_for_color_escape_codes = r"\u001b\[[0-9;]+m" + + +def remove_ansii_color_escapecodes(lines: List[str]) -> List[str]: + return [ re.sub(pattern_for_color_escape_codes, '', line) for line in lines ] + + +################################################################################ +# remove_ignored_lines +# +# Goes through a list of lines and removes any line that matches a pattern in +# the `line_patterns_to_ignore` global variable. +################################################################################ +line_patterns_to_ignore = [ + r"^Loading taco pack .*$", + r"^The taco parse function took [0-9]+ milliseconds to run$", + r"^The xml write function took [0-9]+ milliseconds to run$", + r"^The protobuf read function took [0-9]+ milliseconds to run$", + r"^The protobuf write function took [0-9]+ milliseconds to run$", + r"^$" +] + + +def remove_ignored_lines(lines: List[str]) -> List[str]: + filtered_array = [] + for line in lines: + match_found: bool = False + for pattern in line_patterns_to_ignore: + if re.fullmatch(pattern, line): + match_found = True + break + if not match_found: + filtered_array.append(line) + return filtered_array + + +def main() -> None: + parser = argparse.ArgumentParser(description="A test harness for evaluating the output of the xmlconverter program") + parser.add_argument("-v", "--verbose", help="Prints the results from xmlconverter in JSON format", action="store_true") + args = parser.parse_args() + + output_parent_dirpath = "./outputs" + + # Ensure that the test output directory is empty + if os.path.exists(output_parent_dirpath): + shutil.rmtree(output_parent_dirpath) + + for testcase in testcases: + xml_output_dir_path = os.path.join(output_parent_dirpath, "xml", testcase.name) + proto_output_dir_path = os.path.join(output_parent_dirpath, "proto", testcase.name) + + + os.makedirs(xml_output_dir_path, exist_ok=True) + os.makedirs(proto_output_dir_path, exist_ok=True) + + rawstdout, rawstderr, returncode = run_xml_converter( + input_xml=testcase.xml_input_paths, + output_xml=[xml_output_dir_path], + output_proto=[proto_output_dir_path], + ) + + # Sanitize and denoise the lines + stdout: List[str] = remove_ansii_color_escapecodes(remove_ignored_lines(rawstdout.split("\n"))) + stderr: List[str] = remove_ansii_color_escapecodes(remove_ignored_lines(rawstderr.split("\n"))) + + # Prints the results of xml_converter + if args.verbose: + print(f"Test {testcase.name}") + print(" stdout : {}".format("\n".join(stdout))) + print(" stderr : {}".format("\n".join(stderr))) + print(" return_code : {}".format(returncode)) + + all_tests_passed: bool = True + + stdout_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stdout, stdout)) + if len_diff(stdout_diff) != 0: + print(f"Standard output did not match for test {testcase.name}") + for line in stdout_diff: + print(line) + all_tests_passed = False + + stderr_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stderr, stderr)) + if len_diff(stderr_diff) != 0: + print(f"Standard error did not match for test {testcase.name}") + for line in stderr_diff: + print(line) + all_tests_passed = False + + if testcase.expected_returncode is not None and testcase.expected_returncode != returncode : + print(f"Expected a return code of {testcase.expected_returncode} for {testcase.name} but got {returncode}") + + if testcase.expected_output_xml_path is not None: + # TODO: These paths are directories and `xml_file.xml` is just one + # possible file in the directories. Eventually we should check all + # the files in the directory not just the one. + output_xml_filepath = os.path.join(xml_output_dir_path, "xml_file.xml") + expected_output_xml_filepath = os.path.join(testcase.expected_output_xml_path, "xml_file.xml") + + xml_diff = compare_files(expected_output_xml_filepath , output_xml_filepath) + + if len_diff(xml_diff) != 0: + print(f"XML output was incorrect for test {testcase.name}") + for line in xml_diff: + print(line, end="") + all_tests_passed = False + + if all_tests_passed: + print(f"Success: test {testcase.name}") + + +if __name__ == "__main__": + main() diff --git a/xml_converter/intigration_tests/test_expected_outputs.json b/xml_converter/intigration_tests/test_expected_outputs.json deleted file mode 100644 index 5607991b..00000000 --- a/xml_converter/intigration_tests/test_expected_outputs.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "attribute_name": "can_fade", - "tests": [ - { - "name": "is_correct", - "expected_output_xml_file_name": "can_fade_false.xml", - "expected_stdout" : [ - "Loading taco pack ../test_cases/can_fade/inputs/can_fade_is_correct.xml" - ], - "expected_stderr": [], - "expected_returncode" : 0 - }, - { - "name" : "is_supported_value_not_default", - "expected_output_xml_file_name": "can_fade_false.xml", - "expected_stdout": [ - "Loading taco pack ../test_cases/can_fade/inputs/can_fade_is_supported_value_not_default.xml" - ], - "expected_stderr": [], - "expected_returncode" : 0 - }, - { - "name": "unsupported_value", - "expected_output_xml_file_name": "can_fade_false.xml", - "expected_stdout": [ - "Loading taco pack ../test_cases/can_fade/inputs/can_fade_unsupported_value.xml", - "Error: Found a boolean value that was not a '1', '0', 'true', or 'false'", - "../test_cases/can_fade/inputs/can_fade_unsupported_value.xml", - "6 |", " | ^^^" - ], - "expected_stderr" : [], - "expected_returncode" : 0 - } - ] - } -] diff --git a/xml_converter/intigration_tests/testcases.py b/xml_converter/intigration_tests/testcases.py new file mode 100644 index 00000000..ce3bb720 --- /dev/null +++ b/xml_converter/intigration_tests/testcases.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass, field +from typing import List, Optional + +@dataclass +class Testcase: + name: str + xml_input_paths: List[str] = field(default_factory=list) + proto_input_paths: List[str] = field(default_factory=list) + + # TODO: Eventually the expected output paths wont be optional + expected_output_xml_path: Optional[str] = None + expected_output_proto_path: Optional[str] = None + + expected_stdout: List[str] = field(default_factory=list) + expected_stderr: List[str] = field(default_factory=list) + expected_returncode: int = 0 + + +testcases: List[Testcase] = [ + Testcase( + name="canfade_valid", + xml_input_paths=["./inputs/xml_can_fade_valid"], + expected_output_xml_path="./expected_outputs/xml_can_fade_valid", + ), + Testcase( + name="canfade_invalid", + xml_input_paths=["./inputs/xml_can_fade_invalid"], + expected_output_xml_path="./expected_outputs/xml_can_fade_invalid", + expected_stdout=[ + "Error: Found a boolean value that was not a '1', '0', 'true', or 'false'", + "./inputs/xml_can_fade_invalid/xml_file.xml", + '6 |', + " | ^^^" + ] + ) +] diff --git a/xml_converter/src/xml_converter.cpp b/xml_converter/src/xml_converter.cpp index cb39da84..5d0c9238 100644 --- a/xml_converter/src/xml_converter.cpp +++ b/xml_converter/src/xml_converter.cpp @@ -98,7 +98,7 @@ void read_taco_directory(string directory, map* marker_categor void write_taco_directory(string directory, map* marker_categories, vector* parsed_pois) { // TODO: Exportion of XML Marker Packs File Structure #111 - string xml_filepath = directory + "xml_file.xml"; + string xml_filepath = directory + "/xml_file.xml"; write_xml_file(xml_filepath, marker_categories, parsed_pois); }