From 7f46a28dcc5ebc67c0ace30040eb1d070e2ca652 Mon Sep 17 00:00:00 2001 From: Asher Glick Date: Sat, 4 Nov 2023 20:11:27 -0500 Subject: [PATCH] Adding Proto Tests --- .../proto_can_fade_invalid/markers.bin | 3 + .../proto_can_fade_valid/markers.bin | 3 + .../proto_mount_filter_invalid/markers.bin | 3 + .../proto_mount_filter_valid/markers.bin | 3 + .../xml_can_fade_invalid/xml_file.xml | 4 +- .../xml_can_fade_valid/xml_file.xml | 6 +- .../xml_mount_filter_invalid/xml_file.xml | 10 +-- .../xml_mount_filter_valid/xml_file.xml | 10 +-- .../inputs/xml_can_fade_invalid/xml_file.xml | 10 +-- .../inputs/xml_can_fade_valid/xml_file.xml | 20 ++--- .../xml_mount_filter_invalid/xml_file.xml | 16 ++-- .../xml_mount_filter_valid/xml_file.xml | 16 ++-- .../intigration_tests/proto_utils.py | 80 +++++++++++++++++++ xml_converter/intigration_tests/run_tests.py | 23 ++++-- xml_converter/intigration_tests/testcases.py | 24 +++--- xml_converter/src/packaging_protobin.cpp | 6 +- 16 files changed, 174 insertions(+), 63 deletions(-) create mode 100644 xml_converter/intigration_tests/expected_outputs/proto_can_fade_invalid/markers.bin create mode 100644 xml_converter/intigration_tests/expected_outputs/proto_can_fade_valid/markers.bin create mode 100644 xml_converter/intigration_tests/expected_outputs/proto_mount_filter_invalid/markers.bin create mode 100644 xml_converter/intigration_tests/expected_outputs/proto_mount_filter_valid/markers.bin create mode 100644 xml_converter/intigration_tests/proto_utils.py diff --git a/xml_converter/intigration_tests/expected_outputs/proto_can_fade_invalid/markers.bin b/xml_converter/intigration_tests/expected_outputs/proto_can_fade_invalid/markers.bin new file mode 100644 index 00000000..526c9fe3 --- /dev/null +++ b/xml_converter/intigration_tests/expected_outputs/proto_can_fade_invalid/markers.bin @@ -0,0 +1,3 @@ + +" + My Category 2B \Ï)Cf¦RC{ÔWC \ No newline at end of file diff --git a/xml_converter/intigration_tests/expected_outputs/proto_can_fade_valid/markers.bin b/xml_converter/intigration_tests/expected_outputs/proto_can_fade_valid/markers.bin new file mode 100644 index 00000000..c32cb0ef --- /dev/null +++ b/xml_converter/intigration_tests/expected_outputs/proto_can_fade_valid/markers.bin @@ -0,0 +1,3 @@ + +7 + My Category 2B \Ï)Cf¦RC{ÔWC 2B \Ï)Cf¦RC{ÔWC \ No newline at end of file diff --git a/xml_converter/intigration_tests/expected_outputs/proto_mount_filter_invalid/markers.bin b/xml_converter/intigration_tests/expected_outputs/proto_mount_filter_invalid/markers.bin new file mode 100644 index 00000000..3e5a1112 --- /dev/null +++ b/xml_converter/intigration_tests/expected_outputs/proto_mount_filter_invalid/markers.bin @@ -0,0 +1,3 @@ + +o + My Category 2B \Ï)Cf¦RC{ÔWC 2B \Ï)Cf¦RC{ÔWC 2B \Ï)Cf¦RC{ÔWCê 2B \Ï)Cf¦RC{ÔWCê \ No newline at end of file diff --git a/xml_converter/intigration_tests/expected_outputs/proto_mount_filter_valid/markers.bin b/xml_converter/intigration_tests/expected_outputs/proto_mount_filter_valid/markers.bin new file mode 100644 index 00000000..4236a31f --- /dev/null +++ b/xml_converter/intigration_tests/expected_outputs/proto_mount_filter_valid/markers.bin @@ -0,0 +1,3 @@ + +‹ + My Category 2B \Ï)Cf¦RC{ÔWCê 2B \Ï)Cf¦RC{ÔWCê 2B \Ï)Cf¦RC{ÔWCê* 2B \Ï)Cf¦RC{ÔWCê (08@HP \ No newline at end of file diff --git a/xml_converter/intigration_tests/expected_outputs/xml_can_fade_invalid/xml_file.xml b/xml_converter/intigration_tests/expected_outputs/xml_can_fade_invalid/xml_file.xml index 30e1dcdb..5282685a 100644 --- a/xml_converter/intigration_tests/expected_outputs/xml_can_fade_invalid/xml_file.xml +++ b/xml_converter/intigration_tests/expected_outputs/xml_can_fade_invalid/xml_file.xml @@ -1,8 +1,8 @@ - + - + diff --git a/xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml b/xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml index 4731a19d..fdc5f9f8 100644 --- a/xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml +++ b/xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml @@ -1,9 +1,9 @@ - + - - + + diff --git a/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_invalid/xml_file.xml b/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_invalid/xml_file.xml index 3276bdf2..53b34efe 100644 --- a/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_invalid/xml_file.xml +++ b/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_invalid/xml_file.xml @@ -1,11 +1,11 @@ - + - - - - + + + + diff --git a/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_valid/xml_file.xml b/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_valid/xml_file.xml index 6b80f183..38abbed2 100644 --- a/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_valid/xml_file.xml +++ b/xml_converter/intigration_tests/expected_outputs/xml_mount_filter_valid/xml_file.xml @@ -1,11 +1,11 @@ - + - - - - + + + + diff --git a/xml_converter/intigration_tests/inputs/xml_can_fade_invalid/xml_file.xml b/xml_converter/intigration_tests/inputs/xml_can_fade_invalid/xml_file.xml index 82333328..921fb19a 100644 --- a/xml_converter/intigration_tests/inputs/xml_can_fade_invalid/xml_file.xml +++ b/xml_converter/intigration_tests/inputs/xml_can_fade_invalid/xml_file.xml @@ -1,8 +1,8 @@ - - + + - - - + + + diff --git a/xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml b/xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml index af7cae72..fc1cad1d 100644 --- a/xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml +++ b/xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml @@ -1,13 +1,13 @@ - - + + - - - - - + + + + + diff --git a/xml_converter/intigration_tests/inputs/xml_mount_filter_invalid/xml_file.xml b/xml_converter/intigration_tests/inputs/xml_mount_filter_invalid/xml_file.xml index 3ee980aa..7ce2f5a0 100644 --- a/xml_converter/intigration_tests/inputs/xml_mount_filter_invalid/xml_file.xml +++ b/xml_converter/intigration_tests/inputs/xml_mount_filter_invalid/xml_file.xml @@ -1,11 +1,11 @@ - - + + - - - - - - + + + + + + diff --git a/xml_converter/intigration_tests/inputs/xml_mount_filter_valid/xml_file.xml b/xml_converter/intigration_tests/inputs/xml_mount_filter_valid/xml_file.xml index b3e44ac7..d8472344 100644 --- a/xml_converter/intigration_tests/inputs/xml_mount_filter_valid/xml_file.xml +++ b/xml_converter/intigration_tests/inputs/xml_mount_filter_valid/xml_file.xml @@ -1,11 +1,11 @@ - - + + - - - - - - + + + + + + diff --git a/xml_converter/intigration_tests/proto_utils.py b/xml_converter/intigration_tests/proto_utils.py new file mode 100644 index 00000000..1e1957dc --- /dev/null +++ b/xml_converter/intigration_tests/proto_utils.py @@ -0,0 +1,80 @@ +import os +import subprocess +import difflib + +def compare_protos( + outputs_directory: str, + expected_outputs_directory: str, +) -> bool: + # TODO: These paths are directories and 'markers.bin` is just one + # possible file in the directories. Eventually we should check all + # the files in the directory not just the one. + files_are_equal = compare_binary_file( + os.path.join(expected_outputs_directory, "markers.bin"), + os.path.join(outputs_directory, "markers.bin"), + ) + + if files_are_equal: + return True + + expected_textproto_path = os.path.join(expected_outputs_directory, "markers.bin") + actual_textproto_path = os.path.join(outputs_directory, "markers.bin") + + expected_textproto = get_waypoint_textproto(expected_textproto_path) + actual_textproto = get_waypoint_textproto(actual_textproto_path) + + diff = difflib.unified_diff(expected_textproto.split("\n"), actual_textproto.split("\n"), fromfile=expected_textproto_path, tofile=actual_textproto_path, lineterm="") + + for line in diff: + print(line) + + # TODO: Also might be good to include a HEX diff breakdown because if the + # diff is just ordering then the textproto conversion might correct the + # error and make it look like there is no diff but the test still fails. + + return False + +def compare_binary_file(file_path_1: str, file_path_2: str) -> bool: + if not os.path.exists(file_path_1): + return False + if not os.path.exists(file_path_2): + return False + + with open(file_path_1, 'rb') as file1: + contents_1 = file1.read() + with open(file_path_2, 'rb') as file2: + contents_2 = file2.read() + + return contents_1 == contents_2 + + +################################################################################ +# get_waypoint_textproto +# +# Reads a waypoint protobin and returns a stringy textproto value of the +# contents of the protobin. This makes it easier to diff the contents but also +# can be used to easily inspect the values of the protobin. +################################################################################ +def get_waypoint_textproto(protobin_path: str) -> str: + proto_schema_path = "../proto/waypoint.proto" + proto_schema_basedir = "../proto" + + if not os.path.exists(protobin_path): + return "" + + with open(protobin_path, 'rb') as f: + result = subprocess.run( + [ + "protoc", + "--decode=waypoint.Waypoint", + "--proto_path=" + proto_schema_basedir, + proto_schema_path + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=f, + ) + + # TODO: sanity check return code, stdout, and stderr + + return result.stdout.decode("utf-8") diff --git a/xml_converter/intigration_tests/run_tests.py b/xml_converter/intigration_tests/run_tests.py index 155653ad..6238c448 100755 --- a/xml_converter/intigration_tests/run_tests.py +++ b/xml_converter/intigration_tests/run_tests.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import argparse import difflib import json @@ -7,6 +9,7 @@ from typing import List, Optional, Final, Tuple from testcases import testcases import shutil +from proto_utils import compare_protos # Path to compiled C++ executable xml_converter_binary_path: str = "../build/xml_converter" @@ -46,13 +49,13 @@ def run_xml_converter( return (result.stdout, result.stderr, result.returncode) -def compare_files(file_path1: str, file_path2: str) -> List[str]: +def compare_text_files(file_path1: str, file_path2: str) -> List[str]: with open(file_path1, 'r') as file1: content1 = file1.readlines() with open(file_path2, 'r') as file2: content2 = file2.readlines() - diff = list(difflib.Differ().compare(content1, content2)) + diff = list(difflib.unified_diff(content1, content2, fromfile=file_path1, tofile=file_path2, lineterm="")) return diff @@ -61,7 +64,7 @@ def len_diff(lines: List[str]) -> int: diffcount = 0 for line in lines: - if line.startswith(" "): + if line.startswith(" "): continue diffcount += 1 return diffcount @@ -149,14 +152,14 @@ def main() -> None: all_tests_passed: bool = True - stdout_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stdout, stdout)) + stdout_diff: List[str] = list(difflib.unified_diff(testcase.expected_stdout, stdout, fromfile="Expected stdout", tofile="Actual stdout", lineterm="")) if len_diff(stdout_diff) != 0: print(f"Standard output did not match for test {testcase.name}") for line in stdout_diff: print(line) all_tests_passed = False - stderr_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stderr, stderr)) + stderr_diff: List[str] = list(difflib.unified_diff(testcase.expected_stderr, stderr, fromfile="Expected stderr", tofile="Actual stderr", lineterm="")) if len_diff(stderr_diff) != 0: print(f"Standard error did not match for test {testcase.name}") for line in stderr_diff: @@ -173,7 +176,7 @@ def main() -> None: output_xml_filepath = os.path.join(xml_output_dir_path, "xml_file.xml") expected_output_xml_filepath = os.path.join(testcase.expected_output_xml_path, "xml_file.xml") - xml_diff = compare_files(expected_output_xml_filepath , output_xml_filepath) + xml_diff = compare_text_files(expected_output_xml_filepath , output_xml_filepath) if len_diff(xml_diff) != 0: print(f"XML output was incorrect for test {testcase.name}") @@ -181,6 +184,14 @@ def main() -> None: print(line, end="") all_tests_passed = False + if testcase.expected_output_proto_path is not None: + protos_are_equal = compare_protos( + outputs_directory=proto_output_dir_path, + expected_outputs_directory=testcase.expected_output_proto_path, + ) + + all_tests_passed &= protos_are_equal + if all_tests_passed: print(f"Success: test {testcase.name}") diff --git a/xml_converter/intigration_tests/testcases.py b/xml_converter/intigration_tests/testcases.py index 01c8e01f..297129c5 100644 --- a/xml_converter/intigration_tests/testcases.py +++ b/xml_converter/intigration_tests/testcases.py @@ -21,44 +21,48 @@ class Testcase: name="canfade_valid", xml_input_paths=["./inputs/xml_can_fade_valid"], expected_output_xml_path="./expected_outputs/xml_can_fade_valid", + expected_output_proto_path="./expected_outputs/proto_can_fade_valid", ), Testcase( name="canfade_invalid", xml_input_paths=["./inputs/xml_can_fade_invalid"], expected_output_xml_path="./expected_outputs/xml_can_fade_invalid", + expected_output_proto_path="./expected_outputs/proto_can_fade_invalid", expected_stdout=[ "Error: Found a boolean value that was not a '1', '0', 'true', or 'false'", "./inputs/xml_can_fade_invalid/xml_file.xml", - '6 |', - " | ^^^" + '6 | ', + " | ^^^" ] ), Testcase( name="mountfilter_valid", xml_input_paths=["./inputs/xml_mount_filter_valid"], expected_output_xml_path="./expected_outputs/xml_mount_filter_valid", + expected_output_proto_path="./expected_outputs/proto_mount_filter_valid", ), Testcase( name="mountfilter_invalid", xml_input_paths=["./inputs/xml_mount_filter_invalid"], expected_output_xml_path="./expected_outputs/xml_mount_filter_invalid", + expected_output_proto_path="./expected_outputs/proto_mount_filter_invalid", expected_stdout=[ 'Error: Invalid Filter for MountFilter. Found ', './inputs/xml_mount_filter_invalid/xml_file.xml', - '6 |', - ' | ', + '6 | ', + ' | ', 'Error: Invalid Filter for MountFilter. Found NotAMount', './inputs/xml_mount_filter_invalid/xml_file.xml', - '7 |', - ' | ^^^^^^^^^', + '7 | ', + ' | ^^^^^^^^^', 'Error: Invalid Filter for MountFilter. Found ', './inputs/xml_mount_filter_invalid/xml_file.xml', - '8 |', - ' | ^^^^^^^^^^^^^^^^', + '8 | ', + ' | ^^^^^^^^^^^^^^^^', 'Error: Invalid Filter for MountFilter. Found NotAMount', './inputs/xml_mount_filter_invalid/xml_file.xml', - '9 |', - ' | ^^^^^^^^^^^^^^^^^^^^^^^^^', + '9 | ', + ' | ^^^^^^^^^^^^^^^^^^^^^^^^^', ] ), ] diff --git a/xml_converter/src/packaging_protobin.cpp b/xml_converter/src/packaging_protobin.cpp index a5129bdb..d17a23b2 100644 --- a/xml_converter/src/packaging_protobin.cpp +++ b/xml_converter/src/packaging_protobin.cpp @@ -150,6 +150,10 @@ void _write_protobuf_file( ofstream outfile; outfile.open(filepath, ios::out | ios::binary); + if (!outfile.is_open()) { + cout << "Unable to open " << filepath << endl; + } + waypoint::Waypoint output_message; for (map::const_iterator it = marker_categories->begin(); it != marker_categories->end(); it++) { @@ -195,7 +199,7 @@ void write_protobuf_file( } _write_protobuf_file( - filepath, + filepath + "/markers.bin", category_filter, marker_categories, category_to_pois);