Skip to content

Commit

Permalink
Merge pull request #205 from AsherGlick/proto_integration_tests
Browse files Browse the repository at this point in the history 
Adding Proto Tests
  • Loading branch information
@AsherGlick
AsherGlick authored Nov 5, 2023
2 parents 23ad32c + 7f46a28 commit 7bffa71
Show file tree
Hide file tree
Showing 16 changed files with 174 additions and 63 deletions.
3 changes: 3 additions & 0 deletions xml_converter/intigration_tests/expected_outputs/proto_can_fade_invalid/markers.bin
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

"
My Category 2B\�)Cf�RC{�WC
Expand Down
3 changes: 3 additions & 0 deletions xml_converter/intigration_tests/expected_outputs/proto_can_fade_valid/markers.bin
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

7
My Category 2B\�)Cf�RC{�WC 2B\�)Cf�RC{�WC
Expand Down
3 changes: 3 additions & 0 deletions xml_converter/intigration_tests/expected_outputs/proto_mount_filter_invalid/markers.bin
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

o
My Category 2B\�)Cf�RC{�WC 2B\�)Cf�RC{�WC 2B\�)Cf�RC{�WC� 2B\�)Cf�RC{�WC�
Expand Down
3 changes: 3 additions & 0 deletions xml_converter/intigration_tests/expected_outputs/proto_mount_filter_valid/markers.bin
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

�
My Category 2B\�)Cf�RC{�WC� 2B\�)Cf�RC{�WC� 2B\�)Cf�RC{�WC�* 2B\�)Cf�RC{�WC� (08@HP
Expand Down
4 changes: 2 additions & 2 deletions xml_converter/intigration_tests/expected_outputs/xml_can_fade_invalid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<OverlayData>
<MarkerCategory Name="mycategory">
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI Type="mycategory" IconFile="texture.png" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
</POIs>
</OverlayData>
6 changes: 3 additions & 3 deletions xml_converter/intigration_tests/expected_outputs/xml_can_fade_valid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
<OverlayData>
<MarkerCategory Name="mycategory">
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI CanFade="false" Type="mycategory" IconFile="texture.png" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI CanFade="false" Type="mycategory" IconFile="texture.png" MapID="50" XPos="179.809998" YPos="220.649994" ZPos="225.830002"/>
<POI CanFade="false" Type="mycategory" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI CanFade="false" Type="mycategory" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
</POIs>
</OverlayData>
10 changes: 5 additions & 5 deletions xml_converter/intigration_tests/expected_outputs/xml_mount_filter_invalid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<OverlayData>
<MarkerCategory Name="mycategory">
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
</POIs>
</OverlayData>
10 changes: 5 additions & 5 deletions xml_converter/intigration_tests/expected_outputs/xml_mount_filter_valid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<OverlayData>
<MarkerCategory Name="mycategory">
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="raptor" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" IconFile="texture.png" MapID="50" Mount="raptor,springer,skimmer,jackal,griffon,rollerbeetle,warclaw,skyscale,skiff,seigeturtle" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="raptor" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="raptor,springer" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
<POI Type="mycategory" MapID="50" Mount="raptor,springer,skimmer,jackal,griffon,rollerbeetle,warclaw,skyscale,skiff,seigeturtle" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
</POIs>
</OverlayData>
10 changes: 5 additions & 5 deletions xml_converter/intigration_tests/inputs/xml_can_fade_invalid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<OverlayData>
<MarkerCategory Name="mycategory">
</MarkerCategory>
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI CanFade="yes" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" />
</POIs>
<POIs>
<POI CanFade="yes" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
</POIs>
</OverlayData>
20 changes: 10 additions & 10 deletions xml_converter/intigration_tests/inputs/xml_can_fade_valid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
<OverlayData>
<MarkerCategory Name="mycategory">
</MarkerCategory>
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI CanFade="false" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" />
<POI CanFade="0" Type="mycategory" XPos="179.81" YPos="220.65" ZPos="225.83" MapID="50" IconFile="texture.png" />
<!-- TODO: This value should be elided in the output, but it is not currently
so we are ignoring the test instead of wriitng a bad test
<POI CanFade="true" Type="mycategory"/>
<POI CanFade="1" Type="mycategory"/> -->
</POIs>
<POIs>
<POI CanFade="false" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<POI CanFade="0" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<!-- TODO: This value should be elided in the output, but it is not currently
so we are ignoring the test instead of wriitng a bad test
<POI CanFade="true" Type="mycategory"/>
<POI CanFade="1" Type="mycategory"/> -->
</POIs>
</OverlayData>
16 changes: 8 additions & 8 deletions xml_converter/intigration_tests/inputs/xml_mount_filter_invalid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<OverlayData>
<MarkerCategory Name="mycategory">
</MarkerCategory>
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI Mount="" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
<POI Mount="NotAMount" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
<POI Mount="Raptor,Springer," Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
<POI Mount="Raptor,NotAMount,Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
</POIs>
<POIs>
<POI Mount="" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<POI Mount="NotAMount" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<POI Mount="Raptor,Springer," Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<POI Mount="Raptor,NotAMount,Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
</POIs>
</OverlayData>
16 changes: 8 additions & 8 deletions xml_converter/intigration_tests/inputs/xml_mount_filter_valid/xml_file.xml
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<OverlayData>
<MarkerCategory Name="mycategory">
</MarkerCategory>
<MarkerCategory DisplayName="My Category" Name="mycategory">
</MarkerCategory>

<POIs>
<POI Mount="Raptor" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
<POI Mount="Raptor,Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
<POI Mount="Raptor, Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
<POI Mount="Raptor,Springer,Skimmer,Jackal,Griffon,RollerBeetle,Warclaw,Skyscale,Skiff,SeigeTurtle" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>
</POIs>
<POIs>
<POI Mount="Raptor" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<POI Mount="Raptor,Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<POI Mount="Raptor, Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
<POI Mount="Raptor,Springer,Skimmer,Jackal,Griffon,RollerBeetle,Warclaw,Skyscale,Skiff,SeigeTurtle" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
</POIs>
</OverlayData>
80 changes: 80 additions & 0 deletions xml_converter/intigration_tests/proto_utils.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import os
import subprocess
import difflib

def compare_protos(
outputs_directory: str,
expected_outputs_directory: str,
) -> bool:
# TODO: These paths are directories and 'markers.bin` is just one
# possible file in the directories. Eventually we should check all
# the files in the directory not just the one.
files_are_equal = compare_binary_file(
os.path.join(expected_outputs_directory, "markers.bin"),
os.path.join(outputs_directory, "markers.bin"),
)

if files_are_equal:
return True

expected_textproto_path = os.path.join(expected_outputs_directory, "markers.bin")
actual_textproto_path = os.path.join(outputs_directory, "markers.bin")

expected_textproto = get_waypoint_textproto(expected_textproto_path)
actual_textproto = get_waypoint_textproto(actual_textproto_path)

diff = difflib.unified_diff(expected_textproto.split("\n"), actual_textproto.split("\n"), fromfile=expected_textproto_path, tofile=actual_textproto_path, lineterm="")

for line in diff:
print(line)

# TODO: Also might be good to include a HEX diff breakdown because if the
# diff is just ordering then the textproto conversion might correct the
# error and make it look like there is no diff but the test still fails.

return False

def compare_binary_file(file_path_1: str, file_path_2: str) -> bool:
if not os.path.exists(file_path_1):
return False
if not os.path.exists(file_path_2):
return False

with open(file_path_1, 'rb') as file1:
contents_1 = file1.read()
with open(file_path_2, 'rb') as file2:
contents_2 = file2.read()

return contents_1 == contents_2


################################################################################
# get_waypoint_textproto
#
# Reads a waypoint protobin and returns a stringy textproto value of the
# contents of the protobin. This makes it easier to diff the contents but also
# can be used to easily inspect the values of the protobin.
################################################################################
def get_waypoint_textproto(protobin_path: str) -> str:
proto_schema_path = "../proto/waypoint.proto"
proto_schema_basedir = "../proto"

if not os.path.exists(protobin_path):
return ""

with open(protobin_path, 'rb') as f:
result = subprocess.run(
[
"protoc",
"--decode=waypoint.Waypoint",
"--proto_path=" + proto_schema_basedir,
proto_schema_path
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=f,
)

# TODO: sanity check return code, stdout, and stderr

return result.stdout.decode("utf-8")
23 changes: 17 additions & 6 deletions xml_converter/intigration_tests/run_tests.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#!/usr/bin/env python3

import argparse
import difflib
import json
Expand All @@ -7,6 +9,7 @@
from typing import List, Optional, Final, Tuple
from testcases import testcases
import shutil
from proto_utils import compare_protos

# Path to compiled C++ executable
xml_converter_binary_path: str = "../build/xml_converter"
Expand Down Expand Up @@ -46,13 +49,13 @@ def run_xml_converter(
return (result.stdout, result.stderr, result.returncode)


def compare_files(file_path1: str, file_path2: str) -> List[str]:
def compare_text_files(file_path1: str, file_path2: str) -> List[str]:
with open(file_path1, 'r') as file1:
content1 = file1.readlines()
with open(file_path2, 'r') as file2:
content2 = file2.readlines()

diff = list(difflib.Differ().compare(content1, content2))
diff = list(difflib.unified_diff(content1, content2, fromfile=file_path1, tofile=file_path2, lineterm=""))

return diff

Expand All @@ -61,7 +64,7 @@ def len_diff(lines: List[str]) -> int:
diffcount = 0

for line in lines:
if line.startswith(" "):
if line.startswith(" "):
continue
diffcount += 1
return diffcount
Expand Down Expand Up @@ -149,14 +152,14 @@ def main() -> None:

all_tests_passed: bool = True

stdout_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stdout, stdout))
stdout_diff: List[str] = list(difflib.unified_diff(testcase.expected_stdout, stdout, fromfile="Expected stdout", tofile="Actual stdout", lineterm=""))
if len_diff(stdout_diff) != 0:
print(f"Standard output did not match for test {testcase.name}")
for line in stdout_diff:
print(line)
all_tests_passed = False

stderr_diff: List[str] = list(difflib.Differ().compare(testcase.expected_stderr, stderr))
stderr_diff: List[str] = list(difflib.unified_diff(testcase.expected_stderr, stderr, fromfile="Expected stderr", tofile="Actual stderr", lineterm=""))
if len_diff(stderr_diff) != 0:
print(f"Standard error did not match for test {testcase.name}")
for line in stderr_diff:
Expand All @@ -173,14 +176,22 @@ def main() -> None:
output_xml_filepath = os.path.join(xml_output_dir_path, "xml_file.xml")
expected_output_xml_filepath = os.path.join(testcase.expected_output_xml_path, "xml_file.xml")

xml_diff = compare_files(expected_output_xml_filepath , output_xml_filepath)
xml_diff = compare_text_files(expected_output_xml_filepath , output_xml_filepath)

if len_diff(xml_diff) != 0:
print(f"XML output was incorrect for test {testcase.name}")
for line in xml_diff:
print(line, end="")
all_tests_passed = False

if testcase.expected_output_proto_path is not None:
protos_are_equal = compare_protos(
outputs_directory=proto_output_dir_path,
expected_outputs_directory=testcase.expected_output_proto_path,
)

all_tests_passed &= protos_are_equal

if all_tests_passed:
print(f"Success: test {testcase.name}")

Expand Down
24 changes: 14 additions & 10 deletions xml_converter/intigration_tests/testcases.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,44 +21,48 @@ class Testcase:
name="canfade_valid",
xml_input_paths=["./inputs/xml_can_fade_valid"],
expected_output_xml_path="./expected_outputs/xml_can_fade_valid",
expected_output_proto_path="./expected_outputs/proto_can_fade_valid",
),
Testcase(
name="canfade_invalid",
xml_input_paths=["./inputs/xml_can_fade_invalid"],
expected_output_xml_path="./expected_outputs/xml_can_fade_invalid",
expected_output_proto_path="./expected_outputs/proto_can_fade_invalid",
expected_stdout=[
"Error: Found a boolean value that was not a '1', '0', 'true', or 'false'",
"./inputs/xml_can_fade_invalid/xml_file.xml",
'6 |<POI CanFade="yes" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" />',
" | ^^^"
'6 | <POI CanFade="yes" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />',
" | ^^^"
]
),
Testcase(
name="mountfilter_valid",
xml_input_paths=["./inputs/xml_mount_filter_valid"],
expected_output_xml_path="./expected_outputs/xml_mount_filter_valid",
expected_output_proto_path="./expected_outputs/proto_mount_filter_valid",
),
Testcase(
name="mountfilter_invalid",
xml_input_paths=["./inputs/xml_mount_filter_invalid"],
expected_output_xml_path="./expected_outputs/xml_mount_filter_invalid",
expected_output_proto_path="./expected_outputs/proto_mount_filter_invalid",
expected_stdout=[
'Error: Invalid Filter for MountFilter. Found ',
'./inputs/xml_mount_filter_invalid/xml_file.xml',
'6 |<POI Mount="" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>',
' | ',
'6 | <POI Mount="" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />',
' | ',
'Error: Invalid Filter for MountFilter. Found NotAMount',
'./inputs/xml_mount_filter_invalid/xml_file.xml',
'7 |<POI Mount="NotAMount" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>',
' | ^^^^^^^^^',
'7 | <POI Mount="NotAMount" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />',
' | ^^^^^^^^^',
'Error: Invalid Filter for MountFilter. Found ',
'./inputs/xml_mount_filter_invalid/xml_file.xml',
'8 |<POI Mount="Raptor,Springer," Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>',
' | ^^^^^^^^^^^^^^^^',
'8 | <POI Mount="Raptor,Springer," Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />',
' | ^^^^^^^^^^^^^^^^',
'Error: Invalid Filter for MountFilter. Found NotAMount',
'./inputs/xml_mount_filter_invalid/xml_file.xml',
'9 |<POI Mount="Raptor,NotAMount,Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" IconFile="texture.png" Type="mycategory"/>',
' | ^^^^^^^^^^^^^^^^^^^^^^^^^',
'9 | <POI Mount="Raptor,NotAMount,Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />',
' | ^^^^^^^^^^^^^^^^^^^^^^^^^',
]
),
]
6 changes: 5 additions & 1 deletion xml_converter/src/packaging_protobin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ void _write_protobuf_file(
ofstream outfile;
outfile.open(filepath, ios::out | ios::binary);

if (!outfile.is_open()) {
cout << "Unable to open " << filepath << endl;
}

waypoint::Waypoint output_message;

for (map<string, Category>::const_iterator it = marker_categories->begin(); it != marker_categories->end(); it++) {
Expand Down Expand Up @@ -195,7 +199,7 @@ void write_protobuf_file(
}

_write_protobuf_file(
filepath,
filepath + "/markers.bin",
category_filter,
marker_categories,
category_to_pois);
Expand Down

0 comments on commit 7bffa71

Please sign in to comment.