diff --git a/xml_converter/doc/trail_data/trail_data.md b/xml_converter/doc/trail_data/trail_data.md index 4f31a526..44a41608 100644 --- a/xml_converter/doc/trail_data/trail_data.md +++ b/xml_converter/doc/trail_data/trail_data.md @@ -5,34 +5,38 @@ class: TrailData xml_fields: ["TrailData"] uses_file_path: true protobuf_field: trail_data -side_effects: [Map ID] applies_to: [Trail] +custom_functions: + read.xml: + function: xml_attribute_to_trail_data + side_effects: [Map ID] --- -A binary coordinate array for the path of a trail. -Notes -===== +The coordinates of each point along a trail path. +In the TacO format this attribute is a string that points to a file which contains this information packed in binary. The binary format consists of +- 4 bytes for an integer that represents the `.trl` format version number +- 4 bytes for an integer that represents the Map ID which this trail is for +- Any number of 12 byte sets, containing 3 32 bit floats -TrailData +Example `.trl` parser +-------------------------------------------------------------------------------- +```python +from typing import Tuple, List +import struct -GetFromXML(xml_node) -WriteToXML(self) # not quite right. +with open("./demo.trl", 'rb') as f: + version: int = struct.unpack("* attribute, vec {% for n, attribute_variable in enumerate(attribute_variables) %} {% for i, value in enumerate(attribute_variable.xml_fields) %} {{ "if" if i == n == 0 else "else if" }} (attributename == "{{value}}") { - xml_attribute_to_{{attribute_variable.class_name}}({{", ".join(attribute_variable.args)}}, &(this->{{attribute_variable.attribute_name}}), &(this->{{attribute_variable.attribute_flag_name}})); - {% for side_effect in attribute_variable.side_effects %} - // TOOD: this side effects should be passed into the parse function - this->{{side_effect}} = this->{{attribute_variable.class_name}}.side_effect_{{side_effect}}; - this->{{side_effect}}_is_set = true; - {% endfor %} + {{attribute_variable.deserialize_xml_function}}(attribute, errors, {% if attribute_variable.uses_file_path %}base_dir, {% endif %}&(this->{{attribute_variable.attribute_name}}), &(this->{{attribute_variable.attribute_flag_name}}){% for side_effect in attribute_variable.deserialize_xml_side_effects %}, &(this->{{side_effect}}){% endfor %}); } {% endfor %} {% endfor %} @@ -70,7 +65,7 @@ vector {{cpp_class}}::as_xml() const { {% for attribute_variable in attribute_variables %} {% if attribute_variable.write_to_xml == true %} if (this->{{attribute_variable.attribute_flag_name}}) { - xml_node_contents.push_back({{attribute_variable.class_name}}_to_xml_attribute("{{attribute_variable.default_xml_field}}", &this->{{attribute_variable.attribute_name}})); + xml_node_contents.push_back({{attribute_variable.serialize_xml_function}}("{{attribute_variable.default_xml_field}}", &this->{{attribute_variable.attribute_name}})); } {% endif %} {% endfor %} @@ -99,9 +94,9 @@ waypoint::{{cpp_class}} {{cpp_class}}::as_protobuf() const { {% if attribute_variable.is_component == false %} if (this->{{attribute_variable.attribute_flag_name}}) { {% if not attribute_variable.is_proto_field_scalar %} - proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_allocated_{{attribute_variable.protobuf_field}}(to_proto_{{attribute_variable.class_name}}(this->{{attribute_variable.attribute_name}})); + proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_allocated_{{attribute_variable.protobuf_field}}({{attribute_variable.serialize_proto_function}}(this->{{attribute_variable.attribute_name}})); {% else %} - proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_{{attribute_variable.protobuf_field}}(to_proto_{{attribute_variable.class_name}}(this->{{attribute_variable.attribute_name}})); + proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_{{attribute_variable.protobuf_field}}({{attribute_variable.serialize_proto_function}}(this->{{attribute_variable.attribute_name}})); {% endif %} } {% endif %} @@ -119,7 +114,7 @@ void {{cpp_class}}::parse_protobuf(waypoint::{{cpp_class}} proto_{{cpp_class_hea {% else %} if (proto_{{cpp_class_header}}{{attribute_variable.proto_drilldown_calls}}.{{attribute_variable.protobuf_field}}() != 0) { {% endif %} - this->{{attribute_variable.attribute_name}} = from_proto_{{attribute_variable.class_name}}(proto_{{cpp_class_header}}{{attribute_variable.proto_drilldown_calls}}.{{attribute_variable.protobuf_field}}()); + this->{{attribute_variable.attribute_name}} = {{attribute_variable.deserialize_proto_function}}(proto_{{cpp_class_header}}{{attribute_variable.proto_drilldown_calls}}.{{attribute_variable.protobuf_field}}()); this->{{attribute_variable.attribute_flag_name}} = true; } {% endif %} diff --git a/xml_converter/generators/generate_cpp.py b/xml_converter/generators/generate_cpp.py index d4a473c1..c54a10a6 100644 --- a/xml_converter/generators/generate_cpp.py +++ b/xml_converter/generators/generate_cpp.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Set, List, Dict, Optional, Tuple, Final, TypedDict +from typing import Set, List, Dict, Optional, Tuple, TypedDict, Any from jinja2 import FileSystemLoader, Environment, Template from jinja_helpers import UnindentBlocks from util import lowercase, capitalize, normalize, Document, SchemaType @@ -7,9 +7,6 @@ from protobuf_types import is_proto_field_scalar, get_proto_field_cpp_type, get_proto_field_cpp_prototype -XML_ATTRIBUTE_PARSER_DEFAULT_ARGUMENTS: Final[List[str]] = ["attribute", "errors"] - - ################################################################################ # DocumentationTypeData # @@ -55,6 +52,22 @@ class AttributeVariable: class_name: str xml_fields: List[str] + # The function name and additional side effect pointers for xml serialization. + serialize_xml_function: str + # serialize_xml_side_effects: List[str] + + # The function name and additional side effect pointers for xml deserialization. + deserialize_xml_function: str + deserialize_xml_side_effects: List[str] + + # The function name and additional side effect pointers for proto serialization. + serialize_proto_function: str + # serialize_proto_side_effects: List[str] + + # The function name and additional side effect pointers for proto deserialization. + deserialize_proto_function: str + # deserialize_proto_side_effects: List[str] + # The name of the field in the protobuf that this attribute corresponds to. protobuf_field: str @@ -64,7 +77,6 @@ class AttributeVariable: # Protoc cares if a field is scalar or not in the case of "set" vs "set allocated". is_proto_field_scalar: bool - args: List[str] = field(default_factory=list) default_xml_field: str = "" side_effects: List[str] = field(default_factory=list) xml_bundled_components: List[str] = field(default_factory=list) @@ -176,6 +188,16 @@ def write_cpp_classes( )) +def build_custom_function_data(config: Dict[str, Any]) -> Tuple[str, List[str]]: + function = config["function"] + side_effects = [] + for side_effect in config["side_effects"]: + side_effects.append(attribute_name_from_markdown_data(side_effect)) + side_effects.append(attribute_name_from_markdown_data(side_effect) + "_is_set") + + return function, side_effects + + ############################################################################ # generate_cpp_variable_data # @@ -224,8 +246,6 @@ def generate_cpp_variable_data( write_to_xml: bool = True default_xml_field: str = "" - args: List[str] = XML_ATTRIBUTE_PARSER_DEFAULT_ARGUMENTS.copy() - if fieldval['type'] in documentation_type_data: cpp_type = documentation_type_data[fieldval['type']]["cpp_type"] class_name = documentation_type_data[fieldval['type']]["class_name"] @@ -257,13 +277,6 @@ def generate_cpp_variable_data( protobuf_field: str proto_drilldown_calls, mutable_proto_drilldown_calls, protobuf_field = split_field_into_drilldown(fieldval["protobuf_field"]) - if fieldval.get("uses_file_path", False): - args.append("base_dir") - - if "side_effects" in fieldval: - for side_effect in fieldval['side_effects']: - side_effects.append(attribute_name_from_markdown_data(side_effect)) - # Compound Values are unique in that the components have xml fields in addition to the compound variable if fieldval['type'] in ["CompoundValue", "CompoundCustomClass"]: for component in fieldval['components']: @@ -292,13 +305,67 @@ def generate_cpp_variable_data( attribute_flag_name=attribute_name + "_is_set", write_to_xml=write_to_xml, is_component=True, - args=args, + + serialize_xml_function=component_class_name + "_to_xml_attribute", + # serialize_xml_side_effects=[], + deserialize_xml_function="xml_attribute_to_" + component_class_name, + deserialize_xml_side_effects=[], + serialize_proto_function="to_proto_" + component_class_name, + # serialize_proto_side_effects=[], + deserialize_proto_function="from_proto_" + component_class_name, + # deserialize_proto_side_effects=[], ) attribute_variables.append(component_attribute_variable) # If there aren't any components to bundle, we don't want to render the attribute if fieldval['xml_bundled_components'] == []: write_to_xml = False + serialize_xml_function: str = class_name + "_to_xml_attribute" + serialize_xml_side_effects: List[str] = [] + deserialize_xml_function: str = "xml_attribute_to_" + class_name + deserialize_xml_side_effects: List[str] = [] + serialize_proto_function: str = "to_proto_" + class_name + serialize_proto_side_effects: List[str] = [] + deserialize_proto_function: str = "from_proto_" + class_name + deserialize_proto_side_effects: List[str] = [] + if "custom_functions" in fieldval: + # Overwrite defaults with xml/proto read/write functions + for custom_function in fieldval["custom_functions"]: + if custom_function == "read.xml": + deserialize_xml_function, deserialize_xml_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + elif custom_function == "read.proto": + serialize_xml_function, serialize_xml_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + elif custom_function == "write.xml": + deserialize_proto_function, deserialize_proto_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + elif custom_function == "write.proto": + serialize_proto_function, serialize_proto_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + # Overwrite with marker type specific functions + for custom_function in fieldval["custom_functions"]: + if custom_function == "read.xml." + doc_type: + deserialize_xml_function, deserialize_xml_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + elif custom_function == "read.proto." + doc_type: + serialize_xml_function, serialize_xml_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + elif custom_function == "write.xml." + doc_type: + deserialize_proto_function, deserialize_proto_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + elif custom_function == "write.proto." + doc_type: + serialize_proto_function, serialize_proto_side_effects = build_custom_function_data( + fieldval["custom_functions"][custom_function] + ) + attribute_variable = AttributeVariable( attribute_name=attribute_name, attribute_type=fieldval["type"], @@ -311,10 +378,21 @@ def generate_cpp_variable_data( is_proto_field_scalar=is_proto_field_scalar(doc_type, fieldval["protobuf_field"]), proto_drilldown_calls=proto_drilldown_calls, mutable_proto_drilldown_calls=mutable_proto_drilldown_calls, - args=args, + write_to_xml=write_to_xml, attribute_flag_name=attribute_name + "_is_set", side_effects=side_effects, + + serialize_xml_function=serialize_xml_function, + # serialize_xml_side_effects=serialize_xml_side_effects, + deserialize_xml_function=deserialize_xml_function, + deserialize_xml_side_effects=deserialize_xml_side_effects, + serialize_proto_function=serialize_proto_function, + # serialize_proto_side_effects=serialize_proto_side_effects, + deserialize_proto_function=deserialize_proto_function, + # deserialize_proto_side_effects=deserialize_proto_side_effects, + + uses_file_path=fieldval.get("uses_file_path", False) ) attribute_variables.append(attribute_variable) @@ -394,7 +472,15 @@ def write_attribute(output_directory: str, data: Dict[str, Document]) -> None: proto_drilldown_calls=proto_drilldown_calls, mutable_proto_drilldown_calls=mutable_proto_drilldown_calls, protobuf_cpp_type="", - is_proto_field_scalar=False + is_proto_field_scalar=False, + serialize_xml_function="", + deserialize_xml_function="", + serialize_proto_function="", + deserialize_proto_function="", + # serialize_xml_side_effects=[], + deserialize_xml_side_effects=[], + # serialize_proto_side_effects=[], + # deserialize_proto_side_effects=[], ) attribute_variables.append(attribute_variable) @@ -422,6 +508,14 @@ def write_attribute(output_directory: str, data: Dict[str, Document]) -> None: mutable_proto_drilldown_calls=mutable_proto_drilldown_calls, protobuf_cpp_type="", is_proto_field_scalar=False, + serialize_xml_function="", + deserialize_xml_function="", + serialize_proto_function="", + deserialize_proto_function="", + # serialize_xml_side_effects=[], + deserialize_xml_side_effects=[], + # serialize_proto_side_effects=[], + # deserialize_proto_side_effects=[], ) attribute_variables.append(attribute_variable) @@ -442,6 +536,14 @@ def write_attribute(output_directory: str, data: Dict[str, Document]) -> None: mutable_proto_drilldown_calls=mutable_proto_drilldown_calls, protobuf_cpp_type="", is_proto_field_scalar=False, + serialize_xml_function="", + deserialize_xml_function="", + serialize_proto_function="", + deserialize_proto_function="", + # serialize_xml_side_effects=[], + deserialize_xml_side_effects=[], + # serialize_proto_side_effects=[], + # deserialize_proto_side_effects=[], ) attribute_variables.append(attribute_variable) diff --git a/xml_converter/generators/main.py b/xml_converter/generators/main.py index c0dbb3b8..47352737 100644 --- a/xml_converter/generators/main.py +++ b/xml_converter/generators/main.py @@ -14,30 +14,41 @@ XML_ATTRIBUTE_REGEX: Final[str] = "^[A-Za-z]+$" PROTO_FIELD_REGEX: Final[str] = "^[a-z_.]+$" INTERNAL_VARIABLE_REGEX: Final[str] = "^[a-z_]+$" +ATTRIBUTE_NAME_REGEX: Final[str] = "^[A-Za-z ]+$" +CUSTOM_SERIALIZER_REGEX: Final[str] = r"^(?:read|write)(?:\.xml|\.proto)(?:\.icon|\.trail)?$" shared_field_properties: Dict[str, DefType] = { "type": string_t(), - "name": string_t(), + "name": string_t(pattern=ATTRIBUTE_NAME_REGEX), "applies_to": array_t(enum_t(["Icon", "Trail", "Category"])), "xml_fields": array_t(string_t(pattern=XML_ATTRIBUTE_REGEX)), "protobuf_field": string_t(pattern=PROTO_FIELD_REGEX), } +custom_serial_functions: Dict[str, DefType] = { + "custom_functions": pattern_dictionary_t({CUSTOM_SERIALIZER_REGEX: object_t({ + "function": string_t(), + "side_effects": array_t(string_t()), + })}), +} + schema = union_t({ - "Int32": union_partial_t(required=shared_field_properties), - "Fixed32": union_partial_t(required=shared_field_properties), - "Float32": union_partial_t(required=shared_field_properties), - "String": union_partial_t(required=shared_field_properties), - "Boolean": union_partial_t(required=shared_field_properties), + "Int32": union_partial_t(required=shared_field_properties, optional=custom_serial_functions), + "Fixed32": union_partial_t(required=shared_field_properties, optional=custom_serial_functions), + "Float32": union_partial_t(required=shared_field_properties, optional=custom_serial_functions), + "String": union_partial_t(required=shared_field_properties, optional=custom_serial_functions), + "Boolean": union_partial_t(required=shared_field_properties, optional=custom_serial_functions), "MultiflagValue": union_partial_t( required={**shared_field_properties, **{ "flags": pattern_dictionary_t({INTERNAL_VARIABLE_REGEX: array_t(string_t())}), }}, + optional=custom_serial_functions, ), "Enum": union_partial_t( required={**shared_field_properties, **{ "values": pattern_dictionary_t({INTERNAL_VARIABLE_REGEX: array_t(string_t())}) - }} + }}, + optional=custom_serial_functions, ), "CompoundValue": union_partial_t( required={**shared_field_properties, **{ @@ -49,7 +60,8 @@ "xml_fields": array_t(string_t(XML_ATTRIBUTE_REGEX)), "protobuf_field": string_t(PROTO_FIELD_REGEX), })), - }} + }}, + optional=custom_serial_functions, ), "CompoundCustomClass": union_partial_t( required={**shared_field_properties, **{ @@ -62,13 +74,14 @@ "xml_fields": array_t(string_t(XML_ATTRIBUTE_REGEX)), "protobuf_field": string_t(PROTO_FIELD_REGEX), })), - }} + }}, + optional=custom_serial_functions, ), "Custom": union_partial_t( required={**shared_field_properties, **{"class": string_t()}}, optional={ - "side_effects": array_t(string_t()), - "uses_file_path": boolean_t(), + **custom_serial_functions, + "uses_file_path": boolean_t(), # This will eventually be part of a struct that is passed into everything } ), }) @@ -157,7 +170,7 @@ def write_webdocs(self, output_directory: str) -> None: # Resets the content and metadata to empty for each loop for subpage in categories[page]: - content[subpage] = markdown.markdown(self.data[subpage].content) + content[subpage] = markdown.markdown(self.data[subpage].content, extensions=['extra', 'codehilite']) metadata[subpage] = self.data[subpage].metadata generated_doc, field_rows = self.generate_auto_docs(metadata, content) diff --git a/xml_converter/src/attribute/trail_data.cpp b/xml_converter/src/attribute/trail_data.cpp index 92a7de10..d671cf06 100644 --- a/xml_converter/src/attribute/trail_data.cpp +++ b/xml_converter/src/attribute/trail_data.cpp @@ -23,7 +23,9 @@ void xml_attribute_to_trail_data( vector* errors, string base_dir, TrailData* value, - bool* is_set) { + bool* is_set, + int* map_id_value, + bool* is_map_id_set) { TrailData trail_data; string trail_data_relative_path = get_attribute_value(input); if (base_dir == "") { @@ -50,9 +52,9 @@ void xml_attribute_to_trail_data( } char map_id_char[4]; - trail_data_file.read(map_id_char, 4); - trail_data.side_effect_map_id = *reinterpret_cast(map_id_char); + *map_id_value = *reinterpret_cast(map_id_char); + *is_map_id_set = true; while (trail_data_file.tellg() > 0) { char point_x[4]; diff --git a/xml_converter/src/attribute/trail_data.hpp b/xml_converter/src/attribute/trail_data.hpp index 5bc55a6f..9b4f3d37 100644 --- a/xml_converter/src/attribute/trail_data.hpp +++ b/xml_converter/src/attribute/trail_data.hpp @@ -12,7 +12,6 @@ class TrailData; class TrailData { public: - int side_effect_map_id; std::vector points_x; std::vector points_y; std::vector points_z; @@ -23,7 +22,9 @@ void xml_attribute_to_trail_data( std::vector* errors, std::string base_dir, TrailData* value, - bool* is_set); + bool* is_set, + int* map_id_value, + bool* is_map_id_set); std::string trail_data_to_xml_attribute(const std::string& attribute_name, const TrailData* value); diff --git a/xml_converter/src/trail_gen.cpp b/xml_converter/src/trail_gen.cpp index 5df6d70d..c35758fa 100644 --- a/xml_converter/src/trail_gen.cpp +++ b/xml_converter/src/trail_gen.cpp @@ -137,10 +137,7 @@ bool Trail::init_xml_attribute(rapidxml::xml_attribute<>* attribute, vectortexture), &(this->texture_is_set)); } else if (attributename == "traildata") { - xml_attribute_to_trail_data(attribute, errors, base_dir, &(this->trail_data), &(this->trail_data_is_set)); - // TOOD: this side effects should be passed into the parse function - this->map_id = this->trail_data.side_effect_map_id; - this->map_id_is_set = true; + xml_attribute_to_trail_data(attribute, errors, base_dir, &(this->trail_data), &(this->trail_data_is_set), &(this->map_id), &(this->map_id_is_set)); } else if (attributename == "trailscale") { xml_attribute_to_float(attribute, errors, &(this->trail_scale), &(this->trail_scale_is_set));