Initial Pass of Custom Functions

This adds a bit of interdependent stuff
* Custom Functions for XML Reading
* Custom side effect arguments for XML Reading
* Migration of everything (one thing) that currently uses side effect arguments

Future commits will finish fleshing out the other de/serializer customization.

xml_converter/doc/trail_data/trail_data.md

@@ -5,34 +5,38 @@ class: TrailData
 xml_fields: ["TrailData"]
 uses_file_path: true
 protobuf_field: trail_data
-side_effects: [Map ID]
 applies_to: [Trail]
+custom_functions:
+  read.xml:
+    function: xml_attribute_to_trail_data
+    side_effects: [Map ID]
 ---
-A binary coordinate array for the path of a trail.
 
-Notes
-=====
+The coordinates of each point along a trail path.
 
+In the TacO format this attribute is a string that points to a file which contains this information packed in binary. The binary format consists of
 
+- 4 bytes for an integer that represents the `.trl` format version number
+- 4 bytes for an integer that represents the Map ID which this trail is for
+- Any number of 12 byte sets, containing 3 32 bit floats
 
-TrailData
+Example `.trl` parser
+--------------------------------------------------------------------------------
 
+```python
+from typing import Tuple, List
+import struct
 
-GetFromXML(xml_node)
-WriteToXML(self) # not quite right.
+with open("./demo.trl", 'rb') as f:
+    version: int = struct.unpack("<i", f.read(4))[0]
+    map_id: int = struct.unpack("<i", f.read(4))[0]
 
+    points: List[Tuple[float, float, float]] = []
+    while point_bytes := f.read(12):
+        x, y, z = struct.unpack("<fff", point_bytes)
+        points.append((x, y, z))
 
-GetFromProtobuf(protobuf_node_object) # not quite right
-SetProtobuf() # not quite right
-
-
-
-the return value of the set should be the value that needs to be set.
-But also we can just pass in the number of variables to be "set" or "get"
-
-
-
-so really
-  GetFromProtobuf(arg1type* argument1, arg2type* argument2)
-so really 
-  SetProtobuf(arg1type* argument1, arg2type* argument2)
+print("Version Number:", version)
+print("Map ID:", map_id)
+print("Points:", points)
+```

xml_converter/generators/cpp_templates/class_template.cpp

@@ -40,12 +40,7 @@ bool {{cpp_class}}::init_xml_attribute(rapidxml::xml_attribute<>* attribute, vec
     {% for n, attribute_variable in enumerate(attribute_variables) %}
         {% for i, value in enumerate(attribute_variable.xml_fields) %}
             {{ "if" if i == n == 0 else "else if" }} (attributename == "{{value}}") {
-                xml_attribute_to_{{attribute_variable.class_name}}({{", ".join(attribute_variable.args)}}, &(this->{{attribute_variable.attribute_name}}), &(this->{{attribute_variable.attribute_flag_name}}));
-                {% for side_effect in attribute_variable.side_effects %}
-                    // TOOD: this side effects should be passed into the parse function
-                    this->{{side_effect}} = this->{{attribute_variable.class_name}}.side_effect_{{side_effect}};
-                    this->{{side_effect}}_is_set = true;
-                {% endfor %}
+                {{attribute_variable.deserialize_xml_function}}(attribute, errors, {% if attribute_variable.uses_file_path %}base_dir, {% endif %}&(this->{{attribute_variable.attribute_name}}), &(this->{{attribute_variable.attribute_flag_name}}){% for side_effect in attribute_variable.deserialize_xml_side_effects %}, &(this->{{side_effect}}){% endfor %});
             }
         {% endfor %}
     {% endfor %}
@@ -70,7 +65,7 @@ vector<string> {{cpp_class}}::as_xml() const {
     {% for attribute_variable in attribute_variables %}
         {% if attribute_variable.write_to_xml == true %}
             if (this->{{attribute_variable.attribute_flag_name}}) {
-                xml_node_contents.push_back({{attribute_variable.class_name}}_to_xml_attribute("{{attribute_variable.default_xml_field}}", &this->{{attribute_variable.attribute_name}}));
+                xml_node_contents.push_back({{attribute_variable.serialize_xml_function}}("{{attribute_variable.default_xml_field}}", &this->{{attribute_variable.attribute_name}}));
             }
         {% endif %}
     {% endfor %}
@@ -99,9 +94,9 @@ waypoint::{{cpp_class}} {{cpp_class}}::as_protobuf() const {
         {% if attribute_variable.is_component == false %}
             if (this->{{attribute_variable.attribute_flag_name}}) {
                 {% if not attribute_variable.is_proto_field_scalar %}
-                    proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_allocated_{{attribute_variable.protobuf_field}}(to_proto_{{attribute_variable.class_name}}(this->{{attribute_variable.attribute_name}}));
+                    proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_allocated_{{attribute_variable.protobuf_field}}({{attribute_variable.serialize_proto_function}}(this->{{attribute_variable.attribute_name}}));
                 {% else %}
-                    proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_{{attribute_variable.protobuf_field}}(to_proto_{{attribute_variable.class_name}}(this->{{attribute_variable.attribute_name}}));
+                    proto_{{cpp_class_header}}.{{attribute_variable.mutable_proto_drilldown_calls}}set_{{attribute_variable.protobuf_field}}({{attribute_variable.serialize_proto_function}}(this->{{attribute_variable.attribute_name}}));
                 {% endif %}
             }
         {% endif %}
@@ -119,7 +114,7 @@ void {{cpp_class}}::parse_protobuf(waypoint::{{cpp_class}} proto_{{cpp_class_hea
             {% else %}
                 if (proto_{{cpp_class_header}}{{attribute_variable.proto_drilldown_calls}}.{{attribute_variable.protobuf_field}}() != 0) {
             {% endif %}
-                this->{{attribute_variable.attribute_name}} = from_proto_{{attribute_variable.class_name}}(proto_{{cpp_class_header}}{{attribute_variable.proto_drilldown_calls}}.{{attribute_variable.protobuf_field}}());
+                this->{{attribute_variable.attribute_name}} = {{attribute_variable.deserialize_proto_function}}(proto_{{cpp_class_header}}{{attribute_variable.proto_drilldown_calls}}.{{attribute_variable.protobuf_field}}());
                 this->{{attribute_variable.attribute_flag_name}} = true;
             }
         {% endif %}

xml_converter/generators/generate_cpp.py

@@ -1,15 +1,12 @@
 from dataclasses import dataclass, field
-from typing import Set, List, Dict, Optional, Tuple, Final, TypedDict
+from typing import Set, List, Dict, Optional, Tuple, TypedDict, Any
 from jinja2 import FileSystemLoader, Environment, Template
 from jinja_helpers import UnindentBlocks
 from util import lowercase, capitalize, normalize, Document, SchemaType
 import os
 from protobuf_types import is_proto_field_scalar, get_proto_field_cpp_type, get_proto_field_cpp_prototype
 
 
-XML_ATTRIBUTE_PARSER_DEFAULT_ARGUMENTS: Final[List[str]] = ["attribute", "errors"]
-
-
 ################################################################################
 # DocumentationTypeData
 #
@@ -55,6 +52,22 @@ class AttributeVariable:
     class_name: str
     xml_fields: List[str]
 
+    # The function name and additional side effect pointers for xml serialization.
+    serialize_xml_function: str
+    # serialize_xml_side_effects: List[str]
+
+    # The function name and additional side effect pointers for xml deserialization.
+    deserialize_xml_function: str
+    deserialize_xml_side_effects: List[str]
+
+    # The function name and additional side effect pointers for proto serialization.
+    serialize_proto_function: str
+    # serialize_proto_side_effects: List[str]
+
+    # The function name and additional side effect pointers for proto deserialization.
+    deserialize_proto_function: str
+    # deserialize_proto_side_effects: List[str]
+
     # The name of the field in the protobuf that this attribute corresponds to.
     protobuf_field: str
 
@@ -64,7 +77,6 @@ class AttributeVariable:
     # Protoc cares if a field is scalar or not in the case of "set" vs "set allocated".
     is_proto_field_scalar: bool
 
-    args: List[str] = field(default_factory=list)
     default_xml_field: str = ""
     side_effects: List[str] = field(default_factory=list)
     xml_bundled_components: List[str] = field(default_factory=list)
@@ -176,6 +188,16 @@ def write_cpp_classes(
             ))
 
 
+def build_custom_function_data(config: Dict[str, Any]) -> Tuple[str, List[str]]:
+    function = config["function"]
+    side_effects = []
+    for side_effect in config["side_effects"]:
+        side_effects.append(attribute_name_from_markdown_data(side_effect))
+        side_effects.append(attribute_name_from_markdown_data(side_effect) + "_is_set")
+
+    return function, side_effects
+
+
 ############################################################################
 # generate_cpp_variable_data
 #
@@ -224,8 +246,6 @@ def generate_cpp_variable_data(
             write_to_xml: bool = True
             default_xml_field: str = ""
 
-            args: List[str] = XML_ATTRIBUTE_PARSER_DEFAULT_ARGUMENTS.copy()
-
             if fieldval['type'] in documentation_type_data:
                 cpp_type = documentation_type_data[fieldval['type']]["cpp_type"]
                 class_name = documentation_type_data[fieldval['type']]["class_name"]
@@ -257,13 +277,6 @@ def generate_cpp_variable_data(
             protobuf_field: str
             proto_drilldown_calls, mutable_proto_drilldown_calls, protobuf_field = split_field_into_drilldown(fieldval["protobuf_field"])
 
-            if fieldval.get("uses_file_path", False):
-                args.append("base_dir")
-
-            if "side_effects" in fieldval:
-                for side_effect in fieldval['side_effects']:
-                    side_effects.append(attribute_name_from_markdown_data(side_effect))
-
             # Compound Values are unique in that the components have xml fields in addition to the compound variable
             if fieldval['type'] in ["CompoundValue", "CompoundCustomClass"]:
                 for component in fieldval['components']:
@@ -292,13 +305,67 @@ def generate_cpp_variable_data(
                         attribute_flag_name=attribute_name + "_is_set",
                         write_to_xml=write_to_xml,
                         is_component=True,
-                        args=args,
+
+                        serialize_xml_function=component_class_name + "_to_xml_attribute",
+                        # serialize_xml_side_effects=[],
+                        deserialize_xml_function="xml_attribute_to_" + component_class_name,
+                        deserialize_xml_side_effects=[],
+                        serialize_proto_function="to_proto_" + component_class_name,
+                        # serialize_proto_side_effects=[],
+                        deserialize_proto_function="from_proto_" + component_class_name,
+                        # deserialize_proto_side_effects=[],
                     )
                     attribute_variables.append(component_attribute_variable)
                 # If there aren't any components to bundle, we don't want to render the attribute
                 if fieldval['xml_bundled_components'] == []:
                     write_to_xml = False
 
+            serialize_xml_function: str = class_name + "_to_xml_attribute"
+            serialize_xml_side_effects: List[str] = []
+            deserialize_xml_function: str = "xml_attribute_to_" + class_name
+            deserialize_xml_side_effects: List[str] = []
+            serialize_proto_function: str = "to_proto_" + class_name
+            serialize_proto_side_effects: List[str] = []
+            deserialize_proto_function: str = "from_proto_" + class_name
+            deserialize_proto_side_effects: List[str] = []
+            if "custom_functions" in fieldval:
+                # Overwrite defaults with xml/proto read/write functions
+                for custom_function in fieldval["custom_functions"]:
+                    if custom_function == "read.xml":
+                        deserialize_xml_function, deserialize_xml_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+                    elif custom_function == "read.proto":
+                        serialize_xml_function, serialize_xml_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+                    elif custom_function == "write.xml":
+                        deserialize_proto_function, deserialize_proto_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+                    elif custom_function == "write.proto":
+                        serialize_proto_function, serialize_proto_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+                # Overwrite with marker type specific functions
+                for custom_function in fieldval["custom_functions"]:
+                    if custom_function == "read.xml." + doc_type:
+                        deserialize_xml_function, deserialize_xml_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+                    elif custom_function == "read.proto." + doc_type:
+                        serialize_xml_function, serialize_xml_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+                    elif custom_function == "write.xml." + doc_type:
+                        deserialize_proto_function, deserialize_proto_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+                    elif custom_function == "write.proto." + doc_type:
+                        serialize_proto_function, serialize_proto_side_effects = build_custom_function_data(
+                            fieldval["custom_functions"][custom_function]
+                        )
+
             attribute_variable = AttributeVariable(
                 attribute_name=attribute_name,
                 attribute_type=fieldval["type"],
@@ -311,10 +378,21 @@ def generate_cpp_variable_data(
                 is_proto_field_scalar=is_proto_field_scalar(doc_type, fieldval["protobuf_field"]),
                 proto_drilldown_calls=proto_drilldown_calls,
                 mutable_proto_drilldown_calls=mutable_proto_drilldown_calls,
-                args=args,
+
                 write_to_xml=write_to_xml,
                 attribute_flag_name=attribute_name + "_is_set",
                 side_effects=side_effects,
+
+                serialize_xml_function=serialize_xml_function,
+                # serialize_xml_side_effects=serialize_xml_side_effects,
+                deserialize_xml_function=deserialize_xml_function,
+                deserialize_xml_side_effects=deserialize_xml_side_effects,
+                serialize_proto_function=serialize_proto_function,
+                # serialize_proto_side_effects=serialize_proto_side_effects,
+                deserialize_proto_function=deserialize_proto_function,
+                # deserialize_proto_side_effects=deserialize_proto_side_effects,
+
+                uses_file_path=fieldval.get("uses_file_path", False)
             )
             attribute_variables.append(attribute_variable)
 
@@ -394,7 +472,15 @@ def write_attribute(output_directory: str, data: Dict[str, Document]) -> None:
                     proto_drilldown_calls=proto_drilldown_calls,
                     mutable_proto_drilldown_calls=mutable_proto_drilldown_calls,
                     protobuf_cpp_type="",
-                    is_proto_field_scalar=False
+                    is_proto_field_scalar=False,
+                    serialize_xml_function="",
+                    deserialize_xml_function="",
+                    serialize_proto_function="",
+                    deserialize_proto_function="",
+                    # serialize_xml_side_effects=[],
+                    deserialize_xml_side_effects=[],
+                    # serialize_proto_side_effects=[],
+                    # deserialize_proto_side_effects=[],
                 )
                 attribute_variables.append(attribute_variable)
 
@@ -422,6 +508,14 @@ def write_attribute(output_directory: str, data: Dict[str, Document]) -> None:
                     mutable_proto_drilldown_calls=mutable_proto_drilldown_calls,
                     protobuf_cpp_type="",
                     is_proto_field_scalar=False,
+                    serialize_xml_function="",
+                    deserialize_xml_function="",
+                    serialize_proto_function="",
+                    deserialize_proto_function="",
+                    # serialize_xml_side_effects=[],
+                    deserialize_xml_side_effects=[],
+                    # serialize_proto_side_effects=[],
+                    # deserialize_proto_side_effects=[],
                 )
                 attribute_variables.append(attribute_variable)
 
@@ -442,6 +536,14 @@ def write_attribute(output_directory: str, data: Dict[str, Document]) -> None:
                     mutable_proto_drilldown_calls=mutable_proto_drilldown_calls,
                     protobuf_cpp_type="",
                     is_proto_field_scalar=False,
+                    serialize_xml_function="",
+                    deserialize_xml_function="",
+                    serialize_proto_function="",
+                    deserialize_proto_function="",
+                    # serialize_xml_side_effects=[],
+                    deserialize_xml_side_effects=[],
+                    # serialize_proto_side_effects=[],
+                    # deserialize_proto_side_effects=[],
                 )
                 attribute_variables.append(attribute_variable)