diff --git a/xml_converter/src/packaging_protobin.cpp b/xml_converter/src/packaging_protobin.cpp index f58737af..bd3a38d6 100644 --- a/xml_converter/src/packaging_protobin.cpp +++ b/xml_converter/src/packaging_protobin.cpp @@ -10,6 +10,9 @@ #include "parseable.hpp" #include "string_helper.hpp" #include "waypoint.pb.h" +#include "string_hierarchy.hpp" + +#include // For TextProtos using namespace std; @@ -70,132 +73,157 @@ void read_protobuf_file(string proto_filepath, map* marker_cat //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// -// Adds the name of a category and all of it's parents to a set -// eg. -// { -// "mypath", -// "mypath.easypath", -// "mypath.easypath.trail", -// "mypath.hardpath", -// "mypath.hardpath.trail", -// } -//////////////////////////////////////////////////////////////////////////////// -void populate_categories_to_retain(string category_name, set* categories_to_retain) { - string name; - vector split_categories = split(category_name, "."); - for (unsigned int j = 0; j < split_categories.size(); j++) { - name += split_categories[j]; - categories_to_retain->insert(name); - name += "."; - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Iterates through all children of a Category and removes the categories and -// POIs that do not belong on a particular map +// build_category_object +// +// Builds all the hierarchical category object info as long as the category +// has contents that exist in the category filter. //////////////////////////////////////////////////////////////////////////////// -void remove_waypoint_elements(waypoint::Category* proto_category, set categories_to_retain, string parent_name, int map_id) { - int keep = 0; - for (int i = 0; i < proto_category->icon_size(); i++) { - if (proto_category->icon(i).map_id() == map_id) { - if (keep < i) { - proto_category->mutable_icon()->SwapElements(i, keep); +struct MaybeCategory { + waypoint::Category category; + bool is_category; +}; +MaybeCategory build_category_objects( + const Category* category, + const StringHierarchy &category_filter, + const std::map> &category_to_pois, + vector &category_vector +) { + waypoint::Category category_proto = category->as_protobuf(); + bool has_valid_contents = false; + + vector categories_to_write; + + for (map::const_iterator it = category->children.begin(); it != category->children.end(); it++) { + + // This is currently a copy operation which is kind expensive + category_vector.push_back(it->first); + + if (category_filter.in_hierarchy(category_vector)) { + MaybeCategory child_category = build_category_objects( + &it->second, + category_filter, + category_to_pois, + category_vector + ); + + if (child_category.is_category) { + has_valid_contents = true; + category_proto.add_children()->MergeFrom(child_category.category); } - ++keep; } + category_vector.pop_back(); } - proto_category->mutable_icon()->DeleteSubrange(keep, proto_category->icon_size() - keep); - keep = 0; - for (int i = 0; i < proto_category->trail_size(); i++) { - if (proto_category->trail(i).map_id() == map_id) { - if (keep < i) { - proto_category->mutable_trail()->SwapElements(i, keep); + // This is a pretty expensive operation + string full_category_name = join(category_vector, "."); + auto iterator = category_to_pois.find(full_category_name); + if (iterator != category_to_pois.end()) { + for (size_t i = 0; i < iterator->second.size(); i++) { + Parseable* parsed_poi = iterator->second[i]; + + if (parsed_poi->classname() == "POI") { + Icon* icon = dynamic_cast(parsed_poi); + if (category_filter.in_hierarchy(split(icon->category.category,"."))) { + category_proto.add_icon()->MergeFrom(icon->as_protobuf()); + has_valid_contents = true; + } + } + else if (parsed_poi->classname() == "Trail") { + Trail* trail = dynamic_cast(parsed_poi); + if (category_filter.in_hierarchy(split(trail->category.category,"."))) { + category_proto.add_trail()->MergeFrom(trail->as_protobuf()); + has_valid_contents = true; + } } - ++keep; + else { + std::cout << "Unknown type" << std::endl; + } + } } - proto_category->mutable_trail()->DeleteSubrange(keep, proto_category->trail_size() - keep); - - keep = 0; - for (int i = 0; i < proto_category->children_size(); i++) { - string name = parent_name + "." + proto_category->children(i).name(); - auto pos = categories_to_retain.find(name); - if (pos != categories_to_retain.end()) { - remove_waypoint_elements(proto_category->mutable_children(i), categories_to_retain, name, map_id); - if (keep < i) { - proto_category->mutable_children()->SwapElements(i, keep); - } - ++keep; + + MaybeCategory return_value; + return_value.category = category_proto; + return_value.is_category = has_valid_contents; + return return_value; +} + +void write_protobuf_file( + const string &filepath, + const StringHierarchy &category_filter, + const map* marker_categories, + const vector* parsed_pois +) { + // TODO: call _write_protobuf_file +} + +void _write_protobuf_file( + const string &filepath, + const StringHierarchy &category_filter, + const map* marker_categories, + const std::map> &category_to_pois +){ + ofstream outfile; + outfile.open(filepath, ios::out | ios::binary); + + waypoint::Waypoint output_message; + + for (map::const_iterator it = marker_categories->begin(); it != marker_categories->end(); it++) { + string category_name = it->first; + const Category* category_object = &it->second; + + vector category_vector = {it->first}; + MaybeCategory maybe_category = build_category_objects( + category_object, + category_filter, + category_to_pois, + category_vector + ); + + if (maybe_category.is_category) { + output_message.add_category()->MergeFrom(maybe_category.category); } } - proto_category->mutable_children()->DeleteSubrange(keep, proto_category->children_size() - keep); + + output_message.SerializeToOstream(&outfile); + outfile.close(); } -void write_protobuf_file(string proto_directory, map* marker_categories, vector* parsed_pois) { - waypoint::Waypoint proto_pois; - // Collects a set of map ids from Icon and Trail data - std::set map_ids; - ofstream trail_data_file; - map> map_of_pois; - for (const auto& parsed_poi : *parsed_pois) { +// Write protobuf per map id +void write_protobuf_file_per_map_id( + const string &proto_directory, + const StringHierarchy &category_filter, + const map* marker_categories, + const vector* parsed_pois +) { + + std::map>> mapid_to_category_to_pois; + + for (size_t i = 0; i < parsed_pois->size(); i++) { + Parseable* parsed_poi = (*parsed_pois)[i]; if (parsed_poi->classname() == "POI") { Icon* icon = dynamic_cast(parsed_poi); - map_ids.insert(icon->map_id); - map_of_pois[icon->category.category].push_back(icon); + mapid_to_category_to_pois[icon->map_id][icon->category.category].push_back(parsed_poi); } else if (parsed_poi->classname() == "Trail") { Trail* trail = dynamic_cast(parsed_poi); - map_ids.insert(trail->map_id); - map_of_pois[trail->category.category].push_back(trail); + mapid_to_category_to_pois[trail->map_id][trail->category.category].push_back(parsed_poi); + } + else { + std::cout << "Unknown type" << std::endl; } } - // Creates a Waypoint message that contains all categories - waypoint::Waypoint all_categories; - for (const auto& category : *marker_categories) { - waypoint::Category proto_category = category.second.as_protobuf("", &map_of_pois); - all_categories.add_category()->CopyFrom(proto_category); - } + for (auto iterator = mapid_to_category_to_pois.begin(); iterator != mapid_to_category_to_pois.end(); iterator++) { - waypoint::Waypoint output_message; - std::set categories_to_retain; - for (int map_id : map_ids) { - ofstream outfile; - string output_filepath = proto_directory + "/" + to_string(map_id) + ".data"; - outfile.open(output_filepath, ios::out | ios::binary); - output_message.MergeFrom(all_categories); - - for (const auto& parsed_poi : *parsed_pois) { - if (parsed_poi->classname() == "POI") { - Icon* icon = dynamic_cast(parsed_poi); - if (icon->map_id == map_id) { - populate_categories_to_retain(icon->category.category, &categories_to_retain); - } - } - else if (parsed_poi->classname() == "Trail") { - Trail* trail = dynamic_cast(parsed_poi); - if (trail->map_id == map_id) { - populate_categories_to_retain(trail->category.category, &categories_to_retain); - } - } - } - // In the XML, MarkerCategories have a tree hierarchy while POIS have a - // flat hierarchy. In Waypoint, POIs are elements of the category they - // belong to. We are removing instead of inserting because - // each parent category contains the data for all of its children. It - // would be impractical to include all of the data from each category - // except for the children and then iterating over all the children. - // This pruning method is slower but ensures that information is kept. - for (int i = 0; i < output_message.category_size(); i++) { - remove_waypoint_elements(output_message.mutable_category(i), categories_to_retain, output_message.category(i).name(), map_id); - if (output_message.mutable_category(i)->children_size() == 0) { - output_message.mutable_category(i)->Clear(); - } - } - output_message.SerializeToOstream(&outfile); - outfile.close(); - output_message.Clear(); - categories_to_retain.clear(); + string output_filepath = proto_directory + "/" + to_string(iterator->first) + ".data"; + + _write_protobuf_file( + output_filepath, + category_filter, + marker_categories, + iterator->second + ); } } + diff --git a/xml_converter/src/packaging_protobin.hpp b/xml_converter/src/packaging_protobin.hpp index b50fdcef..c537c8bd 100644 --- a/xml_converter/src/packaging_protobin.hpp +++ b/xml_converter/src/packaging_protobin.hpp @@ -8,6 +8,7 @@ #include "category_gen.hpp" #include "parseable.hpp" #include "waypoint.pb.h" +#include "string_hierarchy.hpp" void read_protobuf_file( std::string proto_filepath, @@ -15,6 +16,13 @@ void read_protobuf_file( std::vector* parsed_pois); void write_protobuf_file( - std::string proto_directory, - std::map* marker_categories, - std::vector* parsed_pois); + const std::string &proto_directory, + const StringHierarchy &category_filter, + const std::map* marker_categories, + const std::vector* parsed_pois); + +void write_protobuf_file_per_map_id( + const std::string &proto_directory, + const StringHierarchy &category_filter, + const std::map* marker_categories, + const std::vector* parsed_pois); diff --git a/xml_converter/src/string_helper.cpp b/xml_converter/src/string_helper.cpp index ccb358fa..a236f6fc 100644 --- a/xml_converter/src/string_helper.cpp +++ b/xml_converter/src/string_helper.cpp @@ -48,6 +48,18 @@ vector split(string input, string delimiter) { return output; } +string join(const vector &input, string delimiter) { + string result; + for(size_t i = 0; i < input.size(); i++) { + result += input[i]; + // Don't add delimiter after the last element + if(i < input.size() - 1) { + result += delimiter; + } + } + return result; +} + //////////////////////////////////////////////////////////////////////////////// // normalize // diff --git a/xml_converter/src/string_helper.hpp b/xml_converter/src/string_helper.hpp index 96333bda..68aa5e1c 100644 --- a/xml_converter/src/string_helper.hpp +++ b/xml_converter/src/string_helper.hpp @@ -12,6 +12,7 @@ bool normalized_matches_any(std::string test, std::vector list); std::string lowercase(std::string); std::vector split(std::string input, std::string delimiter); +std::string join(const std::vector &input, std::string delimiter); std::string normalize(std::string input_string); diff --git a/xml_converter/src/string_hierarchy.cpp b/xml_converter/src/string_hierarchy.cpp index a98870e1..0c907c39 100644 --- a/xml_converter/src/string_hierarchy.cpp +++ b/xml_converter/src/string_hierarchy.cpp @@ -8,7 +8,7 @@ // Returns if the given path is in the hierarchy or not //////////////////////////////////////////////////////////////////////////////// bool StringHierarchy::in_hierarchy( - const std::vector &path) { + const std::vector &path) const { return this->_in_hierarchy(path, 0); } @@ -20,7 +20,7 @@ bool StringHierarchy::in_hierarchy( //////////////////////////////////////////////////////////////////////////////// bool StringHierarchy::_in_hierarchy( const std::vector &path, - const size_t continue_index) { + const size_t continue_index) const { // If all children of this hierarchy node are included then this path exists. if (this->all_children_included) { return true; diff --git a/xml_converter/src/string_hierarchy.hpp b/xml_converter/src/string_hierarchy.hpp index 7047b4b2..9a5587db 100644 --- a/xml_converter/src/string_hierarchy.hpp +++ b/xml_converter/src/string_hierarchy.hpp @@ -7,7 +7,7 @@ class StringHierarchy { public: bool in_hierarchy( - const std::vector &path); + const std::vector &path) const; void add_path( const std::vector &path, @@ -16,7 +16,7 @@ class StringHierarchy { private: bool _in_hierarchy( const std::vector &path, - const size_t continue_index); + const size_t continue_index) const; void _add_path( const std::vector &path, diff --git a/xml_converter/src/xml_converter.cpp b/xml_converter/src/xml_converter.cpp index ea2f9107..cb39da84 100644 --- a/xml_converter/src/xml_converter.cpp +++ b/xml_converter/src/xml_converter.cpp @@ -165,13 +165,17 @@ void process_data( // Write all of the protobin waypoint paths for (size_t i = 0; i < output_waypoint_paths.size(); i++) { - write_protobuf_file(output_waypoint_paths[i], &marker_categories, &parsed_pois); + StringHierarchy category_filter; + category_filter.add_path({}, true); + write_protobuf_file(output_waypoint_paths[i], category_filter, &marker_categories, &parsed_pois); } // Write the special map-split protbin waypoint file begin = chrono::high_resolution_clock::now(); if (output_split_waypoint_dir != "") { - write_protobuf_file(output_split_waypoint_dir, &marker_categories, &parsed_pois); + StringHierarchy category_filter; + category_filter.add_path({}, true); + write_protobuf_file_per_map_id(output_split_waypoint_dir, category_filter, &marker_categories, &parsed_pois); } end = chrono::high_resolution_clock::now(); dur = end - begin;