diff --git a/tests/feature-filter/out/filtered.json.standard b/tests/feature-filter/out/filtered.json.standard index 73fb672d5..c391431c9 100644 --- a/tests/feature-filter/out/filtered.json.standard +++ b/tests/feature-filter/out/filtered.json.standard @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129", -"bounds": "-180.000000,-85.051129,180.000000,85.051129", +"antimeridian_adjusted_bounds": "-100.019531,0.000000,0.966797,0.966751", +"bounds": "-100.019531,0.000000,0.966797,0.966751", "center": "0.000000,0.000000,0", "description": "tests/feature-filter/out/all.mbtiles", "format": "pbf", diff --git a/tests/feature-filter/out/places-filter.mbtiles.json.standard b/tests/feature-filter/out/places-filter.mbtiles.json.standard index 9936c0975..ab8b5a1f5 100644 --- a/tests/feature-filter/out/places-filter.mbtiles.json.standard +++ b/tests/feature-filter/out/places-filter.mbtiles.json.standard @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-175.781250,-42.032974,180.000000,64.168107", -"bounds": "-175.781250,-42.032974,180.000000,64.168107", +"antimeridian_adjusted_bounds": "-175.220947,-41.302571,179.217224,64.168107", +"bounds": "-175.220947,-41.302571,179.217224,64.168107", "center": "-62.578125,17.307462,8", "description": "tests/feature-filter/out/places.mbtiles", "format": "pbf", diff --git a/tests/join-population/concat.mbtiles.json b/tests/join-population/concat.mbtiles.json index 28a2a4025..156596d7a 100644 --- a/tests/join-population/concat.mbtiles.json +++ b/tests/join-population/concat.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129", -"bounds": "-180.000000,-85.051129,180.000000,85.051129", +"antimeridian_adjusted_bounds": "-122.255859,37.718590,-122.080078,37.857507", +"bounds": "-122.255859,37.718590,-122.080078,37.857507", "center": "-122.104097,37.695438,0", "description": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", "format": "pbf", diff --git a/tests/join-population/joined-i.mbtiles.json b/tests/join-population/joined-i.mbtiles.json index 35a15fd7c..6d8dd7636 100644 --- a/tests/join-population/joined-i.mbtiles.json +++ b/tests/join-population/joined-i.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.309418,37.881357,-122.280579,37.900865", +"bounds": "-122.309418,37.881357,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-no-tile-stats.mbtiles.json b/tests/join-population/joined-no-tile-stats.mbtiles.json index 6f7cde119..4e5b7ec0a 100644 --- a/tests/join-population/joined-no-tile-stats.mbtiles.json +++ b/tests/join-population/joined-no-tile-stats.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-null.mbtiles.json b/tests/join-population/joined-null.mbtiles.json index 8339f1946..c8a716765 100644 --- a/tests/join-population/joined-null.mbtiles.json +++ b/tests/join-population/joined-null.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json b/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json index 37b5326a5..c6f6d52c9 100644 --- a/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json +++ b/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json b/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json index 4b217de0f..b3ce37876 100644 --- a/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json +++ b/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-tile-stats-values-limit.mbtiles.json b/tests/join-population/joined-tile-stats-values-limit.mbtiles.json index 9fa160253..7999ec3cc 100644 --- a/tests/join-population/joined-tile-stats-values-limit.mbtiles.json +++ b/tests/join-population/joined-tile-stats-values-limit.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined.mbtiles.json b/tests/join-population/joined.mbtiles.json index 16ad55eea..4d4166184 100644 --- a/tests/join-population/joined.mbtiles.json +++ b/tests/join-population/joined.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/just-macarthur.mbtiles.json b/tests/join-population/just-macarthur.mbtiles.json index 97b1b7e1e..c1a48b330 100644 --- a/tests/join-population/just-macarthur.mbtiles.json +++ b/tests/join-population/just-macarthur.mbtiles.json @@ -1,7 +1,7 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", +"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649", "attribution": "macarthur's attribution", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"bounds": "-122.294655,37.694688,-122.103424,37.833649", "center": "-122.299805,37.892187,11", "description": "macarthur description", "format": "pbf", diff --git a/tests/join-population/macarthur-6-9-exclude.mbtiles.json b/tests/join-population/macarthur-6-9-exclude.mbtiles.json index 0e191929c..f079ea240 100644 --- a/tests/join-population/macarthur-6-9-exclude.mbtiles.json +++ b/tests/join-population/macarthur-6-9-exclude.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.695231,-122.103424,37.833107", +"bounds": "-122.294655,37.695231,-122.103424,37.833107", "center": "-122.167969,37.833010,9", "description": "tests/join-population/macarthur.mbtiles", "format": "pbf", diff --git a/tests/join-population/macarthur-6-9.mbtiles.json b/tests/join-population/macarthur-6-9.mbtiles.json index 3f79712d5..0452f638e 100644 --- a/tests/join-population/macarthur-6-9.mbtiles.json +++ b/tests/join-population/macarthur-6-9.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.695231,-122.103424,37.833107", +"bounds": "-122.294655,37.695231,-122.103424,37.833107", "center": "-122.167969,37.833010,9", "description": "tests/join-population/macarthur.mbtiles", "format": "pbf", diff --git a/tests/join-population/merged-folder.mbtiles.json b/tests/join-population/merged-folder.mbtiles.json index 86337ba1b..3e5f1cd4f 100644 --- a/tests/join-population/merged-folder.mbtiles.json +++ b/tests/join-population/merged-folder.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865", +"bounds": "-122.343750,37.694688,-122.103424,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420-folder", "format": "pbf", diff --git a/tests/join-population/merged.mbtiles.json b/tests/join-population/merged.mbtiles.json index e21069965..17d426098 100644 --- a/tests/join-population/merged.mbtiles.json +++ b/tests/join-population/merged.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865", +"bounds": "-122.343750,37.694688,-122.103424,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/no-macarthur.mbtiles.json b/tests/join-population/no-macarthur.mbtiles.json index 80a22b9e1..5d33240ae 100644 --- a/tests/join-population/no-macarthur.mbtiles.json +++ b/tests/join-population/no-macarthur.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/raw-merged-folder.json b/tests/join-population/raw-merged-folder.json index 2c7607ced..a61b2cee9 100644 --- a/tests/join-population/raw-merged-folder.json +++ b/tests/join-population/raw-merged-folder.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865", +"bounds": "-122.343750,37.694688,-122.103424,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/renamed.mbtiles.json b/tests/join-population/renamed.mbtiles.json index dc74bcfb4..58bcc1530 100644 --- a/tests/join-population/renamed.mbtiles.json +++ b/tests/join-population/renamed.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649", +"bounds": "-122.294655,37.694688,-122.103424,37.833649", "center": "-122.167969,37.828608,10", "description": "tests/join-population/macarthur2.mbtiles", "format": "pbf", diff --git a/tests/join-population/windows.mbtiles.json b/tests/join-population/windows.mbtiles.json index ef2dcc14f..570b17109 100644 --- a/tests/join-population/windows.mbtiles.json +++ b/tests/join-population/windows.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649", +"bounds": "-122.294655,37.694688,-122.103424,37.833649", "center": "-122.167969,37.833010,10", "description": "tests/join-population/macarthur.mbtiles", "format": "pbf", diff --git a/tests/raw-tiles/raw-tiles-z67-join.json b/tests/raw-tiles/raw-tiles-z67-join.json index d82e92ee6..8582128ca 100644 --- a/tests/raw-tiles/raw-tiles-z67-join.json +++ b/tests/raw-tiles/raw-tiles-z67-join.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-123.750000,45.089036,-120.937500,47.040182", -"bounds": "-123.750000,45.089036,-120.937500,47.040182", +"antimeridian_adjusted_bounds": "-122.682953,45.512121,-122.654800,45.569832", +"bounds": "-122.682953,45.512121,-122.654800,45.569832", "center": "-122.662354,45.514045,7", "description": "tests/raw-tiles/raw-tiles", "format": "pbf", diff --git a/tile-join.cpp b/tile-join.cpp index ee8889595..8304ec0f7 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -55,7 +55,12 @@ int maxzoom = 32; int minzoom = 0; std::map renames; bool exclude_all = false; +bool exclude_all_tile_attributes = false; std::vector unidecode_data; +std::string join_tile_column; +std::string join_table_column; +std::string join_table; +std::string attribute_for_id; bool want_overzoom = false; int buffer = 5; @@ -73,7 +78,106 @@ struct stats { std::vector strategies{}; }; -void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter) { +std::vector> get_joined_rows(sqlite3 *db, const std::vector &join_keys) { + std::vector> ret; + ret.resize(join_keys.size()); + + // double quotes for table and column identifiers + const char *s = sqlite3_mprintf("select \"%w\", * from \"%w\" where \"%w\" in (", + join_table_column.c_str(), join_table.c_str(), join_table_column.c_str()); + std::string query = s; + sqlite3_free((void *) s); + + std::map key_to_row; + for (size_t i = 0; i < join_keys.size(); i++) { + const mvt_value &v = join_keys[i]; + + // single quotes for literals + if (v.type == mvt_string) { + s = sqlite3_mprintf("'%q'", v.c_str()); + query += s; + sqlite3_free((void *) s); + key_to_row.emplace(v.get_string_value(), i); + } else { + std::string stringified = v.toString(); + key_to_row.emplace(stringified, i); + query += stringified; + } + + if (i + 1 < join_keys.size()) { + query += ", "; + } + } + + query += ");"; + + sqlite3_stmt *stmt; + if (sqlite3_prepare_v2(db, query.c_str(), -1, &stmt, NULL) != SQLITE_OK) { + fprintf(stderr, "sqlite3 query %s failed: %s\n", query.c_str(), sqlite3_errmsg(db)); + exit(EXIT_SQLITE); + } + while (sqlite3_step(stmt) == SQLITE_ROW) { + int count = sqlite3_column_count(stmt); + std::map row; + + if (count > 0) { + // join key is 0th column of query + std::string key = (const char *) sqlite3_column_text(stmt, 0); + auto f = key_to_row.find(key); + if (f == key_to_row.end()) { + fprintf(stderr, "Unexpected join key: %s\n", key.c_str()); + continue; + } + + for (int i = 1; i < count; i++) { + int type = sqlite3_column_type(stmt, i); + mvt_value v; + v.type = mvt_null; + + if (type == SQLITE_INTEGER || type == SQLITE_FLOAT) { + v = mvt_value(sqlite3_column_double(stmt, i)); + } else if (type == SQLITE_TEXT || type == SQLITE_BLOB) { + v.set_string_value((const char *) sqlite3_column_text(stmt, i)); + } + + const char *name = sqlite3_column_name(stmt, i); + row.emplace(name, v); + } + + ret[f->second] = std::move(row); + } + } + if (sqlite3_finalize(stmt) != SQLITE_OK) { + fprintf(stderr, "sqlite3 finalize failed: %s\n", sqlite3_errmsg(db)); + exit(EXIT_SQLITE); + } + + return ret; +} + +struct arg { + std::map> inputs{}; + std::map outputs{}; + + std::map *layermap = NULL; + + std::vector *header = NULL; + std::map> *mapping = NULL; + sqlite3 *db = NULL; + std::set *exclude = NULL; + std::set *include = NULL; + std::set *keep_layers = NULL; + std::set *remove_layers = NULL; + int ifmatched = 0; + json_object *filter = NULL; + struct tileset_reader *readers = NULL; + + double minlat, minlon; + double maxlat, maxlon; + double minlon2, maxlon2; +}; + +void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter, struct arg *a) { mvt_tile tile; int features_added = 0; bool was_compressed; @@ -139,12 +243,42 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map> joined; + if (db != NULL) { + // collect join keys for sql query + + std::vector join_keys; + join_keys.resize(layer.features.size()); + + for (size_t f = 0; f < layer.features.size(); f++) { + mvt_feature &feat = layer.features[f]; + join_keys[f].type = mvt_no_such_key; + + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + const std::string &key = layer.keys[feat.tags[t]]; + if (key == join_tile_column) { + const mvt_value &val = layer.values[feat.tags[t + 1]]; + join_keys[f] = val; + break; + } + } + } + + joined = get_joined_rows(db, join_keys); + } + auto tilestats = layermap.find(layer.name); + long long minx = LLONG_MAX; + long long miny = LLONG_MAX; + long long maxx = LLONG_MIN; + long long maxy = LLONG_MIN; + bool features_added_to_layer = false; + for (size_t f = 0; f < layer.features.size(); f++) { - mvt_feature feat = layer.features[f]; - std::set exclude_attributes; + mvt_feature &feat = layer.features[f]; + std::set exclude_attributes; if (filter != NULL && !evaluate(feat, layer, filter, exclude_attributes, z, unidecode_data)) { continue; } @@ -161,7 +295,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map key_order; for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { - const char *key = layer.keys[feat.tags[t]].c_str(); + const std::string &key = layer.keys[feat.tags[t]]; mvt_value &val = layer.values[feat.tags[t + 1]]; serial_val sv = mvt_value_to_serial_val(val); @@ -169,12 +303,32 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map>(key, std::pair(val, sv))); - key_order.push_back(key); + if (!exclude_all_tile_attributes) { + if (include.count(key) || (!exclude_all && exclude.count(key) == 0 && exclude_attributes.count(key) == 0)) { + attributes.insert(std::pair>(key, std::pair(val, sv))); + key_order.push_back(key); + } + } + + if (f < joined.size()) { + if (joined[f].size() > 0) { + matched = true; + } + + for (auto const &kv : joined[f]) { + if (kv.first == attribute_for_id) { + outfeature.has_id = true; + outfeature.id = mvt_value_to_long_long(kv.second); + } else if (include.count(kv.first) || (!exclude_all && exclude.count(kv.first) == 0 && exclude_attributes.count(kv.first) == 0)) { + if (kv.second.type != mvt_null) { + attributes.insert(std::pair>(kv.first, std::pair(kv.second, mvt_value_to_serial_val(kv.second)))); + key_order.push_back(kv.first); + } + } + } } - if (header.size() > 0 && strcmp(key, header[0].c_str()) == 0) { + if (header.size() > 0 && key == header[0]) { std::map>::iterator ii = mapping.find(sv.s); if (ii != mapping.end()) { @@ -217,7 +371,6 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map>(joinkey, std::pair(outval, outsv))); @@ -257,7 +410,22 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapsecond.minzoom) { @@ -276,26 +444,42 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapminlat = std::min(a->minlat, std::min(lat1, lat2)); + a->minlon = std::min(a->minlon, std::min(lon1, lon2)); + a->maxlat = std::max(a->maxlat, std::max(lat1, lat2)); + a->maxlon = std::max(a->maxlon, std::max(lon1, lon2)); + + if (lon1 < 0 || lon2 < 0) { + lon1 += 360; + lon2 += 360; + } -double min(double a, double b) { - if (a < b) { - return a; - } else { - return b; + a->minlon2 = std::min(a->minlon2, std::min(lon1, lon2)); + a->maxlon2 = std::max(a->maxlon2, std::max(lon1, lon2)); + } } -} -double max(double a, double b) { - if (a > b) { - return a; - } else { - return b; + if (features_added == 0) { + return; } } @@ -736,23 +920,6 @@ struct tileset_reader *begin_reading(char *fname) { return r; } -struct arg { - std::map> inputs{}; - std::map outputs{}; - - std::map *layermap = NULL; - - std::vector *header = NULL; - std::map> *mapping = NULL; - std::set *exclude = NULL; - std::set *include = NULL; - std::set *keep_layers = NULL; - std::set *remove_layers = NULL; - int ifmatched = 0; - json_object *filter = NULL; - struct tileset_reader *readers = NULL; -}; - void *join_worker(void *v) { arg *a = (arg *) v; @@ -760,7 +927,7 @@ void *join_worker(void *v) { mvt_tile tile; for (size_t i = 0; i < ai->second.size(); i++) { - append_tile(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->include), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter); + append_tile(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), a->db, *(a->exclude), *(a->include), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter, a); } ai->second.clear(); @@ -795,7 +962,7 @@ void *join_worker(void *v) { return NULL; } -void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter, struct tileset_reader *readers) { +void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) { pthread_t pthreads[CPUS]; std::vector args; @@ -805,6 +972,7 @@ void dispatch_tasks(std::map> &tasks, std::vector< args[i].layermap = &layermaps[i]; args[i].header = &header; args[i].mapping = &mapping; + args[i].db = db; args[i].exclude = &exclude; args[i].include = &include; args[i].keep_layers = &keep_layers; @@ -812,6 +980,12 @@ void dispatch_tasks(std::map> &tasks, std::vector< args[i].ifmatched = ifmatched; args[i].filter = filter; args[i].readers = readers; + args[i].minlat = *minlat; + args[i].minlon = *minlon; + args[i].maxlat = *maxlat; + args[i].maxlon = *maxlon; + args[i].minlon2 = *minlon2; + args[i].maxlon2 = *maxlon2; } size_t count = 0; @@ -844,6 +1018,13 @@ void dispatch_tasks(std::map> &tasks, std::vector< perror("pthread_join"); } + *minlat = std::min(*minlat, args[i].minlat); + *minlon = std::min(*minlon, args[i].minlon); + *maxlat = std::max(*maxlat, args[i].maxlat); + *maxlon = std::max(*maxlon, args[i].maxlon); + *minlon2 = std::min(*minlon2, args[i].minlon2); + *maxlon2 = std::max(*maxlon2, args[i].maxlon2); + for (auto ai = args[i].outputs.begin(); ai != args[i].outputs.end(); ++ai) { if (outdb != NULL) { mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size()); @@ -945,7 +1126,7 @@ void handle_vector_layers(json_object *vector_layers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { +void decode(struct tileset_reader *readers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { std::vector> layermaps; for (size_t i = 0; i < CPUS; i++) { layermaps.push_back(std::map()); @@ -958,36 +1139,10 @@ void decode(struct tileset_reader *readers, std::mapall_done()) { std::pair current = readers->current(); - if (current.first.z != zoom_for_bbox) { - // Only use highest zoom for bbox calculation - // to avoid z0 always covering the world - - minlat = minlon = minlon2 = INT_MAX; - maxlat = maxlon = maxlon2 = INT_MIN; - zoom_for_bbox = current.first.z; - } - - double lat1, lon1, lat2, lon2; - tile2lonlat(current.first.x, current.first.y, current.first.z, &lon1, &lat1); - tile2lonlat(current.first.x + 1, current.first.y + 1, current.first.z, &lon2, &lat2); - minlat = min(lat2, minlat); - minlon = min(lon1, minlon); - maxlat = max(lat1, maxlat); - maxlon = max(lon2, maxlon); - - if (lon1 < 0) { - lon1 += 360; - lon2 += 360; - } - - minlon2 = min(lon1, minlon2); - maxlon2 = max(lon2, maxlon2); - if (current.first.z >= minzoom && current.first.z <= maxzoom) { zxy tile = current.first; if (tasks.count(tile) == 0) { @@ -1014,7 +1169,7 @@ void decode(struct tileset_reader *readers, std::mapzoom != current.first.z || readers->x != current.first.x || readers->y != current.first.y) { if (tasks.size() > 100 * CPUS) { - dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers); + dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, db, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers, &minlat, &minlon, &maxlat, &maxlon, &minlon2, &maxlon2); tasks.clear(); } } @@ -1033,18 +1188,18 @@ void decode(struct tileset_reader *readers, std::mapminlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); + dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, db, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers, &minlat, &minlon, &maxlat, &maxlon, &minlon2, &maxlon2); + layermap = merge_layermaps(layermaps); - st->minlon2 = min(minlon2, st->minlon2); - st->maxlon2 = max(maxlon2, st->maxlon2); - st->minlat2 = min(minlat, st->minlat2); - st->maxlat2 = max(maxlat, st->maxlat2); + st->minlon = std::min(minlon, st->minlon); + st->maxlon = std::max(maxlon, st->maxlon); + st->minlat = std::min(minlat, st->minlat); + st->maxlat = std::max(maxlat, st->maxlat); - dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers); - layermap = merge_layermaps(layermaps); + st->minlon2 = std::min(minlon2, st->minlon2); + st->maxlon2 = std::max(maxlon2, st->maxlon2); + st->minlat2 = std::min(minlat, st->minlat2); + st->maxlat2 = std::max(maxlat, st->maxlat2); struct tileset_reader *next; for (struct tileset_reader *r = readers; r != NULL; r = next) { @@ -1054,14 +1209,14 @@ void decode(struct tileset_reader *readers, std::mapdb, "SELECT value from metadata where name = 'minzoom'", -1, &stmt, NULL) == SQLITE_OK) { if (sqlite3_step(stmt) == SQLITE_ROW) { - int minz = max(sqlite3_column_int(stmt, 0), minzoom); - st->minzoom = min(st->minzoom, minz); + int minz = std::max(sqlite3_column_int(stmt, 0), minzoom); + st->minzoom = std::min(st->minzoom, minz); } sqlite3_finalize(stmt); } if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'maxzoom'", -1, &stmt, NULL) == SQLITE_OK) { if (sqlite3_step(stmt) == SQLITE_ROW) { - int maxz = min(sqlite3_column_int(stmt, 0), maxzoom); + int maxz = std::min(sqlite3_column_int(stmt, 0), maxzoom); if (!want_overzoom) { if (st->maxzoom >= 0 && maxz != st->maxzoom) { @@ -1069,7 +1224,7 @@ void decode(struct tileset_reader *readers, std::mapmaxzoom = max(st->maxzoom, maxz); + st->maxzoom = std::max(st->maxzoom, maxz); } sqlite3_finalize(stmt); } @@ -1116,20 +1271,6 @@ void decode(struct tileset_reader *readers, std::mapdb, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(stmt, 0); - if (s != NULL) { - if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { - st->minlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); - } - } - } - sqlite3_finalize(stmt); - } if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'json'", -1, &stmt, NULL) == SQLITE_OK) { if (sqlite3_step(stmt) == SQLITE_ROW) { const unsigned char *s = sqlite3_column_text(stmt, 0); @@ -1198,6 +1339,8 @@ int main(int argc, char **argv) { int filearg = 0; json_object *filter = NULL; + std::string join_sqlite_fname; + struct tileset_reader *readers = NULL; CPUS = get_num_avail_cpus(); @@ -1210,8 +1353,14 @@ int main(int argc, char **argv) { CPUS = 1; } + if (sqlite3_config(SQLITE_CONFIG_SERIALIZED) != SQLITE_OK) { + fprintf(stderr, "Could not enable sqlite3 serialized multithreading\n"); + exit(EXIT_SQLITE); + } + std::vector header; std::map> mapping; + sqlite3 *db = NULL; std::set exclude; std::set include; @@ -1235,6 +1384,7 @@ int main(int argc, char **argv) { {"exclude", required_argument, 0, 'x'}, {"exclude-all", no_argument, 0, 'X'}, {"include", required_argument, 0, 'y'}, + {"exclude-all-tile-attributes", no_argument, 0, '~'}, {"layer", required_argument, 0, 'l'}, {"exclude-layer", required_argument, 0, 'L'}, {"quiet", no_argument, 0, 'q'}, @@ -1245,6 +1395,12 @@ int main(int argc, char **argv) { {"rename-layer", required_argument, 0, 'R'}, {"read-from", required_argument, 0, 'r'}, + {"join-sqlite", required_argument, 0, '~'}, + {"join-tile-column", required_argument, 0, '~'}, + {"join-table-column", required_argument, 0, '~'}, + {"join-table", required_argument, 0, '~'}, + {"use-attribute-for-id", required_argument, 0, '~'}, + {"no-tile-size-limit", no_argument, &pk, 1}, {"no-tile-compression", no_argument, &pC, 1}, {"empty-csv-columns-are-null", no_argument, &pe, 1}, @@ -1429,6 +1585,22 @@ int main(int argc, char **argv) { max_tilestats_values = atoi(optarg); } else if (strcmp(opt, "unidecode-data") == 0) { unidecode_data = read_unidecode(optarg); + } else if (strcmp(opt, "join-sqlite") == 0) { + join_sqlite_fname = optarg; + if (sqlite3_open(optarg, &db) != SQLITE_OK) { + fprintf(stderr, "%s: %s\n", optarg, sqlite3_errmsg(db)); + exit(EXIT_SQLITE); + } + } else if (strcmp(opt, "join-table") == 0) { + join_table = optarg; + } else if (strcmp(opt, "join-table-column") == 0) { + join_table_column = optarg; + } else if (strcmp(opt, "join-tile-column") == 0) { + join_tile_column = optarg; + } else if (strcmp(opt, "use-attribute-for-id") == 0) { + attribute_for_id = optarg; + } else if (strcmp(opt, "exclude-all-tile-attributes") == 0) { + exclude_all_tile_attributes = true; } else { fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt); exit(EXIT_ARGS); @@ -1510,7 +1682,7 @@ int main(int argc, char **argv) { std::string generator_options; std::vector strategies; - decode(readers, layermap, outdb, out_dir, &st, header, mapping, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies); + decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies); if (set_attribution.size() != 0) { attribution = set_attribution; @@ -1541,6 +1713,10 @@ int main(int argc, char **argv) { } } + if (st.maxlon < st.minlon) { + st.maxlon = st.minlon = st.maxlat = st.minlat = st.minlon2 = st.maxlon2 = st.minlat2 = st.maxlat2 = 0; + } + if (st.maxlon - st.minlon <= st.maxlon2 - st.minlon2) { st.minlon2 = st.minlon; st.maxlon2 = st.maxlon;