Skip to content

Commit

Permalink
Use a hash chain when deduplicating tile layer keys and values
Browse files Browse the repository at this point in the history
  • Loading branch information
@e-n-f
e-n-f committed Dec 4, 2024
1 parent 1d81935 commit 95ef343
Show file tree
Hide file tree
Showing 5 changed files with 49 additions and 23 deletions.
32 changes: 22 additions & 10 deletions mvt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -607,22 +607,34 @@ std::string mvt_value::toString() const {

void mvt_layer::tag(mvt_feature &feature, std::string const &key, mvt_value const &value) {
size_t key_hash = fnv1a(key) % key_dedup.size();
if (key_dedup[key_hash] >= 0 &&
keys[key_dedup[key_hash]] == key) {
} else {
key_dedup[key_hash] = keys.size();
ssize_t key_index = -1;

for (auto const &k : key_dedup[key_hash]) {
if (keys[k] == key) {
key_index = k;
break;
}
}
if (key_index < 0) {
key_dedup[key_hash].push_back(key_index = keys.size());
keys.push_back(key);
}
feature.tags.push_back(key_dedup[key_hash]);
feature.tags.push_back(key_index);

size_t value_hash = std::hash<mvt_value>()(value) % value_dedup.size();
if (value_dedup[value_hash] >= 0 &&
values[value_dedup[value_hash]] == value) {
} else {
value_dedup[value_hash] = values.size();
ssize_t value_index = -1;

for (auto const &v : value_dedup[value_hash]) {
if (values[v] == value) {
value_index = v;
break;
}
}
if (value_index < 0) {
value_dedup[value_hash].push_back(value_index = values.size());
values.push_back(value);
}
feature.tags.push_back(value_dedup[value_hash]);
feature.tags.push_back(value_index);
}

bool is_integer(const char *s, long long *v) {
Expand Down
4 changes: 2 additions & 2 deletions mvt.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,8 @@ struct mvt_layer {
void tag(mvt_feature &feature, std::string const &key, mvt_value const &value);

// For tracking the key-value constants already used in this layer
std::vector<ssize_t> key_dedup = std::vector<ssize_t>(65536, -1);
std::vector<ssize_t> value_dedup = std::vector<ssize_t>(65536, -1);
std::vector<std::vector<ssize_t>> key_dedup = std::vector<std::vector<ssize_t>>(500, std::vector<ssize_t>());
std::vector<std::vector<ssize_t>> value_dedup = std::vector<std::vector<ssize_t>>(500, std::vector<ssize_t>());

int detail() const {
return std::round(std::log(extent) / std::log(2));
Expand Down
4 changes: 3 additions & 1 deletion ...M10000_--drop-densest-as-needed_--extend-zooms-if-still-dropping-maximum_2.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"maxzoom": "2",
"minzoom": "0",
"name": "tests/ne_110m_populated_places/out/-z0_-M10000_--drop-densest-as-needed_--extend-zooms-if-still-dropping-maximum_2.json.check.mbtiles",
"strategies": "[{\"dropped_as_needed\":223,\"tile_size_desired\":75535},{\"dropped_as_needed\":230,\"tile_size_desired\":44899},{\"dropped_as_needed\":195,\"tile_size_desired\":35505}]",
"strategies": "[{\"dropped_as_needed\":222,\"tile_size_desired\":74734},{\"dropped_as_needed\":230,\"tile_size_desired\":44415},{\"dropped_as_needed\":195,\"tile_size_desired\":35130}]",
"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
"type": "overlay",
"version": "2"
Expand All @@ -18,6 +18,8 @@
{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
{ "type": "Feature", "properties": { "SCALERANK": 1, "NATSCALE": 300, "LABELRANK": 2, "FEATURECLA": "Populated place", "NAME": "Vancouver", "DIFFASCII": 0, "NAMEASCII": "Vancouver", "ADM0CAP": 0, "CAPALT": 0, "WORLDCITY": 0, "MEGACITY": 1, "SOV0NAME": "Canada", "SOV_A3": "CAN", "ADM0NAME": "Canada", "ADM0_A3": "CAN", "ADM1NAME": "British Columbia", "ISO_A2": "CA", "LATITUDE": 49.273417, "LONGITUDE": -123.121644, "CHANGED": 5, "NAMEDIFF": 0, "DIFFNOTE": "Changed scale rank.", "POP_MAX": 2313328, "POP_MIN": 603502, "POP_OTHER": 482002, "RANK_MAX": 12, "RANK_MIN": 11, "GEONAMEID": 6173331, "MEGANAME": "Vancouver", "LS_NAME": "Vancouver2", "LS_MATCH": 1, "CHECKME": 0, "MAX_POP10": 1590116, "MAX_POP20": 1588839, "MAX_POP50": 1590116, "MAX_POP300": 1590116, "MAX_POP310": 0, "MAX_NATSCA": 100, "MIN_AREAKM": 706, "MAX_AREAKM": 708, "MIN_AREAMI": 273, "MAX_AREAMI": 273, "MIN_PERKM": 398, "MAX_PERKM": 405, "MIN_PERMI": 248, "MAX_PERMI": 251, "MIN_BBXMIN": -123.283333, "MAX_BBXMIN": -123.283333, "MIN_BBXMAX": -122.708333, "MAX_BBXMAX": -122.708333, "MIN_BBYMIN": 49.1, "MAX_BBYMIN": 49.1, "MIN_BBYMAX": 49.383333, "MAX_BBYMAX": 49.383333, "MEAN_BBXC": -122.982768, "MEAN_BBYC": 49.228888, "COMPARE": 0, "GN_ASCII": "Vancouver", "FEATURE_CL": "P", "FEATURE_CO": "PPL", "ADMIN1_COD": 2, "GN_POP": 1837969, "ELEVATION": 0, "GTOPO30": 63, "TIMEZONE": "America/Vancouver", "GEONAMESNO": "Geonames ascii name + lat.d + long.d matching.", "UN_FID": 15, "UN_ADM0": "Canada", "UN_LAT": 49.27, "UN_LONG": -122.96, "POP1950": 556, "POP1955": 588, "POP1960": 620, "POP1965": 836, "POP1970": 1045, "POP1975": 1150, "POP1980": 1247, "POP1985": 1359, "POP1990": 1559, "POP1995": 1789, "POP2000": 1959, "POP2005": 2093, "POP2010": 2146, "POP2015": 2219, "POP2020": 2310, "POP2025": 2380, "POP2050": 2444 }, "geometry": { "type": "Point", "coordinates": [ -123.134766, 49.267805 ] } }
,
{ "type": "Feature", "properties": { "SCALERANK": 1, "NATSCALE": 300, "LABELRANK": 1, "FEATURECLA": "Populated place", "NAME": "Chicago", "DIFFASCII": 0, "NAMEASCII": "Chicago", "ADM0CAP": 0, "CAPALT": 0, "WORLDCITY": 1, "MEGACITY": 1, "SOV0NAME": "United States", "SOV_A3": "USA", "ADM0NAME": "United States of America", "ADM0_A3": "USA", "ADM1NAME": "Illinois", "ISO_A2": "US", "LATITUDE": 41.829991, "LONGITUDE": -87.750055, "CHANGED": 0, "NAMEDIFF": 0, "POP_MAX": 8990000, "POP_MIN": 2841952, "POP_OTHER": 3635101, "RANK_MAX": 13, "RANK_MIN": 12, "GEONAMEID": 4887398, "MEGANAME": "Chicago", "LS_NAME": "Chicago", "LS_MATCH": 1, "CHECKME": 0, "MAX_POP10": 3747798, "MAX_POP20": 5069998, "MAX_POP50": 8416660, "MAX_POP300": 8416660, "MAX_POP310": 8450289, "MAX_NATSCA": 300, "MIN_AREAKM": 1345, "MAX_AREAKM": 4804, "MIN_AREAMI": 519, "MAX_AREAMI": 1855, "MIN_PERKM": 471, "MAX_PERKM": 2946, "MIN_PERMI": 293, "MAX_PERMI": 1830, "MIN_BBXMIN": -88.408333, "MAX_BBXMIN": -88.03629, "MIN_BBXMAX": -87.528138, "MAX_BBXMAX": -87.125, "MIN_BBYMIN": 41.391667, "MAX_BBYMIN": 41.458333, "MIN_BBYMAX": 42.000972, "MAX_BBYMAX": 42.491667, "MEAN_BBXC": -87.85874, "MEAN_BBYC": 41.832719, "COMPARE": 0, "GN_ASCII": "Chicago", "FEATURE_CL": "P", "FEATURE_CO": "PPL", "ADMIN1_COD": 0, "GN_POP": 2841952, "ELEVATION": 179, "GTOPO30": 181, "TIMEZONE": "America/Chicago", "GEONAMESNO": "GeoNames match with ascii name + lat + long whole numbers.", "UN_FID": 531, "UN_ADM0": "United States of America", "UN_LAT": 41.82, "UN_LONG": -87.64, "POP1950": 4999, "POP1955": 5565, "POP1960": 6183, "POP1965": 6639, "POP1970": 7106, "POP1975": 7160, "POP1980": 7216, "POP1985": 7285, "POP1990": 7374, "POP1995": 7839, "POP2000": 8333, "POP2005": 8820, "POP2010": 8990, "POP2015": 9211, "POP2020": 9516, "POP2025": 9756, "POP2050": 9932 }, "geometry": { "type": "Point", "coordinates": [ -87.714844, 41.836828 ] } }
,
{ "type": "Feature", "properties": { "SCALERANK": 3, "NATSCALE": 110, "LABELRANK": 8, "FEATURECLA": "Admin-0 capital", "NAME": "Reykjavík", "DIFFASCII": 1, "NAMEASCII": "Reykjavik", "ADM0CAP": 1, "CAPALT": 0, "WORLDCITY": 0, "MEGACITY": 0, "SOV0NAME": "Iceland", "SOV_A3": "ISL", "ADM0NAME": "Iceland", "ADM0_A3": "ISL", "ADM1NAME": "Suðurnes", "ISO_A2": "IS", "LATITUDE": 64.150024, "LONGITUDE": -21.950014, "CHANGED": 0, "NAMEDIFF": 0, "POP_MAX": 166212, "POP_MIN": 113906, "POP_OTHER": 160116, "RANK_MAX": 9, "RANK_MIN": 9, "GEONAMEID": 3413829, "LS_NAME": "Reykjavik", "LS_MATCH": 1, "CHECKME": 0, "MAX_POP10": 166212, "MAX_POP20": 166212, "MAX_POP50": 166212, "MAX_POP300": 166212, "MAX_POP310": 0, "MAX_NATSCA": 100, "MIN_AREAKM": 75, "MAX_AREAKM": 75, "MIN_AREAMI": 29, "MAX_AREAMI": 29, "MIN_PERKM": 119, "MAX_PERKM": 119, "MIN_PERMI": 74, "MAX_PERMI": 74, "MIN_BBXMIN": -22.008333, "MAX_BBXMIN": -22.008333, "MIN_BBXMAX": -21.75, "MAX_BBXMAX": -21.75, "MIN_BBYMIN": 64.05, "MAX_BBYMIN": 64.05, "MIN_BBYMAX": 64.166667, "MAX_BBYMAX": 64.166667, "MEAN_BBXC": -21.8825, "MEAN_BBYC": 64.116125, "COMPARE": 0, "GN_ASCII": "Reykjavik", "FEATURE_CL": "P", "FEATURE_CO": "PPLC", "ADMIN1_COD": 39, "GN_POP": 113906, "ELEVATION": 0, "GTOPO30": 16, "TIMEZONE": "Atlantic/Reykjavik", "GEONAMESNO": "GeoNames match general.", "UN_FID": 0, "UN_LAT": 0, "UN_LONG": 0, "POP1950": 0, "POP1955": 0, "POP1960": 0, "POP1965": 0, "POP1970": 0, "POP1975": 0, "POP1980": 0, "POP1985": 0, "POP1990": 0, "POP1995": 0, "POP2000": 0, "POP2005": 0, "POP2010": 0, "POP2015": 0, "POP2020": 0, "POP2025": 0, "POP2050": 0 }, "geometry": { "type": "Point", "coordinates": [ -21.972656, 64.168107 ] } }
,
{ "type": "Feature", "properties": { "SCALERANK": 3, "NATSCALE": 110, "LABELRANK": 0, "FEATURECLA": "Admin-0 capital", "NAME": "Praia", "DIFFASCII": 0, "NAMEASCII": "Praia", "ADM0CAP": 1, "CAPALT": 0, "WORLDCITY": 0, "MEGACITY": 0, "SOV0NAME": "Cape Verde", "SOV_A3": "CPV", "ADM0NAME": "Cape Verde", "ADM0_A3": "CPV", "ISO_A2": "CV", "LATITUDE": 14.916698, "LONGITUDE": -23.516689, "CHANGED": 0, "NAMEDIFF": 0, "POP_MAX": 113364, "POP_MIN": 88859, "POP_OTHER": 89205, "RANK_MAX": 9, "RANK_MIN": 8, "GEONAMEID": 3374333, "LS_NAME": "Praia", "LS_MATCH": 1, "CHECKME": 5, "MAX_POP10": 88859, "MAX_POP20": 88859, "MAX_POP50": 88859, "MAX_POP300": 88859, "MAX_POP310": 0, "MAX_NATSCA": 100, "MIN_AREAKM": 37, "MAX_AREAKM": 37, "MIN_AREAMI": 14, "MAX_AREAMI": 14, "MIN_PERKM": 40, "MAX_PERKM": 40, "MIN_PERMI": 25, "MAX_PERMI": 25, "MIN_BBXMIN": -23.541667, "MAX_BBXMIN": -23.541667, "MIN_BBXMAX": -23.483333, "MAX_BBXMAX": -23.483333, "MIN_BBYMIN": 14.9, "MAX_BBYMIN": 14.9, "MIN_BBYMAX": 14.983333, "MAX_BBYMAX": 14.983333, "MEAN_BBXC": -23.514907, "MEAN_BBYC": 14.938056, "COMPARE": 0, "GN_ASCII": "Praia", "FEATURE_CL": "P", "FEATURE_CO": "PPLC", "ADMIN1_COD": 0, "GN_POP": 113364, "ELEVATION": 0, "GTOPO30": -9999, "TIMEZONE": "Atlantic/Cape_Verde", "GEONAMESNO": "GeoNames match general.", "UN_FID": 0, "UN_LAT": 0, "UN_LONG": 0, "POP1950": 0, "POP1955": 0, "POP1960": 0, "POP1965": 0, "POP1970": 0, "POP1975": 0, "POP1980": 0, "POP1985": 0, "POP1990": 0, "POP1995": 0, "POP2000": 0, "POP2005": 0, "POP2010": 0, "POP2015": 0, "POP2020": 0, "POP2025": 0, "POP2050": 0 }, "geometry": { "type": "Point", "coordinates": [ -23.554688, 14.944785 ] } }
Expand Down
Loading

0 comments on commit 95ef343

Please sign in to comment.