Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Working on joining attributes to features in overzoom #225

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

Working on joining attributes to features in overzoom #225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
19b79d0
Scaffolding
e-n-f Mar 26, 2024
2e0f099
Figuring out a little more
e-n-f Mar 26, 2024
8876015
Merge remote-tracking branch 'origin/main' into overzoom-join-attributes
e-n-f Apr 8, 2024
1d48570
Most of the rest of joining JSON attributes onto features in overzoom
e-n-f Apr 8, 2024
20ca314
Test attribute joining in overzoom
e-n-f Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ tippecanoe-enumerate: enumerate.o
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3

tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o
tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o projection.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread

tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
Expand All @@ -76,8 +76,8 @@ tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
unit: unit.o text.o sort.o mvt.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread

tippecanoe-overzoom: overzoom.o mvt.o clip.o evaluator.o jsonpull/jsonpull.o text.o attribute.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
tippecanoe-overzoom: overzoom.o mvt.o clip.o evaluator.o jsonpull/jsonpull.o text.o attribute.o read_json.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread projection.o

-include $(wildcard *.d)

Expand Down Expand Up @@ -345,6 +345,11 @@ overzoom-test: tippecanoe-overzoom
./tippecanoe-decode tests/pbf/12-2145-1391-filter2.pbf 12 2145 1391 > tests/pbf/12-2145-1391-filter2.pbf.json.check
cmp tests/pbf/12-2145-1391-filter2.pbf.json.check tests/pbf/12-2145-1391-filter2.pbf.json
rm tests/pbf/12-2145-1391-filter2.pbf.json.check tests/pbf/12-2145-1391-filter2.pbf
# Test joining attributes from JSON
./tippecanoe-overzoom -o tests/pbf/ne-110m-z3-0-0-0-joined.pbf --join-attributes-json tests/pbf/name-fr.json tests/pbf/ne-110m-z3-0-0-0.pbf 0/0/0 0/0/0
./tippecanoe-decode tests/pbf/ne-110m-z3-0-0-0-joined.pbf 0 0 0 > tests/pbf/ne-110m-z3-0-0-0-joined.pbf.json.check
cmp tests/pbf/ne-110m-z3-0-0-0-joined.pbf.json.check tests/pbf/ne-110m-z3-0-0-0-joined.pbf.json
rm tests/pbf/ne-110m-z3-0-0-0-joined.pbf.json.check tests/pbf/ne-110m-z3-0-0-0-joined.pbf

join-test: tippecanoe tippecanoe-decode tile-join
./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'
Expand Down
78 changes: 72 additions & 6 deletions clip.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "evaluator.hpp"
#include "serial.hpp"
#include "attribute.hpp"
#include "read_json.hpp"

static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<double, double>> &geom,
long long minx, long long miny, long long maxx, long long maxy,
Expand Down Expand Up @@ -758,7 +759,7 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
std::string overzoom(const std::string &s, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map<std::string, attribute_op> const &attribute_accum, std::vector<std::string> const &unidecode_data) {
bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map<std::string, attribute_op> const &attribute_accum, std::vector<std::string> const &unidecode_data, json_object *join_attributes_json) {
mvt_tile tile;

try {
Expand All @@ -772,7 +773,7 @@ std::string overzoom(const std::string &s, int oz, int ox, int oy, int nz, int n
exit(EXIT_PROTOBUF);
}

return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data);
return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, join_attributes_json);
}

struct tile_feature {
Expand Down Expand Up @@ -870,14 +871,75 @@ static struct preservecmp {
}
} preservecmp;

std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int nx, int ny,
/* Format looks like:

{
"id": {
"1234": { "name": "Rome", "population", 567890 },
"15": { "name": "New York", "population": 32, "fictional": false }
}
}

IDs are really unsigned ints, not strings, but it feels like the keys
and values should be written as keys instead of in some other way.

Joined values win over original values, at least for the moment.

*/

void join_by_id(mvt_layer &layer, mvt_feature &feature, const json_object *j, std::shared_ptr<std::string> tile_stringpool) {
if (!feature.has_id) {
return;
}

if (j->type != JSON_HASH) {
fprintf(stderr, "list of attributes to be joined to features is not a json object\n");
exit(EXIT_JSON);
}

for (size_t i = 0; i < j->value.object.length; i++) {
const json_object *key = j->value.object.keys[i];
if (key->type == JSON_STRING) {
unsigned long long feature_id = atoll(j->value.object.keys[i]->value.string.string);
if (feature_id == feature.id) {
const json_object *value = j->value.object.values[i];

if (value->type == JSON_HASH) {
for (size_t a = 0; a < value->value.object.length; a++) {
json_object *k = value->value.object.keys[a];
json_object *v = value->value.object.values[a];

if (k->type != JSON_STRING) {
fprintf(stderr, "Expected string for key in json join, not %s\n", json_stringify(k));
exit(EXIT_JSON);
}

serial_val sv = stringify_value(v, "joined json", 1, value);
mvt_value mv = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool);
layer.tag(feature, k->value.string.string, mv);
}
} else {
fprintf(stderr, "Expected attribute object for value in json join, not %s\n", json_stringify(value));
exit(EXIT_JSON);
}

break;
}
} else {
fprintf(stderr, "Expected string for key in json join, not %s\n", json_stringify(key));
exit(EXIT_JSON);
}
}
}

std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map<std::string, attribute_op> const &attribute_accum, std::vector<std::string> const &unidecode_data) {
bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map<std::string, attribute_op> const &attribute_accum, std::vector<std::string> const &unidecode_data, json_object *join_attributes_json) {
mvt_tile outtile;
std::shared_ptr<std::string> tile_stringpool = std::make_shared<std::string>();

for (auto const &layer : tile.layers) {
for (auto &layer : tile.layers) {
mvt_layer outlayer = mvt_layer();

int det = detail;
Expand All @@ -895,6 +957,10 @@ std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int n
static const std::string retain_points_multiplier_sequence = "tippecanoe:retain_points_multiplier_sequence";

for (auto feature : layer.features) {
if (join_attributes_json != NULL) {
join_by_id(layer, feature, join_attributes_json, tile_stringpool);
}

bool flush_multiplier_cluster = false;
if (demultiply) {
for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) {
Expand Down Expand Up @@ -1046,7 +1112,7 @@ std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int n
std::string child = overzoom(outtile, nz, nx, ny,
nz + 1, nx * 2 + x, ny * 2 + y,
detail, buffer, keep, false, NULL,
demultiply, filter, preserve_input_order, attribute_accum, unidecode_data);
demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, join_attributes_json);
if (child.size() > 0) {
next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
}
Expand Down
8 changes: 5 additions & 3 deletions geometry.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,18 +100,20 @@ void visvalingam(drawvec &ls, size_t start, size_t end, double threshold, size_t
int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, long long testy);
double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y);

std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int nx, int ny,
std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order,
std::unordered_map<std::string, attribute_op> const &attribute_accum,
std::vector<std::string> const &unidecode_data);
std::vector<std::string> const &unidecode_data,
json_object *join_attributes_json);

std::string overzoom(const std::string &s, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order,
std::unordered_map<std::string, attribute_op> const &attribute_accum,
std::vector<std::string> const &unidecode_data);
std::vector<std::string> const &unidecode_data,
json_object *join_attributes_json);

#endif
6 changes: 3 additions & 3 deletions jsonpull/jsonpull.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -905,7 +905,7 @@ void json_disconnect(json_object *o) {
o->parent = NULL;
}

static void json_print_one(struct string *val, json_object *o) {
static void json_print_one(struct string *val, const json_object *o) {
if (o == NULL) {
string_append_string(val, "...");
} else if (o->type == JSON_STRING) {
Expand Down Expand Up @@ -955,7 +955,7 @@ static void json_print_one(struct string *val, json_object *o) {
}
}

static void json_print(struct string *val, json_object *o) {
static void json_print(struct string *val, const json_object *o) {
if (o == NULL) {
// Hash value in incompletely read hash
string_append_string(val, "...");
Expand Down Expand Up @@ -987,7 +987,7 @@ static void json_print(struct string *val, json_object *o) {
}
}

char *json_stringify(json_object *o) {
char *json_stringify(const json_object *o) {
struct string val;
string_init(&val);
json_print(&val, o);
Expand Down
2 changes: 1 addition & 1 deletion jsonpull/jsonpull.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ void json_disconnect(json_object *j);

json_object *json_hash_get(json_object *o, const char *s);

char *json_stringify(json_object *o);
char *json_stringify(const json_object *o);

#ifdef __cplusplus
}
Expand Down
8 changes: 7 additions & 1 deletion overzoom.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ std::string filter;
bool preserve_input_order = false;
std::unordered_map<std::string, attribute_op> attribute_accum;
std::vector<std::string> unidecode_data;
json_object *join_attributes_json = NULL;

std::set<std::string> keep;

Expand All @@ -43,6 +44,7 @@ int main(int argc, char **argv) {
{"preserve-input-order", no_argument, 0, 'o' & 0x1F},
{"accumulate-attribute", required_argument, 0, 'E'},
{"unidecode-data", required_argument, 0, 'u' & 0x1F},
{"join-attributes-json", required_argument, 0, 'c' & 0x1F},

{0, 0, 0, 0},
};
Expand Down Expand Up @@ -97,6 +99,10 @@ int main(int argc, char **argv) {
unidecode_data = read_unidecode(optarg);
break;

case 'c' & 0x1F:
join_attributes_json = read_filter(optarg);
break;

default:
fprintf(stderr, "Unrecognized flag -%c\n", i);
usage(argv);
Expand Down Expand Up @@ -151,7 +157,7 @@ int main(int argc, char **argv) {
json_filter = parse_filter(filter.c_str());
}

std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data);
std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, join_attributes_json);
fwrite(out.c_str(), sizeof(char), out.size(), f);
fclose(f);

Expand Down
4 changes: 2 additions & 2 deletions read_json.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ int mb_geometry[GEOM_TYPES] = {
VT_POLYGON,
};

void json_context(json_object *j) {
void json_context(const json_object *j) {
char *s = json_stringify(j);

if (strlen(s) >= 500) {
Expand Down Expand Up @@ -121,7 +121,7 @@ void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fna
// type and stringified value. All numeric values, even if they are integers,
// even integers that are too large to fit in a double but will still be
// stringified with their original precision, are recorded here as mvt_double.
serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature) {
serial_val stringify_value(json_object const *value, char const *reading, int line, json_object const *feature) {
serial_val sv;

if (value != NULL) {
Expand Down
5 changes: 2 additions & 3 deletions read_json.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ extern const char *geometry_names[GEOM_TYPES];
extern int geometry_within[GEOM_TYPES];
extern int mb_geometry[GEOM_TYPES];

void json_context(json_object *j);
void json_context(const json_object *j);
void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature);

serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature);
serial_val stringify_value(json_object const *, char const *, int, json_object const *);
1 change: 1 addition & 0 deletions tests/pbf/name-fr.json
View file
Open in desktop

Large diffs are not rendered by default.

35 changes: 35 additions & 0 deletions tests/pbf/ne-110m-z3-0-0-0-joined.pbf.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
{ "type": "FeatureCollection", "properties": { "layer": "11", "version": 2, "extent": 4096 }, "features": [
{ "type": "Feature", "id": 6173331, "properties": { "NAME": "Vancouver", "NAME_FR": "Vancouver", "SCALERANK": 1, "NAME_RU": "Ванкувер" }, "geometry": { "type": "Point", "coordinates": [ -123.134766, 49.267805 ] } }
,
{ "type": "Feature", "id": 1690681, "properties": { "NAME": "San Salvador", "NAME_FR": "San Salvador", "SCALERANK": 3, "NAME_RU": "Сан-Сальвадор" }, "geometry": { "type": "Point", "coordinates": [ -89.208984, 13.667338 ] } }
,
{ "type": "Feature", "id": 3579925, "properties": { "NAME": "Saint George's", "NAME_FR": "Saint-Georges", "SCALERANK": 4, "NAME_RU": "Сент-Джорджес" }, "geometry": { "type": "Point", "coordinates": [ -61.699219, 12.039321 ] } }
,
{ "type": "Feature", "id": 2422465, "properties": { "NAME": "Conakry", "NAME_FR": "Conakry", "SCALERANK": 3, "NAME_RU": "Конакри" }, "geometry": { "type": "Point", "coordinates": [ -13.710938, 9.535749 ] } }
,
{ "type": "Feature", "id": 3652462, "properties": { "NAME": "Quito", "NAME_FR": "Quito", "SCALERANK": 3, "NAME_RU": "Кито" }, "geometry": { "type": "Point", "coordinates": [ -78.486328, -0.175781 ] } }
,
{ "type": "Feature", "id": 2660646, "properties": { "NAME": "Geneva", "NAME_FR": "Genève", "SCALERANK": 1, "NAME_RU": "Женева" }, "geometry": { "type": "Point", "coordinates": [ 6.152344, 46.195042 ] } }
,
{ "type": "Feature", "id": 6691831, "properties": { "NAME": "Vatican City", "NAME_FR": "Cité du Vatican", "SCALERANK": 8, "NAME_RU": "Ватикан" }, "geometry": { "type": "Point", "coordinates": [ 12.480469, 41.902277 ] } }
,
{ "type": "Feature", "id": 618426, "properties": { "NAME": "Chișinău", "NAME_FR": "Chișinău", "SCALERANK": 3, "NAME_RU": "Кишинёв" }, "geometry": { "type": "Point", "coordinates": [ 28.828125, 46.980252 ] } }
,
{ "type": "Feature", "id": 2309527, "properties": { "NAME": "Malabo", "NAME_FR": "Malabo", "SCALERANK": 3, "NAME_RU": "Малабо" }, "geometry": { "type": "Point", "coordinates": [ 8.789062, 3.776559 ] } }
,
{ "type": "Feature", "id": 373303, "properties": { "NAME": "Juba", "NAME_FR": "Djouba", "SCALERANK": 4, "NAME_RU": "Джуба" }, "geometry": { "type": "Point", "coordinates": [ 31.552734, 4.828260 ] } }
,
{ "type": "Feature", "id": 285787, "properties": { "NAME": "Kuwait City", "NAME_FR": "Koweït", "SCALERANK": 2, "NAME_RU": "Эль-Кувейт" }, "geometry": { "type": "Point", "coordinates": [ 47.988281, 29.382175 ] } }
,
{ "type": "Feature", "id": 1238992, "properties": { "NAME": "Sri Jayawardenepura Kotte", "NAME_FR": "Sri Jayawardenapura", "SCALERANK": 4, "NAME_RU": "Шри-Джаяварденепура-Котте" }, "geometry": { "type": "Point", "coordinates": [ 79.980469, 6.926427 ] } }
,
{ "type": "Feature", "id": 1735161, "properties": { "NAME": "Kuala Lumpur", "NAME_FR": "Kuala Lumpur", "SCALERANK": 2, "NAME_RU": "Куала-Лумпур" }, "geometry": { "type": "Point", "coordinates": [ 101.689453, 3.162456 ] } }
,
{ "type": "Feature", "id": 2314302, "properties": { "NAME": "Kinshasa", "NAME_FR": "Kinshasa", "SCALERANK": 1, "NAME_RU": "Киншаса" }, "geometry": { "type": "Point", "coordinates": [ 15.292969, -4.302591 ] } }
,
{ "type": "Feature", "id": 921772, "properties": { "NAME": "Moroni", "NAME_FR": "Moroni", "SCALERANK": 6, "NAME_RU": "Морони" }, "geometry": { "type": "Point", "coordinates": [ 43.242188, -11.695273 ] } }
,
{ "type": "Feature", "id": 2108502, "properties": { "NAME": "Honiara", "NAME_FR": "Honiara", "SCALERANK": 3, "NAME_RU": "Хониара" }, "geometry": { "type": "Point", "coordinates": [ 159.960938, -9.449062 ] } }
] }
] }
Binary file added tests/pbf/ne-110m-z3-0-0-0.pbf
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion tile-join.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,7 @@ struct tileset_reader {
}

if (source.layers.size() != 0) {
std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map<std::string, attribute_op>(), unidecode_data);
std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map<std::string, attribute_op>(), unidecode_data, NULL /* XXX join_attributes_json */);
return ret;
}

Expand Down
Loading