Skip to content

Commit

Permalink
Reduce tiling memory (#227)
Browse files Browse the repository at this point in the history
* Trying to reduce memory in tiling

* Let the simplification workers go out of scope earlier

* Bail out quickly once the maximum feature count is reached

* Update changelog and version
  • Loading branch information
e-n-f authored Apr 3, 2024
1 parent 266ef3c commit bb4f220
Show file tree
Hide file tree
Showing 7 changed files with 655 additions and 525 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# 2.53.0

* Stop trying to add features to the tile after the feature limit is reached

# 2.52.0

* Fix accidental loss (at all zooms) of features that specify an explicit minzoom
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"maxzoom": "0",
"minzoom": "0",
"name": "tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json.check.mbtiles",
"strategies": "[{\"coalesced_as_needed\":999,\"feature_count_desired\":1000}]",
"strategies": "[{\"coalesced_as_needed\":999,\"feature_count_desired\":201}]",
"type": "overlay",
"version": "2"
}, "features": [
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/out/-z0_-O200_--drop-densest-as-needed.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"maxzoom": "0",
"minzoom": "0",
"name": "tests/loop/out/-z0_-O200_--drop-densest-as-needed.json.check.mbtiles",
"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":1000}]",
"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":201}]",
"type": "overlay",
"version": "2"
}, "features": [
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/out/-z0_-O200_--drop-fraction-as-needed.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"maxzoom": "0",
"minzoom": "0",
"name": "tests/loop/out/-z0_-O200_--drop-fraction-as-needed.json.check.mbtiles",
"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":1000}]",
"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":201}]",
"type": "overlay",
"version": "2"
}, "features": [
Expand Down
1,116 changes: 619 additions & 497 deletions tests/muni/out/-Z11_-z13_-O100_--cluster-densest-as-needed.json

Large diffs are not rendered by default.

52 changes: 28 additions & 24 deletions tile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1875,7 +1875,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
unsigned long long sfindex = sf.index;

if (sf.geometry.size() > 0) {
if (lead_features_count > max_tile_size) {
if (lead_features_count > max_tile_size || (lead_features_count + other_multiplier_cluster_features_count > max_tile_features && !prevent[P_FEATURE_LIMIT])) {
// Even being maximally conservative, each feature is still going to be
// at least one byte in the output tile, so this can't possibly work.
skipped++;
Expand Down Expand Up @@ -2110,33 +2110,35 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
tasks = 1;
}

pthread_t pthreads[tasks];
std::vector<simplification_worker_arg> args;
args.resize(tasks);
for (int i = 0; i < tasks; i++) {
args[i].task = i;
args[i].tasks = tasks;
args[i].features = &features;
args[i].shared_nodes = &shared_nodes;
args[i].shared_nodes_map = shared_nodes_map;
args[i].nodepos = nodepos;

if (tasks > 1) {
if (thread_create(&pthreads[i], NULL, simplification_worker, &args[i]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
{
pthread_t pthreads[tasks];
std::vector<simplification_worker_arg> args;
args.resize(tasks);
for (int i = 0; i < tasks; i++) {
args[i].task = i;
args[i].tasks = tasks;
args[i].features = &features;
args[i].shared_nodes = &shared_nodes;
args[i].shared_nodes_map = shared_nodes_map;
args[i].nodepos = nodepos;

if (tasks > 1) {
if (thread_create(&pthreads[i], NULL, simplification_worker, &args[i]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
} else {
simplification_worker(&args[i]);
}
} else {
simplification_worker(&args[i]);
}
}

if (tasks > 1) {
for (int i = 0; i < tasks; i++) {
void *retval;
if (tasks > 1) {
for (int i = 0; i < tasks; i++) {
void *retval;

if (pthread_join(pthreads[i], &retval) != 0) {
perror("pthread_join");
if (pthread_join(pthreads[i], &retval) != 0) {
perror("pthread_join");
}
}
}
}
Expand Down Expand Up @@ -2261,6 +2263,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}

if (layer_features[x].geometry.size() == 0) {
layer_features[x] = serial_feature();
continue;
}

Expand Down Expand Up @@ -2301,6 +2304,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}

layer.features.push_back(std::move(feature));
layer_features[x] = serial_feature();
}

if (layer.features.size() > 0) {
Expand Down
2 changes: 1 addition & 1 deletion version.hpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#ifndef VERSION_HPP
#define VERSION_HPP

#define VERSION "v2.52.0"
#define VERSION "v2.53.0"

#endif

0 comments on commit bb4f220

Please sign in to comment.