From 2f86ea12ae67ae0cd7c3571aee611f4990a47fce Mon Sep 17 00:00:00 2001 From: Michael Barry Date: Sun, 24 Sep 2023 08:10:47 -0400 Subject: [PATCH] Utilities to reduce tile size (#669) --- .../onthegomap/planetiler/FeatureMerge.java | 70 +++++++ .../com/onthegomap/planetiler/VectorTile.java | 172 +++++++++++++++++- .../planetiler/archive/TileArchiveWriter.java | 8 +- .../planetiler/collection/FeatureGroup.java | 58 +++--- .../planetiler/config/PlanetilerConfig.java | 10 +- .../planetiler/FeatureMergeTest.java | 171 ++++++++++++++++- .../planetiler/PlanetilerTests.java | 157 ++++++++++++++-- .../onthegomap/planetiler/VectorTileTest.java | 85 +++++++++ .../collection/FeatureGroupTest.java | 18 +- .../reader/SourceFeatureProcessorTest.java | 3 +- 10 files changed, 689 insertions(+), 63 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java index 6b9967d619..56fb3a179b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java @@ -87,6 +87,53 @@ public static List mergeLineStrings(List return mergeLineStrings(features, minLength, tolerance, buffer, false); } + /** Merges points with the same attributes into multipoints. */ + public static List mergeMultiPoint(List features) { + return mergeGeometries(features, GeometryType.POINT); + } + + /** + * Merges polygons with the same attributes into multipolygons. + *

+ * NOTE: This does not attempt to combine overlapping geometries, see {@link #mergeOverlappingPolygons(List, double)} + * or {@link #mergeNearbyPolygons(List, double, double, double, double)} for that. + */ + public static List mergeMultiPolygon(List features) { + return mergeGeometries(features, GeometryType.POLYGON); + } + + /** + * Merges linestrings with the same attributes into multilinestrings. + *

+ * NOTE: This does not attempt to connect linestrings that intersect at endpoints, see + * {@link #mergeLineStrings(List, double, double, double, boolean)} for that. Also, this removes extra detail that was + * preserved to improve connected-linestring merging, so you should only use one or the other. + */ + public static List mergeMultiLineString(List features) { + return mergeGeometries(features, GeometryType.LINE); + } + + private static List mergeGeometries( + List features, + GeometryType geometryType + ) { + List result = new ArrayList<>(features.size()); + var groupedByAttrs = groupByAttrs(features, result, geometryType); + for (List groupedFeatures : groupedByAttrs) { + VectorTile.Feature feature1 = groupedFeatures.get(0); + if (groupedFeatures.size() == 1) { + result.add(feature1); + } else { + VectorTile.VectorGeometryMerger combined = VectorTile.newMerger(geometryType); + for (var feature : groupedFeatures) { + combined.accept(feature.geometry()); + } + result.add(feature1.copyWithNewGeometry(combined.finish())); + } + } + return result; + } + /** * Merges linestrings with the same attributes as {@link #mergeLineStrings(List, Function, double, double, boolean)} * except sets {@code resimplify=false} by default. @@ -485,4 +532,27 @@ private static void depthFirstSearch(int startNode, IntArrayList group, IntObjec } } } + + /** + * Returns a new list of features with points that are more than {@code buffer} pixels outside the tile boundary + * removed, assuming a 256x256px tile. + */ + public static List removePointsOutsideBuffer(List features, double buffer) { + if (!Double.isFinite(buffer)) { + return features; + } + List result = new ArrayList<>(features.size()); + for (var feature : features) { + var geometry = feature.geometry(); + if (geometry.geomType() == GeometryType.POINT) { + var newGeometry = geometry.filterPointsOutsideBuffer(buffer); + if (!newGeometry.isEmpty()) { + result.add(feature.copyWithNewGeometry(newGeometry)); + } + } else { + result.add(feature); + } + } + return result; + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index f115074fb0..567f9bdcad 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -31,7 +31,9 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.concurrent.NotThreadSafe; @@ -176,7 +178,7 @@ private static int[] unscale(int[] commands, int scale, GeometryType geomType) { return result.toArray(); } - private static int zigZagEncode(int n) { + static int zigZagEncode(int n) { // https://developers.google.com/protocol-buffers/docs/encoding#types return (n << 1) ^ (n >> 31); } @@ -206,6 +208,11 @@ private static Geometry decodeCommands(GeometryType geomType, int[] commands, in length = commands[i++]; command = length & ((1 << 3) - 1); length = length >> 3; + assert geomType != GeometryType.POINT || i == 1 : "Invalid multipoint, command found at index %d, expected 0" + .formatted(i); + assert geomType != GeometryType.POINT || + (length * 2 + 1 == geometryCount) : "Invalid multipoint: int[%d] length=%d".formatted(geometryCount, + length); } if (length > 0) { @@ -404,6 +411,14 @@ public static VectorGeometry encodeGeometry(Geometry geometry, int scale) { return new VectorGeometry(getCommands(geometry, scale), GeometryType.typeOf(geometry), scale); } + /** + * Returns a new {@link VectorGeometryMerger} that combines encoded geometries of the same type into a merged + * multipoint, multilinestring, or multipolygon. + */ + public static VectorGeometryMerger newMerger(GeometryType geometryType) { + return new VectorGeometryMerger(geometryType); + } + /** * Adds features in a layer to this tile. * @@ -411,7 +426,7 @@ public static VectorGeometry encodeGeometry(Geometry geometry, int scale) { * @param features features to add to the tile * @return this encoder for chaining */ - public VectorTile addLayerFeatures(String layerName, List features) { + public VectorTile addLayerFeatures(String layerName, List features) { if (features.isEmpty()) { return this; } @@ -548,7 +563,7 @@ public boolean likelyToBeDuplicated() { return layers.values().stream().allMatch(v -> v.encodedFeatures.isEmpty()) || containsOnlyFillsOrEdges(); } - private enum Command { + enum Command { MOVE_TO(1), LINE_TO(2), CLOSE_PATH(7); @@ -560,6 +575,85 @@ private enum Command { } } + /** + * Utility that combines encoded geometries of the same type into a merged multipoint, multilinestring, or + * multipolygon. + */ + public static class VectorGeometryMerger implements Consumer { + // For the most part this just concatenates the individual command arrays together + // EXCEPT we need to adjust the first coordinate of each subsequent linestring to + // be an offset from the end of the previous linestring. + // AND we need to combine all multipoint "move to" commands into one at the start of + // the sequence + + private final GeometryType geometryType; + private int overallX = 0; + private int overallY = 0; + private final IntArrayList result = new IntArrayList(); + + private VectorGeometryMerger(GeometryType geometryType) { + this.geometryType = geometryType; + } + + @Override + public void accept(VectorGeometry vectorGeometry) { + if (vectorGeometry.geomType != geometryType) { + throw new IllegalArgumentException( + "Cannot merge a " + vectorGeometry.geomType.name().toLowerCase(Locale.ROOT) + " geometry into a multi" + + vectorGeometry.geomType.name().toLowerCase(Locale.ROOT)); + } + if (vectorGeometry.isEmpty()) { + return; + } + var commands = vectorGeometry.unscale().commands(); + int x = 0; + int y = 0; + + int geometryCount = commands.length; + int length = 0; + int command = 0; + int i = 0; + + result.ensureCapacity(result.elementsCount + commands.length); + // and multipoints will end up with only one command ("move to" with length=# points) + if (geometryType != GeometryType.POINT || result.isEmpty()) { + result.add(commands[0]); + } + result.add(zigZagEncode(zigZagDecode(commands[1]) - overallX)); + result.add(zigZagEncode(zigZagDecode(commands[2]) - overallY)); + if (commands.length > 3) { + result.add(commands, 3, commands.length - 3); + } + + while (i < geometryCount) { + if (length <= 0) { + length = commands[i++]; + command = length & ((1 << 3) - 1); + length = length >> 3; + } + + if (length > 0) { + length--; + if (command != Command.CLOSE_PATH.value) { + x += zigZagDecode(commands[i++]); + y += zigZagDecode(commands[i++]); + } + } + } + overallX = x; + overallY = y; + } + + /** Returns the merged multi-geometry. */ + public VectorGeometry finish() { + // set the correct "move to" length for multipoints based on how many points were actually added + if (geometryType == GeometryType.POINT) { + result.buffer[0] = Command.MOVE_TO.value | (((result.size() - 1) / 2) << 3); + } + return new VectorGeometry(result.toArray(), geometryType, 0); + } + } + /** * A vector geometry encoded as a list of commands according to the * vector tile @@ -578,6 +672,7 @@ public record VectorGeometry(int[] commands, GeometryType geomType, int scale) { private static final int BOTTOM = 1 << 3; private static final int INSIDE = 0; private static final int ALL = TOP | LEFT | RIGHT | BOTTOM; + private static final VectorGeometry EMPTY_POINT = new VectorGeometry(new int[0], GeometryType.POINT, 0); public VectorGeometry { if (scale < 0) { @@ -759,6 +854,75 @@ public boolean isFillOrEdge(boolean allowEdges) { return visitedEnoughSides(allowEdges, visited); } + /** Returns true if there are no commands in this geometry. */ + public boolean isEmpty() { + return commands.length == 0; + } + + /** + * If this is a point, returns an empty geometry if more than {@code buffer} pixels outside the tile bounds, or if + * it is a multipoint than removes all points outside the buffer. + */ + public VectorGeometry filterPointsOutsideBuffer(double buffer) { + if (geomType != GeometryType.POINT) { + return this; + } + IntArrayList result = null; + + int extent = (EXTENT << scale); + int bufferInt = (int) Math.ceil(buffer * extent / 256); + int min = -bufferInt; + int max = extent + bufferInt; + + int x = 0; + int y = 0; + int lastX = 0; + int lastY = 0; + + int geometryCount = commands.length; + int length = 0; + int i = 0; + + while (i < geometryCount) { + if (length <= 0) { + length = commands[i++] >> 3; + assert i <= 1 : "Bad index " + i; + } + + if (length > 0) { + length--; + x += zigZagDecode(commands[i++]); + y += zigZagDecode(commands[i++]); + if (x < min || y < min || x > max || y > max) { + if (result == null) { + // short-circuit the common case of only a single point that gets filtered-out + if (commands.length == 3) { + return EMPTY_POINT; + } + result = new IntArrayList(commands.length); + result.add(commands, 0, i - 2); + } + } else { + if (result != null) { + result.add(zigZagEncode(x - lastX), zigZagEncode(y - lastY)); + } + lastX = x; + lastY = y; + } + } + } + if (result != null) { + if (result.size() < 3) { + result.elementsCount = 0; + } else { + result.set(0, Command.MOVE_TO.value | (((result.size() - 1) / 2) << 3)); + } + + return new VectorGeometry(result.toArray(), geomType, scale); + } else { + return this; + } + } } /** @@ -807,7 +971,7 @@ public Feature copyWithNewGeometry(Geometry newGeometry) { * Returns a copy of this feature with {@code geometry} replaced with {@code newGeometry}. */ public Feature copyWithNewGeometry(VectorGeometry newGeometry) { - return new Feature( + return newGeometry == geometry ? this : new Feature( layer, id, newGeometry, diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index 60ebcc1a14..afa6048b3c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -277,13 +277,13 @@ private void tileEncoderSink(Iterable prev) throws IOException { layerStats = lastLayerStats; memoizedTiles.inc(); } else { - VectorTile en = tileFeatures.getVectorTileEncoder(); - if (skipFilled && (lastIsFill = en.containsOnlyFills())) { + VectorTile tile = tileFeatures.getVectorTile(); + if (skipFilled && (lastIsFill = tile.containsOnlyFills())) { encoded = null; layerStats = null; bytes = null; } else { - var proto = en.toProto(); + var proto = tile.toProto(); encoded = proto.toByteArray(); bytes = switch (config.tileCompression()) { case GZIP -> gzip(encoded); @@ -301,7 +301,7 @@ private void tileEncoderSink(Iterable prev) throws IOException { lastEncoded = encoded; lastBytes = bytes; last = tileFeatures; - if (archive.deduplicates() && en.likelyToBeDuplicated() && bytes != null) { + if (archive.deduplicates() && tile.likelyToBeDuplicated() && bytes != null) { tileDataHash = generateContentHash(bytes); } else { tileDataHash = null; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index a4601f7daf..d990f62b28 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -60,46 +60,35 @@ public final class FeatureGroup implements Iterable, private final CommonStringEncoder commonValueStrings = new CommonStringEncoder(100_000); private final Stats stats; private final LayerAttrStats layerStats = new LayerAttrStats(); + private final PlanetilerConfig config; private volatile boolean prepared = false; private final TileOrder tileOrder; - FeatureGroup(FeatureSort sorter, TileOrder tileOrder, Profile profile, Stats stats) { + FeatureGroup(FeatureSort sorter, TileOrder tileOrder, Profile profile, PlanetilerConfig config, Stats stats) { this.sorter = sorter; this.tileOrder = tileOrder; this.profile = profile; + this.config = config; this.stats = stats; } /** Returns a feature grouper that stores all feature in-memory. Only suitable for toy use-cases like unit tests. */ - public static FeatureGroup newInMemoryFeatureGroup(TileOrder tileOrder, Profile profile, Stats stats) { - return new FeatureGroup(FeatureSort.newInMemory(), tileOrder, profile, stats); + public static FeatureGroup newInMemoryFeatureGroup(TileOrder tileOrder, Profile profile, PlanetilerConfig config, + Stats stats) { + return new FeatureGroup(FeatureSort.newInMemory(), tileOrder, profile, config, stats); } + /** * Returns a feature grouper that writes all elements to disk in chunks, sorts each chunk, then reads back in order * from those chunks. Suitable for making maps up to planet-scale. */ public static FeatureGroup newDiskBackedFeatureGroup(TileOrder tileOrder, Path tempDir, Profile profile, - PlanetilerConfig config, - Stats stats) { - return new FeatureGroup( - new ExternalMergeSort(tempDir, config, stats), - tileOrder, profile, stats - ); - } - - /** backwards compatibility **/ - public static FeatureGroup newInMemoryFeatureGroup(Profile profile, Stats stats) { - return new FeatureGroup(FeatureSort.newInMemory(), TileOrder.TMS, profile, stats); - } - - /** backwards compatibility **/ - public static FeatureGroup newDiskBackedFeatureGroup(Path tempDir, Profile profile, PlanetilerConfig config, - Stats stats) { + PlanetilerConfig config, Stats stats) { return new FeatureGroup( new ExternalMergeSort(tempDir, config, stats), - TileOrder.TMS, profile, stats + tileOrder, profile, config, stats ); } @@ -206,7 +195,6 @@ private long encodeKey(RenderedFeature feature) { var vectorTileFeature = feature.vectorTileFeature(); byte encodedLayer = commonLayerStrings.encode(vectorTileFeature.layer()); - return encodeKey( this.tileOrder.encode(feature.tile()), encodedLayer, @@ -362,13 +350,23 @@ private TileFeatures(int lastTileId) { this.tileCoord = tileOrder.decode(lastTileId); } - private static void unscale(List features) { + private static void unscaleAndRemovePointsOutsideBuffer(List features, double maxPointBuffer) { + boolean checkPoints = maxPointBuffer <= 256 && maxPointBuffer >= -128; for (int i = 0; i < features.size(); i++) { var feature = features.get(i); if (feature != null) { VectorTile.VectorGeometry geometry = feature.geometry(); + var orig = geometry; if (geometry.scale() != 0) { - features.set(i, feature.copyWithNewGeometry(geometry.unscale())); + geometry = geometry.unscale(); + } + if (checkPoints && geometry.geomType() == GeometryType.POINT && !geometry.isEmpty()) { + geometry = geometry.filterPointsOutsideBuffer(maxPointBuffer); + } + if (geometry.isEmpty()) { + features.set(i, null); + } else if (geometry != orig) { + features.set(i, feature.copyWithNewGeometry(geometry)); } } } @@ -457,8 +455,8 @@ private VectorTile.Feature decodeVectorTileFeature(SortableFeature entry) { } } - public VectorTile getVectorTileEncoder() { - VectorTile encoder = new VectorTile(); + public VectorTile getVectorTile() { + VectorTile tile = new VectorTile(); List items = new ArrayList<>(entries.size()); String currentLayer = null; for (SortableFeature entry : entries) { @@ -468,15 +466,15 @@ public VectorTile getVectorTileEncoder() { if (currentLayer == null) { currentLayer = layer; } else if (!currentLayer.equals(layer)) { - postProcessAndAddLayerFeatures(encoder, currentLayer, items); + postProcessAndAddLayerFeatures(tile, currentLayer, items); currentLayer = layer; items.clear(); } items.add(feature); } - postProcessAndAddLayerFeatures(encoder, currentLayer, items); - return encoder; + postProcessAndAddLayerFeatures(tile, currentLayer, items); + return tile; } private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer, @@ -488,7 +486,9 @@ private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer, // lines are stored using a higher precision so that rounding does not // introduce artificial intersections between endpoints to confuse line merging, // so we have to reduce the precision here, now that line merging is done. - unscale(features); + unscaleAndRemovePointsOutsideBuffer(features, config.maxPointBuffer()); + // also remove points more than --max-point-buffer pixels outside the tile if the + // user has requested a narrower buffer than the profile provides by default } catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors // failures in tile post-processing happen very late so err on the side of caution and // log failures, only throwing when it's a fatal error diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 13ed394b67..0ca347041d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -57,7 +57,8 @@ public record PlanetilerConfig( boolean outputLayerStats, String debugUrlPattern, Path tmpDir, - Path tileWeights + Path tileWeights, + double maxPointBuffer ) { public static final int MIN_MINZOOM = 0; @@ -202,7 +203,12 @@ public static PlanetilerConfig from(Arguments arguments) { "https://onthegomap.github.io/planetiler-demo/#{z}/{lat}/{lon}"), tmpDir, arguments.file("tile_weights", "tsv.gz file with columns z,x,y,loads to generate weighted average tile size stat", - tmpDir.resolveSibling("tile_weights.tsv.gz")) + tmpDir.resolveSibling("tile_weights.tsv.gz")), + arguments.getDouble("max_point_buffer", + "Max tile pixels to include points outside tile bounds. Set to a lower value to reduce tile size for " + + "clients that handle label collisions across tiles (most web and native clients). NOTE: Do not reduce if you need to support " + + "raster tile rendering", + Double.POSITIVE_INFINITY) ); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java index b080abdc0b..32b6527e0e 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java @@ -8,15 +8,23 @@ import com.carrotsearch.hppc.IntObjectMap; import com.onthegomap.planetiler.collection.Hppc; import com.onthegomap.planetiler.geo.GeometryException; +import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.mbtiles.Mbtiles; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.function.UnaryOperator; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryCollection; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,7 +32,7 @@ class FeatureMergeTest { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureMergeTest.class); - private VectorTile.Feature feature(long id, Geometry geom, Map attrs) { + private static VectorTile.Feature feature(long id, Geometry geom, Map attrs) { return new VectorTile.Feature( "layer", id, @@ -684,4 +692,165 @@ void mergeMultiPolygonExcludeSmallInnerRings() throws GeometryException { ) ); } + + @Test + void mergeMultipoints() throws GeometryException { + testMultigeometryMerger( + i -> newPoint(i, 2 * i), + items -> newMultiPoint(items.toArray(Point[]::new)), + rectangle(0, 1), + FeatureMerge::mergeMultiPoint + ); + } + + @Test + void mergeMultipolygons() throws GeometryException { + testMultigeometryMerger( + i -> rectangle(i, i + 1), + items -> newMultiPolygon(items.toArray(Polygon[]::new)), + newPoint(0, 0), + FeatureMerge::mergeMultiPolygon + ); + } + + @Test + void mergeMultiline() throws GeometryException { + testMultigeometryMerger( + i -> newLineString(i, i + 1, i + 2, i + 3), + items -> newMultiLineString(items.toArray(LineString[]::new)), + newPoint(0, 0), + FeatureMerge::mergeMultiLineString + ); + } + + void testMultigeometryMerger( + IntFunction generateGeometry, + Function, M> combineJTS, + Geometry otherGeometry, + UnaryOperator> merge + ) throws GeometryException { + var geom1 = generateGeometry.apply(1); + var geom2 = generateGeometry.apply(2); + var geom3 = generateGeometry.apply(3); + var geom4 = generateGeometry.apply(4); + var geom5 = generateGeometry.apply(5); + + assertTopologicallyEquivalentFeatures( + List.of(), + merge.apply(List.of()) + ); + + assertTopologicallyEquivalentFeatures( + List.of( + feature(1, geom1, Map.of("a", 1)) + ), + merge.apply( + List.of( + feature(1, geom1, Map.of("a", 1)) + ) + ) + ); + + assertTopologicallyEquivalentFeatures( + List.of( + feature(4, otherGeometry, Map.of("a", 1)), + feature(1, combineJTS.apply(List.of(geom1, geom2, geom3, geom4)), Map.of("a", 1)), + feature(3, geom5, Map.of("a", 2)) + ), + merge.apply( + List.of( + feature(1, combineJTS.apply(List.of(geom1, geom2)), Map.of("a", 1)), + feature(2, combineJTS.apply(List.of(geom3, geom4)), Map.of("a", 1)), + feature(3, geom5, Map.of("a", 2)), + feature(4, otherGeometry, Map.of("a", 1)), + new VectorTile.Feature("layer", 5, new VectorTile.VectorGeometry(new int[0], GeometryType.typeOf(geom1), 0), + Map.of("a", 1)) + ) + ) + ); + } + + @Test + void removePointsOutsideBufferEmpty() throws GeometryException { + assertEquals( + List.of(), + FeatureMerge.removePointsOutsideBuffer(List.of(), 4d) + ); + } + + @Test + void removePointsOutsideBufferSinglePoints() throws GeometryException { + assertEquals( + List.of(), + FeatureMerge.removePointsOutsideBuffer(List.of(), 4d) + ); + assertTopologicallyEquivalentFeatures( + List.of( + feature(1, newPoint(0, 0), Map.of()), + feature(1, newPoint(256, 256), Map.of()), + feature(1, newPoint(-4, -4), Map.of()), + feature(1, newPoint(-4, 260), Map.of()), + feature(1, newPoint(260, -4), Map.of()), + feature(1, newPoint(260, 260), Map.of()) + ), + FeatureMerge.removePointsOutsideBuffer( + List.of( + feature(1, newPoint(0, 0), Map.of()), + feature(1, newPoint(256, 256), Map.of()), + feature(1, newPoint(-4, -4), Map.of()), + feature(1, newPoint(-4, 260), Map.of()), + feature(1, newPoint(260, -4), Map.of()), + feature(1, newPoint(260, 260), Map.of()), + feature(1, newPoint(-5, -5), Map.of()), + feature(1, newPoint(-5, 261), Map.of()), + feature(1, newPoint(261, -5), Map.of()), + feature(1, newPoint(261, 261), Map.of()) + ), + 4d + ) + ); + } + + @Test + void removePointsOutsideBufferMultiPoints() throws GeometryException { + assertEquals( + List.of(), + FeatureMerge.removePointsOutsideBuffer(List.of(), 4d) + ); + assertTopologicallyEquivalentFeatures( + List.of( + feature(1, newMultiPoint( + newPoint(0, 0), + newPoint(256, 256), + newPoint(-4, -4), + newPoint(-4, 260), + newPoint(260, -4), + newPoint(260, 260) + ), Map.of()) + ), + FeatureMerge.removePointsOutsideBuffer( + List.of( + feature(1, newMultiPoint( + newPoint(0, 0), + newPoint(256, 256), + newPoint(-4, -4), + newPoint(-4, 260), + newPoint(260, -4), + newPoint(260, 260), + newPoint(-5, -5), + newPoint(-5, 261), + newPoint(261, -5), + newPoint(261, 261) + ), Map.of()), + feature(1, newMultiPoint( + newPoint(-5, -5), + newPoint(-5, 261), + newPoint(261, -5), + newPoint(261, 261) + ), Map.of()) + ), + 4d + ) + ); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 55e7ee2703..bc9f952559 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -61,6 +61,7 @@ import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.MultiPolygon; +import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.io.InputStreamInStream; import org.locationtech.jts.io.WKBReader; @@ -85,6 +86,7 @@ class PlanetilerTests { private static final int Z13_TILES = 1 << 13; private static final double Z13_WIDTH = 1d / Z13_TILES; private static final int Z12_TILES = 1 << 12; + private static final double Z12_WIDTH = 1d / Z12_TILES; private static final int Z4_TILES = 1 << 4; private static final Polygon WORLD_POLYGON = newPolygon( worldCoordinateList( @@ -150,7 +152,7 @@ private PlanetilerResults run( Profile profile ) throws Exception { PlanetilerConfig config = PlanetilerConfig.from(Arguments.of(args)); - FeatureGroup featureGroup = FeatureGroup.newInMemoryFeatureGroup(TileOrder.TMS, profile, stats); + FeatureGroup featureGroup = FeatureGroup.newInMemoryFeatureGroup(TileOrder.TMS, profile, config, stats); runner.run(featureGroup, profile, config); featureGroup.prepare(); try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.arguments())) { @@ -469,6 +471,33 @@ void testLineString() throws Exception { ), results.tiles); } + @Test + void testLineStringDegenerateWhenUnscaled() throws Exception { + double x1 = 0.5 + Z12_WIDTH / 2; + double y1 = 0.5 + Z12_WIDTH / 2; + double x2 = x1 + Z12_WIDTH / 4096 / 3; + double y2 = y1 + Z12_WIDTH / 4096 / 3; + double lat1 = GeoUtils.getWorldLat(y1); + double lng1 = GeoUtils.getWorldLon(x1); + double lat2 = GeoUtils.getWorldLat(y2); + double lng2 = GeoUtils.getWorldLon(x2); + + var results = runWithReaderFeatures( + Map.of("threads", "1"), + List.of( + newReaderFeature(newLineString(lng1, lat1, lng2, lat2), Map.of( + "attr", "value" + )) + ), + (in, features) -> features.line("layer") + .setZoomRange(12, 12) + .setMinPixelSize(0) + .setBufferPixels(4) + ); + + assertSubmap(Map.of(), results.tiles); + } + @Test void testNumPointsAttr() throws Exception { double x1 = 0.5 + Z14_WIDTH / 2; @@ -567,6 +596,13 @@ public List z14CoordinatePixelList(double... coords) { return z14CoordinateList(DoubleStream.of(coords).map(c -> c / 256d).toArray()); } + public Point z14Point(double x, double y) { + return newPoint( + GeoUtils.getWorldLon(0.5 + x * Z14_WIDTH / 256), + GeoUtils.getWorldLat(0.5 + y * Z14_WIDTH / 256) + ); + } + @Test void testPolygonWithHoleSpanningMultipleTiles() throws Exception { List outerPoints = z14CoordinateList( @@ -1204,8 +1240,9 @@ void testPostProcessNodeUseLabelGridRank() throws Exception { ), results.tiles); } - @Test - void testMergeLineStrings() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMergeLineStrings(boolean connectEndpoints) throws Exception { double y = 0.5 + Z15_WIDTH / 2; double lat = GeoUtils.getWorldLat(y); @@ -1237,7 +1274,9 @@ void testMergeLineStrings() throws Exception { .setMinZoom(13) .setAttrWithMinzoom("z14attr", in.getTag("other"), 14) .inheritAttrFromSource("group"), - (layer, zoom, items) -> FeatureMerge.mergeLineStrings(items, 0, 0, 0) + (layer, zoom, items) -> connectEndpoints ? + FeatureMerge.mergeLineStrings(items, 0, 0, 0) : + FeatureMerge.mergeMultiLineString(items) ); assertSubmap(sortListValues(Map.of( @@ -1251,10 +1290,16 @@ void testMergeLineStrings() throws Exception { feature(newLineString(37, 64, 42, 64), Map.of("group", "1", "z14attr", "2")), feature(newLineString(42, 64, 47, 64), Map.of("group", "2", "z14attr", "3")) ), - TileCoord.ofXYZ(Z13_TILES / 2, Z13_TILES / 2, 13), List.of( + TileCoord.ofXYZ(Z13_TILES / 2, Z13_TILES / 2, 13), connectEndpoints ? List.of( // merge 32->37 and 37->42 since they have same attrs feature(newLineString(16, 32, 21, 32), Map.of("group", "1")), feature(newLineString(21, 32, 23.5, 32), Map.of("group", "2")) + ) : List.of( + feature(newMultiLineString( + newLineString(16, 32, 18.5, 32), + newLineString(18.5, 32, 21, 32) + ), Map.of("group", "1")), + feature(newLineString(21, 32, 23.5, 32), Map.of("group", "2")) ) )), sortListValues(results.tiles)); } @@ -1310,8 +1355,9 @@ void testMergeLineStringsIgnoresRoundingIntersections() throws Exception { )), sortListValues(results.tiles)); } - @Test - void testMergePolygons() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMergePolygons(boolean unionOverlapping) throws Exception { var results = runWithReaderFeatures( Map.of("threads", "1"), List.of( @@ -1342,19 +1388,97 @@ void testMergePolygons() throws Exception { (in, features) -> features.polygon("layer") .setZoomRange(14, 14) .inheritAttrFromSource("group"), - (layer, zoom, items) -> FeatureMerge.mergeNearbyPolygons( + (layer, zoom, items) -> unionOverlapping ? FeatureMerge.mergeNearbyPolygons( items, 0, 0, 1, 1 + ) : FeatureMerge.mergeMultiPolygon(items) + ); + + if (unionOverlapping) { + assertSubmap(sortListValues(Map.of( + TileCoord.ofXYZ(Z14_TILES / 2, Z14_TILES / 2, 14), List.of( + feature(rectangle(10, 10, 30, 20), Map.of("group", "1")), + feature(rectangle(10, 20.5, 20, 30), Map.of("group", "2")) + ) + )), sortListValues(results.tiles)); + } else { + assertSubmap(sortListValues(Map.of( + TileCoord.ofXYZ(Z14_TILES / 2, Z14_TILES / 2, 14), List.of( + feature( + newMultiPolygon( + rectangle(10, 10, 20, 20), + rectangle(20.5, 10, 30, 20) + ), Map.of("group", "1")), + feature(rectangle(10, 20.5, 20, 30), Map.of("group", "2")) + ) + )), sortListValues(results.tiles)); + } + } + + @Test + void testCombineMultiPoint() throws Exception { + var results = runWithReaderFeatures( + Map.of("threads", "1"), + List.of( + // merge same group: + newReaderFeature(z14Point(0, 0), Map.of("group", "1")), + newReaderFeature(newMultiPoint( + z14Point(1, 1), + z14Point(2, 2) + ), Map.of("group", "1")), + // don't merge - different group: + newReaderFeature(z14Point(3, 3), Map.of("group", "2")) + ), + (in, features) -> features.point("layer") + .setZoomRange(14, 14) + .setBufferPixels(0) + .inheritAttrFromSource("group"), + (layer, zoom, items) -> FeatureMerge.mergeMultiPoint(items) + ); + + assertSubmap(sortListValues(Map.of( + TileCoord.ofXYZ(Z14_TILES / 2, Z14_TILES / 2, 14), List.of( + feature(newMultiPoint( + newPoint(0, 0), + newPoint(1, 1), + newPoint(2, 2) + ), Map.of("group", "1")), + feature(newPoint(3, 3), Map.of("group", "2")) ) + )), sortListValues(results.tiles)); + } + + @Test + void testReduceMaxPointBuffer() throws Exception { + var results = runWithReaderFeatures( + Map.of( + "threads", "1", + "max-point-buffer", "1" + ), + List.of( + newReaderFeature(z14Point(0, 0), Map.of("group", "1")), + newReaderFeature(newMultiPoint( + z14Point(-1, -1), + z14Point(-2, -2) // should get filtered out + ), Map.of("group", "1")), + // don't merge - different group: + newReaderFeature(z14Point(257, 257), Map.of("group", "2")), + newReaderFeature(z14Point(258, 258), Map.of("group", "3")) // filter out + ), + (in, features) -> features.point("layer") + .setZoomRange(14, 14) + .setBufferPixels(10) + .inheritAttrFromSource("group") ); assertSubmap(sortListValues(Map.of( TileCoord.ofXYZ(Z14_TILES / 2, Z14_TILES / 2, 14), List.of( - feature(rectangle(10, 10, 30, 20), Map.of("group", "1")), - feature(rectangle(10, 20.5, 20, 30), Map.of("group", "2")) + feature(newPoint(-1, -1), Map.of("group", "1")), + feature(newPoint(0, 0), Map.of("group", "1")), + feature(newPoint(257, 257), Map.of("group", "2")) ) )), sortListValues(results.tiles)); } @@ -1809,7 +1933,8 @@ private static TileCompression extractTileCompression(String args) { "--output-format=json", "--tile-compression=none", "--tile-compression=gzip", - "--output-layerstats" + "--output-layerstats", + "--max-point-buffer=1" }) void testPlanetilerRunner(String args) throws Exception { Path originalOsm = TestUtils.pathToResource("monaco-latest.osm.pbf"); @@ -1838,6 +1963,8 @@ void testPlanetilerRunner(String args) throws Exception { public void processFeature(SourceFeature source, FeatureCollector features) { if (source.canBePolygon() && source.hasTag("building", "yes")) { features.polygon("building").setZoomRange(0, 14).setMinPixelSize(1); + } else if (source.isPoint() && source.hasTag("place")) { + features.point("place").setZoomRange(0, 14); } } }) @@ -1863,8 +1990,10 @@ public void processFeature(SourceFeature source, FeatureCollector features) { } } - assertEquals(11, tileMap.size(), "num tiles"); - assertEquals(2146, features, "num buildings"); + int expectedFeatures = args.contains("max-point-buffer=1") ? 2311 : 2313; + + assertEquals(22, tileMap.size(), "num tiles"); + assertEquals(expectedFeatures, features, "num feature"); final boolean checkMetadata = switch (format) { case MBTILES -> true; @@ -1888,7 +2017,7 @@ public void processFeature(SourceFeature source, FeatureCollector features) { byte[] data = Files.readAllBytes(layerstats); byte[] uncompressed = Gzip.gunzip(data); String[] lines = new String(uncompressed, StandardCharsets.UTF_8).split("\n"); - assertEquals(12, lines.length); + assertEquals(33, lines.length); assertEquals(List.of( "z", diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java index e8fe51caf8..46fea78674 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java @@ -19,16 +19,21 @@ package com.onthegomap.planetiler; import static com.onthegomap.planetiler.TestUtils.*; +import static com.onthegomap.planetiler.VectorTile.zigZagEncode; import static com.onthegomap.planetiler.geo.GeoUtils.JTS_FACTORY; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.DynamicTest.dynamicTest; import com.google.common.primitives.Ints; +import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.geo.GeometryException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; @@ -517,6 +522,86 @@ Stream testScaleUnscale() throws NoninvertibleTransformationExcepti )); } + @Test + void testUnscaleDegenerate() throws GeometryException { + var lessThanOnePx = 256d / 4096 / 4; + var encoded = VectorTile.encodeGeometry(newLineString(0, 0, lessThanOnePx, lessThanOnePx), 2); + assertEquals(6, encoded.commands().length); + var unscaled = encoded.unscale(); + assertEquals(0, unscaled.commands().length); + assertFalse(encoded.isEmpty()); + assertTrue(unscaled.isEmpty()); + assertEquals(GeoUtils.EMPTY_GEOMETRY, unscaled.decode()); + var reEncoded = VectorTile.encodeGeometry(unscaled.decode()); + assertEquals(0, reEncoded.commands().length); + } + + @Test + void testFilterPointsOutsideBuffer() { + assertArrayEquals( + new int[0], + VectorTile.encodeGeometry(newPoint(-5, -5)) + .filterPointsOutsideBuffer(4).commands() + ); + assertArrayEquals( + new int[]{ + VectorTile.Command.MOVE_TO.value | (1 << 3), + zigZagEncode((int) (-5d * 4096 / 256)), + zigZagEncode((int) (-5d * 4096 / 256)), + }, + VectorTile.encodeGeometry(newPoint(-5, -5)) + .filterPointsOutsideBuffer(5).commands() + ); + } + + @Test + void testFilterMultiPointsAllOutsideBuffer() { + assertArrayEquals( + new int[0], + VectorTile.encodeGeometry(newMultiPoint( + newPoint(-5, -5), + newPoint(261, 261) + )).filterPointsOutsideBuffer(4).commands() + ); + } + + @Test + void testFilterMultiPointsFirstOutsideBuffer() { + assertArrayEquals( + new int[]{ + VectorTile.Command.MOVE_TO.value | (1 << 3), + zigZagEncode(4096), + zigZagEncode(4096), + }, + VectorTile.encodeGeometry(newMultiPoint( + newPoint(-5, -5), + newPoint(256, 256) + )).filterPointsOutsideBuffer(4).commands() + ); + } + + @Test + void testFilterMultiPointsLastOutsideBuffer() { + assertArrayEquals( + new int[]{ + VectorTile.Command.MOVE_TO.value | (1 << 3), + zigZagEncode(4096), + zigZagEncode(4096), + }, + VectorTile.encodeGeometry(newMultiPoint( + newPoint(256, 256), + newPoint(-5, -5) + )).filterPointsOutsideBuffer(4).commands() + ); + } + + private static void assertArrayEquals(int[] a, int[] b) { + assertEquals( + IntStream.of(a).boxed().toList(), + IntStream.of(b).boxed().toList() + ); + } + private void assertSameGeometry(Geometry expected, Geometry actual) { if (expected.isEmpty() && actual.isEmpty()) { // OK diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java index 240e76a4d1..7543269cc1 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java @@ -11,6 +11,7 @@ import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.archive.TileArchiveWriter; +import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; @@ -40,9 +41,10 @@ class FeatureGroupTest { private final FeatureSort sorter = FeatureSort.newInMemory(); + private final PlanetilerConfig config = PlanetilerConfig.defaults(); private FeatureGroup features = - new FeatureGroup(sorter, TileOrder.TMS, new Profile.NullProfile(), Stats.inMemory()); + new FeatureGroup(sorter, TileOrder.TMS, new Profile.NullProfile(), config, Stats.inMemory()); private CloseableConsumer featureWriter = features.writerForThread(); @Test @@ -90,7 +92,7 @@ private void put(PuTileArgs args) { private Map>> getFeatures() { Map>> map = new TreeMap<>(); for (FeatureGroup.TileFeatures tile : features) { - for (var feature : VectorTile.decode(tile.getVectorTileEncoder().encode())) { + for (var feature : VectorTile.decode(tile.getVectorTile().encode())) { map.computeIfAbsent(tile.tileCoord().encoded(), (i) -> new TreeMap<>()) .computeIfAbsent(feature.layer(), l -> new ArrayList<>()) .add(new Feature(feature.attrs(), decodeSilently(feature.geometry()))); @@ -104,7 +106,7 @@ private Map>> getFeaturesParallel() { Map>> map = new TreeMap<>(); var reader = features.parallelIterator(2); for (FeatureGroup.TileFeatures tile : reader.result()) { - for (var feature : VectorTile.decode(tile.getVectorTileEncoder().encode())) { + for (var feature : VectorTile.decode(tile.getVectorTile().encode())) { map.computeIfAbsent(tile.tileCoord().encoded(), (i) -> new TreeMap<>()) .computeIfAbsent(feature.layer(), l -> new ArrayList<>()) .add(new Feature(feature.attrs(), decodeSilently(feature.geometry()))); @@ -274,7 +276,7 @@ public List postProcessLayerFeatures(String layer, int zoom, Collections.reverse(items); return items; } - }, Stats.inMemory()); + }, config, Stats.inMemory()); featureWriter = features.writerForThread(); putWithGroup( 1, "layer", Map.of("id", 3), newPoint(5, 6), 2, 1, 2 @@ -298,7 +300,7 @@ public List postProcessLayerFeatures(String layer, int zoom, @Test void testHilbertOrdering() { - features = new FeatureGroup(sorter, TileOrder.HILBERT, new Profile.NullProfile() {}, Stats.inMemory()); + features = new FeatureGroup(sorter, TileOrder.HILBERT, new Profile.NullProfile() {}, config, Stats.inMemory()); featureWriter = features.writerForThread(); // Hilbert tile IDs at zoom level 1: @@ -337,7 +339,7 @@ void testHilbertOrdering() { @Test void testTMSOrdering() { - features = new FeatureGroup(sorter, TileOrder.TMS, new Profile.NullProfile() {}, Stats.inMemory()); + features = new FeatureGroup(sorter, TileOrder.TMS, new Profile.NullProfile() {}, config, Stats.inMemory()); featureWriter = features.writerForThread(); // TMS tile IDs at zoom level 1: @@ -447,10 +449,10 @@ void testGenerateContentHash(String testName, boolean expectSame, PuTileArgs arg sorter.sort(); var iter = features.iterator(); var tileHash0 = TileArchiveWriter.generateContentHash( - Gzip.gzip(iter.next().getVectorTileEncoder().encode()) + Gzip.gzip(iter.next().getVectorTile().encode()) ); var tileHash1 = TileArchiveWriter.generateContentHash( - Gzip.gzip(iter.next().getVectorTileEncoder().encode()) + Gzip.gzip(iter.next().getVectorTile().encode()) ); if (expectSame) { assertEquals(tileHash0, tileHash1); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/SourceFeatureProcessorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/SourceFeatureProcessorTest.java index c26f1e19ba..7f36f3efad 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/SourceFeatureProcessorTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/SourceFeatureProcessorTest.java @@ -69,7 +69,8 @@ void testCountFeatures() { void testProcessMultipleInputs() { var profile = new Profile.NullProfile(); var stats = Stats.inMemory(); - var featureGroup = FeatureGroup.newInMemoryFeatureGroup(TileOrder.TMS, profile, stats); + var config = PlanetilerConfig.defaults(); + var featureGroup = FeatureGroup.newInMemoryFeatureGroup(TileOrder.TMS, profile, config, stats); var emittedFeatures = new ArrayList(); var paths = List.of(