diff --git a/README.md b/README.md index ed87f65b6c..36364a15d5 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ for more of the backstory. ## Demo -See the [live demo](https://onthegomap.github.io/planetiler-demo/) of vector tiles created by Planetiler and [hosted by OpenStreetMap US](https://github.com/osmus/tileservice). +See the [live demo](https://onthegomap.github.io/planetiler-demo/) of vector tiles created by Planetiler +and [hosted by OpenStreetMap US](https://github.com/osmus/tileservice). [![Planetiler Demo Screenshot](./diagrams/demo.png)](https://onthegomap.github.io/planetiler-demo/) [© OpenMapTiles](https://www.openmaptiles.org/) [© OpenStreetMap contributors](https://www.openstreetmap.org/copyright) @@ -83,7 +84,7 @@ You will need the full data sources to run anywhere besides Monaco. #### To view tiles locally: -Using [Node.js](https://nodejs.org/en/download/): +Using [Node.js](https://nodejs.org/en/download/package-manager): ```bash npm install -g tileserver-gl-light @@ -102,6 +103,7 @@ Some common arguments: - `--output` tells planetiler where to write output to, and what format to write it in. For example `--output=australia.pmtiles` creates a pmtiles archive named `australia.pmtiles`. + It is best to specify the full path to the file. In docker image you should be using `/data/australia.pmtiles` to let the docker know where to write the file. - `--download` downloads input sources automatically and `--only-download` exits after downloading - `--area=monaco` downloads a `.osm.pbf` extract from [Geofabrik](https://download.geofabrik.de/) - `--osm-path=path/to/file.osm.pbf` points Planetiler at an existing OSM extract on disk diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkLineMerge.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkLineMerge.java new file mode 100644 index 0000000000..05196735cb --- /dev/null +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkLineMerge.java @@ -0,0 +1,114 @@ +package com.onthegomap.planetiler.benchmarks; + +import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.util.Format; +import com.onthegomap.planetiler.util.FunctionThatThrows; +import com.onthegomap.planetiler.util.Gzip; +import com.onthegomap.planetiler.util.LoopLineMerger; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.MathContext; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.operation.linemerge.LineMerger; + +public class BenchmarkLineMerge { + private static int numLines; + + public static void main(String[] args) throws Exception { + for (int i = 0; i < 10; i++) { + time(" JTS", geom -> { + var lm = new LineMerger(); + lm.add(geom); + return lm.getMergedLineStrings(); + }); + time(" loop(0)", geom -> loopMerger(0).add(geom).getMergedLineStrings()); + time(" loop(0.1)", geom -> loopMerger(0.1).add(geom).getMergedLineStrings()); + time("loop(20.0)", geom -> loopMerger(20).add(geom).getMergedLineStrings()); + } + System.err.println(numLines); + } + + private static LoopLineMerger loopMerger(double minLength) { + var lm = new LoopLineMerger(); + lm.setMinLength(minLength); + lm.setStubMinLength(minLength); + lm.setLoopMinLength(minLength); + lm.setTolerance(1); + lm.setMergeStrokes(true); + return lm; + } + + private static void time(String name, FunctionThatThrows> fn) throws Exception { + System.err.println(String.join("\t", + name, + timeMillis(read("mergelines_200433_lines.wkb.gz"), fn), + timeMillis(read("mergelines_239823_lines.wkb.gz"), fn), + "(/s):", + timePerSec(read("mergelines_1759_point_line.wkb.gz"), fn), + timePerSec(makeLines(50, 2), fn), + timePerSec(makeLines(10, 10), fn), + timePerSec(makeLines(2, 50), fn) + )); + } + + private static String timePerSec(Geometry geometry, FunctionThatThrows> fn) + throws Exception { + long start = System.nanoTime(); + long end = start + Duration.ofSeconds(1).toNanos(); + int num = 0; + for (; System.nanoTime() < end;) { + numLines += fn.apply(geometry).size(); + num++; + } + return Format.defaultInstance() + .numeric(Math.round(num * 1d / ((System.nanoTime() - start) * 1d / Duration.ofSeconds(1).toNanos())), true); + } + + private static String timeMillis(Geometry geometry, FunctionThatThrows> fn) + throws Exception { + long start = System.nanoTime(); + long end = start + Duration.ofSeconds(1).toNanos(); + int num = 0; + for (; System.nanoTime() < end;) { + numLines += fn.apply(geometry).size(); + num++; + } + // equivalent of toPrecision(3) + long nanosPer = (System.nanoTime() - start) / num; + var bd = new BigDecimal(nanosPer, new MathContext(3)); + return Format.padRight(Duration.ofNanos(bd.longValue()).toString().replace("PT", ""), 6); + } + + + private static Geometry read(String fileName) throws IOException, ParseException { + var path = Path.of("planetiler-core", "src", "test", "resources", "mergelines", fileName); + byte[] bytes = Gzip.gunzip(Files.readAllBytes(path)); + return new WKBReader().read(bytes); + } + + private static Geometry makeLines(int lines, int parts) { + List result = new ArrayList<>(); + double idx = 0; + for (int i = 0; i < lines; i++) { + Coordinate[] coords = new Coordinate[parts]; + for (int j = 0; j < parts; j++) { + coords[j] = new CoordinateXY(idx, idx); + idx += 0.5; + } + result.add(GeoUtils.JTS_FACTORY.createLineString(coords)); + } + return new GeometryFactory().createMultiLineString(result.toArray(LineString[]::new)); + } +} diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml index ed9403936f..89258c44df 100644 --- a/planetiler-core/pom.xml +++ b/planetiler-core/pom.xml @@ -16,8 +16,8 @@ - 32.0 - 2.24.1 + 32.1 + 2.24.2 0.16.0 4.28.1 @@ -181,6 +181,11 @@ parquet-floor 1.48 + + org.lz4 + lz4-java + 1.8.0 + diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java index 012ddd0e06..4c86a45cbc 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java @@ -4,13 +4,13 @@ import com.carrotsearch.hppc.IntObjectMap; import com.carrotsearch.hppc.IntStack; import com.onthegomap.planetiler.collection.Hppc; -import com.onthegomap.planetiler.geo.DouglasPeuckerSimplifier; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.MutableCoordinateSequence; import com.onthegomap.planetiler.stats.DefaultStats; import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.util.LoopLineMerger; import java.util.ArrayList; import java.util.BitSet; import java.util.Collection; @@ -171,7 +171,12 @@ public static List mergeLineStrings(List if (groupedFeatures.size() == 1 && buffer == 0d && lengthLimit == 0 && (!resimplify || tolerance == 0)) { result.add(feature1); } else { - LineMerger merger = new LineMerger(); + LoopLineMerger merger = new LoopLineMerger() + .setTolerance(tolerance) + .setMergeStrokes(true) + .setMinLength(lengthLimit) + .setLoopMinLength(lengthLimit) + .setStubMinLength(0.5); for (VectorTile.Feature feature : groupedFeatures) { try { merger.add(feature.geometry().decode()); @@ -180,24 +185,14 @@ public static List mergeLineStrings(List } } List outputSegments = new ArrayList<>(); - for (Object merged : merger.getMergedLineStrings()) { - if (merged instanceof LineString line && line.getLength() >= lengthLimit) { - // re-simplify since some endpoints of merged segments may be unnecessary - if (line.getNumPoints() > 2 && tolerance >= 0) { - Geometry simplified = DouglasPeuckerSimplifier.simplify(line, tolerance); - if (simplified instanceof LineString simpleLineString) { - line = simpleLineString; - } else { - LOGGER.warn("line string merge simplify emitted {}", simplified.getGeometryType()); - } - } - if (buffer >= 0) { - removeDetailOutsideTile(line, buffer, outputSegments); - } else { - outputSegments.add(line); - } + for (var line : merger.getMergedLineStrings()) { + if (buffer >= 0) { + removeDetailOutsideTile(line, buffer, outputSegments); + } else { + outputSegments.add(line); } } + if (!outputSegments.isEmpty()) { outputSegments = sortByHilbertIndex(outputSegments); Geometry newGeometry = GeoUtils.combineLineStrings(outputSegments); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index d87fe774d0..47b250879c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -48,6 +48,7 @@ import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.IntStream; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,7 +104,7 @@ public class Planetiler { private boolean overwrite = false; private boolean ran = false; // most common OSM languages - private List languages = List.of( + private List defaultLanguages = List.of( "en", "ru", "ar", "zh", "ja", "ko", "fr", "de", "fi", "pl", "es", "be", "br", "he" ); @@ -547,7 +548,7 @@ public Planetiler addStage(String name, String description, RunnableThatThrows t * @return this runner instance for chaining */ public Planetiler setDefaultLanguages(List languages) { - this.languages = languages; + this.defaultLanguages = languages; return this; } @@ -587,7 +588,13 @@ public Planetiler fetchWikidataNameTranslations(Path defaultWikidataCache) { public Translations translations() { if (translations == null) { boolean transliterate = arguments.getBoolean("transliterate", "attempt to transliterate latin names", true); - List languages = arguments.getList("languages", "languages to use", this.languages); + List languages = arguments.getList("languages", "Languages to include labels for. \"default\" expands to the default set of languages configured by the profile. \"-lang\" excludes \"lang\". \"*\" includes every language not listed.", this.defaultLanguages); + if (languages.contains("default")) { + languages = Stream.concat( + languages.stream().filter(language -> !language.equals("default")), + this.defaultLanguages.stream() + ).toList(); + } translations = Translations.defaultProvider(languages).setShouldTransliterate(transliterate); } return translations; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index 452ee6e269..b72e416851 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -463,6 +463,24 @@ public static int countGeometries(VectorTileProto.Tile.Feature feature) { return result; } + /** + * Returns the encoded geometry for a polygon that fills an entire tile plus {@code buffer} pixels as a shortcut to + * avoid needing to create an extra JTS geometry for encoding. + */ + public static VectorGeometry encodeFill(double buffer) { + int min = (int) Math.round(EXTENT * buffer / 256d); + int width = EXTENT + min + min; + return new VectorGeometry(new int[]{ + CommandEncoder.commandAndLength(Command.MOVE_TO, 1), + zigZagEncode(-min), zigZagEncode(-min), + CommandEncoder.commandAndLength(Command.LINE_TO, 3), + zigZagEncode(width), 0, + 0, zigZagEncode(width), + zigZagEncode(-width), 0, + CommandEncoder.commandAndLength(Command.CLOSE_PATH, 1) + }, GeometryType.POLYGON, 0); + } + /** * Adds features in a layer to this tile. * diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index e304eb6e37..0563485044 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -124,7 +124,7 @@ public static PlanetilerConfig from(Arguments arguments) { int renderMaxzoom = arguments.getInteger("render_maxzoom", "maximum rendering zoom level up to " + MAX_MAXZOOM, Math.max(maxzoom, DEFAULT_MAXZOOM)); - Path tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp")); + Path tmpDir = arguments.file("tmpdir|tmp", "temp directory", Path.of("data", "tmp")); return new PlanetilerConfig( arguments, diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java index db08f14607..d70d14c833 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java @@ -40,12 +40,27 @@ public record MultiExpression(List> expressions) implements Simplifi private static final Logger LOGGER = LoggerFactory.getLogger(MultiExpression.class); private static final Comparator BY_ID = Comparator.comparingInt(WithId::id); + /** + * Returns a new multi-expression from {@code expressions} where multiple expressions for the same key get OR'd + * together. + *

+ * If the order in which expresions match matter, use {@link #ofOrdered(List)} instead. + */ public static MultiExpression of(List> expressions) { LinkedHashMap map = new LinkedHashMap<>(); for (var expression : expressions) { map.merge(expression.result, expression.expression, Expression::or); } - return new MultiExpression<>(map.entrySet().stream().map(e -> entry(e.getKey(), e.getValue())).collect(Collectors.toList())); + return new MultiExpression<>( + map.entrySet().stream().map(e -> entry(e.getKey(), e.getValue())).collect(Collectors.toList())); + } + + /** + * Returns a new multi-expression from {@code expressions} where multiple expressions for the same key stay separate, + * in cases where the order in which expressions matches is important. + */ + public static MultiExpression ofOrdered(List> expressions) { + return new MultiExpression<>(new ArrayList<>(expressions)); } public static Entry entry(T result, Expression expression) { @@ -63,8 +78,8 @@ private static boolean mustAlwaysEvaluate(Expression expression) { case Expression.Not(var child) -> !mustAlwaysEvaluate(child); case Expression.MatchAny any when any.mustAlwaysEvaluate() -> true; case null, default -> !(expression instanceof Expression.MatchAny) && - !(expression instanceof Expression.MatchField) && - !FALSE.equals(expression); + !(expression instanceof Expression.MatchField) && + !FALSE.equals(expression); }; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifier.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifier.java index 6ccb2bca5e..9ff3d0ef10 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifier.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifier.java @@ -11,6 +11,8 @@ */ package com.onthegomap.planetiler.geo; +import java.util.ArrayList; +import java.util.List; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.Geometry; @@ -45,6 +47,22 @@ public static Geometry simplify(Geometry geom, double distanceTolerance) { return (new DPTransformer(distanceTolerance)).transform(geom); } + /** + * Returns a copy of {@code coords}, simplified using Douglas Peucker Algorithm. + * + * @param coords the coordinate list to simplify + * @param distanceTolerance the threshold below which we discard points + * @param area true if this is a polygon to retain at least 4 points to avoid collapse + * @return the simplified coordinate list + */ + public static List simplify(List coords, double distanceTolerance, boolean area) { + if (coords.isEmpty()) { + return List.of(); + } + + return (new DPTransformer(distanceTolerance)).transformCoordinateList(coords, area); + } + private static class DPTransformer extends GeometryTransformer { private final double sqTolerance; @@ -84,6 +102,42 @@ private static double getSqSegDist(double px, double py, double p1x, double p1y, return dx * dx + dy * dy; } + private void subsimplify(List in, List out, int first, int last, int numForcedPoints) { + // numForcePoints lets us keep some points even if they are below simplification threshold + boolean force = numForcedPoints > 0; + double maxSqDist = force ? -1 : sqTolerance; + int index = -1; + Coordinate p1 = in.get(first); + Coordinate p2 = in.get(last); + double p1x = p1.x; + double p1y = p1.y; + double p2x = p2.x; + double p2y = p2.y; + + int i = first + 1; + Coordinate furthest = null; + for (Coordinate coord : in.subList(first + 1, last)) { + double sqDist = getSqSegDist(coord.x, coord.y, p1x, p1y, p2x, p2y); + + if (sqDist > maxSqDist) { + index = i; + furthest = coord; + maxSqDist = sqDist; + } + i++; + } + + if (force || maxSqDist > sqTolerance) { + if (index - first > 1) { + subsimplify(in, out, first, index, numForcedPoints - 1); + } + out.add(furthest); + if (last - index > 1) { + subsimplify(in, out, index, last, numForcedPoints - 2); + } + } + } + private void subsimplify(CoordinateSequence in, MutableCoordinateSequence out, int first, int last, int numForcedPoints) { // numForcePoints lets us keep some points even if they are below simplification threshold @@ -117,6 +171,20 @@ private void subsimplify(CoordinateSequence in, MutableCoordinateSequence out, i } } + protected List transformCoordinateList(List coords, boolean area) { + if (coords.isEmpty()) { + return coords; + } + // make sure we include the first and last points even if they are closer than the simplification threshold + List result = new ArrayList<>(); + result.add(coords.getFirst()); + // for polygons, additionally keep at least 2 intermediate points even if they are below simplification threshold + // to avoid collapse. + subsimplify(coords, result, 0, coords.size() - 1, area ? 2 : 0); + result.add(coords.getLast()); + return result; + } + @Override protected CoordinateSequence transformCoordinates(CoordinateSequence coords, Geometry parent) { boolean area = parent instanceof LinearRing; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java index dc6c925552..ddfad722e1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java @@ -22,6 +22,9 @@ import java.util.function.IntUnaryOperator; import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import net.jpountz.lz4.LZ4Exception; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; import org.locationtech.jts.geom.Envelope; /** @@ -74,8 +77,24 @@ private static byte[] readBlobContent(Fileformat.Blob blob) { throw new FileFormatException("PBF blob contains incomplete compressed data."); } inflater.end(); + } else if (blob.hasLz4Data()) { + final int decompressedLength = blob.getRawSize(); + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4FastDecompressor decompressor = factory.fastDecompressor(); + blobData = new byte[decompressedLength]; + try { + int compressedBytesRead = + decompressor.decompress(blob.getLz4Data().toByteArray(), 0, blobData, 0, decompressedLength); + int compressedBytesExpected = blob.getLz4Data().size(); + if (compressedBytesRead != compressedBytesExpected) { + throw new FileFormatException("Unable to decompress PBF blob. read %d compressed bytes but expected %d" + .formatted(decompressedLength, compressedBytesExpected)); + } + } catch (LZ4Exception e) { + throw new FileFormatException("Unable to decompress PBF blob.", e); + } } else { - throw new FileFormatException("PBF blob uses unsupported compression, only raw or zlib may be used."); + throw new FileFormatException("PBF blob uses unsupported compression, only lz4, zlib, or raw may be used."); } return blobData; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java index 930e21972f..bf1acd0390 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java @@ -30,7 +30,6 @@ import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.Polygonal; -import org.locationtech.jts.geom.impl.PackedCoordinateSequence; import org.locationtech.jts.geom.util.AffineTransformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,14 +40,6 @@ */ public class FeatureRenderer implements Consumer, Closeable { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureRenderer.class); - private static final VectorTile.VectorGeometry FILL = VectorTile.encodeGeometry(GeoUtils.JTS_FACTORY - .createPolygon(GeoUtils.JTS_FACTORY.createLinearRing(new PackedCoordinateSequence.Double(new double[]{ - -5, -5, - 261, -5, - 261, 261, - -5, 261, - -5, -5 - }, 2, 0)))); private final PlanetilerConfig config; private final Consumer consumer; private final Stats stats; @@ -293,13 +284,13 @@ private void writeTileFeatures(int zoom, long id, FeatureCollector.Feature featu // polygons that span multiple tiles contain detail about the outer edges separate from the filled tiles, so emit // filled tiles now if (feature.isPolygon()) { - emitted += emitFilledTiles(id, feature, sliced); + emitted += emitFilledTiles(zoom, id, feature, sliced); } stats.emittedFeatures(zoom, feature.getLayer(), emitted); } - private int emitFilledTiles(long id, FeatureCollector.Feature feature, TiledGeometry sliced) { + private int emitFilledTiles(int zoom, long id, FeatureCollector.Feature feature, TiledGeometry sliced) { Optional groupInfo = Optional.empty(); /* * Optimization: large input polygons that generate many filled interior tiles (i.e. the ocean), the encoder avoids @@ -309,7 +300,7 @@ private int emitFilledTiles(long id, FeatureCollector.Feature feature, TiledGeom VectorTile.Feature vectorTileFeature = new VectorTile.Feature( feature.getLayer(), id, - FILL, + VectorTile.encodeFill(feature.getBufferPixelsAtZoom(zoom)), feature.getAttrsAtZoom(sliced.zoomLevel()) ); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LoopLineMerger.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LoopLineMerger.java new file mode 100644 index 0000000000..865c96de04 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LoopLineMerger.java @@ -0,0 +1,609 @@ +package com.onthegomap.planetiler.util; + +import com.onthegomap.planetiler.geo.DouglasPeuckerSimplifier; +import com.onthegomap.planetiler.geo.GeoUtils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import org.locationtech.jts.algorithm.Angle; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryComponentFilter; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.PrecisionModel; +import org.locationtech.jts.operation.linemerge.LineMerger; + +/** + * A utility class for merging, simplifying, and connecting linestrings and removing small loops. + *

+ * Compared to JTS {@link LineMerger} which only connects when 2 lines meet at a single point, this utility: + *

    + *
  • snap-rounds points to a grid + *
  • splits lines that intersect at a midpoint + *
  • breaks small loops less than {@code loopMinLength} so only the shortest path connects both endpoints of the loop + *
  • removes short "hair" edges less than {@code stubMinLength} coming off the side of longer segments + *
  • simplifies linestrings, without touching the points shared between multiple lines to avoid breaking connections + *
  • removes any duplicate edges + *
  • at any remaining 3+ way intersections, connect pairs of edges that form the straightest path through the node + *
  • remove any remaining edges shorter than {@code minLength} + *
+ * + * @see Improving Linestring + * Merging in Planetiler + */ +public class LoopLineMerger { + private final List input = new ArrayList<>(); + private final List output = new ArrayList<>(); + private int numNodes = 0; + private int numEdges = 0; + private PrecisionModel precisionModel = new PrecisionModel(GeoUtils.TILE_PRECISION); + private GeometryFactory factory = new GeometryFactory(precisionModel); + private double minLength = 0.0; + private double loopMinLength = 0.0; + private double stubMinLength = 0.0; + private double tolerance = -1.0; + private boolean mergeStrokes = false; + + /** + * Sets the precision model used to snap points to a grid. + *

+ * Use {@link PrecisionModel#FLOATING} to not snap points at all, or {@code new PrecisionModel(4)} to snap to a 0.25px + * grid. + */ + public LoopLineMerger setPrecisionModel(PrecisionModel precisionModel) { + this.precisionModel = precisionModel; + factory = new GeometryFactory(precisionModel); + return this; + } + + /** + * Sets the minimum length for retaining linestrings in the resulting geometry. + *

+ * Linestrings shorter than this value will be removed. {@code minLength <= 0} disables min length removal. + */ + public LoopLineMerger setMinLength(double minLength) { + this.minLength = minLength; + return this; + } + + /** + * Sets the minimum loop length for breaking loops in the merged geometry. + *

+ * Loops that are shorter than loopMinLength are broken up so that only the shortest path between loop endpoints + * remains. This should be {@code >= minLength}. {@code loopMinLength <= 0} disables loop removal. + */ + public LoopLineMerger setLoopMinLength(double loopMinLength) { + this.loopMinLength = loopMinLength; + return this; + } + + /** + * Sets the minimum length of stubs to be removed during processing. + *

+ * Stubs are short "hair" line segments that hang off of a longer linestring without connecting to anything else. + * {@code stubMinLength <= 0} disables stub removal. + */ + public LoopLineMerger setStubMinLength(double stubMinLength) { + this.stubMinLength = stubMinLength; + return this; + } + + /** + * Sets the tolerance for simplifying linestrings during processing. Lines are simplified between endpoints to avoid + * breaking intersections. + *

+ * {@code tolerance = 0} still removes collinear points, so you need to set {@code tolerance <= 0} to disable + * simplification. + */ + public LoopLineMerger setTolerance(double tolerance) { + this.tolerance = tolerance; + return this; + } + + /** + * Enables or disables stroke merging. Stroke merging connects the straightest pairs of linestrings at junctions with + * 3 or more attached linestrings based on the angle between them. + */ + public LoopLineMerger setMergeStrokes(boolean mergeStrokes) { + this.mergeStrokes = mergeStrokes; + return this; + } + + /** + * Adds a geometry to the merger. Only linestrings from the input geometry are considered. + */ + public LoopLineMerger add(Geometry geometry) { + geometry.apply((GeometryComponentFilter) component -> { + if (component instanceof LineString lineString) { + input.add(lineString); + } + }); + return this; + } + + private void degreeTwoMerge() { + for (var node : output) { + degreeTwoMerge(node); + } + output.removeIf(node -> node.getEdges().isEmpty()); + assert valid(); + } + + private boolean valid() { + // when run from a unit test, ensure some basic conditions always hold... + for (var node : output) { + for (var edge : node.getEdges()) { + assert edge.isLoop() || edge.to.getEdges().contains(edge.reversed) : edge.to + " does not contain " + + edge.reversed; + for (var other : node.getEdges()) { + if (edge != other) { + assert edge != other.reversed : "node contained edge and its reverse " + node; + assert !edge.coordinates.equals(other.coordinates) : "duplicate edges " + edge + " and " + other; + } + } + } + assert node.getEdges().size() != 2 || node.getEdges().stream().anyMatch(Edge::isLoop) : "degree 2 node found " + + node; + } + return true; + } + + private Edge degreeTwoMerge(Node node) { + if (node.getEdges().size() == 2) { + Edge a = node.getEdges().getFirst(); + Edge b = node.getEdges().get(1); + // if one side is a loop, degree is actually > 2 + if (!a.isLoop() && !b.isLoop()) { + return mergeTwoEdges(node, a, b); + } + } + return null; + } + + private Edge mergeTwoEdges(Node node, Edge edge1, Edge edge2) { + // attempt to preserve segment directions from the original line + // when: A << N -- B then output C reversed from B to A + // when: A >> N -- B then output C from A to B + Edge a = edge1.main ? edge2 : edge1; + Edge b = edge1.main ? edge1 : edge2; + node.getEdges().remove(a); + node.getEdges().remove(b); + List coordinates = new ArrayList<>(); + coordinates.addAll(a.coordinates.reversed()); + coordinates.addAll(b.coordinates.subList(1, b.coordinates.size())); + Edge c = new Edge(a.to, b.to, coordinates, a.length + b.length); + a.to.removeEdge(a.reversed); + b.to.removeEdge(b.reversed); + a.to.addEdge(c); + if (a.to != b.to) { + b.to.addEdge(c.reversed); + } + return c; + } + + private void strokeMerge() { + for (var node : output) { + List edges = List.copyOf(node.getEdges()); + if (edges.size() >= 2) { + record AngledPair(Edge a, Edge b, double angle) {} + List angledPairs = new ArrayList<>(); + for (var i = 0; i < edges.size(); ++i) { + var edgei = edges.get(i); + for (var j = i + 1; j < edges.size(); ++j) { + var edgej = edges.get(j); + if (edgei != edgej.reversed) { + double angle = edgei.angleTo(edgej); + angledPairs.add(new AngledPair(edgei, edgej, angle)); + } + } + } + angledPairs.sort(Comparator.comparingDouble(angledPair -> angledPair.angle)); + List merged = new ArrayList<>(); + for (var angledPair : angledPairs.reversed()) { + if (merged.contains(angledPair.a) || merged.contains(angledPair.b)) { + continue; + } + mergeTwoEdges(angledPair.a.from, angledPair.a, angledPair.b); + merged.add(angledPair.a); + merged.add(angledPair.b); + } + } + } + } + + private void breakLoops() { + for (var node : output) { + if (node.getEdges().size() <= 1) { + continue; + } + for (var current : List.copyOf(node.getEdges())) { + record HasLoop(Edge edge, double distance) {} + List loops = new ArrayList<>(); + if (!node.getEdges().contains(current)) { + continue; + } + for (var other : node.getEdges()) { + double distance = other.length + + shortestDistanceAStar(other.to, current.to, current.from, loopMinLength - other.length); + if (distance <= loopMinLength) { + loops.add(new HasLoop(other, distance)); + } + } + if (loops.size() > 1) { + HasLoop min = loops.stream().min(Comparator.comparingDouble(HasLoop::distance)).get(); + for (var loop : loops) { + if (loop != min) { + loop.edge.remove(); + } + } + } + } + } + } + + private double shortestDistanceAStar(Node start, Node end, Node exclude, double maxLength) { + Map bestDistance = new HashMap<>(); + record Candidate(Node node, double length, double minTotalLength) {} + PriorityQueue frontier = new PriorityQueue<>(Comparator.comparingDouble(Candidate::minTotalLength)); + if (exclude != start) { + frontier.offer(new Candidate(start, 0, start.distance(end))); + } + while (!frontier.isEmpty()) { + Candidate candidate = frontier.poll(); + Node current = candidate.node; + if (current == end) { + return candidate.length; + } + + for (var edge : current.getEdges()) { + var neighbor = edge.to; + if (neighbor != exclude) { + double newDist = candidate.length + edge.length; + double prev = bestDistance.getOrDefault(neighbor.id, Double.POSITIVE_INFINITY); + if (newDist < prev) { + bestDistance.put(neighbor.id, newDist); + double minTotalLength = newDist + neighbor.distance(end); + if (minTotalLength <= maxLength) { + frontier.offer(new Candidate(neighbor, newDist, minTotalLength)); + } + } + } + } + } + return Double.POSITIVE_INFINITY; + } + + private void removeShortStubEdges() { + PriorityQueue toCheck = new PriorityQueue<>(Comparator.comparingDouble(Edge::length)); + for (var node : output) { + for (var edge : node.getEdges()) { + if (isShortStubEdge(edge)) { + toCheck.offer(edge); + } + } + } + while (!toCheck.isEmpty()) { + var edge = toCheck.poll(); + if (edge.removed) { + continue; + } + if (isShortStubEdge(edge)) { + edge.remove(); + } + if (degreeTwoMerge(edge.from) instanceof Edge merged) { + toCheck.offer(merged); + } + if (edge.from.getEdges().size() == 1) { + var other = edge.from.getEdges().getFirst(); + if (isShortStubEdge(other)) { + toCheck.offer(other); + } + } + if (edge.from != edge.to) { + if (degreeTwoMerge(edge.to) instanceof Edge merged) { + toCheck.offer(merged); + } + if (edge.to.getEdges().size() == 1) { + var other = edge.to.getEdges().getFirst(); + if (isShortStubEdge(other)) { + toCheck.offer(other); + } + } + } + } + } + + private boolean isShortStubEdge(Edge edge) { + return edge != null && !edge.removed && edge.length < stubMinLength && + (edge.from.getEdges().size() == 1 || edge.to.getEdges().size() == 1 || edge.isLoop()); + } + + private void removeShortEdges() { + for (var node : output) { + for (var edge : List.copyOf(node.getEdges())) { + if (edge.length < minLength) { + edge.remove(); + } + } + } + } + + private void simplify() { + List toRemove = new ArrayList<>(); + for (var node : output) { + for (var edge : node.getEdges()) { + if (edge.main) { + edge.simplify(); + if (edge.isCollapsed()) { + toRemove.add(edge); + } + } + } + } + toRemove.forEach(Edge::remove); + } + + private void removeDuplicatedEdges() { + for (var node : output) { + List toRemove = new ArrayList<>(); + for (var i = 0; i < node.getEdges().size(); ++i) { + Edge a = node.getEdges().get(i); + for (var j = i + 1; j < node.getEdges().size(); ++j) { + Edge b = node.getEdges().get(j); + if (b.to == a.to && a.coordinates.equals(b.coordinates)) { + toRemove.add(b); + } + } + } + for (var edge : toRemove) { + edge.remove(); + } + } + } + + /** + * Processes the added geometries and returns the merged linestrings. + *

+ * Can be called more than once. + */ + public List getMergedLineStrings() { + output.clear(); + List> edges = nodeLines(input); + buildNodes(edges); + + degreeTwoMerge(); + + if (loopMinLength > 0.0) { + breakLoops(); + degreeTwoMerge(); + } + + if (stubMinLength > 0.0) { + removeShortStubEdges(); + // removeShortStubEdges does degreeTwoMerge internally + } + + if (tolerance >= 0.0) { + simplify(); + removeDuplicatedEdges(); + degreeTwoMerge(); + } + + if (mergeStrokes) { + strokeMerge(); + degreeTwoMerge(); + } + + if (minLength > 0) { + removeShortEdges(); + } + + List result = new ArrayList<>(); + + for (var node : output) { + for (var edge : node.getEdges()) { + if (edge.main) { + result.add(factory.createLineString(edge.coordinates.toArray(Coordinate[]::new))); + } + } + } + + return result; + } + + private static double length(List edge) { + Coordinate last = null; + double length = 0; + for (Coordinate coord : edge) { + if (last != null) { + length += last.distance(coord); + } + last = coord; + } + return length; + } + + private void buildNodes(List> edges) { + Map nodes = new HashMap<>(); + for (var coordinateSequence : edges) { + Coordinate first = coordinateSequence.getFirst(); + Node firstNode = nodes.get(first); + if (firstNode == null) { + firstNode = new Node(first); + nodes.put(first, firstNode); + output.add(firstNode); + } + + Coordinate last = coordinateSequence.getLast(); + Node lastNode = nodes.get(last); + if (lastNode == null) { + lastNode = new Node(last); + nodes.put(last, lastNode); + output.add(lastNode); + } + + double length = length(coordinateSequence); + + Edge edge = new Edge(firstNode, lastNode, coordinateSequence, length); + + firstNode.addEdge(edge); + if (firstNode != lastNode) { + lastNode.addEdge(edge.reversed); + } + } + } + + private List> nodeLines(List input) { + Map nodeCounts = new HashMap<>(); + List> coords = new ArrayList<>(input.size()); + for (var line : input) { + var coordinateSequence = line.getCoordinateSequence(); + List snapped = new ArrayList<>(); + Coordinate last = null; + for (int i = 0; i < coordinateSequence.size(); i++) { + Coordinate current = new CoordinateXY(coordinateSequence.getX(i), coordinateSequence.getY(i)); + precisionModel.makePrecise(current); + if (last == null || !last.equals(current)) { + snapped.add(current); + nodeCounts.merge(current, 1, Integer::sum); + } + last = current; + } + if (snapped.size() >= 2) { + coords.add(snapped); + } + } + + List> result = new ArrayList<>(input.size()); + for (var coordinateSequence : coords) { + int start = 0; + for (int i = 0; i < coordinateSequence.size(); i++) { + Coordinate coordinate = coordinateSequence.get(i); + if (i > 0 && i < coordinateSequence.size() - 1 && nodeCounts.get(coordinate) > 1) { + result.add(coordinateSequence.subList(start, i + 1)); + start = i; + } + } + if (start < coordinateSequence.size()) { + var sublist = start == 0 ? coordinateSequence : coordinateSequence.subList(start, coordinateSequence.size()); + result.add(sublist); + } + } + return result; + } + + private class Node { + final int id = numNodes++; + final List edge = new ArrayList<>(); + Coordinate coordinate; + + Node(Coordinate coordinate) { + this.coordinate = coordinate; + } + + void addEdge(Edge edge) { + for (Edge other : this.edge) { + if (other.coordinates.equals(edge.coordinates)) { + return; + } + } + this.edge.add(edge); + } + + List getEdges() { + return edge; + } + + void removeEdge(Edge edge) { + this.edge.remove(edge); + } + + @Override + public String toString() { + return "Node{" + id + ": " + edge + '}'; + } + + double distance(Node end) { + return coordinate.distance(end.coordinate); + } + } + + private class Edge { + + final int id; + final Node from; + final Node to; + final double length; + final boolean main; + boolean removed; + + Edge reversed; + List coordinates; + + + private Edge(Node from, Node to, List coordinateSequence, double length) { + this(numEdges, from, to, length, coordinateSequence, true, null); + reversed = new Edge(numEdges, to, from, length, coordinateSequence.reversed(), false, this); + numEdges++; + } + + private Edge(int id, Node from, Node to, double length, List coordinates, boolean main, Edge reversed) { + this.id = id; + this.from = from; + this.to = to; + this.length = length; + this.coordinates = coordinates; + this.main = main; + this.reversed = reversed; + } + + void remove() { + if (!removed) { + from.removeEdge(this); + to.removeEdge(reversed); + removed = true; + } + } + + double angleTo(Edge other) { + assert from.equals(other.from); + assert coordinates.size() >= 2; + + double angle = Angle.angle(coordinates.get(0), coordinates.get(1)); + double angleOther = Angle.angle(other.coordinates.get(0), other.coordinates.get(1)); + + return Math.abs(Angle.normalize(angle - angleOther)); + } + + double length() { + return length; + } + + void simplify() { + coordinates = DouglasPeuckerSimplifier.simplify(coordinates, tolerance, false); + if (reversed != null) { + reversed.coordinates = coordinates.reversed(); + } + } + + boolean isCollapsed() { + return coordinates.size() < 2 || + (coordinates.size() == 2 && coordinates.getFirst().equals(coordinates.getLast())); + } + + boolean isLoop() { + return from == to; + } + + @Override + public String toString() { + return "Edge{" + from.id + "->" + to.id + (main ? "" : "(R)") + ": [" + coordinates.getFirst() + ".." + + coordinates.getLast() + "], length=" + length + '}'; + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java index 7e3c60b0f8..85c87deecf 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Translations.java @@ -22,15 +22,31 @@ public class Translations { ThreadLocal.withInitial(() -> new ThreadLocalTransliterator().getInstance("Any-Latin")); private boolean shouldTransliterate = true; - private final Set languageSet; private final List providers = new ArrayList<>(); + private final Set includeLanguages; + private final Set excludeLanguages; + private boolean defaultInclude = false; private Translations(List languages) { - this.languageSet = new HashSet<>(); + this.includeLanguages = new HashSet<>(); + this.excludeLanguages = new HashSet<>(); + for (String language : languages) { + if (language.equals("*")) { + defaultInclude = true; + continue; + } + + boolean include = true; + if (language.startsWith("-")) { + language = language.replaceFirst("^-", ""); + include = false; + } + String withoutPrefix = language.replaceFirst("^name:", ""); - languageSet.add(withoutPrefix); - languageSet.add("name:" + withoutPrefix); + Set set = include ? this.includeLanguages : this.excludeLanguages; + set.add(withoutPrefix); + set.add("name:" + withoutPrefix); } } @@ -85,7 +101,7 @@ public void addTranslations(Map output, Map inpu if (translations != null && !translations.isEmpty()) { for (var entry : translations.entrySet()) { String key = entry.getKey(); - if (languageSet.contains(key)) { + if (careAboutLanguage(key)) { output.putIfAbsent(key.startsWith("name:") ? key : "name:" + key, entry.getValue()); } } @@ -105,7 +121,11 @@ public Translations setShouldTransliterate(boolean shouldTransliterate) { /** Returns true if {@code language} is in the set of language translations to use. */ public boolean careAboutLanguage(String language) { - return languageSet.contains(language); + if (excludeLanguages.contains(language)) + return false; + if (includeLanguages.contains(language)) + return true; + return defaultInclude; } /** A source of name translations. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java index 18ae737f0a..cc89fbc655 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java @@ -62,8 +62,8 @@ public class Wikidata { private static final ObjectMapper objectMapper = new ObjectMapper(); private static final Logger LOGGER = LoggerFactory.getLogger(Wikidata.class); - private static final Pattern wikidataIRIMatcher = Pattern.compile("http://www.wikidata.org/entity/Q([0-9]+)"); - private static final Pattern qidPattern = Pattern.compile("Q([0-9]+)"); + private static final Pattern wikidataIRIMatcher = Pattern.compile("http://www.wikidata.org/entity/Q(\\d+)"); + private static final Pattern qidPattern = Pattern.compile("Q(\\d+)"); private final Counter.Readable blocks = Counter.newMultiThreadCounter(); private final Counter.Readable nodes = Counter.newMultiThreadCounter(); private final Counter.Readable ways = Counter.newMultiThreadCounter(); @@ -128,7 +128,7 @@ public static void fetch(OsmInputFile infile, Path outfile, PlanetilerConfig con var timer = stats.startStage("wikidata"); int processThreads = Math.max(1, config.threads() - 1); - LOGGER.info("Starting with " + processThreads + " process threads"); + LOGGER.info("Starting with {} process threads", processThreads); WikidataTranslations oldMappings = load(outfile, maxAge, updateLimit); try ( @@ -166,7 +166,7 @@ public static void fetch(OsmInputFile infile, Path outfile, PlanetilerConfig con .addPipelineStats(pipeline); pipeline.awaitAndLog(loggers, config.logInterval()); - LOGGER.info("DONE fetched:" + fetcher.wikidatas.get()); + LOGGER.info("DONE fetched: {}", fetcher.wikidatas.get()); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -192,10 +192,14 @@ private static WikidataTranslations load(Path path, Duration maxAge, int updateL try (BufferedReader fis = Files.newBufferedReader(path)) { WikidataTranslations result = load(fis, maxAge, updateLimit, Clock.systemUTC()); LOGGER.info( - "loaded from " + result.getAll().size() + " mappings from " + path.toAbsolutePath() + " in " + timer.stop()); + "loaded from {} mappings from {} in {}", + result.getAll().size(), + path.toAbsolutePath(), + timer.stop() + ); return result; } catch (IOException e) { - LOGGER.info("error loading " + path.toAbsolutePath() + ": " + e); + LOGGER.info("error loading {}: {}", path.toAbsolutePath(), e); return new WikidataTranslations(); } } @@ -247,7 +251,7 @@ private static long extractIdFromWikidataIRI(String iri) { String idText = matcher.group(1); return Long.parseLong(idText); } else { - throw new RuntimeException("Unexpected response IRI: " + iri); + throw new IllegalStateException("Unexpected response IRI: " + iri); } } @@ -277,14 +281,13 @@ private void filter(Iterable prev, Consumer next) { blockRelations++; } Object wikidata = elem.getString("wikidata"); - if (wikidata instanceof String wikidataString) { - if (profile.caresAboutWikidataTranslation(elem)) { - long qid = parseQid(wikidataString); - if (qid > 0) { - next.accept(qid); - } + if (wikidata instanceof String wikidataString && profile.caresAboutWikidataTranslation(elem)) { + long qid = parseQid(wikidataString); + if (qid > 0) { + next.accept(qid); } } + } blocks.inc(); nodes.incBy(blockNodes); @@ -356,12 +359,12 @@ private LongObjectMap> queryWikidata(List qidsToFetch) } catch (IOException e) { boolean lastTry = i == config.httpRetries(); if (!lastTry) { - LOGGER.warn("sparql query failed, retrying: " + e); + LOGGER.warn("sparql query failed, retrying: {}", e.toString()); } else { - LOGGER.error("sparql query failed, exhausted retries: " + e); + LOGGER.error("sparql query failed, exhausted retries: {}", e.toString()); throw e; } - Thread.sleep(config.httpRetryWait()); + sleep(config.httpRetryWait()); } } @@ -372,11 +375,15 @@ private LongObjectMap> queryWikidata(List qidsToFetch) } } + protected void sleep(Duration duration) throws InterruptedException { + Thread.sleep(duration.toMillis()); + } + void loadExisting(WikidataTranslations oldMappings) throws IOException { LongObjectMap> alreadyHave = oldMappings.getAll(); LongObjectMap alreadyHaveUpdateTimes = oldMappings.getUpdateTimes(); if (!alreadyHave.isEmpty()) { - LOGGER.info("skipping " + alreadyHave.size() + " mappings we already have"); + LOGGER.info("skipping {} mappings we already have", alreadyHave.size()); writeTranslations(alreadyHave, alreadyHaveUpdateTimes); for (LongObjectCursor> cursor : alreadyHave) { visited.add(cursor.key); @@ -405,12 +412,12 @@ private void writeTranslations(LongObjectMap> results, LongO interface Client { static Client wrap(HttpClient client) { - return (req) -> { + return req -> { var response = client.send(req, BodyHandlers.ofInputStream()); if (response.statusCode() >= 400) { String body; - try { - body = new String(response.body().readAllBytes(), StandardCharsets.UTF_8); + try (var responseBody = response.body()) { + body = new String(responseBody.readAllBytes(), StandardCharsets.UTF_8); } catch (IOException e) { body = "Error reading body: " + e; } @@ -435,8 +442,6 @@ public static class WikidataTranslations implements Translations.TranslationProv private final LongObjectMap> data = Hppc.newLongObjectHashMap(); private final LongObjectMap updateTimes = Hppc.newLongObjectHashMap(); - public WikidataTranslations() {} - /** Returns a map from language code to translated name for {@code qid}. */ public Map get(long qid) { return data.get(qid); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java index 5a1b713ac0..e27feda284 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureMergeTest.java @@ -638,6 +638,7 @@ void testExtractConnectedComponents() { ); } + @Slow @ParameterizedTest @CsvSource({ "bostonbuildings.mbtiles, 2477, 3028, 13, 1141", diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 9359047427..eb6b9ba5ce 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -73,6 +73,7 @@ /** * In-memory tests with fake data and profiles to ensure all features work end-to-end. */ +@Slow class PlanetilerTests { private static final Logger LOGGER = LoggerFactory.getLogger(PlanetilerTests.class); @@ -782,7 +783,7 @@ void testPolygonWithHoleSpanningMultipleTiles(boolean anyGeom) throws Exception ), Map.of()) )), newTileEntry(Z14_TILES / 2 + 2, Z14_TILES / 2 + 1, 14, List.of( - feature(newPolygon(tileFill(5), List.of()), Map.of()) + feature(newPolygon(tileFill(4), List.of()), Map.of()) )), newTileEntry(Z14_TILES / 2 + 3, Z14_TILES / 2 + 1, 14, List.of( feature(tileLeft(4), Map.of()) @@ -826,7 +827,7 @@ void testZ15Fill() throws Exception { ); assertEquals(List.of( - feature(newPolygon(tileFill(5)), Map.of()) + feature(newPolygon(tileFill(4)), Map.of()) ), results.tiles.get(TileCoord.ofXYZ(Z15_TILES / 2, Z15_TILES / 2, 15))); } @@ -844,7 +845,7 @@ void testFullWorldPolygon() throws Exception { assertEquals(5461, results.tiles.size()); // spot-check one filled tile - assertEquals(List.of(rectangle(-5, 256 + 5).norm()), results.tiles.get(TileCoord.ofXYZ( + assertEquals(List.of(rectangle(-4, 256 + 4).norm()), results.tiles.get(TileCoord.ofXYZ( Z4_TILES / 2, Z4_TILES / 2, 4 )).stream().map(d -> d.geometry().geom().norm()).toList()); } @@ -2099,10 +2100,13 @@ private static TileCompression extractTileCompression(String args) { "--tile-compression=none", "--tile-compression=gzip", "--output-layerstats", - "--max-point-buffer=1" + "--max-point-buffer=1", + "--osm-test-path=monaco-latest.lz4.osm.pbf", }) void testPlanetilerRunner(String args) throws Exception { - Path originalOsm = TestUtils.pathToResource("monaco-latest.osm.pbf"); + var argParsed = Arguments.fromArgs(args.split(" ")); + Path originalOsm = + TestUtils.pathToResource(argParsed.getString("osm-test-path", "osm-test-path", "monaco-latest.osm.pbf")); Path tempOsm = tempDir.resolve("monaco-temp.osm.pbf"); final TileCompression tileCompression = extractTileCompression(args); @@ -2461,7 +2465,7 @@ private PlanetilerResults runForCompactTest(boolean compactDbEnabled) throws Exc ), (in, features) -> features.polygon("layer") .setZoomRange(0, 2) - .setBufferPixels(0) + .setBufferPixels(1) ); } @@ -2729,6 +2733,17 @@ void testBoundFiltersFill() throws Exception { assertTrue(polyResultz8.tiles.containsKey(TileCoord.ofXYZ(z8tiles * 3 / 4, z8tiles * 7 / 8, 8))); } + @Test + void testDefaultLanguages() { + var planetiler = Planetiler.create(Arguments.of("languages", "default,en")) + .setDefaultLanguages(List.of("jbo", "tlh")); + var translations = planetiler.translations(); + assertTrue(translations.careAboutLanguage("jbo")); + assertTrue(translations.careAboutLanguage("tlh")); + assertTrue(translations.careAboutLanguage("en")); + assertFalse(translations.careAboutLanguage("fr")); + } + @FunctionalInterface private interface ReadableTileArchiveFactory { ReadableTileArchive create(Path p) throws IOException; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/Slow.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/Slow.java new file mode 100644 index 0000000000..241462a873 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/Slow.java @@ -0,0 +1,14 @@ +package com.onthegomap.planetiler; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Tag; + +/** Add to any junit test classes or methods to exclude when run with {@code -Pfast} maven argument. */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Tag("slow") +@Retention(RetentionPolicy.RUNTIME) +public @interface Slow { +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java index 203ac7e41d..d3f9346958 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java @@ -692,6 +692,18 @@ void firstCoordinateOfPolygon(double x, double y) { } } + @ParameterizedTest + @CsvSource({ + "0, 0, 256", + "1, -1, 257", + "10, -10, 266", + }) + void testFill(double buffer, double min, double max) throws GeometryException { + var geom = VectorTile.encodeFill(buffer); + assertSameGeometry(rectangle(min, max), geom.decode()); + assertArrayEquals(VectorTile.encodeGeometry(rectangle(min, max)).commands(), geom.commands()); + } + private static void assertArrayEquals(int[] a, int[] b) { assertEquals( IntStream.of(a).boxed().toList(), diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/AppendStoreTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/AppendStoreTest.java index 0c2b38f5dd..626e339b84 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/AppendStoreTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/AppendStoreTest.java @@ -3,22 +3,29 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.io.IOException; import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -class AppendStoreTest { +abstract class AppendStoreTest { - static abstract class IntsTest { + abstract static class IntsTest { protected AppendStore.Ints store; + @AfterEach + void close() throws IOException { + store.close(); + } + @ParameterizedTest @ValueSource(ints = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - public void writeThenRead(int num) { + void writeThenRead(int num) { for (int i = 0; i < num; i++) { store.appendInt(i + 1); } @@ -30,7 +37,7 @@ public void writeThenRead(int num) { } @Test - public void readBig() { + void readBig() { store.appendInt(Integer.MAX_VALUE); store.appendInt(Integer.MAX_VALUE - 1); store.appendInt(Integer.MAX_VALUE - 2); @@ -40,13 +47,18 @@ public void readBig() { } } - static abstract class LongsTest { + abstract static class LongsTest { protected AppendStore.Longs store; + @AfterEach + void close() throws IOException { + store.close(); + } + @ParameterizedTest @ValueSource(ints = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) - public void writeThenRead(int num) { + void writeThenRead(int num) { for (int i = 0; i < num; i++) { store.appendLong(i + 1); } @@ -57,87 +69,88 @@ public void writeThenRead(int num) { assertThrows(IndexOutOfBoundsException.class, () -> store.getLong(num + 1)); } - private static final long maxInt = Integer.MAX_VALUE; + private static final long MAX_INT = Integer.MAX_VALUE; @ParameterizedTest - @ValueSource(longs = {maxInt - 1, maxInt, maxInt + 1, 2 * maxInt - 1, 2 * maxInt, 5 * maxInt - 1, 5 * maxInt + 1}) - public void readBig(long value) { + @ValueSource(longs = {MAX_INT - 1, + MAX_INT, MAX_INT + 1, 2 * MAX_INT - 1, 2 * MAX_INT, 5 * MAX_INT - 1, 5 * MAX_INT + 1}) + void readBig(long value) { store.appendLong(value); assertEquals(value, store.getLong(0)); } } - static class RamInt extends IntsTest { + static class RamIntTest extends IntsTest { @BeforeEach - public void setup() { + void setup() { this.store = new AppendStoreRam.Ints(false, 4 << 2); } } - static class MMapInt extends IntsTest { + static class MMapIntTest extends IntsTest { @BeforeEach - public void setup(@TempDir Path path) { + void setup(@TempDir Path path) { this.store = new AppendStoreMmap.Ints(path.resolve("ints"), 4 << 2, true); } } - static class DirectInt extends IntsTest { + static class DirectIntTest extends IntsTest { @BeforeEach - public void setup() { + void setup() { this.store = new AppendStoreRam.Ints(true, 4 << 2); } } - static class RamLong extends LongsTest { + static class RamLongTest extends LongsTest { @BeforeEach - public void setup() { + void setup() { this.store = new AppendStoreRam.Longs(false, 4 << 2); } } - static class MMapLong extends LongsTest { + static class MMapLongTest extends LongsTest { @BeforeEach - public void setup(@TempDir Path path) { + void setup(@TempDir Path path) { this.store = new AppendStoreMmap.Longs(path.resolve("longs"), 4 << 2, true); } } - static class DirectLong extends LongsTest { + static class DirectLongTest extends LongsTest { @BeforeEach - public void setup() { + void setup() { this.store = new AppendStoreRam.Longs(true, 4 << 2); } } - static class MMapSmallLong extends LongsTest { + static class MMapSmallLongTest extends LongsTest { @BeforeEach - public void setup(@TempDir Path path) { + void setup(@TempDir Path path) { this.store = new AppendStore.SmallLongs( - (i) -> new AppendStoreMmap.Ints(path.resolve("smalllongs" + i), 4 << 2, true)); + i -> new AppendStoreMmap.Ints(path.resolve("smalllongs" + i), 4 << 2, true)); } } - static class RamSmallLong extends LongsTest { + static class RamSmallLongTest extends LongsTest { @BeforeEach - public void setup() { - this.store = new AppendStore.SmallLongs((i) -> new AppendStoreRam.Ints(false, 4 << 2)); + void setup() { + this.store = new AppendStore.SmallLongs(i -> new AppendStoreRam.Ints(false, 4 << 2)); } } - static class DirectSmallLong extends LongsTest { + static class DirectSmallLongTest extends LongsTest { @BeforeEach - public void setup() { - this.store = new AppendStore.SmallLongs((i) -> new AppendStoreRam.Ints(true, 4 << 2)); + void setup() { + this.store = new AppendStore.SmallLongs(i -> new AppendStoreRam.Ints(true, 4 << 2)); } } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongLongMultimapTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongLongMultimapTest.java index e51367aeda..ffb943692f 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongLongMultimapTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongLongMultimapTest.java @@ -11,18 +11,18 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -public abstract class LongLongMultimapTest { +abstract class LongLongMultimapTest { protected LongLongMultimap map; protected boolean retainInputOrder = false; @Test - public void missingValue() { + void missingValue() { assertTrue(map.get(0).isEmpty()); } @Test - public void oneValue() { + void oneValue() { put(1, 1); assertResultLists(LongArrayList.from(), map.get(0)); assertResultLists(LongArrayList.from(1), map.get(1)); @@ -50,7 +50,7 @@ private void putAll(long k, LongArrayList vs) { } @Test - public void twoConsecutiveValues() { + void twoConsecutiveValues() { put(1, 1); put(2, 2); assertResultLists(LongArrayList.from(), map.get(0)); @@ -60,7 +60,7 @@ public void twoConsecutiveValues() { } @Test - public void twoNonconsecutiveValues() { + void twoNonconsecutiveValues() { put(1, 1); put(3, 3); assertResultLists(LongArrayList.from(), map.get(0)); @@ -71,7 +71,7 @@ public void twoNonconsecutiveValues() { } @Test - public void returnToFirstKey() { + void returnToFirstKey() { if (retainInputOrder) { return; } @@ -91,7 +91,7 @@ public void returnToFirstKey() { } @Test - public void manyInsertsOrdered() { + void manyInsertsOrdered() { long[] toInsert = new long[10]; for (int i = 0; i < 100; i++) { for (int j = 0; j < 10; j++) { @@ -128,7 +128,7 @@ private void assertResultLists(LongArrayList expected, LongArrayList actual) { } @Test - public void manyInsertsUnordered() { + void manyInsertsUnordered() { for (long i = 99; i >= 0; i--) { putAll(i, LongArrayList.from( i * 10 + 10, @@ -160,7 +160,7 @@ public void manyInsertsUnordered() { } @Test - public void multiInsert() { + void multiInsert() { putAll(1, LongArrayList.from(1, 2, 3)); put(0, 3); assertResultLists(LongArrayList.from(3), map.get(0)); @@ -168,35 +168,35 @@ public void multiInsert() { assertResultLists(LongArrayList.from(), map.get(2)); } - public static class SparseUnorderedTest extends LongLongMultimapTest { + static class SparseUnorderedTest extends LongLongMultimapTest { @BeforeEach - public void setup() { + void setup() { this.map = LongLongMultimap.newAppendableMultimap(); } } - public static class DenseOrderedTest extends LongLongMultimapTest { + static class DenseOrderedTest extends LongLongMultimapTest { @BeforeEach - public void setup() { + void setup() { retainInputOrder = true; this.map = LongLongMultimap.newInMemoryReplaceableMultimap(); } } - public static class DenseOrderedMmapTest extends LongLongMultimapTest { + static class DenseOrderedMmapTest extends LongLongMultimapTest { @BeforeEach - public void setup(@TempDir Path dir) { + void setup(@TempDir Path dir) { retainInputOrder = true; this.map = LongLongMultimap.newReplaceableMultimap(Storage.MMAP, new Storage.Params(dir.resolve("multimap"), true)); } @AfterEach - public void teardown() { + void teardown() { this.map.close(); } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java index e7f5b678e2..83fdf36a0e 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java @@ -729,33 +729,24 @@ private static SourceFeature point(String source, String layer, Map void assertSameElements(List a, List b) { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifierTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifierTest.java index e2a18899c9..857a00bc04 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifierTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifierTest.java @@ -4,9 +4,13 @@ import static com.onthegomap.planetiler.TestUtils.newLineString; import static com.onthegomap.planetiler.TestUtils.newPolygon; import static com.onthegomap.planetiler.TestUtils.rectangle; +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.List; import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.Polygonal; import org.locationtech.jts.geom.util.AffineTransformation; class DouglasPeuckerSimplifierTest { @@ -16,10 +20,18 @@ class DouglasPeuckerSimplifierTest { private void testSimplify(Geometry in, Geometry expected, double amount) { for (int rotation : rotations) { var rotate = AffineTransformation.rotationInstance(Math.PI * rotation / 180); + var expRot = rotate.transform(expected); + var inRot = rotate.transform(in); assertSameNormalizedFeature( - rotate.transform(expected), - DouglasPeuckerSimplifier.simplify(rotate.transform(in), amount) + expRot, + DouglasPeuckerSimplifier.simplify(inRot, amount) ); + + // ensure the List version also works... + List inList = List.of(inRot.getCoordinates()); + List expList = List.of(expRot.getCoordinates()); + List actual = DouglasPeuckerSimplifier.simplify(inList, amount, in instanceof Polygonal); + assertEquals(expList, actual); } } @@ -65,8 +77,8 @@ void testPolygonLeaveAPoint() { rectangle(0, 10), newPolygon( 0, 0, - 10, 10, 10, 0, + 10, 10, 0, 0 ), 20 diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java index 315b03c4e9..a1a3dab4dd 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java @@ -814,7 +814,7 @@ void testFill() { tileRight(1) ), TileCoord.ofXYZ(Z14_TILES / 2, Z14_TILES / 2, 14), List.of( - newPolygon(tileFill(5), List.of()) // <<<<---- the filled tile! + newPolygon(tileFill(1), List.of()) // <<<<---- the filled tile! ), TileCoord.ofXYZ(Z14_TILES / 2 + 1, Z14_TILES / 2, 14), List.of( tileLeft(1) @@ -1173,7 +1173,7 @@ void testNestedMultipolygonFill() { var rendered = renderGeometry(feature); var innerTile = rendered.get(TileCoord.ofXYZ(Z14_TILES / 2, Z14_TILES / 2, 14)); assertEquals(1, innerTile.size()); - assertEquals(new TestUtils.NormGeometry(rectangle(-5, 256 + 5)), + assertEquals(new TestUtils.NormGeometry(rectangle(-1, 256 + 1)), new TestUtils.NormGeometry(innerTile.iterator().next())); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LoopLineMergerTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LoopLineMergerTest.java new file mode 100644 index 0000000000..59fa2e2215 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LoopLineMergerTest.java @@ -0,0 +1,518 @@ +package com.onthegomap.planetiler.util; + +import static com.onthegomap.planetiler.TestUtils.newLineString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.geo.GeoUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.operation.linemerge.LineMerger; + +class LoopLineMergerTest { + + @Test + void testMergeTouchingLinestrings() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setStubMinLength(-1) + .setTolerance(-1) + .setLoopMinLength(-1); + + merger.add(newLineString(10, 10, 20, 20)); + merger.add(newLineString(20, 20, 30, 30)); + assertEquals( + List.of(newLineString(10, 10, 20, 20, 30, 30)), + merger.getMergedLineStrings() + ); + } + + @Test + void testKeepTwoSeparateLinestring() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setLoopMinLength(-1); + + merger.add(newLineString(10, 10, 20, 20)); + merger.add(newLineString(30, 30, 40, 40)); + assertEquals( + List.of( + newLineString(10, 10, 20, 20), + newLineString(30, 30, 40, 40) + ), + merger.getMergedLineStrings() + ); + } + + @Test + void testDoesNotOvercountAlreadyAddedLines() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setTolerance(-1) + .setStubMinLength(-1) + .setLoopMinLength(-1); + + merger.add(newLineString(10, 10, 20, 20)); + merger.add(newLineString(20, 20, 30, 30)); + merger.add(newLineString(20, 20, 30, 30)); + assertEquals( + List.of(newLineString(10, 10, 20, 20, 30, 30)), + merger.getMergedLineStrings() + ); + } + + @Test + void testSplitLinestringsBeforeMerging() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setLoopMinLength(-1) + .setStubMinLength(-1) + .setTolerance(-1); + + merger.add(newLineString(10, 10, 20, 20, 30, 30)); + merger.add(newLineString(20, 20, 30, 30, 40, 40)); + assertEquals( + List.of(newLineString(10, 10, 20, 20, 30, 30, 40, 40)), + merger.getMergedLineStrings() + ); + } + + @Test + void testProgressiveStubRemoval() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setStubMinLength(4) + .setLoopMinLength(-1) + .setTolerance(-1); + + merger.add(newLineString(0, 0, 5, 0)); // stub length 5 + merger.add(newLineString(5, 0, 6, 0)); // mid piece + merger.add(newLineString(6, 0, 8, 0)); // stub length 2 + merger.add(newLineString(5, 0, 5, 1)); // stub length 1 + merger.add(newLineString(6, 0, 6, 1)); // stub length 1 + + assertEquals( + List.of(newLineString(0, 0, 5, 0, 6, 0, 8, 0)), + merger.getMergedLineStrings() + ); + } + + @Test + void testRoundCoordinatesBeforeMerging() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setLoopMinLength(-1) + .setStubMinLength(-1) + .setTolerance(-1); + + merger.add(newLineString(10.00043983098, 10, 20, 20)); + merger.add(newLineString(20, 20, 30, 30)); + assertEquals( + List.of(newLineString(10, 10, 20, 20, 30, 30)), + merger.getMergedLineStrings() + ); + } + + @Test + void testRemoveSmallLoops() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setStubMinLength(-1) + .setTolerance(-1) + .setLoopMinLength(100); + + merger.add(newLineString( + 10, 10, + 20, 10, + 30, 10, + 30, 20, + 40, 20 + )); + merger.add(newLineString( + 20, 10, + 30, 20 + )); + assertEquals( + List.of( + newLineString( + 10, 10, + 20, 10, + 30, 20, + 40, 20 + ) + ), + merger.getMergedLineStrings() + ); + } + + @Test + void testRemoveSelfClosingLoops() { + // Note that self-closing loops are considered stubs. + // They are removed by stubMinLength, not loopMinLength... + var merger = new LoopLineMerger() + .setMinLength(-1) + .setTolerance(-1) + .setStubMinLength(5) + .setLoopMinLength(-1); + + merger.add(newLineString( + 1, -10, + 1, 1, + 1, 2, + 0, 2, + 0, 1, + 1, 1, + 10, 1)); + assertEquals( + List.of(newLineString(1, -10, 1, 1, 10, 1)), + merger.getMergedLineStrings() + ); + } + + @Test + void testDoNotRemoveLargeLoops() { + var merger = new LoopLineMerger() + .setMinLength(-1) + .setLoopMinLength(0.001); + + merger.add(newLineString( + 10, 10, + 20, 10, + 30, 10, + 30, 20, + 40, 20 + )); + merger.add(newLineString( + 20, 10, + 30, 20 + )); + assertEquals( + List.of( + newLineString( + 10, 10, + 20, 10 + ), + newLineString( + 20, 10, + 30, 10, + 30, 20 + ), + newLineString( + 20, 10, + 30, 20 + ), + newLineString( + 30, 20, + 40, 20 + ) + ), + merger.getMergedLineStrings() + ); + } + + @Test + void testRemoveShortLine() { + var merger = new LoopLineMerger() + .setMinLength(10) + .setStubMinLength(-1) + .setTolerance(-1) + .setLoopMinLength(-1); + + merger.add(newLineString(10, 10, 11, 11)); + merger.add(newLineString(20, 20, 30, 30)); + merger.add(newLineString(30, 30, 40, 40)); + assertEquals( + List.of(newLineString(20, 20, 30, 30, 40, 40)), + merger.getMergedLineStrings() + ); + } + + @Test + void testRemovesShortStubsTheNonStubsThatAreTooShort() { + var merger = new LoopLineMerger() + .setMinLength(0) + .setLoopMinLength(-1) + .setStubMinLength(15) + .setTolerance(-1); + + merger.add(newLineString(0, 0, 20, 0)); + merger.add(newLineString(20, 0, 30, 0)); + merger.add(newLineString(30, 0, 50, 0)); + merger.add(newLineString(20, 0, 20, 10)); + merger.add(newLineString(30, 0, 30, 10)); + + assertEquals( + List.of(newLineString(0, 0, 20, 0, 30, 0, 50, 0)), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeCarriagewaysWithOneSplitShorterThanLoopMinLength() { + var merger = new LoopLineMerger() + .setMinLength(20) + .setMergeStrokes(true) + .setLoopMinLength(20); + + merger.add(newLineString(0, 0, 10, 0, 20, 0, 30, 0)); + merger.add(newLineString(30, 0, 20, 0, 15, 1, 10, 0, 0, 0)); + + assertEquals( + List.of(newLineString(0, 0, 10, 0, 20, 0, 30, 0)), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeCarriagewaysWithOneSplitLongerThanLoopMinLength() { + var merger = new LoopLineMerger() + .setMinLength(5) + .setMergeStrokes(true) + .setLoopMinLength(5); + + merger.add(newLineString(0, 0, 10, 0, 20, 0, 30, 0)); + merger.add(newLineString(30, 0, 20, 0, 15, 1, 10, 0, 0, 0)); + + assertEquals( + // ideally loop merging should connect long line strings and represent loops as separate segments off of the edges + List.of(newLineString(0, 0, 10, 0, 20, 0, 30, 0), newLineString(20, 0, 15, 1, 10, 0)), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeCarriagewaysWithTwoSplits() { + var merger = new LoopLineMerger() + .setMinLength(20) + .setMergeStrokes(true) + .setLoopMinLength(20); + + merger.add(newLineString(0, 0, 10, 0, 20, 0, 30, 0, 40, 0)); + merger.add(newLineString(40, 0, 30, 0, 25, 5, 20, 0, 15, 5, 10, 0, 0, 0)); + + assertEquals( + List.of(newLineString(0, 0, 10, 0, 20, 0, 30, 0, 40, 0)), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeLoopAttachedToStub() { + var merger = new LoopLineMerger() + .setMinLength(10) + .setLoopMinLength(10) + .setStubMinLength(10) + .setTolerance(-1); + + merger.add(newLineString(-20, 0, 0, 0, 20, 0)); + merger.add(newLineString(0, 0, 0, 1)); + merger.add(newLineString(0, 1, 1, 2, 1, 1, 0, 1)); + + assertEquals( + List.of(newLineString(-20, 0, 0, 0, 20, 0)), + merger.getMergedLineStrings() + ); + } + + @Test + void testRealWorldHarkingen() { + var merger = new LoopLineMerger() + .setMinLength(4 * 0.0625) + .setLoopMinLength(8 * 0.0625); + + merger.add( + newLineString(99.185791015625, 109.83056640625, 99.202392578125, 109.8193359375, 99.21337890625, 109.810302734375, + 99.222412109375, 109.8017578125, 99.229736328125, 109.793701171875, 99.241943359375, 109.779541015625)); + merger.add(newLineString(98.9931640625, 109.863525390625, 99.005126953125, 109.862060546875, 99.01708984375, + 109.86083984375, 99.028564453125, 109.85986328125, 99.040283203125, 109.859375, 99.0712890625, 109.85791015625, + 99.08203125, 109.857421875, 99.093017578125, 109.856689453125, 99.104248046875, 109.855712890625, 99.115478515625, + 109.8544921875, 99.12646484375, 109.852783203125, 99.1376953125, 109.850341796875, 99.1474609375, 109.84765625, + 99.15673828125, 109.844482421875, 99.166748046875, 109.84033203125, 99.175537109375, 109.836181640625, + 99.185791015625, 109.83056640625)); + merger.add(newLineString(99.162841796875, 109.812744140625, 99.0966796875, 109.824462890625, 99.055419921875, + 109.832275390625, 99.008544921875, 109.842041015625, 98.967529296875, 109.8525390625, 98.8818359375, + 109.875244140625)); + merger.add(newLineString(98.879150390625, 109.885498046875, 98.94091796875, 109.86572265625, 98.968017578125, + 109.859130859375, 99.017578125, 109.847412109375, 99.056396484375, 109.83984375, 99.09814453125, 109.831298828125, + 99.163330078125, 109.81982421875)); + var merged = merger.getMergedLineStrings(); + + assertEquals( + 1, + merged.size() + ); + } + + @ParameterizedTest + @CsvSource({ + "mergelines_1759_point_line.wkb.gz,0,false,3", + "mergelines_1759_point_line.wkb.gz,1,false,2", + "mergelines_1759_point_line.wkb.gz,1,true,2", + + "mergelines_200433_lines.wkb.gz,0,false,9103", + "mergelines_200433_lines.wkb.gz,0.1,false,8834", + "mergelines_200433_lines.wkb.gz,1,false,861", + "mergelines_200433_lines.wkb.gz,1,true,508", + + "mergelines_239823_lines.wkb.gz,0,false,6188", + "mergelines_239823_lines.wkb.gz,0.1,false,5941", + "mergelines_239823_lines.wkb.gz,1,false,826", + "mergelines_239823_lines.wkb.gz,1,true,681", + + "i90.wkb.gz,0,false,17", + "i90.wkb.gz,1,false,18", + "i90.wkb.gz,20,false,4", + "i90.wkb.gz,30,false,1", + }) + void testOnRealWorldData(String file, double minLengths, boolean simplify, int expected) + throws IOException, ParseException { + Geometry geom = new WKBReader(GeoUtils.JTS_FACTORY).read( + Gzip.gunzip(Files.readAllBytes(TestUtils.pathToResource("mergelines").resolve(file)))); + var merger = new LoopLineMerger(); + merger.setMinLength(minLengths); + merger.setLoopMinLength(minLengths); + merger.setStubMinLength(minLengths); + merger.setMergeStrokes(true); + merger.setTolerance(simplify ? 1 : -1); + merger.add(geom); + var merged = merger.getMergedLineStrings(); + Set> lines = new HashSet<>(); + var merger2 = new LineMerger(); + for (var line : merged) { + merger2.add(line); + assertTrue(lines.add(Arrays.asList(line.getCoordinates())), "contained duplicate: " + line); + if (minLengths > 0 && !simplify) { // simplification can make an edge < min length + assertTrue(line.getLength() >= minLengths, "line < " + minLengths + ": " + line); + } + } + // ensure there are no more opportunities for simplification found by JTS: + List loop = List.copyOf(merged); + List jts = merger2.getMergedLineStrings().stream().map(LineString.class::cast).toList(); + List missing = jts.stream().filter(l -> !loop.contains(l)).toList(); + List extra = loop.stream().filter(l -> !jts.contains(l)).toList(); + assertEquals(List.of(), missing, "missing edges"); + assertEquals(List.of(), extra, "extra edges"); + assertEquals(merged.size(), merger2.getMergedLineStrings().size()); + assertEquals(expected, merged.size()); + } + + @Test + void testMergeStrokesAt3WayIntersectionWithLoop() { + var merger = new LoopLineMerger() + .setMinLength(1) + .setLoopMinLength(1) + .setStubMinLength(1) + .setMergeStrokes(true); + + merger.add(newLineString(-5, 0, 0, 0)); + merger.add(newLineString(0, 0, 5, 0, 5, 5, 0, 5, 0, 0)); + + assertEquals( + List.of( + newLineString(-5, 0, 0, 0, 5, 0, 5, 5, 0, 5, 0, 0) + ), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeStrokesAt3WayIntersectionWithLoop2() { + var merger = new LoopLineMerger() + .setMinLength(1) + .setLoopMinLength(1) + .setStubMinLength(1) + .setMergeStrokes(true); + + merger.add(newLineString(-5, 0, 0, 0)); + merger.add(newLineString(0, 0, 0, -1, 5, 0, 5, 5, 0, 5, 0, 0)); + + assertEquals( + List.of( + newLineString( + -5, 0, 0, 0, 0, -1, 5, 0, 5, 5, 0, 5, 0, 0 + ) + ), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeStrokesAt3WayIntersection() { + var merger = new LoopLineMerger() + .setMinLength(1) + .setLoopMinLength(1) + .setStubMinLength(1) + .setMergeStrokes(true); + + merger.add(newLineString(-5, 0, 0, 0)); + merger.add(newLineString(0, 0, 5, 0)); + merger.add(newLineString(0, 0, 0, 5)); + + assertEquals( + List.of( + newLineString(-5, 0, 0, 0, 5, 0), + newLineString(0, 0, 0, 5) + ), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeStrokesAt4WayIntersection() { + var merger = new LoopLineMerger() + .setMinLength(1) + .setLoopMinLength(1) + .setStubMinLength(1) + .setMergeStrokes(true); + + merger.add(newLineString(-5, 0, 0, 0)); + merger.add(newLineString(0, 0, 5, 0)); + merger.add(newLineString(0, 0, 0, 5)); + merger.add(newLineString(0, 0, 0, -5)); + + assertEquals( + List.of( + newLineString(-5, 0, 0, 0, 5, 0), + newLineString(0, -5, 0, 0, 0, 5) + ), + merger.getMergedLineStrings() + ); + } + + @Test + void testMergeStrokesAt5WayIntersection() { + var merger = new LoopLineMerger() + .setMinLength(1) + .setLoopMinLength(1) + .setStubMinLength(1) + .setMergeStrokes(true); + + merger.add(newLineString(-5, 0, 0, 0)); + merger.add(newLineString(0, 0, 5, 0)); + merger.add(newLineString(0, 0, 0, 5)); + merger.add(newLineString(0, 0, 0, -5)); + merger.add(newLineString(0, 0, 5, 5)); + + assertEquals( + List.of( + newLineString(-5, 0, 0, 0, 5, 0), + newLineString(0, 0, 5, 5), + newLineString(0, -5, 0, 0, 0, 5) + ), + merger.getMergedLineStrings() + ); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TranslationsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TranslationsTest.java index 14291c8d8c..091efd95af 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TranslationsTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TranslationsTest.java @@ -1,10 +1,16 @@ package com.onthegomap.planetiler.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class TranslationsTest { @@ -33,4 +39,25 @@ void testTwoProvidersPrefersFirst() { void testTransliterate() { assertEquals("rì běn", Translations.transliterate("日本")); } + + @ParameterizedTest + @MethodSource("includeExcludeCases") + void testIncludeExclude(List languages, List shouldCare, List shouldNotCare) { + var translations = Translations.nullProvider(languages); + for (var lang : shouldCare) { + assertTrue(translations.careAboutLanguage(lang)); + } + for (var lang : shouldNotCare) { + assertFalse(translations.careAboutLanguage(lang)); + } + } + + private static Stream includeExcludeCases() { + return Stream.of( + Arguments.of(List.of("jbo", "tlh"), List.of("jbo", "tlh"), List.of("en", "fr")), + Arguments.of(List.of("*"), List.of("jbo", "tlh", "en", "fr"), List.of()), + Arguments.of(List.of("*", "-tlh"), List.of("jbo", "fr"), List.of("tlh")), + Arguments.of(List.of("tlh", "-tlh"), List.of(), List.of("tlh", "en")) + ); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java index bec46b968c..7d3a2e0994 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/WikidataTest.java @@ -111,7 +111,7 @@ void testWikidataTranslations() { List testFetchWikidata() throws IOException, InterruptedException { StringWriter writer = new StringWriter(); Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS); - Wikidata fixture = new Wikidata(writer, client, 2, profile, config); + Wikidata fixture = createFixture(writer, client, 2); fixture.fetch(1L); Mockito.verifyNoInteractions(client); Mockito.when(client.send(Mockito.any())) @@ -142,7 +142,7 @@ List testFetchWikidata() throws IOException, InterruptedException { dynamicTest("do not re-request on subsequent loads", () -> { StringWriter writer2 = new StringWriter(); Wikidata.Client client2 = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS); - Wikidata fixture2 = new Wikidata(writer2, client2, 2, profile, config); + Wikidata fixture2 = createFixture(writer2, client2, 2); fixture2.loadExisting(Wikidata.load(new BufferedReader(new StringReader(writer.toString())))); fixture2.fetch(1L); fixture2.fetch(2L); @@ -157,7 +157,7 @@ List testFetchWikidata() throws IOException, InterruptedException { void testRetryFailedRequestOnce() throws IOException, InterruptedException { StringWriter writer = new StringWriter(); Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS); - Wikidata fixture = new Wikidata(writer, client, 1, profile, config); + Wikidata fixture = createFixture(writer, client, 1); Mockito.when(client.send(Mockito.any())) // fail once then succeed .thenThrow(IOException.class) @@ -236,4 +236,13 @@ public void onComplete() { })); return stringSubscriber.getBody().toCompletableFuture().join(); } + + private Wikidata createFixture(StringWriter writer, Wikidata.Client client, int batchSize) { + return new Wikidata(writer, client, batchSize, profile, config) { + @Override + protected void sleep(Duration duration) { + // don't sleep in tests + } + }; + } } diff --git a/planetiler-core/src/test/resources/mergelines/i90.wkb.gz b/planetiler-core/src/test/resources/mergelines/i90.wkb.gz new file mode 100644 index 0000000000..10671bbb1e Binary files /dev/null and b/planetiler-core/src/test/resources/mergelines/i90.wkb.gz differ diff --git a/planetiler-core/src/test/resources/mergelines/mergelines_1759_point_line.wkb.gz b/planetiler-core/src/test/resources/mergelines/mergelines_1759_point_line.wkb.gz new file mode 100644 index 0000000000..8813e1c180 Binary files /dev/null and b/planetiler-core/src/test/resources/mergelines/mergelines_1759_point_line.wkb.gz differ diff --git a/planetiler-core/src/test/resources/mergelines/mergelines_200433_lines.wkb.gz b/planetiler-core/src/test/resources/mergelines/mergelines_200433_lines.wkb.gz new file mode 100644 index 0000000000..eb0ee1fd58 Binary files /dev/null and b/planetiler-core/src/test/resources/mergelines/mergelines_200433_lines.wkb.gz differ diff --git a/planetiler-core/src/test/resources/mergelines/mergelines_239823_lines.wkb.gz b/planetiler-core/src/test/resources/mergelines/mergelines_239823_lines.wkb.gz new file mode 100644 index 0000000000..5a863f85c5 Binary files /dev/null and b/planetiler-core/src/test/resources/mergelines/mergelines_239823_lines.wkb.gz differ diff --git a/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf b/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf new file mode 100644 index 0000000000..9d4e0a34f0 Binary files /dev/null and b/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf differ diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java index 1137bfffe9..8db83addf1 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfigExpressionParser.java @@ -146,7 +146,7 @@ private ConfigExpression.Match parseMatch(Object match, boolean allowE } else { throw new ParseException("Invalid match block. Expected a list or map, but got: " + match); } - return ConfigExpression.match(signature(output), MultiExpression.of(List.copyOf(conditions)), fallback); + return ConfigExpression.match(signature(output), MultiExpression.ofOrdered(List.copyOf(conditions)), fallback); } private Signature signature(Class outputClass) { diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java index 4e4853bbd2..65cb28700c 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/expression/ConfigExpression.java @@ -152,7 +152,7 @@ public ConfigExpression simplifyOnce() { if (Expression.TRUE.equals(expression.expression())) { return new Match<>( signature, - MultiExpression.of(expressions.stream().limit(i).toList()), + MultiExpression.ofOrdered(expressions.stream().limit(i).toList()), expression.result() ); } diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java index 4a9d2fd82e..3ad107562c 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java @@ -1602,4 +1602,38 @@ void testGeometryAttributesArea(String expression, double expected) { testFeature(config, sfMatch, any -> assertEquals(expected, (Double) any.getAttrsAtZoom(14).get("attr"), expected / 1e3), 1); } + + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMatchOrdering(boolean withFallback) { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf + layers: + - id: testLayer + features: + - source: osm + attributes: + - key: attr + value: + - if: {natural: tree} + value: green + - if: {historic: memorial} + value: black + - if: {tourism: viewpoint} + value: green + - if: ${%s} + value: fallback + + """.formatted(withFallback ? "true" : "false"); + testFeature(config, SimpleFeature.createFakeOsmFeature(newPoint(0, 0), Map.of( + "historic", "memorial", + "tourism", "viewpoint" + ), "osm", null, 1, emptyList(), OSM_INFO), feature -> assertEquals("black", feature.getAttrsAtZoom(14).get("attr")), + 1); + } } diff --git a/planetiler-dist/pom.xml b/planetiler-dist/pom.xml index 945759c742..53b45702ea 100644 --- a/planetiler-dist/pom.xml +++ b/planetiler-dist/pom.xml @@ -73,7 +73,7 @@ false - eclipse-temurin:21-jre + eclipse-temurin:23-jdk diff --git a/planetiler-examples/README.md b/planetiler-examples/README.md index e7faea3949..f3c92383f5 100644 --- a/planetiler-examples/README.md +++ b/planetiler-examples/README.md @@ -8,7 +8,7 @@ Requirements: - on mac: `brew install --cask temurin` - [Maven](https://maven.apache.org/install.html) - on mac: `brew install maven` -- [Node.js](https://nodejs.org/en/download/) +- [Node.js](https://nodejs.org/en/download/package-manager) - on mac: `brew install node` - [TileServer GL](https://github.com/maptiler/tileserver-gl) - `npm install -g tileserver-gl-light` diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java index 7dbc25ef11..77b2eff33e 100644 --- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java +++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java @@ -96,7 +96,7 @@ void integrationTest(@TempDir Path tmpDir) throws Exception { // Override input source locations "osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"), // Override temp dir location - "tmp", tmpDir.toString(), + "tmpdir", tmpDir.toString(), // Override output location "output", dbPath.toString() )); @@ -109,7 +109,7 @@ void integrationTest(@TempDir Path tmpDir) throws Exception { .assertNumFeatures(mbtiles, "bicycle-route-international", 14, Map.of( "name", "EuroVelo 8 - Mediterranean Route - part Monaco", "ref", "EV8" - ), GeoUtils.WORLD_LAT_LON_BOUNDS, 25, LineString.class); + ), GeoUtils.WORLD_LAT_LON_BOUNDS, 13, LineString.class); TestUtils.assertTileDuplicates(mbtiles, 0); } diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java index 656695084c..808791f058 100644 --- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java +++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java @@ -91,7 +91,7 @@ void integrationTest(@TempDir Path tmpDir) throws Exception { // Override input source locations "osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"), // Override temp dir location - "tmp", tmpDir.toString(), + "tmpdir", tmpDir.toString(), // Override output location "output", dbPath.toString() )); diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java index 691cbf992c..8f3bc74652 100644 --- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java +++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java @@ -56,7 +56,7 @@ void integrationTest(@TempDir Path tmpDir) throws Exception { // Override input source locations "osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"), // Override temp dir location - "tmp", tmpDir.toString(), + "tmpdir", tmpDir.toString(), // Override output location "output", dbPath.toString() )); diff --git a/planetiler-openmaptiles b/planetiler-openmaptiles index 2957c27075..0818cb981f 160000 --- a/planetiler-openmaptiles +++ b/planetiler-openmaptiles @@ -1 +1 @@ -Subproject commit 2957c27075c54c765b99e288df9d202520b96b2c +Subproject commit 0818cb981fb9b7ead2c0a093794ccda07bb02c31 diff --git a/pom.xml b/pom.xml index 6cca7720b6..116f3749e5 100644 --- a/pom.xml +++ b/pom.xml @@ -186,7 +186,7 @@ - 4.29 + 4.33 ${maven.multiModuleProjectDirectory}/eclipse-formatter.xml @@ -236,11 +236,15 @@ org.apache.maven.plugins maven-surefire-plugin 3.5.2 - + + + + **/*.java + @@ -441,5 +445,21 @@ + + + fast + + + + org.apache.maven.plugins + maven-surefire-plugin + + !slow + 1C + + + + + diff --git a/scripts/fasttests.sh b/scripts/fasttests.sh new file mode 100755 index 0000000000..9a8674e765 --- /dev/null +++ b/scripts/fasttests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +./mvnw -T 1C -Pfast clean test