diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml index d2b7766f8b..945bc8ac94 100644 --- a/planetiler-core/pom.xml +++ b/planetiler-core/pom.xml @@ -124,6 +124,10 @@ jackson-datatype-jsr310 ${jackson.version} + + org.snakeyaml + snakeyaml-engine + io.prometheus simpleclient diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java index 71862fab50..734f24b0ea 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java @@ -86,7 +86,7 @@ public Feature point(String layer) { return geometry(layer, source.worldGeometry()); } catch (GeometryException e) { e.log(stats, "feature_point", "Error getting point geometry for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } @@ -106,7 +106,7 @@ public Feature line(String layer) { return geometry(layer, source.line()); } catch (GeometryException e) { e.log(stats, "feature_line", "Error constructing line for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } @@ -126,7 +126,7 @@ public Feature partialLine(String layer, double start, double end) { return geometry(layer, source.partialLine(start, end)); } catch (GeometryException e) { e.log(stats, "feature_partial_line", "Error constructing partial line for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } @@ -146,10 +146,27 @@ public Feature polygon(String layer) { return geometry(layer, source.polygon()); } catch (GeometryException e) { e.log(stats, "feature_polygon", "Error constructing polygon for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } + /** + * Starts building a new polygon, line, or point map feature based on the geometry type of the input feature. + * + * @param layer the output vector tile layer this feature will be written to + * @return a feature that can be configured further. + */ + public Feature anyGeometry(String layer) { + return source.canBePolygon() ? polygon(layer) : + source.canBeLine() ? line(layer) : + source.isPoint() ? point(layer) : + empty(layer); + } + + private Feature empty(String layer) { + return new Feature(layer, EMPTY_GEOM, source.id()); + } + /** * Starts building a new point map feature with geometry from {@link Geometry#getCentroid()} of the source feature. * @@ -161,7 +178,7 @@ public Feature centroid(String layer) { return geometry(layer, source.centroid()); } catch (GeometryException e) { e.log(stats, "feature_centroid", "Error getting centroid for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } @@ -178,7 +195,7 @@ public Feature centroidIfConvex(String layer) { return geometry(layer, source.centroidIfConvex()); } catch (GeometryException e) { e.log(stats, "feature_centroid_if_convex", "Error constructing centroid if convex for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } @@ -194,7 +211,7 @@ public Feature pointOnSurface(String layer) { return geometry(layer, source.pointOnSurface()); } catch (GeometryException e) { e.log(stats, "feature_point_on_surface", "Error constructing point on surface for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } @@ -216,7 +233,7 @@ public Feature innermostPoint(String layer, double tolerance) { return geometry(layer, source.innermostPoint(tolerance)); } catch (GeometryException e) { e.log(stats, "feature_innermost_point", "Error constructing innermost point for " + source); - return new Feature(layer, EMPTY_GEOM, source.id()); + return empty(layer); } } @@ -297,6 +314,23 @@ default T inheritAttrFromSource(String key) { return setAttr(key, collector().source.getTag(key)); } + /** Copies the values for {@code keys} attributes from source feature to the output feature. */ + default T inheritAttrsFromSource(String... keys) { + for (var key : keys) { + inheritAttrFromSource(key); + } + return self(); + } + + + /** Copies the values for {@code keys} attributes from source feature to the output feature. */ + default T inheritAttrsFromSourceWithMinzoom(int minzoom, String... keys) { + for (var key : keys) { + setAttrWithMinzoom(key, collector().source.getTag(key), minzoom); + } + return self(); + } + /** * Sets an attribute on the output feature to either a string, number, boolean, or instance of {@link ZoomFunction} * to change the value for {@code key} by zoom-level. @@ -319,7 +353,7 @@ default T setAttrWithMinzoom(String key, Object value, int minzoom) { * size. */ default T setAttrWithMinSize(String key, Object value, double minPixelSize) { - return setAttrWithMinzoom(key, value, collector().getMinZoomForPixelSize(minPixelSize)); + return setAttrWithMinzoom(key, value, getMinZoomForPixelSize(minPixelSize)); } /** @@ -334,7 +368,11 @@ default T setAttrWithMinSize(String key, Object value, double minPixelSize) { default T setAttrWithMinSize(String key, Object value, double minPixelSize, int minZoomIfBigEnough, int minZoomToShowAlways) { return setAttrWithMinzoom(key, value, - Math.clamp(collector().getMinZoomForPixelSize(minPixelSize), minZoomIfBigEnough, minZoomToShowAlways)); + Math.clamp(getMinZoomForPixelSize(minPixelSize), minZoomIfBigEnough, minZoomToShowAlways)); + } + + default int getMinZoomForPixelSize(double minPixelSize) { + return collector().getMinZoomForPixelSize(minPixelSize); } /** @@ -410,7 +448,7 @@ public final class Feature implements WithZoomRange, WithAttrs private ZoomFunction labelGridPixelSize = null; private ZoomFunction labelGridLimit = null; - private boolean attrsChangeByZoom = false; + private boolean mustUnwrapValues = false; private CacheByZoom> attrCache = null; private CacheByZoom> partialRangeCache = null; @@ -817,18 +855,21 @@ private Map computeAttrsAtZoom(int zoom) { } private static Object unwrap(Object object, int zoom) { - if (object instanceof ZoomFunction fn) { - object = fn.apply(zoom); - } - if (object instanceof Struct struct) { - object = struct.rawValue(); + for (int i = 0; i < 100; i++) { + switch (object) { + case ZoomFunction fn -> object = fn.apply(zoom); + case Struct struct -> object = struct.rawValue(); + case null, default -> { + return object; + } + } } - return object; + throw new IllegalStateException("Failed to unwrap at z" + zoom + ": " + object); } /** Returns the attribute to put on all output vector tile features at a zoom level. */ public Map getAttrsAtZoom(int zoom) { - if (!attrsChangeByZoom) { + if (!mustUnwrapValues) { return attrs; } if (attrCache == null) { @@ -840,8 +881,8 @@ public Map getAttrsAtZoom(int zoom) { @Override public Feature setAttr(String key, Object value) { - if (value instanceof ZoomFunction) { - attrsChangeByZoom = true; + if (value instanceof ZoomFunction || value instanceof Struct) { + mustUnwrapValues = true; } if (value != null) { attrs.put(key, value); @@ -852,8 +893,8 @@ public Feature setAttr(String key, Object value) { @Override public Feature putAttrs(Map attrs) { for (Object value : attrs.values()) { - if (value instanceof ZoomFunction) { - attrsChangeByZoom = true; + if (value instanceof ZoomFunction || value instanceof Struct) { + mustUnwrapValues = true; break; } } @@ -957,7 +998,7 @@ Partial withAttr(String key, Object value) { return new Partial(omit, MapUtil.with(attrs, key, value)); } } - MergingRangeMap result = MergingRangeMap.unit(new Partial(false, attrs), Partial::merge); + MergingRangeMap result = MergingRangeMap.unit(new Partial(false, getAttrsAtZoom(zoom)), Partial::merge); for (var override : partialOverrides) { result.update(override.range(), m -> switch (override) { case Attr attr -> m.withAttr(attr.key, unwrap(attr.value, zoom)); @@ -1018,6 +1059,9 @@ public LinearRange setMaxZoom(int max) { @Override public LinearRange setAttr(String key, Object value) { + if (value instanceof ZoomFunction || value instanceof Struct) { + mustUnwrapValues = true; + } return add(new Attr(range, key, value)); } @@ -1049,6 +1093,12 @@ public LinearRange linearRange(Range range) { return entireLine().linearRange(range); } + + @Override + public int getMinZoomForPixelSize(double minPixelSize) { + return WithAttrs.super.getMinZoomForPixelSize(minPixelSize / (range.upperEndpoint() - range.lowerEndpoint())); + } + @Override public FeatureCollector collector() { return FeatureCollector.this; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureProcessor.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureProcessor.java new file mode 100644 index 0000000000..614bf9bc6b --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureProcessor.java @@ -0,0 +1,24 @@ +package com.onthegomap.planetiler; + +import com.onthegomap.planetiler.reader.SourceFeature; + +/** + * Subcomponent of {@link Profile} that handles processing layers from a feature, and optionally when that source is + * finished. + */ +@FunctionalInterface +public interface FeatureProcessor { + + /** + * Generates output features for any input feature that should appear in the map. + *

+ * Multiple threads may invoke this method concurrently for a single data source so implementations should ensure + * thread-safe access to any shared data structures. Separate data sources are processed sequentially. + *

+ * All OSM nodes are processed first, then ways, then relations. + * + * @param sourceFeature the input feature from a source dataset (OSM element, shapefile element, etc.) + * @param features a collector for generating output map features to emit + */ + void processFeature(T sourceFeature, FeatureCollector features); +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/ForwardingProfile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/ForwardingProfile.java index e919b41d13..0cdc5f88e9 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/ForwardingProfile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/ForwardingProfile.java @@ -1,5 +1,8 @@ package com.onthegomap.planetiler; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.expression.Expression; +import com.onthegomap.planetiler.expression.MultiExpression; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.reader.SourceFeature; @@ -9,6 +12,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Consumer; /** @@ -41,8 +45,41 @@ public abstract class ForwardingProfile implements Profile { private final Map> layerPostProcessors = new HashMap<>(); /** List of handlers that implement {@link TilePostProcessor}. */ private final List tilePostProcessors = new ArrayList<>(); - /** Map from source ID to its handler if it implements {@link FeatureProcessor}. */ - private final Map> sourceElementProcessors = new HashMap<>(); + /** List of handlers that implements {@link FeatureProcessor} along with a filter expression. */ + private final List> sourceElementProcessors = new CopyOnWriteArrayList<>(); + private final List onlyLayers; + private final List excludeLayers; + @SuppressWarnings("java:S3077") + private volatile MultiExpression.Index indexedSourceElementProcessors = null; + + protected ForwardingProfile(PlanetilerConfig config, Handler... handlers) { + onlyLayers = config.arguments().getList("only_layers", "Include only certain layers", List.of()); + excludeLayers = config.arguments().getList("exclude_layers", "Exclude certain layers", List.of()); + for (var handler : handlers) { + registerHandler(handler); + } + } + + protected ForwardingProfile() { + onlyLayers = List.of(); + excludeLayers = List.of(); + } + + protected ForwardingProfile(Handler... handlers) { + onlyLayers = List.of(); + excludeLayers = List.of(); + for (var handler : handlers) { + registerHandler(handler); + } + } + + private boolean caresAboutLayer(String layer) { + return (onlyLayers.isEmpty() || onlyLayers.contains(layer)) && !excludeLayers.contains(layer); + } + + private boolean caresAboutLayer(Object obj) { + return !(obj instanceof HandlerForLayer l) || caresAboutLayer(l.name()); + } /** * Call {@code processor} for every element in {@code source}. @@ -51,8 +88,29 @@ public abstract class ForwardingProfile implements Profile { * @param processor handler that will process elements in that source */ public void registerSourceHandler(String source, FeatureProcessor processor) { - sourceElementProcessors.computeIfAbsent(source, name -> new ArrayList<>()) - .add(processor); + if (!caresAboutLayer(processor)) { + return; + } + sourceElementProcessors + .add(MultiExpression.entry(processor, Expression.and(Expression.matchSource(source), processor.filter()))); + synchronized (sourceElementProcessors) { + indexedSourceElementProcessors = null; + } + } + + /** + * Call {@code processor} for every element. + * + * @param processor handler that will process elements in that source + */ + public void registerFeatureHandler(FeatureProcessor processor) { + if (!caresAboutLayer(processor)) { + return; + } + sourceElementProcessors.add(MultiExpression.entry(processor, processor.filter())); + synchronized (sourceElementProcessors) { + indexedSourceElementProcessors = null; + } } /** @@ -60,6 +118,9 @@ public void registerSourceHandler(String source, FeatureProcessor processor) { * {@link OsmRelationPreprocessor}, {@link FinishHandler}, {@link TilePostProcessor} or {@link LayerPostProcesser}. */ public void registerHandler(Handler handler) { + if (!caresAboutLayer(handler)) { + return; + } this.handlers.add(handler); if (handler instanceof OsmNodePreprocessor osmNodePreprocessor) { osmNodePreprocessors.add(osmNodePreprocessor); @@ -80,6 +141,9 @@ public void registerHandler(Handler handler) { if (handler instanceof TilePostProcessor postProcessor) { tilePostProcessors.add(postProcessor); } + if (handler instanceof FeatureProcessor processor) { + registerFeatureHandler(processor); + } } @Override @@ -117,19 +181,34 @@ public List preprocessOsmRelation(OsmElement.Relation relation) @Override public void processFeature(SourceFeature sourceFeature, FeatureCollector features) { // delegate source feature processing to each handler for that source - var handlers = sourceElementProcessors.get(sourceFeature.getSource()); - if (handlers != null) { - for (var handler : handlers) { - handler.processFeature(sourceFeature, features); - // TODO extract common handling for expression-based filtering from basemap to this - // common profile when we have another use-case for it. + for (var handler : getHandlerIndex().getMatches(sourceFeature)) { + handler.processFeature(sourceFeature, features); + } + } + + private MultiExpression.Index getHandlerIndex() { + MultiExpression.Index result = indexedSourceElementProcessors; + if (result == null) { + synchronized (sourceElementProcessors) { + result = indexedSourceElementProcessors; + if (result == null) { + indexedSourceElementProcessors = result = MultiExpression.of(sourceElementProcessors).index(); + } } } + return result; } @Override public boolean caresAboutSource(String name) { - return sourceElementProcessors.containsKey(name); + return caresAbout(Expression.PartialInput.ofSource(name)); + } + + @Override + public boolean caresAbout(Expression.PartialInput input) { + return sourceElementProcessors.stream().anyMatch(e -> e.expression() + .partialEvaluate(input) + .simplify() != Expression.FALSE); } @Override @@ -271,13 +350,12 @@ Map> postProcessTile(TileCoord tileCoord, } /** Handlers should implement this interface to process input features from a given source ID. */ - public interface FeatureProcessor { + @FunctionalInterface + public interface FeatureProcessor extends com.onthegomap.planetiler.FeatureProcessor, Handler { - /** - * Process an input element from a source feature. - * - * @see Profile#processFeature(SourceFeature, FeatureCollector) - */ - void processFeature(SourceFeature feature, FeatureCollector features); + /** Returns an {@link Expression} that limits the features that this processor gets called for. */ + default Expression filter() { + return Expression.TRUE; + } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index 59371024a5..e0fb09463d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -34,6 +34,7 @@ import com.onthegomap.planetiler.util.TopOsmTiles; import com.onthegomap.planetiler.util.Translations; import com.onthegomap.planetiler.util.Wikidata; +import com.onthegomap.planetiler.validator.JavaProfileValidator; import com.onthegomap.planetiler.worker.RunnableThatThrows; import java.io.IOException; import java.nio.file.FileSystem; @@ -87,7 +88,8 @@ public class Planetiler { private final Path nodeDbPath; private final Path multipolygonPath; private final Path featureDbPath; - private final boolean downloadSources; + private final Path onlyRunTests; + private boolean downloadSources; private final boolean refreshSources; private final boolean onlyDownloadSources; private final boolean parseNodeBounds; @@ -122,6 +124,7 @@ private Planetiler(Arguments arguments) { } tmpDir = config.tmpDir(); onlyDownloadSources = arguments.getBoolean("only_download", "download source data then exit", false); + onlyRunTests = arguments.file("tests", "run test cases in a yaml then quit", null); downloadSources = onlyDownloadSources || arguments.getBoolean("download", "download sources", false); refreshSources = arguments.getBoolean("refresh_sources", "download new version of source files if they have changed", false); @@ -670,7 +673,11 @@ public Planetiler overwriteOutput(String defaultOutputUri) { return setOutput(defaultOutputUri); } - /** Alias for {@link #overwriteOutput(String)} which infers the output type based on extension. */ + /** + * Alias for {@link #overwriteOutput(String)} which infers the output type based on extension. + *

+ * This will override the value returned by + */ public Planetiler overwriteOutput(Path defaultOutput) { return overwriteOutput(defaultOutput.toString()); } @@ -680,9 +687,8 @@ public Planetiler overwriteOutput(Path defaultOutput) { * writes the rendered tiles to the output archive. * * @throws IllegalArgumentException if expected inputs have not been provided - * @throws Exception if an error occurs while processing */ - public void run() throws Exception { + public void run() { var showVersion = arguments.getBoolean("version", "show version then exit", false); var buildInfo = BuildInfo.get(); if (buildInfo != null && LOGGER.isInfoEnabled()) { @@ -709,6 +715,9 @@ public void run() throws Exception { if (arguments.getBoolean("help", "show arguments then exit", false)) { System.exit(0); + } else if (onlyRunTests != null) { + boolean success = JavaProfileValidator.validate(profile(), onlyRunTests, config()); + System.exit(success ? 0 : 1); } else if (onlyDownloadSources) { // don't check files if not generating map } else if (config.append()) { @@ -772,7 +781,7 @@ public void run() throws Exception { // in case any temp files are left from a previous run... FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath); - Files.createDirectories(tmpDir); + FileUtils.createDirectory(tmpDir); FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalBasePath()); if (!toDownload.isEmpty()) { @@ -814,7 +823,11 @@ public void run() throws Exception { stats.monitorFile("archive", output.getLocalPath(), archive::bytesWritten); for (Stage stage : stages) { - stage.task.run(); + try { + stage.task.run(); + } catch (Exception e) { + throw new PlanetilerException("Error occurred during stage " + stage.id, e); + } } LOGGER.info("Deleting node.db to make room for output file"); @@ -831,13 +844,17 @@ public void run() throws Exception { TileArchiveWriter.writeOutput(featureGroup, archive, archive::bytesWritten, tileArchiveMetadata, layerStatsPath, config, stats); } catch (IOException e) { - throw new IllegalStateException("Unable to write to " + output, e); + throw new PlanetilerException("Unable to write to " + output, e); } overallTimer.stop(); LOGGER.info("FINISHED!"); stats.printSummary(); - stats.close(); + try { + stats.close(); + } catch (Exception e) { + throw new PlanetilerException(e); + } } private void checkDiskSpace() { @@ -990,4 +1007,15 @@ private record Stage(String id, List details, RunnableThatThrows task) { private record ToDownload(String id, String url, Path path) {} private record InputPath(String id, Path path, boolean freeAfterReading) {} + + /** An exception that occurs while running planetiler. */ + public static class PlanetilerException extends RuntimeException { + public PlanetilerException(String message, Exception e) { + super(message, e); + } + + public PlanetilerException(Exception e) { + super(e); + } + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java index 4d8ae8d4ac..8a7dc8f53b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java @@ -1,5 +1,6 @@ package com.onthegomap.planetiler; +import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.mbtiles.Mbtiles; @@ -32,9 +33,18 @@ * For complex profiles, {@link ForwardingProfile} provides a framework for splitting the logic up into several handlers * (i.e. one per layer) and forwarding each element/event to the handlers that care about it. */ -public interface Profile { +public interface Profile extends FeatureProcessor { // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and TileArchiveMetadata) can use too + /** + * Default attribution recommended for profiles using OpenStreetMap data + * + * @see www.openstreetmap.org/copyright + */ + String OSM_ATTRIBUTION = """ + © OpenStreetMap contributors + """.trim(); + /** * Allows profile to extract any information it needs from a {@link OsmElement.Node} during the first pass through OSM * elements. @@ -71,19 +81,6 @@ default List preprocessOsmRelation(OsmElement.Relation relation return null; } - /** - * Generates output features for any input feature that should appear in the map. - *

- * Multiple threads may invoke this method concurrently for a single data source so implementations should ensure - * thread-safe access to any shared data structures. Separate data sources are processed sequentially. - *

- * All OSM nodes are processed first, then ways, then relations. - * - * @param sourceFeature the input feature from a source dataset (OSM element, shapefile element, etc.) - * @param features a collector for generating output map features to emit - */ - void processFeature(SourceFeature sourceFeature, FeatureCollector features); - /** Free any resources associated with this profile (i.e. shared data structures) */ default void release() {} @@ -143,7 +140,9 @@ default Map> postProcessTileFeatures(TileCoord * * @see MBTiles specification */ - String name(); + default String name() { + return getClass().getSimpleName(); + } /** * Returns the description of the generated tileset to put into {@link Mbtiles} metadata @@ -248,6 +247,14 @@ default long estimateRamRequired(long osmFileSize) { return 0L; } + /** + * Returns false if this profile will ignore every feature in a set where {@linkplain Expression.PartialInput partial + * attributes} are known ahead of time. + */ + default boolean caresAbout(Expression.PartialInput input) { + return true; + } + /** * A default implementation of {@link Profile} that emits no output elements. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java index 879e20e3bf..9124552c29 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java @@ -546,4 +546,9 @@ public Arguments subset(String... allowedKeys) { () -> keys.get().stream().filter(key -> allowed.contains(normalize(key))).toList() ); } + + /** Returns a new arguments instance where the value for {@code key} defaults to {@code value}. */ + public Arguments withDefault(Object key, Object value) { + return orElse(Arguments.of(key.toString().replaceFirst("^-*", ""), value)); + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java index 3d24c1875a..79a51f140f 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/Expression.java @@ -2,15 +2,19 @@ import static com.onthegomap.planetiler.expression.DataType.GET_TAG; -import com.google.common.base.Joiner; +import com.onthegomap.planetiler.geo.GeometryType; +import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.reader.WithGeometryType; import com.onthegomap.planetiler.reader.WithTags; import com.onthegomap.planetiler.util.Format; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -28,6 +32,7 @@ * } */ // TODO rename to BooleanExpression +@FunctionalInterface public interface Expression extends Simplifiable { Logger LOGGER = LoggerFactory.getLogger(Expression.class); @@ -123,6 +128,32 @@ static MatchType matchType(String type) { return new MatchType(type); } + /** + * Returns an expression that evaluates to true if the geometry of an element matches {@code type}. + */ + static MatchType matchGeometryType(GeometryType type) { + return new MatchType(switch (type) { + case POINT -> POINT_TYPE; + case LINE -> LINESTRING_TYPE; + case POLYGON -> POLYGON_TYPE; + case null, default -> throw new IllegalArgumentException("Unsupported type: " + type); + }); + } + + /** + * Returns an expression that evaluates to true if the source of an element matches {@code source}. + */ + static MatchSource matchSource(String source) { + return new MatchSource(source); + } + + /** + * Returns an expression that evaluates to true if the source layer of an element matches {@code layer}. + */ + static MatchSourceLayer matchSourceLayer(String layer) { + return new MatchSourceLayer(layer); + } + private static String generateJavaCodeList(List items) { return items.stream().map(Expression::generateJavaCode).collect(Collectors.joining(", ")); } @@ -148,6 +179,19 @@ default Expression replace(Predicate replace, Expression b) { } } + /** Calls {@code fn} for every expression within the current one. */ + default void visit(Consumer fn) { + fn.accept(this); + switch (this) { + case Not(var child) -> child.visit(fn); + case Or(var children) -> children.forEach(child -> child.visit(fn)); + case And(var children) -> children.forEach(child -> child.visit(fn)); + default -> { + // already called fn, and no nested children + } + } + } + /** Returns true if this expression or any subexpression matches {@code filter}. */ default boolean contains(Predicate filter) { if (filter.test(this)) { @@ -162,6 +206,10 @@ default boolean contains(Predicate filter) { } } + private static Expression constBool(boolean value) { + return value ? TRUE : FALSE; + } + /** * Returns true if this expression matches an input element. * @@ -171,6 +219,20 @@ default boolean contains(Predicate filter) { */ boolean evaluate(WithTags input, List matchKeys); + /** + * Returns a copy of this expression where any parts that the value is known replaced with {@link #TRUE} or + * {@link #FALSE} for a set of features where {@link PartialInput} are known ahead of time (ie. a hive-partitioned + * parquet input file). + */ + default Expression partialEvaluate(PartialInput input) { + return switch (this) { + case Not(var expr) -> not(expr.partialEvaluate(input)); + case And(var exprs) -> and(exprs.stream().map(e -> e.partialEvaluate(input)).toList()); + case Or(var exprs) -> or(exprs.stream().map(e -> e.partialEvaluate(input)).toList()); + default -> this; + }; + } + //A list that silently drops all additions class NoopList extends ArrayList { @Override @@ -190,7 +252,9 @@ default boolean evaluate(WithTags input) { } /** Returns Java code that can be used to reconstruct this expression. */ - String generateJavaCode(); + default String generateJavaCode() { + throw new UnsupportedOperationException(); + } /** A constant boolean value. */ record Constant(boolean value, @Override String generateJavaCode) implements Expression { @@ -329,8 +393,8 @@ public Expression simplifyOnce() { } /** - * Evaluates to true if the value for {@code field} tag is any of {@code exactMatches} or contains any of {@code - * wildcards}. + * Evaluates to true if the value for {@code field} tag is any of {@code exactMatches} or contains any of + * {@code wildcards}. * * @param values all raw string values that were initially provided * @param exactMatches the input {@code values} that should be treated as exact matches @@ -405,8 +469,31 @@ private static String unescape(String input) { @Override public boolean evaluate(WithTags input, List matchKeys) { Object value = valueGetter.apply(input, field); + return evaluate(matchKeys, value); + } + + @Override + public Expression partialEvaluate(PartialInput input) { + Object value = input.getTag(field); + return value == null ? this : constBool(evaluate(new ArrayList<>(), value)); + } + + private boolean evaluate(List matchKeys, Object value) { if (value == null || "".equals(value)) { return matchWhenMissing; + } else if (value instanceof Collection c) { + if (c.isEmpty()) { + return matchWhenMissing; + } else { + for (var item : c) { + if (evaluate(matchKeys, item)) { + return true; + } + } + return false; + } + } else if (value instanceof Map) { + return false; } else { String str = value.toString(); if (exactMatches.contains(str)) { @@ -483,12 +570,17 @@ public String generateJavaCode() { @Override public boolean evaluate(WithTags input, List matchKeys) { Object value = input.getTag(field); - if (value != null && !"".equals(value)) { + if (value != null && !"".equals(value) && !(value instanceof Collection c && c.isEmpty())) { matchKeys.add(field); return true; } return false; } + + @Override + public Expression partialEvaluate(PartialInput input) { + return input.hasTag(field) ? TRUE : this; + } } /** @@ -514,5 +606,90 @@ public boolean evaluate(WithTags input, List matchKeys) { return false; } } + + @Override + public Expression partialEvaluate(PartialInput input) { + return input.types.isEmpty() || input.types.contains(GeometryType.UNKNOWN) ? this : + partialEvaluateContains(input.types, switch (type) { + case LINESTRING_TYPE -> GeometryType.LINE; + case POLYGON_TYPE -> GeometryType.POLYGON; + case POINT_TYPE -> GeometryType.POINT; + default -> GeometryType.UNKNOWN; + }, this); + } + } + + /** + * Evaluates to true if an input element has source matching {@code source}. + */ + record MatchSource(String source) implements Expression { + + @Override + public String generateJavaCode() { + return "matchSource(" + Format.quote(source) + ")"; + } + + @Override + public boolean evaluate(WithTags input, List matchKeys) { + return input instanceof SourceFeature feature && source.equals(feature.getSource()); + } + + @Override + public Expression partialEvaluate(PartialInput input) { + return partialEvaluateContains(input.source, source, this); + } + } + + /** + * Evaluates to true if an input element has source layer matching {@code layer}. + */ + record MatchSourceLayer(String layer) implements Expression { + + @Override + public String generateJavaCode() { + return "matchSourceLayer(" + Format.quote(layer) + ")"; + } + + @Override + public boolean evaluate(WithTags input, List matchKeys) { + return input instanceof SourceFeature feature && layer.equals(feature.getSourceLayer()); + } + + @Override + public Expression partialEvaluate(PartialInput input) { + return partialEvaluateContains(input.layer, layer, this); + } + } + + private static Expression partialEvaluateContains(Set set, T item, Expression self) { + if (set.isEmpty()) { + return self; + } else if (set.size() == 1) { + return constBool(set.contains(item)); + } else if (set.contains(item)) { + return self; + } else { + return FALSE; + } + } + + /** + * Partial attributes of a set of features that are known ahead of time, for example a hive-partitioned parquet input + * file. + *

+ * Features within this set will only add tags but not change them, and the source/layer/geometry type will be one of + * the values specified in those sets. If the set is empty, the values are not known ahead of time. + */ + record PartialInput(Set source, Set layer, Map tags, Set types) + implements WithTags { + + public static PartialInput ofSource(String source) { + return new PartialInput(Set.of(source), Set.of(), Map.of(), Set.of()); + } + + @Override + public Map tags() { + return tags == null ? Map.of() : tags; + } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java index d1e86d944c..0f6d9691c3 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/expression/MultiExpression.java @@ -4,6 +4,7 @@ import static com.onthegomap.planetiler.expression.Expression.TRUE; import static com.onthegomap.planetiler.expression.Expression.matchType; +import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.reader.WithGeometryType; import com.onthegomap.planetiler.reader.WithTags; import java.util.ArrayList; @@ -32,7 +33,7 @@ * * @param type of data value associated with each expression */ -public record MultiExpression (List> expressions) implements Simplifiable> { +public record MultiExpression(List> expressions) implements Simplifiable> { private static final Logger LOGGER = LoggerFactory.getLogger(MultiExpression.class); private static final Comparator BY_ID = Comparator.comparingInt(WithId::id); @@ -98,9 +99,18 @@ private Index index(boolean warn) { if (expressions.isEmpty()) { return new EmptyIndex<>(); } - boolean caresAboutGeometryType = - expressions.stream().anyMatch(entry -> entry.expression.contains(exp -> exp instanceof Expression.MatchType)); - return caresAboutGeometryType ? new GeometryTypeIndex<>(this, warn) : new KeyIndex<>(simplify(), warn); + if (contains(Expression.MatchSource.class::isInstance)) { + return new SourceIndex<>(this, warn); + } else if (contains(Expression.MatchSourceLayer.class::isInstance)) { + return new SourceLayerIndex<>(this, warn); + } else if (contains(Expression.MatchType.class::isInstance)) { + return new GeometryTypeIndex<>(this, warn); + } + return new KeyIndex<>(simplify(), warn); + } + + private boolean contains(Predicate test) { + return expressions.stream().anyMatch(entry -> entry.expression.contains(test)); } /** Returns a copy of this multi-expression that replaces every expression using {@code mapper}. */ @@ -203,7 +213,7 @@ private static class EmptyIndex implements Index { @Override public List> getMatchesWithTriggers(WithTags input) { - return List.of(); + return new ArrayList<>(); } @Override @@ -238,7 +248,12 @@ private KeyIndex(MultiExpression expressions, boolean warn) { always.add(expressionValue); } else { getRelevantKeys(expression, - key -> keyToExpressions.computeIfAbsent(key, k -> new HashSet<>()).add(expressionValue)); + key -> { + while (!key.isBlank()) { + keyToExpressions.computeIfAbsent(key, k -> new HashSet<>()).add(expressionValue); + key = key.replaceAll("(^|(\\[])?\\.)[^.]*$", ""); + } + }); } } // create immutable copies for fast iteration at matching time @@ -302,10 +317,10 @@ public List> getMatchesWithTriggers(WithTags input) { /** Index that limits the search space of expressions based on geometry type of an input element. */ private static class GeometryTypeIndex implements Index { - private final KeyIndex pointIndex; - private final KeyIndex lineIndex; - private final KeyIndex polygonIndex; - private final KeyIndex otherIndex; + private final Index pointIndex; + private final Index lineIndex; + private final Index polygonIndex; + private final Index otherIndex; private GeometryTypeIndex(MultiExpression expressions, boolean warn) { // build an index per type then search in each of those indexes based on the geometry type of each input element @@ -316,14 +331,12 @@ private GeometryTypeIndex(MultiExpression expressions, boolean warn) { otherIndex = indexForType(expressions, Expression.UNKNOWN_GEOMETRY_TYPE, warn); } - private KeyIndex indexForType(MultiExpression expressions, String type, boolean warn) { - return new KeyIndex<>( - expressions - .replace(matchType(type), TRUE) - .replace(e -> e instanceof Expression.MatchType, FALSE) - .simplify(), - warn - ); + private Index indexForType(MultiExpression expressions, String type, boolean warn) { + return expressions + .replace(matchType(type), TRUE) + .replace(e -> e instanceof Expression.MatchType, FALSE) + .simplify() + .index(warn); } /** @@ -354,14 +367,97 @@ public List> getMatchesWithTriggers(WithTags input) { } } + private abstract static class StringFieldIndex implements Index { + + private final Map> sourceIndex; + private final Index allSourcesIndex; + + private StringFieldIndex(MultiExpression expressions, boolean warn, Function extract, + Function make) { + Set sources = new HashSet<>(); + for (var expression : expressions.expressions) { + expression.expression.visit(e -> { + String key = extract.apply(e); + if (key != null) { + sources.add(key); + } + }); + } + sourceIndex = HashMap.newHashMap(sources.size()); + for (var source : sources) { + var forThisSource = expressions + .replace(make.apply(source), TRUE) + .replace(e -> extract.apply(e) != null, FALSE) + .simplify() + .index(warn); + if (!forThisSource.isEmpty()) { + sourceIndex.put(source, forThisSource); + } + } + allSourcesIndex = expressions.replace(e -> extract.apply(e) != null, FALSE).simplify().index(warn); + } + + abstract String extract(WithTags input); + + /** + * Returns all data values associated with expressions that match an input element, along with the tag keys that + * caused the match. + */ + public List> getMatchesWithTriggers(WithTags input) { + List> result = null; + String key = extract(input); + if (key != null) { + var index = sourceIndex.get(key); + if (index != null) { + result = index.getMatchesWithTriggers(input); + } + } + if (result == null) { + result = allSourcesIndex.getMatchesWithTriggers(input); + } + result.sort(BY_ID); + return result; + } + } + + /** Index that limits the search space of expressions based on geometry type of an input element. */ + private static class SourceLayerIndex extends StringFieldIndex { + + private SourceLayerIndex(MultiExpression expressions, boolean warn) { + super(expressions, warn, + e -> e instanceof Expression.MatchSourceLayer(var layer) ? layer : null, + Expression::matchSourceLayer); + } + + @Override + String extract(WithTags input) { + return input instanceof SourceFeature feature ? feature.getSourceLayer() : null; + } + } + + /** Index that limits the search space of expressions based on geometry type of an input element. */ + private static class SourceIndex extends StringFieldIndex { + + private SourceIndex(MultiExpression expressions, boolean warn) { + super(expressions, warn, + e -> e instanceof Expression.MatchSource(var source) ? source : null, + Expression::matchSource); + } + + @Override + String extract(WithTags input) { + return input instanceof SourceFeature feature ? feature.getSource() : null; + } + } + /** An expression/value pair with unique ID to store whether we evaluated it yet. */ - private record EntryWithId (T result, Expression expression, @Override int id) implements WithId {} + private record EntryWithId(T result, Expression expression, @Override int id) implements WithId {} /** * An {@code expression} to evaluate on input elements and {@code result} value to return when the element matches. */ - public record Entry (T result, Expression expression) {} + public record Entry(T result, Expression expression) {} /** The result when an expression matches, along with the input element tag {@code keys} that triggered the match. */ - public record Match (T match, List keys, @Override int id) implements WithId {} + public record Match(T match, List keys, @Override int id) implements WithId {} } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java index 4f32a398db..b5c71f0f6a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/SourceFeature.java @@ -65,19 +65,6 @@ protected SourceFeature(Map tags, String source, String sourceLa this.id = id; } - // slight optimization: replace default implementation with direct access to the tags - // map to get slightly improved performance when matching elements against expressions - - @Override - public Object getTag(String key) { - return tags.get(key); - } - - @Override - public boolean hasTag(String key) { - return tags.containsKey(key); - } - @Override public Map tags() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/Struct.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/Struct.java index cb86c7fe15..c5e599e1d5 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/Struct.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/Struct.java @@ -56,14 +56,14 @@ static Struct of(Object o) { result.put(e.getKey(), v); } } - yield new MapStruct(result); + yield new MapStruct(result, map); } case Collection collection -> { List result = new ArrayList<>(collection.size()); for (var d : collection) { result.add(of(d)); } - yield new ListStruct(result); + yield new ListStruct(result, collection); } default -> throw new IllegalArgumentException("Unable to convert " + o + " (" + o.getClass() + ")"); }; @@ -253,22 +253,29 @@ default Struct flatMap(UnaryOperator mapper) { .flatMap(item -> mapper.apply(item).asList().stream()) .map(Struct::of) .toList(); - return list.isEmpty() ? NULL : new ListStruct(list); + var raw = list.stream().map(Struct::rawValue).toList(); + return list.isEmpty() ? NULL : new ListStruct(list, raw); } class PrimitiveStruct implements Struct { final T value; + private final Object raw; private String asJson; - PrimitiveStruct(T value) { + PrimitiveStruct(T value, Object raw) { this.value = value; + this.raw = raw; + } + + PrimitiveStruct(T value) { + this(value, value); } @Override public final Object rawValue() { - return value; + return raw; } @Override @@ -521,8 +528,8 @@ public String asString() { class MapStruct extends PrimitiveStruct> { - MapStruct(Map value) { - super(value); + MapStruct(Map value, Map raw) { + super(value, raw); } @Override @@ -562,8 +569,8 @@ public Map asMap() { } class ListStruct extends PrimitiveStruct> { - ListStruct(List value) { - super(value); + ListStruct(List value, Collection raw) { + super(value, raw); } @Override @@ -578,6 +585,9 @@ public Struct get(int index) { @Override public Struct get(Object key) { + if (key instanceof String k) { + return flatMap(v -> v.get(k.replaceAll("^\\[]\\.?", ""))); + } return key instanceof Number n ? get(n.intValue()) : NULL; } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/WithTags.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/WithTags.java index 1fcdfd25a1..242b85a811 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/WithTags.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/WithTags.java @@ -3,6 +3,7 @@ import com.onthegomap.planetiler.util.Imposm3Parsers; import com.onthegomap.planetiler.util.Parse; import java.util.Arrays; +import java.util.Collection; import java.util.Map; import java.util.Objects; @@ -26,7 +27,7 @@ default Object getTag(String key) { } private Struct getDotted(String key) { - String[] parts = key.split("\\.", 2); + String[] parts = key.split("(\\[])?\\.", 2); if (parts.length == 2) { return getStruct(parts[0]).get(parts[1]); } @@ -46,8 +47,25 @@ default boolean hasTag(String key) { return contains || (key.contains(".") && !getDotted(key).isNull()); } + private static boolean contains(Object actual, Object expected) { + if (actual instanceof Collection actualList) { + if (expected instanceof Collection expectedList) { + for (var elem : expectedList) { + if (actualList.contains(elem)) { + return true; + } + } + } else { + return actualList.contains(expected); + } + } else if (expected instanceof Collection expectedList) { + return expectedList.contains(actual); + } + return expected.equals(actual); + } + default boolean hasTag(String key, Object value) { - return value.equals(getTag(key)); + return contains(getTag(key), value); } /** @@ -61,7 +79,7 @@ default boolean hasTag(String key, Object value1, Object value2) { if (actual == null) { return false; } else { - return value1.equals(actual) || value2.equals(actual); + return contains(actual, value1) || contains(actual, value2); } } @@ -69,11 +87,11 @@ default boolean hasTag(String key, Object value1, Object value2) { default boolean hasTag(String key, Object value1, Object value2, Object... others) { Object actual = getTag(key); if (actual != null) { - if (value1.equals(actual) || value2.equals(actual)) { + if (contains(actual, value1) || contains(actual, value2)) { return true; } for (Object value : others) { - if (value.equals(actual)) { + if (contains(actual, value)) { return true; } } @@ -155,7 +173,7 @@ default T as(Class clazz) { /** * Serializes the properties on this feature as a JSON object. */ - default String asJson() { + default String tagsAsJson() { return JsonConversion.writeValueAsString(tags()); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/GeoParquetMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/GeoParquetMetadata.java index bbf4dab036..694625f70d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/GeoParquetMetadata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/GeoParquetMetadata.java @@ -7,10 +7,13 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.onthegomap.planetiler.config.Bounds; import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.geo.GeometryType; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.filter2.predicate.FilterPredicate; @@ -40,6 +43,34 @@ public record GeoParquetMetadata( private static final ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + /** + * Returns the {@link GeometryType GeometryTypes} that can be contained in this geoparquet file or an empty set if + * unknown. + *

+ * These can come from geoarrow encoding type, or the {@code geometry_types} attributes. + */ + public Set geometryTypes() { + Set types = new HashSet<>(); + for (var type : primaryColumnMetadata().geometryTypes()) { + types.add(switch (type) { + case "Point", "MultiPoint" -> GeometryType.POINT; + case "LineString", "MultiLineString" -> GeometryType.LINE; + case "Polygon", "MultiPolygon" -> GeometryType.POLYGON; + case null, default -> GeometryType.UNKNOWN; + }); + } + // geoarrow + String encoding = primaryColumnMetadata().encoding(); + if (encoding.contains("polygon")) { + types.add(GeometryType.POLYGON); + } else if (encoding.contains("point")) { + types.add(GeometryType.POINT); + } else if (encoding.contains("linestring")) { + types.add(GeometryType.LINE); + } + return types; + } + public record CoveringBbox( List xmin, List ymin, diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetFeature.java index 225ab08238..b7893d1ce7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetFeature.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetFeature.java @@ -109,15 +109,7 @@ public Struct getStruct(Object key, Object... others) { @Override public Object getTag(String key) { - var value = tags().get(key); - if (value == null) { - String[] parts = key.split("\\.", 2); - if (parts.length == 2) { - return getStruct(parts[0]).get(parts[1]).rawValue(); - } - return getStruct(parts[0]).rawValue(); - } - return value; + return cachedStruct().get(key).rawValue(); } @Override @@ -131,7 +123,7 @@ public Object getTag(String key, Object defaultValue) { @Override public boolean hasTag(String key) { - return super.hasTag(key) || getTag(key) != null; + return !cachedStruct().get(key).isNull(); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java index 28f560729e..1f861e42e3 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java @@ -2,8 +2,11 @@ import blue.strategic.parquet.ParquetReader; import com.google.common.collect.Iterators; +import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.config.Bounds; +import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.geo.GeometryException; +import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.util.Hashing; import java.io.Closeable; @@ -17,6 +20,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.function.ToLongFunction; import java.util.stream.IntStream; @@ -52,6 +56,7 @@ public class ParquetInputFile { private final int blockCount; final GeometryReader geometryReader; private final Map extraFields; + private final Set geometryTypes; private Envelope postFilterBounds = null; private boolean outOfBounds = false; @@ -72,6 +77,7 @@ public ParquetInputFile(String source, String layer, Path path, FilterPredicate var fileMetadata = metadata.getFileMetaData(); var geoparquet = GeoParquetMetadata.parse(fileMetadata); this.geometryReader = new GeometryReader(geoparquet); + this.geometryTypes = geoparquet.geometryTypes(); if (!bounds.isWorld()) { if (!geoparquet.primaryColumnMetadata().envelope().intersects(bounds.latLon())) { outOfBounds = true; @@ -110,8 +116,9 @@ public boolean hasFilter() { return FilterCompat.isFilteringRequired(filter); } - public boolean isOutOfBounds() { - return outOfBounds; + public boolean shouldSkip(Profile profile) { + return outOfBounds || + !profile.caresAbout(new Expression.PartialInput(Set.of(source), Set.of(layer), extraFields, geometryTypes)); } public BlockReader get() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetReader.java index ff24ce025a..0fe63b62df 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetReader.java @@ -13,6 +13,7 @@ import com.onthegomap.planetiler.stats.ProgressLoggers; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.Format; +import com.onthegomap.planetiler.worker.Distributor; import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.IOException; import java.io.UncheckedIOException; @@ -90,7 +91,7 @@ public void process(List sourcePath, FeatureGroup writer, PlanetilerConfig String layer = getLayerName(path); return new ParquetInputFile(sourceName, layer, path, null, config.bounds(), hivePartitionFields, idGenerator); }) - .filter(file -> !file.isOutOfBounds()) + .filter(file -> !file.shouldSkip(profile)) .toList(); // don't show % complete on features when a filter is present because to determine total # elements would // take an expensive initial query, and % complete on blocks gives a good enough proxy @@ -112,6 +113,7 @@ public void process(List sourcePath, FeatureGroup writer, PlanetilerConfig throw new UncheckedIOException(e); } }).toList(); + Distributor distributor = Distributor.createWithCapacity(1_000); var pipeline = WorkerPipeline.start(sourceName, stats) .readFromTiny("blocks", inputBlocks) @@ -120,24 +122,28 @@ public void process(List sourcePath, FeatureGroup writer, PlanetilerConfig var elements = featuresRead.counterForThread(); var featureCollectors = new FeatureCollector.Factory(config, stats); try (FeatureRenderer renderer = newFeatureRenderer(writer, config, next)) { + var consumer = distributor.forThread(sourceFeature -> { + FeatureCollector features = featureCollectors.get(sourceFeature); + try { + profile.processFeature(sourceFeature, features); + for (FeatureCollector.Feature renderable : features) { + renderer.accept(renderable); + } + } catch (Exception e) { + LOGGER.error("Error processing {}", sourceFeature, e); + } + elements.inc(); + }); for (var block : prev) { String layer = block.layer(); workingOn.merge(layer, 1, Integer::sum); for (var sourceFeature : block) { - FeatureCollector features = featureCollectors.get(sourceFeature); - try { - profile.processFeature(sourceFeature, features); - for (FeatureCollector.Feature renderable : features) { - renderer.accept(renderable); - } - } catch (Exception e) { - LOGGER.error("Error processing {}", sourceFeature, e); - } - elements.inc(); + consumer.accept(sourceFeature); } blocks.inc(); workingOn.merge(layer, -1, Integer::sum); } + consumer.close(); } }) .addBuffer("write_queue", 50_000, 1_000) diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/YAML.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java similarity index 97% rename from planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/YAML.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java index 2b98417be7..780eaff1ed 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/YAML.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/YAML.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.custommap; +package com.onthegomap.planetiler.util; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/BaseSchemaValidator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/BaseSchemaValidator.java new file mode 100644 index 0000000000..41af894cac --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/BaseSchemaValidator.java @@ -0,0 +1,279 @@ +package com.onthegomap.planetiler.validator; + +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.geo.GeometryType; +import com.onthegomap.planetiler.reader.SimpleFeature; +import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.util.AnsiColors; +import com.onthegomap.planetiler.util.FileWatcher; +import com.onthegomap.planetiler.util.Format; +import com.onthegomap.planetiler.util.Try; +import java.io.PrintStream; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; + +/** + * Verifies that a profile maps input elements map to expected output vector tile features as defined by a + * {@link SchemaSpecification} instance. + */ +public abstract class BaseSchemaValidator { + + private static final String PASS_BADGE = AnsiColors.greenBackground(" PASS "); + private static final String FAIL_BADGE = AnsiColors.redBackground(" FAIL "); + protected final PrintStream output; + private final boolean watch; + + protected BaseSchemaValidator(Arguments args, PrintStream output) { + this( + args.getBoolean("watch", "Watch files for changes and re-run validation when schema or spec changes", false), + output + ); + } + + protected BaseSchemaValidator(boolean watch, PrintStream output) { + this.watch = watch; + this.output = output; + } + + protected static boolean hasCause(Throwable t, Class cause) { + return t != null && (cause.isInstance(t) || hasCause(t.getCause(), cause)); + } + + protected boolean runOrWatch() { + output.println("OK"); + var response = validateFromCli(); + + if (watch) { + output.println(); + output.println("Watching filesystem for changes..."); + var watcher = FileWatcher.newWatcher(response.paths.toArray(Path[]::new)); + watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli().paths); + } + + return response.result.ok(); + } + + public record TestSummary(Result result, Set paths) {} + + public TestSummary validateFromCli() { + Set pathsToWatch = ConcurrentHashMap.newKeySet(); + output.println(); + output.println("Validating..."); + output.println(); + BaseSchemaValidator.Result result; + result = validate(pathsToWatch); + int failed = 0; + if (result != null) { + + int passed = 0; + List failures = new ArrayList<>(); + for (var example : result.results) { + if (example.ok()) { + passed++; + output.printf("%s %s%n", PASS_BADGE, example.example().name()); + } else { + failed++; + printFailure(example, output); + failures.add(example); + } + } + if (!failures.isEmpty()) { + output.println(); + output.println("Summary of failures:"); + for (var failure : failures) { + printFailure(failure, output); + } + } + List summary = new ArrayList<>(); + boolean none = (passed + failed) == 0; + if (none || failed > 0) { + summary.add(AnsiColors.redBold(failed + " failed")); + } + if (none || passed > 0) { + summary.add(AnsiColors.greenBold(passed + " passed")); + } + if (none || passed > 0 && failed > 0) { + summary.add((failed + passed) + " total"); + } + output.println(); + output.println(String.join(", ", summary)); + } + return new TestSummary(result, pathsToWatch); + } + + protected abstract Result validate(Set pathsToWatch); + + private static void printFailure(ExampleResult example, PrintStream output) { + if (example.example() != null) { + output.printf("%s %s%n", FAIL_BADGE, example.example().name()); + } + if (example.issues.isFailure()) { + output.println(ExceptionUtils.getStackTrace(example.issues.exception()).indent(4).stripTrailing()); + } else { + for (var issue : example.issues().get()) { + output.println(" ● " + issue.indent(4).strip()); + } + } + } + + private static Geometry parseGeometry(String geometry) { + String wkt = switch (geometry.toLowerCase(Locale.ROOT).trim()) { + case "point" -> "POINT (0 0)"; + case "line" -> "LINESTRING (0 0, 1 1)"; + case "polygon" -> "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"; + default -> geometry; + }; + try { + return new WKTReader().read(wkt); + } catch (ParseException e) { + throw new IllegalArgumentException(""" + Bad geometry: "%s", must be "point" "line" "polygon" or a valid WKT string. + """.formatted(geometry)); + } + } + + private record FeatureWithOverrides(FeatureCollector.Feature feature, FeatureCollector.RangeWithTags overrides) {} + + /** Returns the result of validating {@code profile} against the examples in {@code specification}. */ + public static Result validate(Profile profile, SchemaSpecification specification, PlanetilerConfig config) { + var featureCollectorFactory = new FeatureCollector.Factory(config, Stats.inMemory()); + return new Result(specification.examples().stream().map(example -> new ExampleResult(example, Try.apply(() -> { + List issues = new ArrayList<>(); + var input = example.input(); + var expectedFeatures = example.output(); + var geometry = parseGeometry(input.geometry()); + var feature = SimpleFeature.create(geometry, input.tags(), input.source(), null, 0); + var collector = featureCollectorFactory.get(feature); + profile.processFeature(feature, collector); + List result = new ArrayList<>(); + Set zooms = expectedFeatures.stream().map(f -> f.atZoom()).collect(Collectors.toSet()); + for (var outputFeature : collector) { + if (outputFeature.hasLinearRanges()) { + for (var zoom : zooms) { + for (var range : outputFeature.getLinearRangesAtZoom(zoom)) { + result.add(new FeatureWithOverrides(outputFeature, range)); + } + } + } else { + result.add(new FeatureWithOverrides(outputFeature, null)); + } + } + if (result.size() != expectedFeatures.size()) { + issues.add( + "Different number of elements, expected=%s actual=%s".formatted(expectedFeatures.size(), result.size())); + } else { + // TODO print a diff of the input and output feature YAML representations + for (int i = 0; i < expectedFeatures.size(); i++) { + var expected = expectedFeatures.get(i); + var actual = result.stream().max(proximityTo(expected)).orElseThrow(); + result.remove(actual); + var actualTags = + actual.overrides != null ? actual.overrides.attrs() : actual.feature.getAttrsAtZoom(expected.atZoom()); + String prefix = "feature[%d]".formatted(i); + validate(prefix + ".layer", issues, expected.layer(), actual.feature.getLayer()); + validate(prefix + ".minzoom", issues, expected.minZoom(), actual.feature.getMinZoom()); + validate(prefix + ".maxzoom", issues, expected.maxZoom(), actual.feature.getMaxZoom()); + validate(prefix + ".minsize", issues, expected.minSize(), + actual.feature.getMinPixelSizeAtZoom(expected.atZoom())); + validate(prefix + ".geometry", issues, expected.geometry(), + GeometryType.typeOf(actual.feature.getGeometry())); + Set tags = new TreeSet<>(actualTags.keySet()); + expected.tags().forEach((tag, value) -> { + validate(prefix + ".tags[\"%s\"]".formatted(tag), issues, value, actualTags.get(tag), false); + tags.remove(tag); + }); + if (Boolean.FALSE.equals(expected.allowExtraTags())) { + for (var tag : tags) { + validate(prefix + ".tags[\"%s\"]".formatted(tag), issues, null, actualTags.get(tag), false); + } + } + } + } + return issues; + }))).toList()); + } + + private static Comparator proximityTo(SchemaSpecification.OutputFeature expected) { + return Comparator.comparingInt(item -> (Objects.equals(item.feature.getLayer(), expected.layer()) ? 2 : 0) + + (Objects.equals(GeometryType.typeOf(item.feature.getGeometry()), expected.geometry()) ? 1 : 0) + + mapProximity(expected, item, expected.allowExtraTags())); + } + + private static int mapProximity(SchemaSpecification.OutputFeature expected, FeatureWithOverrides actualFeature, + Boolean allowExtra) { + int zoom = expected.atZoom(); + var expectedMap = expected.tags(); + var actualMap = + actualFeature.overrides != null ? actualFeature.overrides.attrs() : actualFeature.feature.getAttrsAtZoom(zoom); + int result = 0; + for (var entry : expectedMap.entrySet()) { + if (actualMap.containsKey(entry.getKey())) { + result++; + if (actualMap.get(entry.getKey()).equals(entry.getValue())) { + result++; + } + } + } + + return result; + } + + private static void validate(String field, List issues, T expected, T actual, boolean ignoreWhenNull) { + if ((!ignoreWhenNull || expected != null) && !Objects.equals(expected, actual)) { + // handle when expected and actual are int/long or long/int + if (expected instanceof Number && actual instanceof Number && expected.toString().equals(actual.toString())) { + return; + } + issues.add("%s: expected <%s> actual <%s>".formatted(field, format(expected), format(actual))); + } + } + + private static String format(Object o) { + if (o == null) { + return "null"; + } else if (o instanceof String s) { + return Format.quote(s); + } else { + return o.toString(); + } + } + + private static void validate(String field, List issues, T expected, T actual) { + validate(field, issues, expected, actual, true); + } + + /** Result of comparing the output vector tile feature to what was expected. */ + public record ExampleResult( + SchemaSpecification.Example example, + // TODO include a symmetric diff so we can pretty-print the expected/actual output diff + Try> issues + ) { + + public boolean ok() { + return issues.isSuccess() && issues.get().isEmpty(); + } + } + + public record Result(List results) { + + public boolean ok() { + return results.stream().allMatch(ExampleResult::ok); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/JavaProfileValidator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/JavaProfileValidator.java new file mode 100644 index 0000000000..ec303d557a --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/JavaProfileValidator.java @@ -0,0 +1,59 @@ +package com.onthegomap.planetiler.validator; + +import com.fasterxml.jackson.core.JacksonException; +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.util.AnsiColors; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.Set; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.snakeyaml.engine.v2.exceptions.YamlEngineException; + +/** + * Validates a java profile against a yaml set of example source features and the vector tile features they should map + * to. + */ +public class JavaProfileValidator extends BaseSchemaValidator { + + private final Profile profile; + private final Path specPath; + private final PlanetilerConfig config; + + JavaProfileValidator(PlanetilerConfig config, Path specPath, Profile profile, PrintStream output) { + super(config.arguments(), output); + this.config = config; + this.profile = profile; + this.specPath = specPath; + } + + /** + * Validates that {@code profile} maps input features to expected output features as defined in {@code specPath} and + * returns true if successful, false if failed. + */ + public static boolean validate(Profile profile, Path specPath, PlanetilerConfig config) { + return new JavaProfileValidator(config, specPath, profile, System.out).runOrWatch(); + } + + @Override + protected Result validate(Set pathsToWatch) { + Result result = null; + try { + SchemaSpecification spec; + pathsToWatch.add(specPath); + spec = SchemaSpecification.load(specPath); + result = validate(profile, spec, config); + } catch (Exception exception) { + Throwable rootCause = ExceptionUtils.getRootCause(exception); + if (hasCause(exception, YamlEngineException.class) || hasCause(exception, JacksonException.class)) { + output.println(AnsiColors.red("Malformed yaml input:\n\n" + rootCause.toString().indent(4))); + } else { + output.println(AnsiColors.red( + "Unexpected exception thrown:\n" + rootCause.toString().indent(4) + "\n" + + String.join("\n", ExceptionUtils.getStackTrace(rootCause))) + .indent(4)); + } + } + return result; + } +} diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaSpecification.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/SchemaSpecification.java similarity index 88% rename from planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaSpecification.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/validator/SchemaSpecification.java index 65385adc91..2014e9b9af 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaSpecification.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/validator/SchemaSpecification.java @@ -1,17 +1,21 @@ -package com.onthegomap.planetiler.custommap.validator; +package com.onthegomap.planetiler.validator; import static com.onthegomap.planetiler.config.PlanetilerConfig.MAX_MAXZOOM; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.onthegomap.planetiler.custommap.YAML; import com.onthegomap.planetiler.geo.GeometryType; +import com.onthegomap.planetiler.util.YAML; import java.nio.file.Path; import java.util.List; import java.util.Map; -/** A model of example input source features and expected output vector tile features that a schema should produce. */ +/** + * A model of example input source features and expected output vector tile features that a schema should produce. + *

+ * Executed by a subclass of {@link BaseSchemaValidator}. + */ @JsonIgnoreProperties(ignoreUnknown = true) public record SchemaSpecification(List examples) { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java index 7d894717f6..daab902c6a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java @@ -783,4 +783,73 @@ void testSetAttrPartial() { "d", true ), subFeature.attrs()); } + + @Test + void testSetAttrPartialWithMinzoom() { + var collector = factory.get(newReaderFeature(newLineString(0, 0, 1, 1), Map.of())); + collector.line("layername") + .setAttrWithMinzoom("full", 1, 2) + .setAttrWithMinzoom("fullstruct", Struct.of(2), 2) + .linearRange(0, 0.5) + .setAttr("a", Struct.of(1)) + .setAttrWithMinzoom("b", Struct.of(2d), 9) + .putAttrs(Map.of("c", Struct.of("3"), "d", ZoomFunction.minZoom(9, Struct.of(true)))); + var feature = collector.iterator().next(); + var subFeature = feature.getLinearRangesAtZoom(14).getFirst(); + assertEquals(Map.of( + "a", 1, + "b", 2d, + "c", "3", + "d", true, + "full", 1, + "fullstruct", 2 + ), subFeature.attrs()); + } + + @Test + void testUnwrapStruct() { + var collector = factory.get(newReaderFeature(newLineString(0, 0, 1, 1), Map.of())); + collector.line("layername") + .setAttr("full", Struct.of(1)) + .linearRange(0, 0.5) + .setAttr("partial", Struct.of(2)); + var feature = collector.iterator().next(); + var subFeature = feature.getLinearRangesAtZoom(14).getFirst(); + assertEquals(Map.of( + "full", 1, + "partial", 2 + ), subFeature.attrs()); + } + + @Test + void testUnwrapStructFull() { + var collector = factory.get(newReaderFeature(newLineString(0, 0, 1, 1), Map.of())); + collector.line("layername") + .setAttr("full", Struct.of(1)); + var feature = collector.iterator().next(); + assertEquals(Map.of( + "full", 1 + ), feature.getAttrsAtZoom(14)); + } + + @Test + void testUnwrapStructFullWithMinzoom() { + var collector = factory.get(newReaderFeature(newLineString(0, 0, 1, 1), Map.of())); + collector.line("layername") + .setAttrWithMinzoom("full", Struct.of(1), 2); + var feature = collector.iterator().next(); + assertEquals(Map.of( + "full", 1 + ), feature.getAttrsAtZoom(14)); + } + + @Test + void testSetAttrPartialWithMinSize() { + var collector = factory.get(newReaderFeature(newLineString(0, 0, 1, 1), Map.of())); + var line = collector.line("layername"); + + assertEquals(7, line.getMinZoomForPixelSize(100)); + assertEquals(7, line.linearRange(0, 0.5).getMinZoomForPixelSize(50)); + assertEquals(7, line.linearRange(0, 0.25).getMinZoomForPixelSize(25)); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/ForwardingProfileTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/ForwardingProfileTests.java index 51fc73705a..015a2806c3 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/ForwardingProfileTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/ForwardingProfileTests.java @@ -2,8 +2,13 @@ import static com.onthegomap.planetiler.TestUtils.assertSubmap; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; @@ -18,15 +23,12 @@ import java.util.Set; import java.util.TreeSet; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class ForwardingProfileTests { - private final ForwardingProfile profile = new ForwardingProfile() { - @Override - public String name() { - return "test"; - } - }; + private ForwardingProfile profile = new ForwardingProfile() {}; @Test void testPreprocessOsmNode() { @@ -112,6 +114,28 @@ void testProcessFeature() { )), b); } + @Test + void testProcessFeatureWithFilter() { + SourceFeature a = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of("key", "value"), "srca", null, 1); + SourceFeature b = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of(), "srcb", null, 1); + + profile.registerSourceHandler(a.getSource(), new ForwardingProfile.FeatureProcessor() { + @Override + public void processFeature(SourceFeature elem, FeatureCollector features) { + features.point("a"); + } + + @Override + public Expression filter() { + return Expression.matchAny("key", "value"); + } + }); + testFeatures(List.of(Map.of( + "_layer", "a" + )), a); + testFeatures(List.of(), b); + } + @Test void testFinishHandler() { Set finished = new TreeSet<>(); @@ -260,4 +284,103 @@ public Map> postProcessTile(TileCoord tileCoord profile.postProcessTileFeatures(TileCoord.ofXYZ(0, 0, 0), Map.of("c", List.of(feature, feature, feature, feature)))); } + + @Test + void testCaresAboutSource() { + profile.registerSourceHandler("a", (x, y) -> { + }); + assertTrue(profile.caresAboutSource("a")); + assertFalse(profile.caresAboutSource("b")); + + profile.registerSourceHandler("b", (x, y) -> { + }); + assertTrue(profile.caresAboutSource("a")); + assertTrue(profile.caresAboutSource("b")); + assertFalse(profile.caresAboutSource("c")); + + class C implements ForwardingProfile.Handler, ForwardingProfile.FeatureProcessor { + + @Override + public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {} + + @Override + public Expression filter() { + return Expression.matchSource("c"); + } + } + profile.registerHandler(new C()); + assertTrue(profile.caresAboutSource("a")); + assertTrue(profile.caresAboutSource("b")); + assertTrue(profile.caresAboutSource("c")); + assertFalse(profile.caresAboutSource("d")); + + profile.registerFeatureHandler((x, y) -> { + }); + assertTrue(profile.caresAboutSource("d")); + assertTrue(profile.caresAboutSource("e")); + } + + @Test + void registerAnySourceFeatureHandler() { + SourceFeature a = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of(), "srca", null, 1); + SourceFeature b = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of(), "srcb", null, 1); + testFeatures(List.of(), a); + testFeatures(List.of(), b); + + profile.registerFeatureHandler((elem, features) -> features.point("a")); + testFeatures(List.of(Map.of( + "_layer", "a" + )), a); + testFeatures(List.of(Map.of( + "_layer", "a" + )), b); + } + + @Test + void registerHandlerWithFilter() { + SourceFeature a = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of("key", "value"), "srca", null, 1); + SourceFeature b = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of(), "srcb", null, 1); + testFeatures(List.of(), a); + testFeatures(List.of(), b); + + profile.registerFeatureHandler(new ForwardingProfile.FeatureProcessor() { + @Override + public void processFeature(SourceFeature elem, FeatureCollector features) { + features.point("a"); + } + + @Override + public Expression filter() { + return Expression.matchAny("key", "value"); + } + }); + testFeatures(List.of(Map.of( + "_layer", "a" + )), a); + testFeatures(List.of(), b); + } + + @ParameterizedTest + @ValueSource(strings = { + "--only-layers=water", + "--exclude-layers=land", + "--exclude-layers=land --only-layers=water,land", + }) + void testLayerCliArgFilter(String args) { + profile = new ForwardingProfile(PlanetilerConfig.from(Arguments.fromArgs(args.split(" ")))) {}; + record Processor(String name) implements ForwardingProfile.HandlerForLayer, ForwardingProfile.FeatureProcessor { + + @Override + public void processFeature(SourceFeature sourceFeature, FeatureCollector features) { + features.point(name); + } + } + + SourceFeature a = SimpleFeature.create(GeoUtils.EMPTY_POINT, Map.of("key", "value"), "source", "source layer", 1); + profile.registerHandler(new Processor("water")); + profile.registerHandler(new Processor("land")); + testFeatures(List.of(Map.of( + "_layer", "water" + )), a); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 1f57d9e915..05437a51b8 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -205,17 +205,6 @@ private PlanetilerResults runWithReaderFeaturesProfile( ); } - private PlanetilerResults runWithOsmElements( - Map args, - List features, - BiConsumer profileFunction - ) throws Exception { - return run( - args, - (featureGroup, profile, config) -> processOsmFeatures(featureGroup, profile, config, features), - TestProfile.processSourceFeatures(profileFunction) - ); - } private PlanetilerResults runWithOsmElements( Map args, @@ -300,8 +289,9 @@ void testOverrideMetadata() throws Exception { ), results.metadata); } - @Test - void testSinglePoint() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testSinglePoint(boolean anyGeom) throws Exception { double x = 0.5 + Z14_WIDTH / 4; double y = 0.5 + Z14_WIDTH / 4; double lat = GeoUtils.getWorldLat(y); @@ -314,7 +304,7 @@ void testSinglePoint() throws Exception { "attr", "value" )) ), - (in, features) -> features.point("layer") + (in, features) -> (anyGeom ? features.anyGeometry("layer") : features.point("layer")) .setZoomRange(13, 15) .setAttr("name", "name value") .inheritAttrFromSource("attr") @@ -438,8 +428,9 @@ void testLabelGridLimit() throws Exception { ), results.tiles); } - @Test - void testLineString() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testLineString(boolean anyGeom) throws Exception { double x1 = 0.5 + Z14_WIDTH / 2; double y1 = 0.5 + Z14_WIDTH / 2; double x2 = x1 + Z14_WIDTH; @@ -456,7 +447,7 @@ void testLineString() throws Exception { "attr", "value" )) ), - (in, features) -> features.line("layer") + (in, features) -> (anyGeom ? features.anyGeometry("layer") : features.line("layer")) .setZoomRange(13, 14) .setBufferPixels(4) ); @@ -690,8 +681,9 @@ public Point z14Point(double x, double y) { ); } - @Test - void testPolygonWithHoleSpanningMultipleTiles() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testPolygonWithHoleSpanningMultipleTiles(boolean anyGeom) throws Exception { List outerPoints = z14CoordinateList( 0.5, 0.5, 3.5, 0.5, @@ -715,7 +707,7 @@ void testPolygonWithHoleSpanningMultipleTiles() throws Exception { List.of(innerPoints) ), Map.of()) ), - (in, features) -> features.polygon("layer") + (in, features) -> (anyGeom ? features.anyGeometry("layer") : features.polygon("layer")) .setZoomRange(12, 14) .setBufferPixels(4) ); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 8e2729dca9..ba7e3161f7 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -29,6 +30,8 @@ import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.LayerAttrStats; +import com.onthegomap.planetiler.validator.BaseSchemaValidator; +import com.onthegomap.planetiler.validator.SchemaSpecification; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -52,7 +55,9 @@ import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.DynamicNode; import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; @@ -821,4 +826,18 @@ public static void assertTileDuplicates(Mbtiles db, int expected) { fail(e); } } + + public static Stream validateProfile(Profile profile, String spec) { + return validateProfile(profile, SchemaSpecification.load(spec)); + } + + public static Stream validateProfile(Profile profile, SchemaSpecification spec) { + var result = BaseSchemaValidator.validate(profile, spec, PlanetilerConfig.defaults()); + return result.results().stream().map(test -> dynamicTest(test.example().name(), () -> { + var issues = test.issues().get(); + if (!issues.isEmpty()) { + fail("Failed with " + issues.size() + " issues:\n" + String.join("\n", issues)); + } + })); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java index a51e4f3aaf..9f9e51dd72 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java @@ -38,6 +38,20 @@ void testOrElse() { assertEquals("fallback", args.getString("key4", "key", "fallback")); } + @Test + void testWithFallback() { + Arguments args = Arguments.of("key1", "value1a", "key2", "value2a") + .withDefault("key2", "value2b") + .withDefault("--key3", "value3b") + .withDefault("key_5", true); + + assertEquals("value1a", args.getString("key1", "key", "fallback")); + assertEquals("value2a", args.getString("key2", "key", "fallback")); + assertEquals("value3b", args.getString("key3", "key", "fallback")); + assertEquals("fallback", args.getString("key4", "key", "fallback")); + assertTrue(args.getBoolean("key-5", "key", false)); + } + @Test void testConfigFileParsing() { Arguments args = Arguments.fromConfigFile(TestUtils.pathToResource("test.properties")); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java index 74f2b7db52..6d931a5159 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/ExpressionTest.java @@ -1,12 +1,16 @@ package com.onthegomap.planetiler.expression; +import static com.onthegomap.planetiler.TestUtils.newPoint; import static com.onthegomap.planetiler.expression.Expression.*; import static com.onthegomap.planetiler.expression.ExpressionTestUtil.featureWithTags; import static org.junit.jupiter.api.Assertions.*; +import com.onthegomap.planetiler.geo.GeometryType; +import com.onthegomap.planetiler.reader.SimpleFeature; import com.onthegomap.planetiler.reader.WithTags; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Test; @@ -160,6 +164,31 @@ void testWildcardStartsWith() { assertFalse(matcher.evaluate(featureWithTags("key", "cba"))); } + @Test + void testMatchNested() { + assertTrue(matchAny("key", "a%").evaluate(WithTags.from(Map.of("key", List.of("abc"))))); + assertTrue(matchAny("key", "abc").evaluate(WithTags.from(Map.of("key", List.of("abc"))))); + assertTrue(matchField("key").evaluate(WithTags.from(Map.of("key", List.of("abc"))))); + assertFalse(matchField("key").evaluate(WithTags.from(Map.of("key", List.of())))); + assertTrue(matchAny("key", "abc").evaluate(WithTags.from(Map.of("key", List.of("abc"))))); + assertFalse( + matchAny("key", "a%").evaluate(WithTags.from(Map.of("key", Map.of("key2", "abc"))))); + assertTrue(matchAny("key", "a%").evaluate(WithTags.from(Map.of("key", List.of("a"))))); + assertFalse(matchAny("key", "a%").evaluate(WithTags.from(Map.of("key", List.of("cba"))))); + } + + @Test + void testNestedQuery() { + assertFalse( + matchAny("key.key2", "a").evaluate(WithTags.from(Map.of("other", "value")))); + assertFalse( + matchAny("key.key2", "a").evaluate(WithTags.from(Map.of("key", "value")))); + assertTrue( + matchAny("key.key2", "a").evaluate(WithTags.from(Map.of("key", Map.of("key2", "a"))))); + assertFalse( + matchAny("key.key2", "a").evaluate(WithTags.from(Map.of("key", Map.of("key2", "b"))))); + } + @Test void testWildcardEndsWith() { var matcher = matchAny("key", "%a"); @@ -261,17 +290,7 @@ void testEvaluate() { @Test void testCustomExpression() { - Expression custom = new Expression() { - @Override - public boolean evaluate(WithTags input, List matchKeys) { - return input.hasTag("abc"); - } - - @Override - public String generateJavaCode() { - return null; - } - }; + Expression custom = (input, matchKeys) -> input.hasTag("abc"); WithTags matching = featureWithTags("abc", "123"); WithTags notMatching = featureWithTags("abcd", "123"); @@ -289,4 +308,81 @@ public String generateJavaCode() { assertFalse(and(TRUE, custom).evaluate(notMatching)); assertFalse(or(FALSE, custom).evaluate(notMatching)); } + + @Test + void testSourceFilter() { + assertTrue( + Expression.matchSource("source").evaluate( + SimpleFeature.create(newPoint(0, 0), Map.of(), "source", "layer", 1) + )); + assertFalse( + Expression.matchSource("source").evaluate( + SimpleFeature.create(newPoint(0, 0), Map.of(), "other source", "layer", 1) + )); + assertTrue( + Expression.matchSourceLayer("layer").evaluate( + SimpleFeature.create(newPoint(0, 0), Map.of(), "source", "layer", 1) + )); + assertFalse( + Expression.matchSourceLayer("layer").evaluate( + SimpleFeature.create(newPoint(0, 0), Map.of(), "other source", "other layer", 1) + )); + } + + @Test + void testPartialEvaluateMatchField() { + assertEquals(matchField("field"), matchField("field").partialEvaluate( + new PartialInput(Set.of(), Set.of(), Map.of("other", "value"), Set.of()))); + assertEquals(TRUE, matchField("field").partialEvaluate( + new PartialInput(Set.of(), Set.of(), Map.of("field", "value"), Set.of()))); + } + + @Test + void testPartialEvaluateMatchAny() { + var expr = matchAny("field", "value1", "other%"); + assertEquals(expr, expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of("other", "value"), Set.of()))); + assertEquals(expr, expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), null, Set.of()))); + assertEquals(TRUE, expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of("field", "value1"), Set.of()))); + assertEquals(TRUE, expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of("field", "other"), Set.of()))); + assertEquals(TRUE, + expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of("field", "other..."), Set.of()))); + assertEquals(FALSE, expr.partialEvaluate( + new PartialInput(Set.of(), Set.of(), Map.of("field", "not a value"), Set.of()))); + } + + @Test + void testPartialEvaluateMatchGeometryType() { + var expr = matchGeometryType(GeometryType.POINT); + assertEquals(expr, expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of(), Set.of()))); + assertEquals(expr, expr.partialEvaluate( + new PartialInput(Set.of(), Set.of(), Map.of(), Set.of(GeometryType.POINT, GeometryType.UNKNOWN)))); + assertEquals(expr, + expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of(), Set.of(GeometryType.UNKNOWN)))); + assertEquals(expr, + expr.partialEvaluate( + new PartialInput(Set.of(), Set.of(), Map.of(), Set.of(GeometryType.POINT, GeometryType.POLYGON)))); + assertEquals(TRUE, + expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of(), Set.of(GeometryType.POINT)))); + assertEquals(FALSE, + expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of(), Set.of(GeometryType.POLYGON)))); + } + + @Test + void testPartialEvaluateMatchSource() { + var expr = matchSource("source"); + assertEquals(expr, + expr.partialEvaluate(new PartialInput(Set.of("source", "others"), Set.of(), Map.of(), Set.of()))); + assertEquals(expr, expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of(), Set.of()))); + assertEquals(TRUE, expr.partialEvaluate(new PartialInput(Set.of("source"), Set.of(), Map.of(), Set.of()))); + assertEquals(FALSE, expr.partialEvaluate(new PartialInput(Set.of("other source"), Set.of(), Map.of(), Set.of()))); + } + + @Test + void testPartialEvaluateMatchSourceLayer() { + var expr = matchSourceLayer("layer"); + assertEquals(expr, expr.partialEvaluate(new PartialInput(Set.of(), Set.of("layer", "others"), Map.of(), Set.of()))); + assertEquals(expr, expr.partialEvaluate(new PartialInput(Set.of(), Set.of(), Map.of(), Set.of()))); + assertEquals(TRUE, expr.partialEvaluate(new PartialInput(Set.of(), Set.of("layer"), Map.of(), Set.of()))); + assertEquals(FALSE, expr.partialEvaluate(new PartialInput(Set.of(), Set.of("other layer"), Map.of(), Set.of()))); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java index 3872fdf983..8ed580ccb2 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/expression/MultiExpressionTest.java @@ -12,6 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.planetiler.expression.MultiExpression.Index; +import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.reader.SimpleFeature; import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.reader.WithTags; @@ -625,28 +626,10 @@ void testMatchNotMissing() { @Test void testCustomExpression() { - Expression dontEvaluate = new Expression() { - @Override - public boolean evaluate(WithTags input, List matchKeys) { - throw new AssertionError("should not evaluate"); - } - - @Override - public String generateJavaCode() { - return null; - } - }; - Expression matchAbc = new Expression() { - @Override - public boolean evaluate(WithTags input, List matchKeys) { - return input.hasTag("abc"); - } - - @Override - public String generateJavaCode() { - return null; - } + Expression dontEvaluate = (input, matchKeys) -> { + throw new AssertionError("should not evaluate"); }; + Expression matchAbc = (input, matchKeys) -> input.hasTag("abc"); var index = MultiExpression.of(List.of( entry("a", matchAbc), entry("b", and(matchField("def"), dontEvaluate)), @@ -695,6 +678,76 @@ void testAndOrMatch() { assertFalse(expr.evaluate(featureWithTags(), list)); } + + @Test + void testSourceFilter() { + var index = MultiExpression.of(List.of( + entry("a", matchSource("a")), + entry("b", and(matchSource("b"), matchField("field"))), + entry("c", or(matchField("abc"), matchSourceLayer("c"))) + )).index(); + + assertSameElements(List.of(), index.getMatches(point("source", "layer", Map.of()))); + assertSameElements(List.of("a"), index.getMatches(point("a", "layer", Map.of()))); + assertSameElements(List.of(), index.getMatches(point("b", "layer", Map.of()))); + assertSameElements(List.of("b"), index.getMatches(point("b", "layer", Map.of("field", "value")))); + assertSameElements(List.of("c"), index.getMatches(point("source", "layer", Map.of("abc", "value")))); + assertSameElements(List.of("c"), index.getMatches(point("source", "c", Map.of("abc", "value")))); + assertSameElements(List.of("c"), index.getMatches(point("source", "c", Map.of()))); + } + + @Test + void testSourceAndGeomFilter() { + var index = MultiExpression.of(List.of( + entry("a", and(matchGeometryType(GeometryType.POINT), matchSource("a"))), + entry("b", and(matchGeometryType(GeometryType.POLYGON), and(matchSource("b"), matchField("field")))), + entry("c", and(matchGeometryType(GeometryType.POLYGON), or(matchField("abc"), matchSourceLayer("c")))), + entry("d", and(matchField("field2"))) + )).index(); + + assertSameElements(List.of(), index.getMatches(point("source", "layer", Map.of()))); + assertSameElements(List.of("a"), index.getMatches(point("a", "layer", Map.of()))); + assertSameElements(List.of(), index.getMatches(point("b", "layer", Map.of()))); + assertSameElements(List.of(), index.getMatches(point("b", "layer", Map.of("field", "value")))); + assertSameElements(List.of(), index.getMatches(point("source", "layer", Map.of("abc", "value")))); + assertSameElements(List.of(), index.getMatches(point("source", "c", Map.of("abc", "value")))); + assertSameElements(List.of(), index.getMatches(point("source", "c", Map.of()))); + assertSameElements(List.of("d"), index.getMatches(point("source", "layer", Map.of("field2", "value")))); + } + + private static SourceFeature point(String source, String layer, Map tags) { + return SimpleFeature.create(newPoint(0, 0), tags, source, layer, 1); + } + + + @Test + void testNestedMatching() { + var index = MultiExpression.of(List.of( + entry("a", matchField("a.b")), + entry("b", matchAny("a.b", "c", "d")), + entry("c", matchAny("a", "e")), + entry("d", matchAny("a[].b", "c")) + )).index(); + + assertSameElements(List.of(), index.getMatches(WithTags.from(Map.of("k", "v")))); + assertSameElements(List.of("c"), index.getMatches(WithTags.from(Map.of("a", "e")))); + assertSameElements(List.of("c"), index.getMatches(WithTags.from(Map.of("a", List.of("e"))))); + assertSameElements(List.of("c"), index.getMatches(WithTags.from(Map.of("a", List.of("e", "f"))))); + assertSameElements(List.of(), index.getMatches(WithTags.from(Map.of("a", List.of("g", "f"))))); + + + assertSameElements(List.of("a"), index.getMatches(WithTags.from(Map.of("a.b", "e")))); + assertSameElements(List.of("a", "b"), index.getMatches(WithTags.from(Map.of("a.b", "c")))); + assertSameElements(List.of(), index.getMatches(WithTags.from(Map.of("a", Map.of("b", List.of()))))); + assertSameElements(List.of("a", "b", "d"), index.getMatches(WithTags.from(Map.of("a", Map.of("b", List.of("c")))))); + assertSameElements(List.of("a", "b"), index.getMatches(WithTags.from(Map.of("a", Map.of("b", List.of("d")))))); + assertSameElements(List.of("a"), index.getMatches(WithTags.from(Map.of("a", Map.of("b", List.of("e")))))); + assertSameElements(List.of("a"), index.getMatches(WithTags.from(Map.of("a", Map.of("b", Map.of("c", "e")))))); + + assertSameElements(List.of("a", "b", "c", "d"), + index.getMatches(WithTags.from(Map.of("a", List.of("e", Map.of("b", List.of("c"))))))); + } + private static void assertSameElements(List a, List b) { assertEquals( a.stream().sorted(Comparator.comparing(Object::toString)).toList(), diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/StructTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/StructTest.java index 8e3c9ce675..496daab2ab 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/StructTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/StructTest.java @@ -79,6 +79,15 @@ void testListQuery() { assertEquals(Struct.of(List.of("c", "d")), struct.get("a[].b")); } + @Test + void testListQueryOnList() { + var struct = Struct.of(List.of(Map.of("b", "c"), Map.of("b", "d"))); + assertEquals("d", struct.flatMap(elem -> elem.get("b")).get(1).asString()); + assertEquals(Struct.of(List.of("c", "d")), struct.get("[].b")); + assertEquals(Struct.of(List.of("c", "d")), struct.get("b")); + assertEquals(Struct.NULL, struct.get("e")); + } + @Test void testListGet() { var struct = Struct.of(List.of(1, 2, 3)); @@ -340,7 +349,7 @@ void testAsJson() { )); assertEquals(""" {"a":{"b":"c"}} - """.strip(), struct.asJson()); + """.strip(), struct.tagsAsJson()); assertEquals(""" {"b":"c"} """.strip(), struct.getStruct("a").asJson()); @@ -348,4 +357,22 @@ void testAsJson() { "c" """.strip(), struct.getStruct("a").get("b").asJson()); } + + @Test + void testRawValueList() { + var struct = Struct.of(List.of(1, 2, 3)); + assertEquals(List.of(1, 2, 3), struct.rawValue()); + } + + @Test + void testRawValueListFlatmap() { + assertEquals(List.of(1, 3), Struct.of(List.of(1, List.of(3))).flatMap(d -> d).rawValue()); + assertEquals(List.of(3), Struct.of(List.of(1, List.of(3))).flatMap(d -> d.get(0)).rawValue()); + } + + @Test + void testRawValueMap() { + var struct = Struct.of(Map.of(1, 2)); + assertEquals(Map.of(1, 2), struct.rawValue()); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetFeatureTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetFeatureTest.java new file mode 100644 index 0000000000..e7038b0836 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetFeatureTest.java @@ -0,0 +1,46 @@ +package com.onthegomap.planetiler.reader.parquet; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.hadoop.metadata.FileMetaData; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +class ParquetFeatureTest { + private static ParquetFeature feature(Map tags) throws IOException { + var schema = Types.buildMessage().addField(Types.required(PrimitiveType.PrimitiveTypeName.BINARY).named("geometry")) + .named("root"); + return new ParquetFeature("overture", "layer", 1, + new GeometryReader(GeoParquetMetadata.parse(new FileMetaData(schema, Map.of(), "geometry"))), + new HashMap<>(tags), + Path.of(""), schema); + } + + @Test + void testHasTag() throws IOException { + var feature = feature(Map.of("names", Map.of("primary", "name"))); + assertTrue(feature.hasTag("names.primary")); + assertTrue(feature.hasTag("names[].primary")); + assertTrue(feature.hasTag("names.primary", "name")); + assertTrue(feature.hasTag("names[].primary", "name")); + assertTrue(feature.hasTag("names[].primary", List.of("name", "name2"))); + assertFalse(feature.hasTag("names.primary", "not name")); + assertFalse(feature.hasTag("names.primary", List.of("not name", "not name 2"))); + } + + @Test + void testHasTagWithArg() throws IOException { + var feature = feature(Map.of("names", Map.of("primary", "name"))); + assertTrue(feature.hasTag("names.primary", "name1", "name")); + assertTrue(feature.hasTag("names.primary", "name", "name1")); + assertTrue(feature.hasTag("names.primary", List.of("name"), "name1")); + assertTrue(feature.hasTag("names.primary", "name1", List.of("name"))); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFileTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFileTest.java index 5920c43ca1..d3d30148f0 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFileTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFileTest.java @@ -3,8 +3,12 @@ import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.DynamicTest.dynamicTest; +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.ForwardingProfile; import com.onthegomap.planetiler.TestUtils; import com.onthegomap.planetiler.config.Bounds; +import com.onthegomap.planetiler.expression.Expression; +import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.util.Glob; import java.nio.file.Path; import java.time.Instant; @@ -47,6 +51,25 @@ void testReadBoston(Path path) { } } + @Test + void testReadBostonWithProfileFilter() { + Path path = TestUtils.pathToResource("parquet").resolve("boston.parquet"); + var profile = new ForwardingProfile() {}; + profile.registerFeatureHandler(new ForwardingProfile.FeatureProcessor() { + @Override + public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {} + + @Override + public Expression filter() { + return Expression.matchAny("type", "value"); + } + }); + var file1 = new ParquetInputFile("parquet", "layer", path, null, Bounds.WORLD, Map.of("type", "value"), null); + var file2 = new ParquetInputFile("parquet", "layer", path, null, Bounds.WORLD, Map.of("type", "other"), null); + assertFalse(file1.shouldSkip(profile)); + assertTrue(file2.shouldSkip(profile)); + } + @ParameterizedTest @MethodSource("bostons") void testReadBostonWithBboxFilterCovering(Path path) { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/validator/BaseSchemaValidatorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/validator/BaseSchemaValidatorTest.java new file mode 100644 index 0000000000..efd13bf31c --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/validator/BaseSchemaValidatorTest.java @@ -0,0 +1,235 @@ +package com.onthegomap.planetiler.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.reader.SourceFeature; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class BaseSchemaValidatorTest { + + private final String goodSpecString = """ + examples: + - name: test output + input: + source: osm + geometry: polygon + tags: + natural: water + output: + - layer: water + geometry: polygon + tags: + natural: water + """; + private final SchemaSpecification goodSpec = SchemaSpecification.load(goodSpecString); + + private final Profile waterSchema = new Profile() { + @Override + public void processFeature(SourceFeature sourceFeature, FeatureCollector features) { + if (sourceFeature.canBePolygon() && sourceFeature.hasTag("natural", "water")) { + features.polygon("water") + .setMinPixelSize(10) + .inheritAttrFromSource("natural"); + } + } + + @Override + public String name() { + return "test profile"; + } + }; + + private Result validate(Profile profile, String spec) { + var result = BaseSchemaValidator.validate( + profile, + SchemaSpecification.load(spec), + PlanetilerConfig.defaults() + ); + for (var example : result.results()) { + if (example.issues().isFailure()) { + assertNotNull(example.issues().get()); + } + } + // also exercise the cli writer and return what it would have printed to stdout + var cliOutput = validateCli(profile, SchemaSpecification.load(spec)); + return new Result(result, cliOutput); + } + + private String validateCli(Profile profile, SchemaSpecification spec) { + try ( + var baos = new ByteArrayOutputStream(); + var printStream = new PrintStream(baos, true, StandardCharsets.UTF_8) + ) { + new BaseSchemaValidator(Arguments.of(), printStream) { + @Override + protected Result validate(Set pathsToWatch) { + return validate(profile, spec, PlanetilerConfig.defaults()); + } + }.validateFromCli(); + return baos.toString(StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private Result validateWater(String layer, String geometry, String tags, String allowExtraTags) throws IOException { + return validate( + waterSchema, + """ + examples: + - name: test output + input: + source: osm + geometry: polygon + tags: + natural: water + output: + layer: %s + geometry: %s + %s + tags: + %s + """.formatted(layer, geometry, allowExtraTags == null ? "" : allowExtraTags, + tags == null ? "" : tags.indent(6).strip()) + ); + } + + @ParameterizedTest + @CsvSource(value = { + "true,water,polygon,natural: water,", + "true,water,polygon,,", + "true,water,polygon,'natural: water\nother: null',", + "false,water,polygon,natural: null,", + "false,water2,polygon,natural: water,", + "false,water,line,natural: water,", + "false,water,line,natural: water,", + "false,water,polygon,natural: water2,", + "false,water,polygon,'natural: water\nother: value',", + + "true,water,polygon,natural: water,allow_extra_tags: true", + "true,water,polygon,natural: water,allow_extra_tags: false", + "true,water,polygon,,allow_extra_tags: true", + "false,water,polygon,,allow_extra_tags: false", + + "true,water,polygon,,min_size: 10", + "false,water,polygon,,min_size: 9", + }) + void testValidateWaterPolygon(boolean shouldBeOk, String layer, String geometry, String tags, String allowExtraTags) + throws IOException { + var results = validateWater(layer, geometry, tags, allowExtraTags); + assertEquals(1, results.output.results().size()); + assertEquals("test output", results.output.results().get(0).example().name()); + if (shouldBeOk) { + assertTrue(results.output.ok(), results.toString()); + assertFalse(results.cliOutput.contains("FAIL"), "contained FAIL but should not have: " + results.cliOutput); + } else { + assertFalse(results.output.ok(), "Expected an issue, but there were none"); + assertTrue(results.cliOutput.contains("FAIL"), "did not contain FAIL but should have: " + results.cliOutput); + } + } + + @Test + void testPartialLengthAttribute() { + var results = validate( + (sourceFeature, features) -> features.line("layer") + .linearRange(0, 0.6).setAttr("a", 1) + .linearRange(0.4, 1).setAttr("b", 2), + """ + examples: + - name: test output + input: + source: osm + geometry: line + tags: + natural: water + output: + - layer: layer + geometry: line + tags: {a:1} + - layer: layer + geometry: line + tags: {a:1,b:2} + - layer: layer + geometry: line + tags: {b:2} + """ + ); + assertEquals(1, results.output.results().size()); + assertEquals("test output", results.output.results().getFirst().example().name()); + assertTrue(results.output.results().getFirst().ok()); + } + + @Test + void testValidationFailsWrongNumberOfFeatures() { + var results = validate( + waterSchema, + """ + examples: + - name: test output + input: + source: osm + geometry: polygon + tags: + natural: water + output: + """ + ); + assertFalse(results.output.ok(), results.toString()); + + results = validate( + waterSchema, + """ + examples: + - name: test output + input: + source: osm + geometry: polygon + tags: + natural: water + output: + - layer: water + geometry: polygon + tags: + natural: water + - layer: water2 + geometry: polygon + tags: + natural: water2 + """ + ); + assertFalse(results.output.ok(), results.toString()); + } + + @TestFactory + Stream testJunitAdapterSpec() { + return TestUtils.validateProfile(waterSchema, goodSpec); + } + + @TestFactory + Stream testJunitAdapterString() { + return TestUtils.validateProfile(waterSchema, goodSpecString); + } + + + record Result(BaseSchemaValidator.Result output, String cliOutput) {} +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/validator/JavaProfileValidatorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/validator/JavaProfileValidatorTest.java new file mode 100644 index 0000000000..6b88b6c29d --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/validator/JavaProfileValidatorTest.java @@ -0,0 +1,98 @@ +package com.onthegomap.planetiler.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class JavaProfileValidatorTest { + private final Arguments args = Arguments.of(); + @TempDir + Path tmpDir; + + record Result(BaseSchemaValidator.Result output, String cliOutput) {} + + private Result validate(Profile profile, String spec) throws IOException { + var specPath = Files.writeString(tmpDir.resolve("spec.yaml"), spec); + try ( + var baos = new ByteArrayOutputStream(); + var printStream = new PrintStream(baos, true, StandardCharsets.UTF_8) + ) { + var validator = new JavaProfileValidator(PlanetilerConfig.from(args), specPath, profile, printStream); + var summary = validator.validateFromCli(); + assertEquals(Set.of(specPath), summary.paths()); + return new Result(summary.result(), baos.toString(StandardCharsets.UTF_8)); + } + } + + @ParameterizedTest + @CsvSource(value = { + "true,water,polygon,natural: water,", + "true,water,polygon,,", + "true,water,polygon,'natural: water\nother: null',", + "false,water,polygon,natural: null,", + "false,water2,polygon,natural: water,", + "false,water,line,natural: water,", + "false,water,line,natural: water,", + "false,water,polygon,natural: water2,", + "false,water,polygon,'natural: water\nother: value',", + + "true,water,polygon,natural: water,allow_extra_tags: true", + "true,water,polygon,natural: water,allow_extra_tags: false", + "true,water,polygon,,allow_extra_tags: true", + "false,water,polygon,,allow_extra_tags: false", + + "true,water,polygon,,min_size: 10", + "false,water,polygon,,min_size: 9", + }) + void testValidateWaterPolygon(boolean shouldBeOk, String layer, String geometry, String tags, String allowExtraTags) + throws IOException { + var results = validate( + (sourceFeature, features) -> { + if (sourceFeature.canBePolygon() && sourceFeature.hasTag("natural", "water")) { + features.polygon("water") + .inheritAttrFromSource("natural") + .setMinPixelSize(10); + } + }, + """ + examples: + - name: test output + input: + source: osm + geometry: polygon + tags: + natural: water + output: + layer: %s + geometry: %s + %s + tags: + %s + """.formatted(layer, geometry, allowExtraTags == null ? "" : allowExtraTags, + tags == null ? "" : tags.indent(6).strip()) + ); + assertEquals(1, results.output.results().size()); + assertEquals("test output", results.output.results().getFirst().example().name()); + if (shouldBeOk) { + assertTrue(results.output.ok(), results.toString()); + assertFalse(results.cliOutput.contains("FAIL"), "contained FAIL but should not have: " + results.cliOutput); + } else { + assertFalse(results.output.ok(), "Expected an issue, but there were none"); + assertTrue(results.cliOutput.contains("FAIL"), "did not contain FAIL but should have: " + results.cliOutput); + } + } +} diff --git a/planetiler-custommap/pom.xml b/planetiler-custommap/pom.xml index a4ce251d76..0e25d0c6ef 100644 --- a/planetiler-custommap/pom.xml +++ b/planetiler-custommap/pom.xml @@ -18,10 +18,6 @@ planetiler-core ${project.parent.version} - - org.snakeyaml - snakeyaml-engine - org.commonmark commonmark diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java index 23283e531d..ff73a1f56c 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java @@ -5,6 +5,7 @@ import com.onthegomap.planetiler.custommap.configschema.DataSourceType; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import com.onthegomap.planetiler.custommap.expression.ParseException; +import com.onthegomap.planetiler.util.YAML; import java.nio.file.Files; import java.nio.file.Path; diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java index c4b70912a9..fc11d488cc 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/SchemaConfig.java @@ -1,7 +1,7 @@ package com.onthegomap.planetiler.custommap.configschema; import com.fasterxml.jackson.annotation.JsonProperty; -import com.onthegomap.planetiler.custommap.YAML; +import com.onthegomap.planetiler.util.YAML; import java.nio.file.Path; import java.util.Collection; import java.util.Map; diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java index b112a189fb..b6fe4dcd70 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/validator/SchemaValidator.java @@ -1,44 +1,31 @@ package com.onthegomap.planetiler.custommap.validator; import com.fasterxml.jackson.core.JacksonException; -import com.onthegomap.planetiler.FeatureCollector; -import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.custommap.ConfiguredProfile; import com.onthegomap.planetiler.custommap.Contexts; -import com.onthegomap.planetiler.custommap.YAML; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; -import com.onthegomap.planetiler.geo.GeometryType; -import com.onthegomap.planetiler.reader.SimpleFeature; -import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.AnsiColors; -import com.onthegomap.planetiler.util.FileWatcher; -import com.onthegomap.planetiler.util.Format; -import com.onthegomap.planetiler.util.Try; +import com.onthegomap.planetiler.util.YAML; +import com.onthegomap.planetiler.validator.BaseSchemaValidator; +import com.onthegomap.planetiler.validator.SchemaSpecification; import java.io.PrintStream; import java.nio.file.Path; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; import java.util.List; -import java.util.Locale; -import java.util.Objects; import java.util.Set; -import java.util.TreeSet; import java.util.stream.Stream; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.io.ParseException; -import org.locationtech.jts.io.WKTReader; import org.snakeyaml.engine.v2.exceptions.YamlEngineException; -/** Verifies that a profile maps input elements map to expected output vector tile features. */ -public class SchemaValidator { +public class SchemaValidator extends BaseSchemaValidator { - private static final String PASS_BADGE = AnsiColors.greenBackground(" PASS "); - private static final String FAIL_BADGE = AnsiColors.redBackground(" FAIL "); + private final Path schemaPath; + + SchemaValidator(Arguments args, String schemaFile, PrintStream output) { + super(args, output); + schemaPath = schemaFile == null ? args.inputFile("schema", "Schema file") : + args.inputFile("schema", "Schema file", Path.of(schemaFile)); + } public static void main(String[] args) { // let users run `verify schema.yml` as a shortcut @@ -48,36 +35,23 @@ public static void main(String[] args) { args = Stream.of(args).skip(1).toArray(String[]::new); } var arguments = Arguments.fromEnvOrArgs(args); - var schema = schemaFile == null ? arguments.inputFile("schema", "Schema file") : - arguments.inputFile("schema", "Schema file", Path.of(schemaFile)); - var watch = - arguments.getBoolean("watch", "Watch files for changes and re-run validation when schema or spec changes", false); - - - PrintStream output = System.out; - output.println("OK"); - var paths = validateFromCli(schema, output); - - if (watch) { - output.println(); - output.println("Watching filesystem for changes..."); - var watcher = FileWatcher.newWatcher(paths.toArray(Path[]::new)); - watcher.pollForChanges(Duration.ofMillis(300), changed -> validateFromCli(schema, output)); - } + new SchemaValidator(arguments, schemaFile, System.out).runOrWatch(); } - private static boolean hasCause(Throwable t, Class cause) { - return t != null && (cause.isInstance(t) || hasCause(t.getCause(), cause)); + /** + * Returns the result of validating the profile defined by {@code schema} against the examples in + * {@code specification}. + */ + public static Result validate(SchemaConfig schema, SchemaSpecification specification) { + var context = Contexts.buildRootContext(Arguments.of().silence(), schema.args()); + return validate(new ConfiguredProfile(schema, context), specification, context.config()); } - static Set validateFromCli(Path schemaPath, PrintStream output) { - Set pathsToWatch = new HashSet<>(); - pathsToWatch.add(schemaPath); - output.println(); - output.println("Validating..."); - output.println(); - SchemaValidator.Result result; + @Override + protected Result validate(Set pathsToWatch) { + Result result = null; try { + pathsToWatch.add(schemaPath); var schema = SchemaConfig.load(schemaPath); var examples = schema.examples(); // examples can either be embedded in the yaml file, or referenced @@ -108,169 +82,8 @@ static Set validateFromCli(Path schemaPath, PrintStream output) { String.join("\n", ExceptionUtils.getStackTrace(rootCause))) .indent(4)); } - return pathsToWatch; - } - int failed = 0, passed = 0; - List failures = new ArrayList<>(); - for (var example : result.results) { - if (example.ok()) { - passed++; - output.printf("%s %s%n", PASS_BADGE, example.example().name()); - } else { - failed++; - printFailure(example, output); - failures.add(example); - } - } - if (!failures.isEmpty()) { - output.println(); - output.println("Summary of failures:"); - for (var failure : failures) { - printFailure(failure, output); - } - } - List summary = new ArrayList<>(); - boolean none = (passed + failed) == 0; - if (none || failed > 0) { - summary.add(AnsiColors.redBold(failed + " failed")); - } - if (none || passed > 0) { - summary.add(AnsiColors.greenBold(passed + " passed")); - } - if (none || passed > 0 && failed > 0) { - summary.add((failed + passed) + " total"); - } - output.println(); - output.println(String.join(", ", summary)); - return pathsToWatch; - } - - private static void printFailure(ExampleResult example, PrintStream output) { - output.printf("%s %s%n", FAIL_BADGE, example.example().name()); - if (example.issues.isFailure()) { - output.println(ExceptionUtils.getStackTrace(example.issues.exception()).indent(4).stripTrailing()); - } else { - for (var issue : example.issues().get()) { - output.println(" ● " + issue.indent(4).strip()); - } - } - } - - private static Geometry parseGeometry(String geometry) { - String wkt = switch (geometry.toLowerCase(Locale.ROOT).trim()) { - case "point" -> "POINT (0 0)"; - case "line" -> "LINESTRING (0 0, 1 1)"; - case "polygon" -> "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"; - default -> geometry; - }; - try { - return new WKTReader().read(wkt); - } catch (ParseException e) { - throw new IllegalArgumentException(""" - Bad geometry: "%s", must be "point" "line" "polygon" or a valid WKT string. - """.formatted(geometry)); - } - } - - /** - * Returns the result of validating the profile defined by {@code schema} against the examples in - * {@code specification}. - */ - public static Result validate(SchemaConfig schema, SchemaSpecification specification) { - var context = Contexts.buildRootContext(Arguments.of().silence(), schema.args()); - return validate(new ConfiguredProfile(schema, context), specification, context.config()); - } - - /** Returns the result of validating {@code profile} against the examples in {@code specification}. */ - public static Result validate(Profile profile, SchemaSpecification specification, PlanetilerConfig config) { - var featureCollectorFactory = new FeatureCollector.Factory(config, Stats.inMemory()); - return new Result(specification.examples().stream().map(example -> new ExampleResult(example, Try.apply(() -> { - List issues = new ArrayList<>(); - var input = example.input(); - var expectedFeatures = example.output(); - var geometry = parseGeometry(input.geometry()); - var feature = SimpleFeature.create(geometry, input.tags(), input.source(), null, 0); - var collector = featureCollectorFactory.get(feature); - profile.processFeature(feature, collector); - List result = new ArrayList<>(); - collector.forEach(result::add); - if (result.size() != expectedFeatures.size()) { - issues.add( - "Different number of elements, expected=%s actual=%s".formatted(expectedFeatures.size(), result.size())); - } else { - // TODO print a diff of the input and output feature YAML representations - for (int i = 0; i < expectedFeatures.size(); i++) { - var expected = expectedFeatures.get(i); - var actual = result.stream().max(proximityTo(expected)).orElseThrow(); - result.remove(actual); - var actualTags = actual.getAttrsAtZoom(expected.atZoom()); - String prefix = "feature[%d]".formatted(i); - validate(prefix + ".layer", issues, expected.layer(), actual.getLayer()); - validate(prefix + ".minzoom", issues, expected.minZoom(), actual.getMinZoom()); - validate(prefix + ".maxzoom", issues, expected.maxZoom(), actual.getMaxZoom()); - validate(prefix + ".minsize", issues, expected.minSize(), actual.getMinPixelSizeAtZoom(expected.atZoom())); - validate(prefix + ".geometry", issues, expected.geometry(), GeometryType.typeOf(actual.getGeometry())); - Set tags = new TreeSet<>(actualTags.keySet()); - expected.tags().forEach((tag, value) -> { - validate(prefix + ".tags[\"%s\"]".formatted(tag), issues, value, actualTags.get(tag), false); - tags.remove(tag); - }); - if (Boolean.FALSE.equals(expected.allowExtraTags())) { - for (var tag : tags) { - validate(prefix + ".tags[\"%s\"]".formatted(tag), issues, null, actualTags.get(tag), false); - } - } - } - } - return issues; - }))).toList()); - } - - private static Comparator proximityTo(SchemaSpecification.OutputFeature expected) { - return Comparator.comparingInt(item -> (Objects.equals(item.getLayer(), expected.layer()) ? 2 : 0) + - (Objects.equals(GeometryType.typeOf(item.getGeometry()), expected.geometry()) ? 1 : 0)); - } - - private static void validate(String field, List issues, T expected, T actual, boolean ignoreWhenNull) { - if ((!ignoreWhenNull || expected != null) && !Objects.equals(expected, actual)) { - // handle when expected and actual are int/long or long/int - if (expected instanceof Number && actual instanceof Number && expected.toString().equals(actual.toString())) { - return; - } - issues.add("%s: expected <%s> actual <%s>".formatted(field, format(expected), format(actual))); - } - } - - private static String format(Object o) { - if (o == null) { - return "null"; - } else if (o instanceof String s) { - return Format.quote(s); - } else { - return o.toString(); - } - } - - private static void validate(String field, List issues, T expected, T actual) { - validate(field, issues, expected, actual, true); - } - - /** Result of comparing the output vector tile feature to what was expected. */ - public record ExampleResult( - SchemaSpecification.Example example, - // TODO include a symmetric diff so we can pretty-print the expected/actual output diff - Try> issues - ) { - - public boolean ok() { - return issues.isSuccess() && issues.get().isEmpty(); - } - } - - public record Result(List results) { - - public boolean ok() { - return results.stream().allMatch(ExampleResult::ok); } + return result; } } + diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java index b7dec4a8d8..5b17b81b59 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/BooleanExpressionParserTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.onthegomap.planetiler.expression.Expression; +import com.onthegomap.planetiler.util.YAML; import java.util.Map; import org.junit.jupiter.api.Test; diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java index 78f8804e25..46c25652ea 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfigExpressionParserTest.java @@ -8,6 +8,7 @@ import com.onthegomap.planetiler.expression.DataType; import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.expression.MultiExpression; +import com.onthegomap.planetiler.util.YAML; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java index 3400fe668d..0906ce4861 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/SchemaTests.java @@ -3,8 +3,8 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; -import com.onthegomap.planetiler.custommap.validator.SchemaSpecification; import com.onthegomap.planetiler.custommap.validator.SchemaValidator; +import com.onthegomap.planetiler.validator.SchemaSpecification; import java.nio.file.Path; import java.util.List; import org.junit.jupiter.api.DynamicTest; diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java index eff36cc9c9..309b3a6cca 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/validator/SchemaValidatorTest.java @@ -5,7 +5,10 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; +import com.onthegomap.planetiler.validator.BaseSchemaValidator; +import com.onthegomap.planetiler.validator.SchemaSpecification; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -22,7 +25,7 @@ class SchemaValidatorTest { @TempDir Path tmpDir; - record Result(SchemaValidator.Result output, String cliOutput) {} + record Result(BaseSchemaValidator.Result output, String cliOutput) {} private Result validate(String schema, String spec) throws IOException { var result = SchemaValidator.validate( @@ -57,55 +60,13 @@ private String validateCli(Path path) { var baos = new ByteArrayOutputStream(); var printStream = new PrintStream(baos, true, StandardCharsets.UTF_8) ) { - SchemaValidator.validateFromCli( - path, - printStream - ); + new SchemaValidator(Arguments.of(), path.toString(), printStream).validateFromCli(); return baos.toString(StandardCharsets.UTF_8); } catch (IOException e) { throw new UncheckedIOException(e); } } - String waterSchema = """ - sources: - osm: - type: osm - url: geofabrik:rhode-island - layers: - - id: water - features: - - source: osm - geometry: polygon - min_size: 10 - include_when: - natural: water - attributes: - - key: natural - """; - - private Result validateWater(String layer, String geometry, String tags, String allowExtraTags) throws IOException { - return validate( - waterSchema, - """ - examples: - - name: test output - input: - source: osm - geometry: polygon - tags: - natural: water - output: - layer: %s - geometry: %s - %s - tags: - %s - """.formatted(layer, geometry, allowExtraTags == null ? "" : allowExtraTags, - tags == null ? "" : tags.indent(6).strip()) - ); - } - @ParameterizedTest @CsvSource(value = { "true,water,polygon,natural: water,", @@ -128,37 +89,23 @@ private Result validateWater(String layer, String geometry, String tags, String }) void testValidateWaterPolygon(boolean shouldBeOk, String layer, String geometry, String tags, String allowExtraTags) throws IOException { - var results = validateWater(layer, geometry, tags, allowExtraTags); - assertEquals(1, results.output.results().size()); - assertEquals("test output", results.output.results().get(0).example().name()); - if (shouldBeOk) { - assertTrue(results.output.ok(), results.toString()); - assertFalse(results.cliOutput.contains("FAIL"), "contained FAIL but should not have: " + results.cliOutput); - } else { - assertFalse(results.output.ok(), "Expected an issue, but there were none"); - assertTrue(results.cliOutput.contains("FAIL"), "did not contain FAIL but should have: " + results.cliOutput); - } - } - - @Test - void testValidationFailsWrongNumberOfFeatures() throws IOException { var results = validate( - waterSchema, """ - examples: - - name: test output - input: - source: osm + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: water + features: + - source: osm geometry: polygon - tags: + min_size: 10 + include_when: natural: water - output: - """ - ); - assertFalse(results.output.ok(), results.toString()); - - results = validate( - waterSchema, + attributes: + - key: natural + """, """ examples: - name: test output @@ -168,17 +115,23 @@ void testValidationFailsWrongNumberOfFeatures() throws IOException { tags: natural: water output: - - layer: water - geometry: polygon - tags: - natural: water - - layer: water2 - geometry: polygon + layer: %s + geometry: %s + %s tags: - natural: water2 - """ + %s + """.formatted(layer, geometry, allowExtraTags == null ? "" : allowExtraTags, + tags == null ? "" : tags.indent(6).strip()) ); - assertFalse(results.output.ok(), results.toString()); + assertEquals(1, results.output.results().size()); + assertEquals("test output", results.output.results().getFirst().example().name()); + if (shouldBeOk) { + assertTrue(results.output.ok(), results.toString()); + assertFalse(results.cliOutput.contains("FAIL"), "contained FAIL but should not have: " + results.cliOutput); + } else { + assertFalse(results.output.ok(), "Expected an issue, but there were none"); + assertTrue(results.cliOutput.contains("FAIL"), "did not contain FAIL but should have: " + results.cliOutput); + } } @Test