Skip to content

Commit

Permalink
Merge branch 'main' into vw-simplifier
Browse files Browse the repository at this point in the history
  • Loading branch information
msbarry committed Nov 25, 2024
2 parents d234b2e + 292dc78 commit 5891376
Show file tree
Hide file tree
Showing 42 changed files with 1,705 additions and 164 deletions.
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ for more of the backstory.

## Demo

See the [live demo](https://onthegomap.github.io/planetiler-demo/) of vector tiles created by Planetiler and [hosted by OpenStreetMap US](https://github.com/osmus/tileservice).
See the [live demo](https://onthegomap.github.io/planetiler-demo/) of vector tiles created by Planetiler
and [hosted by OpenStreetMap US](https://github.com/osmus/tileservice).

[![Planetiler Demo Screenshot](./diagrams/demo.png)](https://onthegomap.github.io/planetiler-demo/)
[© OpenMapTiles](https://www.openmaptiles.org/) [© OpenStreetMap contributors](https://www.openstreetmap.org/copyright)
Expand Down Expand Up @@ -83,7 +84,7 @@ You will need the full data sources to run anywhere besides Monaco.

#### To view tiles locally:

Using [Node.js](https://nodejs.org/en/download/):
Using [Node.js](https://nodejs.org/en/download/package-manager):

```bash
npm install -g tileserver-gl-light
Expand All @@ -102,6 +103,7 @@ Some common arguments:

- `--output` tells planetiler where to write output to, and what format to write it in. For
example `--output=australia.pmtiles` creates a pmtiles archive named `australia.pmtiles`.
It is best to specify the full path to the file. In docker image you should be using `/data/australia.pmtiles` to let the docker know where to write the file.
- `--download` downloads input sources automatically and `--only-download` exits after downloading
- `--area=monaco` downloads a `.osm.pbf` extract from [Geofabrik](https://download.geofabrik.de/)
- `--osm-path=path/to/file.osm.pbf` points Planetiler at an existing OSM extract on disk
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package com.onthegomap.planetiler.benchmarks;

import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.util.Format;
import com.onthegomap.planetiler.util.FunctionThatThrows;
import com.onthegomap.planetiler.util.Gzip;
import com.onthegomap.planetiler.util.LoopLineMerger;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.MathContext;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.CoordinateXY;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.LineString;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKBReader;
import org.locationtech.jts.operation.linemerge.LineMerger;

public class BenchmarkLineMerge {
private static int numLines;

public static void main(String[] args) throws Exception {
for (int i = 0; i < 10; i++) {
time(" JTS", geom -> {
var lm = new LineMerger();
lm.add(geom);
return lm.getMergedLineStrings();
});
time(" loop(0)", geom -> loopMerger(0).add(geom).getMergedLineStrings());
time(" loop(0.1)", geom -> loopMerger(0.1).add(geom).getMergedLineStrings());
time("loop(20.0)", geom -> loopMerger(20).add(geom).getMergedLineStrings());
}
System.err.println(numLines);
}

private static LoopLineMerger loopMerger(double minLength) {
var lm = new LoopLineMerger();
lm.setMinLength(minLength);
lm.setStubMinLength(minLength);
lm.setLoopMinLength(minLength);
lm.setTolerance(1);
lm.setMergeStrokes(true);
return lm;
}

private static void time(String name, FunctionThatThrows<Geometry, Collection<LineString>> fn) throws Exception {
System.err.println(String.join("\t",
name,
timeMillis(read("mergelines_200433_lines.wkb.gz"), fn),
timeMillis(read("mergelines_239823_lines.wkb.gz"), fn),
"(/s):",
timePerSec(read("mergelines_1759_point_line.wkb.gz"), fn),
timePerSec(makeLines(50, 2), fn),
timePerSec(makeLines(10, 10), fn),
timePerSec(makeLines(2, 50), fn)
));
}

private static String timePerSec(Geometry geometry, FunctionThatThrows<Geometry, Collection<LineString>> fn)
throws Exception {
long start = System.nanoTime();
long end = start + Duration.ofSeconds(1).toNanos();
int num = 0;
for (; System.nanoTime() < end;) {
numLines += fn.apply(geometry).size();
num++;
}
return Format.defaultInstance()
.numeric(Math.round(num * 1d / ((System.nanoTime() - start) * 1d / Duration.ofSeconds(1).toNanos())), true);
}

private static String timeMillis(Geometry geometry, FunctionThatThrows<Geometry, Collection<LineString>> fn)
throws Exception {
long start = System.nanoTime();
long end = start + Duration.ofSeconds(1).toNanos();
int num = 0;
for (; System.nanoTime() < end;) {
numLines += fn.apply(geometry).size();
num++;
}
// equivalent of toPrecision(3)
long nanosPer = (System.nanoTime() - start) / num;
var bd = new BigDecimal(nanosPer, new MathContext(3));
return Format.padRight(Duration.ofNanos(bd.longValue()).toString().replace("PT", ""), 6);
}


private static Geometry read(String fileName) throws IOException, ParseException {
var path = Path.of("planetiler-core", "src", "test", "resources", "mergelines", fileName);
byte[] bytes = Gzip.gunzip(Files.readAllBytes(path));
return new WKBReader().read(bytes);
}

private static Geometry makeLines(int lines, int parts) {
List<LineString> result = new ArrayList<>();
double idx = 0;
for (int i = 0; i < lines; i++) {
Coordinate[] coords = new Coordinate[parts];
for (int j = 0; j < parts; j++) {
coords[j] = new CoordinateXY(idx, idx);
idx += 0.5;
}
result.add(GeoUtils.JTS_FACTORY.createLineString(coords));
}
return new GeometryFactory().createMultiLineString(result.toArray(LineString[]::new));
}
}
9 changes: 7 additions & 2 deletions planetiler-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
</parent>

<properties>
<geotools.version>32.0</geotools.version>
<log4j.version>2.24.1</log4j.version>
<geotools.version>32.1</geotools.version>
<log4j.version>2.24.2</log4j.version>
<prometheus.version>0.16.0</prometheus.version>
<!-- needs to match CEL -->
<protobuf.version>4.28.1</protobuf.version>
Expand Down Expand Up @@ -181,6 +181,11 @@
<artifactId>parquet-floor</artifactId>
<version>1.48</version>
</dependency>
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>1.8.0</version>
</dependency>

</dependencies>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
import com.carrotsearch.hppc.IntObjectMap;
import com.carrotsearch.hppc.IntStack;
import com.onthegomap.planetiler.collection.Hppc;
import com.onthegomap.planetiler.geo.DouglasPeuckerSimplifier;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.geo.GeometryException;
import com.onthegomap.planetiler.geo.GeometryType;
import com.onthegomap.planetiler.geo.MutableCoordinateSequence;
import com.onthegomap.planetiler.stats.DefaultStats;
import com.onthegomap.planetiler.stats.Stats;
import com.onthegomap.planetiler.util.LoopLineMerger;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
Expand Down Expand Up @@ -171,7 +171,12 @@ public static List<VectorTile.Feature> mergeLineStrings(List<VectorTile.Feature>
if (groupedFeatures.size() == 1 && buffer == 0d && lengthLimit == 0 && (!resimplify || tolerance == 0)) {
result.add(feature1);
} else {
LineMerger merger = new LineMerger();
LoopLineMerger merger = new LoopLineMerger()
.setTolerance(tolerance)
.setMergeStrokes(true)
.setMinLength(lengthLimit)
.setLoopMinLength(lengthLimit)
.setStubMinLength(0.5);
for (VectorTile.Feature feature : groupedFeatures) {
try {
merger.add(feature.geometry().decode());
Expand All @@ -180,24 +185,14 @@ public static List<VectorTile.Feature> mergeLineStrings(List<VectorTile.Feature>
}
}
List<LineString> outputSegments = new ArrayList<>();
for (Object merged : merger.getMergedLineStrings()) {
if (merged instanceof LineString line && line.getLength() >= lengthLimit) {
// re-simplify since some endpoints of merged segments may be unnecessary
if (line.getNumPoints() > 2 && tolerance >= 0) {
Geometry simplified = DouglasPeuckerSimplifier.simplify(line, tolerance);
if (simplified instanceof LineString simpleLineString) {
line = simpleLineString;
} else {
LOGGER.warn("line string merge simplify emitted {}", simplified.getGeometryType());
}
}
if (buffer >= 0) {
removeDetailOutsideTile(line, buffer, outputSegments);
} else {
outputSegments.add(line);
}
for (var line : merger.getMergedLineStrings()) {
if (buffer >= 0) {
removeDetailOutsideTile(line, buffer, outputSegments);
} else {
outputSegments.add(line);
}
}

if (!outputSegments.isEmpty()) {
outputSegments = sortByHilbertIndex(outputSegments);
Geometry newGeometry = GeoUtils.combineLineStrings(outputSegments);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -103,7 +104,7 @@ public class Planetiler {
private boolean overwrite = false;
private boolean ran = false;
// most common OSM languages
private List<String> languages = List.of(
private List<String> defaultLanguages = List.of(
"en", "ru", "ar", "zh", "ja", "ko", "fr",
"de", "fi", "pl", "es", "be", "br", "he"
);
Expand Down Expand Up @@ -547,7 +548,7 @@ public Planetiler addStage(String name, String description, RunnableThatThrows t
* @return this runner instance for chaining
*/
public Planetiler setDefaultLanguages(List<String> languages) {
this.languages = languages;
this.defaultLanguages = languages;
return this;
}

Expand Down Expand Up @@ -587,7 +588,13 @@ public Planetiler fetchWikidataNameTranslations(Path defaultWikidataCache) {
public Translations translations() {
if (translations == null) {
boolean transliterate = arguments.getBoolean("transliterate", "attempt to transliterate latin names", true);
List<String> languages = arguments.getList("languages", "languages to use", this.languages);
List<String> languages = arguments.getList("languages", "Languages to include labels for. \"default\" expands to the default set of languages configured by the profile. \"-lang\" excludes \"lang\". \"*\" includes every language not listed.", this.defaultLanguages);
if (languages.contains("default")) {
languages = Stream.concat(
languages.stream().filter(language -> !language.equals("default")),
this.defaultLanguages.stream()
).toList();
}
translations = Translations.defaultProvider(languages).setShouldTransliterate(transliterate);
}
return translations;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,24 @@ public static int countGeometries(VectorTileProto.Tile.Feature feature) {
return result;
}

/**
* Returns the encoded geometry for a polygon that fills an entire tile plus {@code buffer} pixels as a shortcut to
* avoid needing to create an extra JTS geometry for encoding.
*/
public static VectorGeometry encodeFill(double buffer) {
int min = (int) Math.round(EXTENT * buffer / 256d);
int width = EXTENT + min + min;
return new VectorGeometry(new int[]{
CommandEncoder.commandAndLength(Command.MOVE_TO, 1),
zigZagEncode(-min), zigZagEncode(-min),
CommandEncoder.commandAndLength(Command.LINE_TO, 3),
zigZagEncode(width), 0,
0, zigZagEncode(width),
zigZagEncode(-width), 0,
CommandEncoder.commandAndLength(Command.CLOSE_PATH, 1)
}, GeometryType.POLYGON, 0);
}

/**
* Adds features in a layer to this tile.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public static PlanetilerConfig from(Arguments arguments) {
int renderMaxzoom =
arguments.getInteger("render_maxzoom", "maximum rendering zoom level up to " + MAX_MAXZOOM,
Math.max(maxzoom, DEFAULT_MAXZOOM));
Path tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp"));
Path tmpDir = arguments.file("tmpdir|tmp", "temp directory", Path.of("data", "tmp"));

return new PlanetilerConfig(
arguments,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,27 @@ public record MultiExpression<T>(List<Entry<T>> expressions) implements Simplifi
private static final Logger LOGGER = LoggerFactory.getLogger(MultiExpression.class);
private static final Comparator<WithId> BY_ID = Comparator.comparingInt(WithId::id);

/**
* Returns a new multi-expression from {@code expressions} where multiple expressions for the same key get OR'd
* together.
* <p>
* If the order in which expresions match matter, use {@link #ofOrdered(List)} instead.
*/
public static <T> MultiExpression<T> of(List<Entry<T>> expressions) {
LinkedHashMap<T, Expression> map = new LinkedHashMap<>();
for (var expression : expressions) {
map.merge(expression.result, expression.expression, Expression::or);
}
return new MultiExpression<>(map.entrySet().stream().map(e -> entry(e.getKey(), e.getValue())).collect(Collectors.toList()));
return new MultiExpression<>(
map.entrySet().stream().map(e -> entry(e.getKey(), e.getValue())).collect(Collectors.toList()));
}

/**
* Returns a new multi-expression from {@code expressions} where multiple expressions for the same key stay separate,
* in cases where the order in which expressions matches is important.
*/
public static <T> MultiExpression<T> ofOrdered(List<Entry<T>> expressions) {
return new MultiExpression<>(new ArrayList<>(expressions));
}

public static <T> Entry<T> entry(T result, Expression expression) {
Expand All @@ -63,8 +78,8 @@ private static boolean mustAlwaysEvaluate(Expression expression) {
case Expression.Not(var child) -> !mustAlwaysEvaluate(child);
case Expression.MatchAny any when any.mustAlwaysEvaluate() -> true;
case null, default -> !(expression instanceof Expression.MatchAny) &&
!(expression instanceof Expression.MatchField) &&
!FALSE.equals(expression);
!(expression instanceof Expression.MatchField) &&
!FALSE.equals(expression);
};
}

Expand Down
Loading

0 comments on commit 5891376

Please sign in to comment.