From ec9c05c177b6b9dc4a13bbb0642635cc222e2da2 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Tue, 19 Sep 2023 20:36:47 -0400 Subject: [PATCH] comments --- NOTICE.md | 1 + .../com/onthegomap/planetiler/Planetiler.java | 3 +- .../onthegomap/planetiler/archive/Tile.java | 5 +++- .../planetiler/archive/TileArchiveWriter.java | 30 +++++++++---------- .../archive/TileEncodingResult.java | 4 +-- .../stream/WriteableCsvArchive.java | 1 - .../planetiler/util/TileSizeStats.java | 7 ++--- .../planetiler/geo/TileCoordTest.java | 13 ++++++++ 8 files changed, 38 insertions(+), 26 deletions(-) diff --git a/NOTICE.md b/NOTICE.md index abd9d1bb15..25f81916d1 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -28,6 +28,7 @@ The `planetiler-core` module includes the following software: - mil.nga.geopackage:geopackage (MIT license) - org.snakeyaml:snakeyaml-engine (Apache license) - org.commonmark:commonmark (BSD 2-clause license) + - org.tukaani:xz (public domain) - Adapted code: - `DouglasPeuckerSimplifier` from [JTS](https://github.com/locationtech/jts) (EDL) - `OsmMultipolygon` from [imposm3](https://github.com/omniscale/imposm3) (Apache license) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index 0685e5ff5e..74072ade2c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -92,7 +92,6 @@ public class Planetiler { private FeatureGroup featureGroup; private OsmInputFile osmInputFile; private TileArchiveConfig output; - private Path layerStatsPath; private boolean overwrite = false; private boolean ran = false; // most common OSM languages @@ -672,7 +671,7 @@ public void run() throws Exception { output.uri() + " already exists, use the --force argument to overwrite or --append."); } - layerStatsPath = arguments.file("layer_stats", "layer stats output path", + Path layerStatsPath = arguments.file("layer_stats", "layer stats output path", // default to .layerstats.tsv.gz TileSizeStats.getOutputPath(Optional.ofNullable(output.getLocalPath()).orElse(Path.of("output")))); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/Tile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/Tile.java index ca5de060f5..b580069a76 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/Tile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/Tile.java @@ -2,12 +2,15 @@ import com.onthegomap.planetiler.geo.TileCoord; import java.util.Arrays; +import java.util.Objects; +/** A tile stored in an archive with coordinate {@code coord} and archived {@code bytes}. */ public record Tile(TileCoord coord, byte[] bytes) implements Comparable { @Override public boolean equals(Object o) { - return (this == o) || (o instanceof Tile other && coord.equals(other.coord) && Arrays.equals(bytes, other.bytes)); + return (this == o) || + (o instanceof Tile other && Objects.equals(coord, other.coord) && Arrays.equals(bytes, other.bytes)); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index e69ca5ee9d..681d2b82fd 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -177,7 +177,6 @@ public static void writeOutput(FeatureGroup features, WriteableTileArchive outpu loggers.addPipelineStats(layerStatsBranch); } loggers.newLine() - .newLine() .add(writer::getLastTileLogDetails); var doneFuture = joinFutures( @@ -309,22 +308,21 @@ private void tileEncoderSink(Iterable prev) throws IOException { } lastTileDataHash = tileDataHash; } - if ((skipFilled && lastIsFill) || bytes == null) { - continue; + if ((!skipFilled || !lastIsFill) && bytes != null) { + tileStatsUpdater.recordTile(tileFeatures.tileCoord(), bytes.length, layerStats); + List layerStatsRows = config.outputLayerStats() ? + TileSizeStats.formatOutputRows(tileFeatures.tileCoord(), bytes.length, layerStats) : + List.of(); + result.add( + new TileEncodingResult( + tileFeatures.tileCoord(), + bytes, + encoded.length, + tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash), + layerStatsRows + ) + ); } - tileStatsUpdater.recordTile(tileFeatures.tileCoord(), bytes.length, layerStats); - List layerStatsRows = config.outputLayerStats() ? - TileSizeStats.formatOutputRows(tileFeatures.tileCoord(), bytes.length, layerStats) : - List.of(); - result.add( - new TileEncodingResult( - tileFeatures.tileCoord(), - bytes, - encoded.length, - tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash), - layerStatsRows - ) - ); } // hand result off to writer batch.out.complete(result); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java index b1730069ea..8716c214fe 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java @@ -11,7 +11,7 @@ public record TileEncodingResult( TileCoord coord, @Nonnull byte[] tileData, int rawTileSize, - /** will always be empty in non-compact mode and might also be empty in compact mode */ + /* will always be empty in non-compact mode and might also be empty in compact mode */ OptionalLong tileDataHash, List layerStats ) { @@ -20,7 +20,7 @@ public TileEncodingResult( byte[] tileData, OptionalLong tileDataHash ) { - this(coord, tileData, 0, tileDataHash, List.of()); + this(coord, tileData, tileData.length, tileDataHash, List.of()); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java index e887a8f872..5f9d549e6b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java @@ -115,7 +115,6 @@ private static class CsvTileWriter implements TileWriter { CsvTileWriter(Writer writer, String columnSeparator, String lineSeparator, Function tileDataEncoder) { - this.writer = writer; this.columnSeparator = columnSeparator; this.lineSeparator = lineSeparator; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java index 02f3bf6e0d..f951d06c61 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java @@ -35,6 +35,9 @@ import org.slf4j.LoggerFactory; import vector_tile.VectorTileProto; +/** + * + */ public class TileSizeStats { private static final int BATCH_SIZE = 1_000; @@ -47,10 +50,6 @@ public class TileSizeStats { .withLineSeparator("\n"); public static final ObjectWriter WRITER = MAPPER.writer(SCHEMA); - public TileSizeStats() { - // TODO load OSM tile weights - } - public static Path getOutputPath(Path output) { return output.resolveSibling(output.getFileName() + ".layerstats.tsv.gz"); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/TileCoordTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/TileCoordTest.java index e06a997e1a..30e561da18 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/TileCoordTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/TileCoordTest.java @@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.locationtech.jts.geom.Envelope; class TileCoordTest { @@ -130,4 +131,16 @@ void testTileProgressOnLevelHilbert(int x, int y, int z, double p) { void testDebugUrl(int x, int y, int z, String expected) { assertEquals(expected, TileCoord.ofXYZ(x, y, z).getDebugUrl("{z}/{lat}/{lon}")); } + + @Test + void testEnvelope() { + assertEquals(new Envelope( + -180, 180, + -85.0511287798066, 85.0511287798066 + ), TileCoord.ofXYZ(0, 0, 0).getEnvelope()); + assertEquals(new Envelope( + 0, 180, + -85.0511287798066, 0 + ), TileCoord.ofXYZ(1, 1, 1).getEnvelope()); + } }