From fc2a806fd968a0048577a33b54236941d27cf3ba Mon Sep 17 00:00:00 2001 From: bbilger Date: Sat, 6 Jan 2024 20:58:49 +0100 Subject: [PATCH] add tile-copy utility to copy tiles from one archive into another e.g. mbtiles to files, ... --- .../com/onthegomap/planetiler/Planetiler.java | 34 +-- .../planetiler/archive/TileArchiveConfig.java | 41 ++- .../archive/TileArchiveMetadata.java | 5 + .../planetiler/archive/TileArchiveWriter.java | 10 +- .../planetiler/archive/TileArchives.java | 43 +++- .../planetiler/archive/TileCompression.java | 5 - .../planetiler/archive/TileCopy.java | 236 ++++++++++++++++++ .../archive/TileEncodingResult.java | 6 +- .../planetiler/config/CommonConfigs.java | 31 +++ .../planetiler/config/PlanetilerConfig.java | 15 +- .../files/WriteableFilesArchive.java | 11 +- .../planetiler/mbtiles/Mbtiles.java | 7 +- .../onthegomap/planetiler/mbtiles/Verify.java | 7 +- .../planetiler/pmtiles/WriteablePmtiles.java | 9 +- .../planetiler/stream/CsvBinaryEncoding.java | 50 ++++ .../stream/JsonStreamArchiveEntry.java | 46 ++++ .../planetiler/stream/ReadableCsvArchive.java | 108 ++++++++ .../stream/ReadableJsonStreamArchive.java | 82 ++++++ .../stream/ReadableProtoStreamArchive.java | 160 ++++++++++++ .../stream/ReadableStreamArchive.java | 94 +++++++ .../stream/StreamArchiveConfig.java | 8 +- .../planetiler/stream/StreamArchiveUtils.java | 50 ++++ .../stream/WriteableCsvArchive.java | 58 +---- .../stream/WriteableJsonStreamArchive.java | 92 +------ .../stream/WriteableProtoStreamArchive.java | 12 +- .../planetiler/util/CloseableIterator.java | 40 +++ .../com/onthegomap/planetiler/util/Gzip.java | 26 +- .../onthegomap/planetiler/util/Hashing.java | 3 + .../src/main/proto/stream_archive_proto.proto | 16 +- .../planetiler/PlanetilerTests.java | 14 +- .../com/onthegomap/planetiler/TestUtils.java | 2 +- .../planetiler/archive/TileCopyTest.java | 200 +++++++++++++++ .../stream/InMemoryStreamArchive.java | 119 --------- .../stream/ReadableCsvStreamArchiveTest.java | 75 ++++++ .../stream/ReadableJsonStreamArchiveTest.java | 48 ++++ .../ReadableProtoStreamArchiveTest.java | 60 +++++ .../stream/WriteableCsvArchiveTest.java | 6 +- .../WriteableProtoStreamArchiveTest.java | 20 +- .../util/CloseableIteratorTest.java | 36 +++ .../java/com/onthegomap/planetiler/Main.java | 5 +- 40 files changed, 1512 insertions(+), 378 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java delete mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index b2503d76e3..09ac706f48 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -41,7 +41,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; -import java.util.stream.IntStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -656,18 +655,8 @@ public void run() throws Exception { System.exit(0); } else if (onlyDownloadSources) { // don't check files if not generating map - } else if (config.append()) { - if (!output.format().supportsAppend()) { - throw new IllegalArgumentException("cannot append to " + output.format().id()); - } - if (!output.exists()) { - throw new IllegalArgumentException(output.uri() + " must exist when appending"); - } - } else if (overwrite || config.force()) { - output.delete(); - } else if (output.exists()) { - throw new IllegalArgumentException( - output.uri() + " already exists, use the --force argument to overwrite or --append."); + } else { + output.setup(config.force() || overwrite, config.append(), config.tileWriteThreads()); } Path layerStatsPath = arguments.file("layer_stats", "layer stats output path", @@ -677,23 +666,6 @@ public void run() throws Exception { if (config.tileWriteThreads() < 1) { throw new IllegalArgumentException("require tile_write_threads >= 1"); } - if (config.tileWriteThreads() > 1) { - if (!output.format().supportsConcurrentWrites()) { - throw new IllegalArgumentException(output.format() + " doesn't support concurrent writes"); - } - IntStream.range(1, config.tileWriteThreads()) - .mapToObj(output::getPathForMultiThreadedWriter) - .forEach(p -> { - if (!config.append() && (overwrite || config.force())) { - FileUtils.delete(p); - } - if (config.append() && !output.exists(p)) { - throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending"); - } else if (!config.append() && output.exists(p)) { - throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending"); - } - }); - } LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri()); @@ -718,7 +690,7 @@ public void run() throws Exception { // in case any temp files are left from a previous run... FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath); Files.createDirectories(tmpDir); - FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalBasePath()); + FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath); if (!toDownload.isEmpty()) { download(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java index 0e10afa7f5..5c4575cbe7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java @@ -15,6 +15,7 @@ import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import java.util.stream.IntStream; import java.util.stream.Stream; /** @@ -157,7 +158,7 @@ public Path getLocalPath() { /** * Returns the local base path for this archive, for which directories should be pre-created for. */ - public Path getLocalBasePath() { + Path getLocalBasePath() { Path p = getLocalPath(); if (format() == Format.FILES) { p = FilesArchiveUtils.cleanBasePath(p); @@ -165,7 +166,6 @@ public Path getLocalBasePath() { return p; } - /** * Deletes the archive if possible. */ @@ -186,7 +186,7 @@ public boolean exists() { * @param p path to the archive * @return {@code true} if the archive already exists, {@code false} otherwise. */ - public boolean exists(Path p) { + private boolean exists(Path p) { if (p == null) { return false; } @@ -228,6 +228,41 @@ public Path getPathForMultiThreadedWriter(int index) { }; } + public void setup(boolean force, boolean append, int tileWriteThreads) { + if (append) { + if (!format().supportsAppend()) { + throw new IllegalArgumentException("cannot append to " + format().id()); + } + if (!exists()) { + throw new IllegalArgumentException(uri() + " must exist when appending"); + } + } else if (force) { + delete(); + } else if (exists()) { + throw new IllegalArgumentException(uri() + " already exists, use the --force argument to overwrite or --append."); + } + + if (tileWriteThreads > 1) { + if (!format().supportsConcurrentWrites()) { + throw new IllegalArgumentException(format() + " doesn't support concurrent writes"); + } + IntStream.range(1, tileWriteThreads) + .mapToObj(this::getPathForMultiThreadedWriter) + .forEach(p -> { + if (!append && force) { + FileUtils.delete(p); + } + if (append && !exists(p)) { + throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending"); + } else if (!append && exists(p)) { + throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending"); + } + }); + } + + FileUtils.createParentDirectories(getLocalBasePath()); + } + public enum Format { MBTILES("mbtiles", false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */, diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java index 1c5c83a14f..12f04cec20 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java @@ -169,6 +169,11 @@ public TileArchiveMetadata withJson(TileArchiveMetadataJson json) { maxzoom, json, others, tileCompression); } + public TileArchiveMetadata withTileCompression(TileCompression tileCompression) { + return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, minzoom, + maxzoom, json, others, tileCompression); + } + /* * few workarounds to make collect unknown fields to others work, * because @JsonAnySetter does not yet work on constructor/creator arguments diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index 7b932d5587..d5aa7d1c9e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -260,7 +260,7 @@ private void tileEncoderSink(Iterable prev) throws IOException { * To optimize emitting many identical consecutive tiles (like large ocean areas), memoize output to avoid * recomputing if the input hasn't changed. */ - byte[] lastBytes = null, lastEncoded = null; + byte[] lastBytes = null; Long lastTileDataHash = null; boolean lastIsFill = false; List lastLayerStats = null; @@ -275,24 +275,22 @@ private void tileEncoderSink(Iterable prev) throws IOException { for (int i = 0; i < batch.in.size(); i++) { FeatureGroup.TileFeatures tileFeatures = batch.in.get(i); featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed()); - byte[] bytes, encoded; + byte[] bytes; List layerStats; Long tileDataHash; if (tileFeatures.hasSameContents(last)) { bytes = lastBytes; - encoded = lastEncoded; tileDataHash = lastTileDataHash; layerStats = lastLayerStats; memoizedTiles.inc(); } else { VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater); if (skipFilled && (lastIsFill = tile.containsOnlyFills())) { - encoded = null; layerStats = null; bytes = null; } else { var proto = tile.toProto(); - encoded = proto.toByteArray(); + var encoded = proto.toByteArray(); bytes = switch (config.tileCompression()) { case GZIP -> gzip(encoded); case NONE -> encoded; @@ -306,7 +304,6 @@ private void tileEncoderSink(Iterable prev) throws IOException { } } lastLayerStats = layerStats; - lastEncoded = encoded; lastBytes = bytes; last = tileFeatures; if (archive.deduplicates() && tile.likelyToBeDuplicated() && bytes != null) { @@ -325,7 +322,6 @@ private void tileEncoderSink(Iterable prev) throws IOException { new TileEncodingResult( tileFeatures.tileCoord(), bytes, - encoded.length, tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash), layerStatsRows ) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java index 8ee03b4f1d..638922e3ed 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java @@ -1,17 +1,23 @@ package com.onthegomap.planetiler.archive; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.CommonConfigs; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.files.ReadableFilesArchive; import com.onthegomap.planetiler.files.WriteableFilesArchive; import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; import com.onthegomap.planetiler.pmtiles.WriteablePmtiles; +import com.onthegomap.planetiler.stream.ReadableCsvArchive; +import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive; +import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive; import com.onthegomap.planetiler.stream.StreamArchiveConfig; import com.onthegomap.planetiler.stream.WriteableCsvArchive; import com.onthegomap.planetiler.stream.WriteableJsonStreamArchive; import com.onthegomap.planetiler.stream.WriteableProtoStreamArchive; import java.io.IOException; import java.nio.file.Path; +import java.util.function.Supplier; /** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */ public class TileArchives { @@ -37,45 +43,58 @@ public static ReadableTileArchive newReader(String archive, PlanetilerConfig con return newReader(TileArchiveConfig.from(archive), config); } + public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config) + throws IOException { + return newWriter(archive, config.arguments()); + } + /** * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}. * * @throws IOException if an error occurs creating the resource. */ - public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config) + public static WriteableTileArchive newWriter(TileArchiveConfig archive, Arguments baseArguments) throws IOException { - var options = archive.applyFallbacks(config.arguments()); + var options = archive.applyFallbacks(baseArguments); var format = archive.format(); return switch (format) { case MBTILES -> // pass-through legacy arguments for fallback - Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments() + Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(baseArguments .subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION))); case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath()); case CSV, TSV -> WriteableCsvArchive.newWriteToFile(format, archive.getLocalPath(), - new StreamArchiveConfig(config, options)); + new StreamArchiveConfig(baseArguments, options)); case PROTO, PBF -> WriteableProtoStreamArchive.newWriteToFile(archive.getLocalPath(), - new StreamArchiveConfig(config, options)); + new StreamArchiveConfig(baseArguments, options)); case JSON -> WriteableJsonStreamArchive.newWriteToFile(archive.getLocalPath(), - new StreamArchiveConfig(config, options)); - case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, config.force() || config.append()); + new StreamArchiveConfig(baseArguments, options)); + case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, + CommonConfigs.appendToArchive(baseArguments) || CommonConfigs.force(baseArguments)); }; } + public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config) + throws IOException { + return newReader(archive, config.arguments()); + } + /** * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}. * * @throws IOException if an error occurs opening the resource. */ - public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config) + public static ReadableTileArchive newReader(TileArchiveConfig archive, Arguments baseArguments) throws IOException { - var options = archive.applyFallbacks(config.arguments()); + var options = archive.applyFallbacks(baseArguments); + Supplier streamArchiveConfig = () -> new StreamArchiveConfig(baseArguments, options); return switch (archive.format()) { case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options); case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath()); - case CSV, TSV -> throw new UnsupportedOperationException("reading CSV is not supported"); - case PROTO, PBF -> throw new UnsupportedOperationException("reading PROTO is not supported"); - case JSON -> throw new UnsupportedOperationException("reading JSON is not supported"); + case CSV, TSV -> + ReadableCsvArchive.newReader(archive.format(), archive.getLocalPath(), streamArchiveConfig.get()); + case PROTO, PBF -> ReadableProtoStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get()); + case JSON -> ReadableJsonStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get()); case FILES -> ReadableFilesArchive.newReader(archive.getLocalPath(), options); }; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java index 1277b31355..247abd5d43 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java @@ -54,10 +54,5 @@ static class Deserializer extends JsonDeserializer { public TileCompression deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { return findById(p.getValueAsString()).orElse(TileCompression.UNKNWON); } - - @Override - public TileCompression getNullValue(DeserializationContext ctxt) { - return TileCompression.GZIP; - } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java new file mode 100644 index 0000000000..03d8a13941 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java @@ -0,0 +1,236 @@ +package com.onthegomap.planetiler.archive; + +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.CommonConfigs; +import com.onthegomap.planetiler.stats.Counter; +import com.onthegomap.planetiler.stats.ProcessInfo; +import com.onthegomap.planetiler.stats.ProgressLoggers; +import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.util.Gzip; +import com.onthegomap.planetiler.util.Hashing; +import com.onthegomap.planetiler.worker.WorkerPipeline; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.function.Function; +import java.util.function.UnaryOperator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility to copy/convert tiles and metadata from one archive into another. + *

+ * Example usages: + * + * 

+ * --input=tiles.mbtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
+ * --input=tiles.pmtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles/
+ * --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
+ * --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
+ * --input=tiles.mbtiles --output=tiles.proto
+ *
+ */ +public class TileCopy { + + private static final Logger LOGGER = LoggerFactory.getLogger(TileCopy.class); + + private final TileCopyConfig config; + + private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter(); + + + TileCopy(TileCopyConfig config) { + this.config = config; + } + + public void run() throws IOException { + + if (!config.inArchive().exists()) { + throw new IllegalArgumentException("the input archive does not exist"); + } + + config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads()); + + final var loggers = ProgressLoggers.create() + .addRateCounter("tiles", tilesWrittenOverall::get); + + try ( + var reader = TileArchives.newReader(config.inArchive(), config.inArguments()); + var writer = TileArchives.newWriter(config.outArchive(), config.outArguments()) + ) { + + final TileArchiveMetadata inMetadata = getInMetadata(reader); + final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata); + + writer.initialize(); + try ( + var rawTiles = reader.getAllTiles(); + var it = config.skipEmpty() ? rawTiles.filter(t -> t.bytes() != null && t.bytes().length > 0) : rawTiles + ) { + + final var tileConverter = tileConverter(inMetadata.tileCompression(), outMetadata.tileCompression(), writer); + var pipeline = WorkerPipeline.start("archive", config.stats()) + .readFrom("tiles", () -> it) + .addBuffer("buffer", config.queueSize()) + .sinkTo("write", config.tileWriterThreads(), itt -> tileWriter(writer, itt, tileConverter)); + + final var f = pipeline.done().thenRun(() -> writer.finish(outMetadata)); + + loggers.awaitAndLog(f, config.logInterval()); + } + } + } + + private void tileWriter(WriteableTileArchive archive, Iterable itt, + Function tileConverter) { + + final Counter tilesWritten = tilesWrittenOverall.counterForThread(); + try (var tileWriter = archive.newTileWriter()) { + for (Tile t : itt) { + tileWriter.write(tileConverter.apply(t)); + tilesWritten.inc(); + } + } + } + + private static Function tileConverter(TileCompression inCompression, + TileCompression outCompression, WriteableTileArchive writer) { + + final UnaryOperator bytesReEncoder = bytesReEncoder(inCompression, outCompression); + final Function hasher = + writer.deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) : + b -> OptionalLong.empty(); + + return t -> new TileEncodingResult(t.coord(), bytesReEncoder.apply(t.bytes()), hasher.apply(t.bytes())); + } + + private static UnaryOperator bytesReEncoder(TileCompression inCompression, TileCompression outCompression) { + if (inCompression == outCompression) { + return UnaryOperator.identity(); + } else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) { + return Gzip::gunzip; + } else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) { + return Gzip::gzip; + } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.GZIP) { + return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b); + } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.NONE) { + return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b; + } else { + throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression); + } + } + + private TileArchiveMetadata getInMetadata(ReadableTileArchive reader) { + TileArchiveMetadata inMetadata = config.inMetadata(); + if (inMetadata == null) { + inMetadata = reader.metadata(); + if (inMetadata == null) { + LOGGER.atWarn() + .log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata"); + inMetadata = fallbackMetadata(); + } + } + if (inMetadata.tileCompression() == null) { + inMetadata = inMetadata.withTileCompression(config.inCompression()); + } + + return inMetadata; + } + + private TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata) { + if (config.outCompression() == TileCompression.UNKNWON && inMetadata.tileCompression() == TileCompression.UNKNWON) { + return inMetadata.withTileCompression(TileCompression.GZIP); + } else if (config.outCompression() != TileCompression.UNKNWON) { + return inMetadata.withTileCompression(config.outCompression()); + } else { + return inMetadata; + } + } + + private static TileArchiveMetadata fallbackMetadata() { + return new TileArchiveMetadata( + "unknown", + null, + null, + null, + null, + TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf + null, + null, + null, + null, + new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers + Map.of(), + null + ); + } + + record TileCopyConfig( + TileArchiveConfig inArchive, + TileArchiveConfig outArchive, + Arguments inArguments, + Arguments outArguments, + TileCompression inCompression, + TileCompression outCompression, + int tileWriterThreads, + Duration logInterval, + Stats stats, + int queueSize, + boolean append, + boolean force, + TileArchiveMetadata inMetadata, + boolean skipEmpty + ) { + static TileCopyConfig fromArguments(Arguments baseArguments) { + + final Arguments inArguments = baseArguments.withPrefix("in"); + final Arguments outArguments = baseArguments.withPrefix("out"); + final Arguments baseOrOutArguments = outArguments.orElse(baseArguments); + + final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null); + final TileArchiveMetadata inMetadata; + if (inMetadataPath != null) { + try { + inMetadata = + TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + inMetadata = null; + } + + return new TileCopyConfig( + TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")), + TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")), + inArguments, + outArguments, + getTileCompressionArg(inArguments, "the input tile compression"), + getTileCompressionArg(outArguments, "the output tile compression"), + CommonConfigs.tileWriterThreads(baseOrOutArguments), + CommonConfigs.logInterval(baseArguments), + baseArguments.getStats(), + Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)), + CommonConfigs.appendToArchive(baseOrOutArguments), + CommonConfigs.force(baseOrOutArguments), + inMetadata, + baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", false) + ); + } + } + + private static TileCompression getTileCompressionArg(Arguments args, String description) { + return args.getObject("tile_compression", description, TileCompression.UNKNWON, + v -> TileCompression.findById(v).orElseThrow()); + } + + public static void main(String[] args) throws IOException { + new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run(); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java index 8716c214fe..c6f5889691 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java @@ -5,12 +5,10 @@ import java.util.List; import java.util.Objects; import java.util.OptionalLong; -import javax.annotation.Nonnull; public record TileEncodingResult( TileCoord coord, - @Nonnull byte[] tileData, - int rawTileSize, + byte[] tileData, /* will always be empty in non-compact mode and might also be empty in compact mode */ OptionalLong tileDataHash, List layerStats @@ -20,7 +18,7 @@ public TileEncodingResult( byte[] tileData, OptionalLong tileDataHash ) { - this(coord, tileData, tileData.length, tileDataHash, List.of()); + this(coord, tileData, tileDataHash, List.of()); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java new file mode 100644 index 0000000000..1e30db5955 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java @@ -0,0 +1,31 @@ +package com.onthegomap.planetiler.config; + +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import java.time.Duration; +import java.util.stream.Stream; + +public final class CommonConfigs { + private CommonConfigs() {} + + public static boolean force(Arguments arguments) { + return arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false); + } + + public static boolean appendToArchive(Arguments arguments) { + return arguments.getBoolean("append", + "append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values()) + .filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(), + false); + } + + public static int tileWriterThreads(Arguments arguments) { + return arguments.getInteger("tile_write_threads", + "number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values()) + .filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(), + 1); + } + + public static Duration logInterval(Arguments arguments) { + return arguments.getDuration("loginterval", "time between logs", "10s"); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index b9a1463024..32d02fc166 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -1,6 +1,5 @@ package com.onthegomap.planetiler.config; -import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.archive.TileCompression; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.Storage; @@ -132,19 +131,13 @@ public static PlanetilerConfig from(Arguments arguments) { featureProcessThreads, arguments.getInteger("feature_read_threads", "number of threads to use when reading features at tile write time", threads < 32 ? 1 : 2), - arguments.getInteger("tile_write_threads", - "number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values()) - .filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(), - 1), - arguments.getDuration("loginterval", "time between logs", "10s"), + CommonConfigs.tileWriterThreads(arguments), + CommonConfigs.logInterval(arguments), minzoom, maxzoom, renderMaxzoom, - arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false), - arguments.getBoolean("append", - "append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values()) - .filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(), - false), + CommonConfigs.force(arguments), + CommonConfigs.appendToArchive(arguments), arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false), arguments.getBoolean("mmap_temp", "use memory-mapped IO for temp feature files", true), arguments.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks", diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java index ee4d70c00f..b8b9c47bfa 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java @@ -159,12 +159,17 @@ public final void write(TileEncodingResult encodingResult) { } lastCheckedFolder = folder; try { - Files.write(file, data); + if (data == null) { + Files.createFile(file); + } else { + Files.write(file, data); + } } catch (IOException e) { throw new UncheckedIOException(e); } - - bytesWritten.incBy(data.length); + if (data != null) { + bytesWritten.incBy(data.length); + } } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index e68e1f7fdc..e1e9449a0b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -6,6 +6,7 @@ import com.onthegomap.planetiler.archive.Tile; import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer; +import com.onthegomap.planetiler.archive.TileCompression; import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.config.Arguments; @@ -847,7 +848,11 @@ public Metadata set(TileArchiveMetadata tileArchiveMetadata) { */ public TileArchiveMetadata get() { Map map = new HashMap<>(getAll()); - return TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class); + var metadata = TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class); + if (metadata.tileCompression() == null) { + metadata = metadata.withTileCompression(TileCompression.GZIP); + } + return metadata; } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java index 99a43f15de..d0a18ed75b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java @@ -6,7 +6,6 @@ import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -92,11 +91,7 @@ private static int getGeometryCounts(Geometry geom, Class cl } private static List decode(byte[] zipped) { - try { - return VectorTile.decode(gunzip(zipped)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + return VectorTile.decode(gunzip(zipped)); } /** diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java index 63f42ec3bc..504d83522d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java @@ -57,8 +57,7 @@ private WriteablePmtiles(SeekableByteChannel channel, LongSupplier bytesWritten) this.bytesWritten = bytesWritten; } - private static Directories makeDirectoriesWithLeaves(List subEntries, int leafSize, int attemptNum) - throws IOException { + private static Directories makeDirectoriesWithLeaves(List subEntries, int leafSize, int attemptNum) { LOGGER.info("Building directories with {} entries per leaf, attempt {}...", leafSize, attemptNum); ArrayList rootEntries = new ArrayList<>(); ByteArrayList leavesOutputStream = new ByteArrayList(); @@ -91,9 +90,8 @@ private static Directories makeDirectoriesWithLeaves(List subEntr * * @param entries a sorted ObjectArrayList of all entries in the tileset. * @return byte arrays of the root and all leaf directories, and the # of leaves. - * @throws IOException if compression fails */ - static Directories makeDirectories(List entries) throws IOException { + static Directories makeDirectories(List entries) { int maxEntriesRootOnly = 16384; int attemptNum = 1; if (entries.size() < maxEntriesRootOnly) { @@ -301,6 +299,9 @@ public void write(TileEncodingResult encodingResult) { long offset; OptionalLong tileDataHashOpt = encodingResult.tileDataHash(); var data = encodingResult.tileData(); + if (data == null) { + return; + } TileCoord coord = encodingResult.coord(); long tileId = coord.hilbertEncoded(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java new file mode 100644 index 0000000000..506aa02d43 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java @@ -0,0 +1,50 @@ +package com.onthegomap.planetiler.stream; + +import com.google.common.base.Suppliers; +import java.util.Base64; +import java.util.HexFormat; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; + +enum CsvBinaryEncoding { + + BASE64("base64", () -> Base64.getEncoder()::encodeToString, () -> Base64.getDecoder()::decode), + HEX("hex", () -> HexFormat.of()::formatHex, () -> HexFormat.of()::parseHex); + + private final String id; + private final Supplier> encoder; + private final Supplier> decoder; + + private CsvBinaryEncoding(String id, Supplier> encoder, + Supplier> decoder) { + this.id = id; + this.encoder = Suppliers.memoize(encoder::get); + this.decoder = Suppliers.memoize(decoder::get); + } + + String encode(byte[] b) { + return encoder.get().apply(b); + } + + byte[] decode(String s) { + return decoder.get().apply(s); + } + + static List ids() { + return Stream.of(CsvBinaryEncoding.values()).map(CsvBinaryEncoding::id).toList(); + } + + static CsvBinaryEncoding fromId(String id) { + return Stream.of(CsvBinaryEncoding.values()) + .filter(de -> de.id().equals(id)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "unexpected binary encoding - expected one of " + ids() + " but got " + id)); + } + + String id() { + return id; + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java new file mode 100644 index 0000000000..2f0808f7c9 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java @@ -0,0 +1,46 @@ +package com.onthegomap.planetiler.stream; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import java.util.Arrays; +import java.util.Objects; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = JsonStreamArchiveEntry.TileEntry.class, name = "tile"), + @JsonSubTypes.Type(value = JsonStreamArchiveEntry.InitializationEntry.class, name = "initialization"), + @JsonSubTypes.Type(value = JsonStreamArchiveEntry.FinishEntry.class, name = "finish") +}) +sealed interface JsonStreamArchiveEntry { + record TileEntry(int x, int y, int z, byte[] encodedData) implements JsonStreamArchiveEntry { + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(encodedData); + result = prime * result + Objects.hash(x, y, z); + return result; + } + + @Override + public boolean equals(Object obj) { + return this == obj || (obj instanceof JsonStreamArchiveEntry.TileEntry tileEntry && + Arrays.equals(encodedData, tileEntry.encodedData) && x == tileEntry.x && y == tileEntry.y && z == tileEntry.z); + } + + @Override + public String toString() { + return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]"; + } + } + + record InitializationEntry() implements JsonStreamArchiveEntry {} + + + record FinishEntry(TileArchiveMetadata metadata) implements JsonStreamArchiveEntry {} +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java new file mode 100644 index 0000000000..f6df44e42d --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java @@ -0,0 +1,108 @@ +package com.onthegomap.planetiler.stream; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Scanner; +import java.util.function.Function; +import java.util.regex.Pattern; + +/** + * Reads tiles from a CSV file. Counterpart to {@link WriteableCsvArchive}. + *

+ * Supported arguments: + *

+ *
column_separator
+ *
The column separator e.g. ",", ";", "\t"
+ *
line_separator
+ *
The line separator e.g. "\n", "\r", "\r\n"
+ *
+ * + * @see WriteableCsvArchive + */ +public class ReadableCsvArchive extends ReadableStreamArchive { + + private final Pattern columnSeparatorPattern; + private final Pattern lineSeparatorPattern; + private final Function tileDataDecoder; + + private ReadableCsvArchive(TileArchiveConfig.Format format, Path basePath, StreamArchiveConfig config) { + super(basePath, config); + this.columnSeparatorPattern = + Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format))); + this.lineSeparatorPattern = + Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format))); + final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions()); + this.tileDataDecoder = binaryEncoding::decode; + } + + public static ReadableCsvArchive newReader(TileArchiveConfig.Format format, Path basePath, + StreamArchiveConfig config) { + return new ReadableCsvArchive(format, basePath, config); + } + + @Override + CloseableIterator createIterator() { + try { + @SuppressWarnings("java:S2095") final Scanner s = + new Scanner(basePath.toFile()).useDelimiter(lineSeparatorPattern); + return new CloseableIterator<>() { + @Override + public void close() { + s.close(); + } + + @Override + public boolean hasNext() { + return s.hasNext(); + } + + @Override + public String next() { + return s.next(); + } + }; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + } + + @Override + Optional mapEntryToTile(String entry) { + final String[] splits = columnSeparatorPattern.split(entry); + final byte[] bytes; + if (splits.length == 4) { + bytes = tileDataDecoder.apply(splits[3].strip()); + } else if (splits.length == 3) { + bytes = null; + } else { + throw new InvalidCsvFormat(entry.length() > 20 ? entry.substring(0, 20) + "..." : entry); + } + return Optional.of(new Tile( + TileCoord.ofXYZ( + Integer.parseInt(splits[0].strip()), + Integer.parseInt(splits[1].strip()), + Integer.parseInt(splits[2].strip()) + ), + bytes + )); + } + + @Override + Optional mapEntryToMetadata(String entry) { + return Optional.empty(); + } + + static class InvalidCsvFormat extends RuntimeException { + InvalidCsvFormat(String message) { + super(message); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java new file mode 100644 index 0000000000..640adf0bee --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java @@ -0,0 +1,82 @@ +package com.onthegomap.planetiler.stream; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +/** + * Reads tiles and metadata from a delimited JSON file. Counterpart to {@link WriteableJsonStreamArchive}. + * + * @see WriteableJsonStreamArchive + */ +public class ReadableJsonStreamArchive extends ReadableStreamArchive { + + private ReadableJsonStreamArchive(Path basePath, StreamArchiveConfig config) { + super(basePath, config); + } + + public static ReadableJsonStreamArchive newReader(Path basePath, StreamArchiveConfig config) { + return new ReadableJsonStreamArchive(basePath, config); + } + + @Override + CloseableIterator createIterator() { + BufferedReader reader = null; + try { + reader = Files.newBufferedReader(basePath); + final var readerFinal = reader; + final var it = StreamArchiveUtils.jsonMapperJsonStreamArchive + .readerFor(JsonStreamArchiveEntry.class) + .readValues(readerFinal); + return new CloseableIterator<>() { + @Override + public void close() { + try { + readerFinal.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public JsonStreamArchiveEntry next() { + return it.next(); + } + }; + } catch (IOException e) { + closeSilentlyOnError(reader); + throw new UncheckedIOException(e); + } + } + + @Override + Optional mapEntryToTile(JsonStreamArchiveEntry entry) { + if (entry instanceof JsonStreamArchiveEntry.TileEntry tileEntry) { + return Optional.of(new Tile( + TileCoord.ofXYZ(tileEntry.x(), tileEntry.y(), tileEntry.z()), + tileEntry.encodedData() + )); + } + return Optional.empty(); + } + + @Override + Optional mapEntryToMetadata(JsonStreamArchiveEntry entry) { + if (entry instanceof JsonStreamArchiveEntry.FinishEntry finishEntry) { + return Optional.ofNullable(finishEntry.metadata()); + } + return Optional.empty(); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java new file mode 100644 index 0000000000..98fb968122 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java @@ -0,0 +1,160 @@ +package com.onthegomap.planetiler.stream; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.proto.StreamArchiveProto; +import com.onthegomap.planetiler.util.CloseableIterator; +import com.onthegomap.planetiler.util.LayerAttrStats; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Envelope; + +/** + * Reads tiles and metadata from a delimited protobuf file. Counterpart to {@link WriteableProtoStreamArchive}. + * + * @see WriteableProtoStreamArchive + */ +public class ReadableProtoStreamArchive extends ReadableStreamArchive { + + private ReadableProtoStreamArchive(Path basePath, StreamArchiveConfig config) { + super(basePath, config); + } + + public static ReadableProtoStreamArchive newReader(Path basePath, StreamArchiveConfig config) { + return new ReadableProtoStreamArchive(basePath, config); + } + + @Override + CloseableIterator createIterator() { + try { + @SuppressWarnings("java:S2095") var in = new FileInputStream(basePath.toFile()); + return new CloseableIterator<>() { + private StreamArchiveProto.Entry nextValue; + + @Override + public void close() { + closeUnchecked(in); + } + + @Override + public boolean hasNext() { + if (nextValue != null) { + return true; + } + try { + nextValue = StreamArchiveProto.Entry.parseDelimitedFrom(in); + return nextValue != null; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public StreamArchiveProto.Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final StreamArchiveProto.Entry returnValue = nextValue; + nextValue = null; + return returnValue; + } + }; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + Optional mapEntryToTile(StreamArchiveProto.Entry entry) { + if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.TILE) { + return Optional.empty(); + } + final StreamArchiveProto.TileEntry tileEntry = entry.getTile(); + return Optional.of(new Tile( + TileCoord.ofXYZ(tileEntry.getX(), tileEntry.getY(), tileEntry.getZ()), + tileEntry.getEncodedData().toByteArray() + )); + } + + @Override + Optional mapEntryToMetadata(StreamArchiveProto.Entry entry) { + if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.FINISH) { + return Optional.empty(); + } + final StreamArchiveProto.Metadata metadata = entry.getFinish().getMetadata(); + return Optional.of(new TileArchiveMetadata( + StringUtils.trimToNull(metadata.getName()), + StringUtils.trimToNull(metadata.getDescription()), + StringUtils.trimToNull(metadata.getAttribution()), + StringUtils.trimToNull(metadata.getVersion()), + StringUtils.trimToNull(metadata.getType()), + StringUtils.trimToNull(metadata.getFormat()), + deserializeEnvelope(metadata.hasBounds() ? metadata.getBounds() : null), + deserializeCoordinate(metadata.hasCenter() ? metadata.getCenter() : null), + metadata.hasMinZoom() ? metadata.getMinZoom() : null, + metadata.hasMaxZoom() ? metadata.getMaxZoom() : null, + extractMetadataJson(metadata), + metadata.getOthersMap(), + deserializeTileCompression(metadata.getTileCompression()) + )); + } + + private Envelope deserializeEnvelope(StreamArchiveProto.Envelope s) { + return s == null ? null : new Envelope(s.getMinX(), s.getMaxX(), s.getMinY(), s.getMaxY()); + } + + private Coordinate deserializeCoordinate(StreamArchiveProto.Coordinate s) { + if (s == null) { + return null; + } + return s.hasZ() ? new Coordinate(s.getX(), s.getY(), s.getZ()) : new CoordinateXY(s.getX(), s.getY()); + } + + private TileCompression deserializeTileCompression(StreamArchiveProto.TileCompression s) { + return switch (s) { + case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNWON; + case TILE_COMPRESSION_GZIP -> TileCompression.GZIP; + case TILE_COMPRESSION_NONE -> TileCompression.NONE; + }; + } + + private TileArchiveMetadata.TileArchiveMetadataJson extractMetadataJson(StreamArchiveProto.Metadata s) { + final List vl = deserializeVectorLayers(s.getVectorLayersList()); + return vl.isEmpty() ? null : new TileArchiveMetadata.TileArchiveMetadataJson(vl); + } + + private List deserializeVectorLayers(List s) { + return s.stream() + .map(vl -> new LayerAttrStats.VectorLayer( + vl.getId(), + vl.getFieldsMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> deserializeFieldType(e.getValue()))), + Optional.ofNullable(StringUtils.trimToNull(vl.getDescription())), + vl.hasMinZoom() ? OptionalInt.of(vl.getMinZoom()) : OptionalInt.empty(), + vl.hasMaxZoom() ? OptionalInt.of(vl.getMaxZoom()) : OptionalInt.empty() + )) + .toList(); + } + + private LayerAttrStats.FieldType deserializeFieldType(StreamArchiveProto.VectorLayer.FieldType s) { + return switch (s) { + case FIELD_TYPE_UNSPECIFIED, UNRECOGNIZED -> throw new IllegalArgumentException("unknown type"); + case FIELD_TYPE_NUMBER -> LayerAttrStats.FieldType.NUMBER; + case FIELD_TYPE_BOOLEAN -> LayerAttrStats.FieldType.BOOLEAN; + case FIELD_TYPE_STRING -> LayerAttrStats.FieldType.STRING; + }; + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java new file mode 100644 index 0000000000..f6f9ecff56 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java @@ -0,0 +1,94 @@ +package com.onthegomap.planetiler.stream; + +import com.google.common.base.Suppliers; +import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Supplier; + +abstract class ReadableStreamArchive implements ReadableTileArchive { + + private final Supplier cachedMetadata = Suppliers.memoize(this::loadMetadata); + + final Path basePath; + final StreamArchiveConfig config; + + ReadableStreamArchive(Path basePath, StreamArchiveConfig config) { + this.basePath = basePath; + this.config = config; + } + + @Override + public final byte[] getTile(TileCoord coord) { + return getAllTiles().stream().filter(c -> c.coord().equals(coord)).map(Tile::bytes).findFirst().orElse(null); + } + + @Override + public final byte[] getTile(int x, int y, int z) { + return getTile(TileCoord.ofXYZ(x, y, z)); + } + + @Override + public final CloseableIterator getAllTileCoords() { + return getAllTiles().map(Tile::coord); + } + + @Override + public final CloseableIterator getAllTiles() { + return createIterator() + .map(this::mapEntryToTile) + .filter(Optional::isPresent) + .map(Optional::get); + } + + @Override + public final TileArchiveMetadata metadata() { + return cachedMetadata.get(); + } + + private TileArchiveMetadata loadMetadata() { + try (var it = createIterator()) { + return it.stream().map(this::mapEntryToMetadata).flatMap(Optional::stream).findFirst().orElse(null); + } + } + + @Override + public void close() throws IOException { + // nothing to close + } + + abstract CloseableIterator createIterator(); + + abstract Optional mapEntryToTile(E entry); + + abstract Optional mapEntryToMetadata(E entry); + + void closeSilentlyOnError(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (Exception ignored) { + // ignore + } + } + + @SuppressWarnings("java:S112") + void closeUnchecked(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java index 6f85ccb5e9..08b8967c46 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java @@ -1,10 +1,10 @@ package com.onthegomap.planetiler.stream; import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.config.CommonConfigs; -public record StreamArchiveConfig(boolean appendToFile, Arguments moreOptions) { - public StreamArchiveConfig(PlanetilerConfig planetilerConfig, Arguments moreOptions) { - this(planetilerConfig.append(), moreOptions); +public record StreamArchiveConfig(boolean appendToFile, Arguments formatOptions) { + public StreamArchiveConfig(Arguments baseArguments, Arguments formatOptions) { + this(CommonConfigs.appendToArchive(baseArguments), formatOptions); } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java index 8d23ef6fa2..b1ff2e20dc 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java @@ -1,5 +1,8 @@ package com.onthegomap.planetiler.stream; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.google.common.net.UrlEscapers; import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.config.Arguments; @@ -12,8 +15,25 @@ public final class StreamArchiveUtils { + /** + * exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases + * => allow to output tiles, only + */ + private static final String JSON_OPTION_WRITE_TILES_ONLY = "tiles_only"; + + private static final String JSON_OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator"; + + static final String CSV_OPTION_COLUMN_SEPARATOR = "column_separator"; + static final String CSV_OPTION_LINE_SEPARATOR = "line_separator"; + static final String CSV_OPTION_BINARY_ENCODING = "binary_encoding"; + private static final Pattern quotedPattern = Pattern.compile("^'(.+?)'$"); + static final JsonMapper jsonMapperJsonStreamArchive = JsonMapper.builder() + .serializationInclusion(JsonInclude.Include.NON_ABSENT) + .addModule(new Jdk8Module()) + .build(); + private StreamArchiveUtils() {} public static Path constructIndexedPath(Path basePath, int index) { @@ -39,6 +59,36 @@ static String getEscapedString(Arguments options, TileArchiveConfig.Format forma .translateEscapes(); } + static String jsonOptionRootValueSeparator(Arguments formatOptions) { + return getEscapedString(formatOptions, TileArchiveConfig.Format.JSON, + JSON_OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " ")); + } + + static boolean jsonOptionWriteTilesOnly(Arguments formatOptions) { + return formatOptions.getBoolean(JSON_OPTION_WRITE_TILES_ONLY, "write tiles, only", false); + } + + static String csvOptionColumnSeparator(Arguments formatOptions, TileArchiveConfig.Format format) { + final String defaultColumnSeparator = switch (format) { + case CSV -> "','"; + case TSV -> "'\\t'"; + default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id()); + }; + return getEscapedString(formatOptions, format, + CSV_OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " ")); + } + + static String csvOptionLineSeparator(Arguments formatOptions, TileArchiveConfig.Format format) { + return StreamArchiveUtils.getEscapedString(formatOptions, format, + CSV_OPTION_LINE_SEPARATOR, "line separator", "'\\n'", List.of("\n", "\r\n")); + } + + static CsvBinaryEncoding csvOptionBinaryEncoding(Arguments formatOptions) { + return CsvBinaryEncoding.fromId(formatOptions.getString(CSV_OPTION_BINARY_ENCODING, + "binary (tile) data encoding - one of " + CsvBinaryEncoding.ids(), "base64")); + } + + private static String escapeJava(String s) { if (!s.trim().equals(s)) { s = "'" + s + "'"; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java index 5f9d549e6b..be0bc51316 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java @@ -11,11 +11,7 @@ import java.io.Writer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.Base64; -import java.util.HexFormat; -import java.util.List; import java.util.function.Function; -import java.util.stream.Stream; /** * Writes tile data into a CSV file (or pipe). @@ -67,31 +63,16 @@ */ public final class WriteableCsvArchive extends WriteableStreamArchive { - static final String OPTION_COLUMN_SEPARATOR = "column_separator"; - static final String OPTION_LINE_SEPARTATOR = "line_separator"; - static final String OPTION_BINARY_ENCODING = "binary_encoding"; - private final String columnSeparator; private final String lineSeparator; private final Function tileDataEncoder; private WriteableCsvArchive(TileArchiveConfig.Format format, Path p, StreamArchiveConfig config) { super(p, config); - final String defaultColumnSeparator = switch (format) { - case CSV -> "','"; - case TSV -> "'\\t'"; - default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id()); - }; - this.columnSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format, - OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " ")); - this.lineSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format, - OPTION_LINE_SEPARTATOR, "line separator", "'\\n'", List.of("\n", "\r\n")); - final BinaryEncoding binaryEncoding = BinaryEncoding.fromId(config.moreOptions().getString(OPTION_BINARY_ENCODING, - "binary (tile) data encoding - one of " + BinaryEncoding.ids(), "base64")); - this.tileDataEncoder = switch (binaryEncoding) { - case BASE64 -> Base64.getEncoder()::encodeToString; - case HEX -> HexFormat.of()::formatHex; - }; + this.columnSeparator = StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format); + this.lineSeparator = StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format); + final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions()); + this.tileDataEncoder = binaryEncoding::encode; } public static WriteableCsvArchive newWriteToFile(TileArchiveConfig.Format format, Path path, @@ -131,7 +112,8 @@ private static class CsvTileWriter implements TileWriter { @Override public void write(TileEncodingResult encodingResult) { final TileCoord coord = encodingResult.coord(); - final String tileDataEncoded = tileDataEncoder.apply(encodingResult.tileData()); + final byte[] data = encodingResult.tileData(); + final String tileDataEncoded = data == null ? "" : tileDataEncoder.apply(encodingResult.tileData()); try { // x | y | z | encoded data writer.write("%d%s%d%s%d%s%s%s".formatted(coord.x(), columnSeparator, coord.y(), columnSeparator, coord.z(), @@ -150,32 +132,4 @@ public void close() { } } } - - private enum BinaryEncoding { - - BASE64("base64"), - HEX("hex"); - - private final String id; - - private BinaryEncoding(String id) { - this.id = id; - } - - static List ids() { - return Stream.of(BinaryEncoding.values()).map(BinaryEncoding::id).toList(); - } - - static BinaryEncoding fromId(String id) { - return Stream.of(BinaryEncoding.values()) - .filter(de -> de.id().equals(id)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "unexpected binary encoding - expected one of " + ids() + " but got " + id)); - } - - String id() { - return id; - } - } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java index 1fad9f0b23..671a8676e6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java @@ -1,14 +1,8 @@ package com.onthegomap.planetiler.stream; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonSubTypes.Type; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.databind.json.JsonMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.geo.TileCoord; @@ -19,41 +13,26 @@ import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Writes JSON-serialized tile data as well as meta data into file(s). The entries are of type - * {@link WriteableJsonStreamArchive.Entry} are separated by newline (by default). + * {@link JsonStreamArchiveEntry} are separated by newline (by default). */ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { private static final Logger LOGGER = LoggerFactory.getLogger(WriteableJsonStreamArchive.class); - /** - * exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases - * => allow to output tiles, only - */ - private static final String OPTION_WRITE_TILES_ONLY = "tiles_only"; - - private static final String OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator"; - - static final JsonMapper jsonMapper = JsonMapper.builder() - .serializationInclusion(Include.NON_ABSENT) - .addModule(new Jdk8Module()) - .build(); + private static final JsonMapper jsonMapper = StreamArchiveUtils.jsonMapperJsonStreamArchive; private final boolean writeTilesOnly; private final String rootValueSeparator; private WriteableJsonStreamArchive(Path p, StreamArchiveConfig config) { super(p, config); - this.writeTilesOnly = config.moreOptions().getBoolean(OPTION_WRITE_TILES_ONLY, "write tiles, only", false); - this.rootValueSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), TileArchiveConfig.Format.JSON, - OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " ")); + this.writeTilesOnly = StreamArchiveUtils.jsonOptionWriteTilesOnly(config.formatOptions()); + this.rootValueSeparator = StreamArchiveUtils.jsonOptionRootValueSeparator(config.formatOptions()); } public static WriteableJsonStreamArchive newWriteToFile(Path path, StreamArchiveConfig config) { @@ -70,7 +49,7 @@ public void initialize() { if (writeTilesOnly) { return; } - writeEntryFlush(new InitializationEntry()); + writeEntryFlush(new JsonStreamArchiveEntry.InitializationEntry()); } @Override @@ -78,13 +57,13 @@ public void finish(TileArchiveMetadata metadata) { if (writeTilesOnly) { return; } - writeEntryFlush(new FinishEntry(metadata)); + writeEntryFlush(new JsonStreamArchiveEntry.FinishEntry(metadata)); } - private void writeEntryFlush(Entry entry) { + private void writeEntryFlush(JsonStreamArchiveEntry entry) { try (var out = new OutputStreamWriter(getPrimaryOutputStream(), StandardCharsets.UTF_8.newEncoder())) { jsonMapper - .writerFor(Entry.class) + .writerFor(JsonStreamArchiveEntry.class) .withoutFeatures(JsonGenerator.Feature.AUTO_CLOSE_TARGET) .writeValue(out, entry); out.write(rootValueSeparator); @@ -104,7 +83,8 @@ private static class JsonTileWriter implements TileWriter { this.rootValueSeparator = rootValueSeparator; try { this.jsonWriter = - jsonMapper.writerFor(Entry.class).withRootValueSeparator(rootValueSeparator).writeValues(outputStream); + jsonMapper.writerFor(JsonStreamArchiveEntry.class).withRootValueSeparator(rootValueSeparator) + .writeValues(outputStream); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -114,7 +94,8 @@ private static class JsonTileWriter implements TileWriter { public void write(TileEncodingResult encodingResult) { final TileCoord coord = encodingResult.coord(); try { - jsonWriter.write(new TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData())); + jsonWriter + .write(new JsonStreamArchiveEntry.TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData())); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -150,53 +131,4 @@ public void close() { } } - - @JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "type") - @JsonSubTypes({ - @Type(value = TileEntry.class, name = "tile"), - @Type(value = InitializationEntry.class, name = "initialization"), - @Type(value = FinishEntry.class, name = "finish") - }) - sealed interface Entry { - - } - - - record TileEntry(int x, int y, int z, byte[] encodedData) implements Entry { - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Arrays.hashCode(encodedData); - result = prime * result + Objects.hash(x, y, z); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof TileEntry)) { - return false; - } - TileEntry other = (TileEntry) obj; - return Arrays.equals(encodedData, other.encodedData) && x == other.x && y == other.y && z == other.z; - } - - @Override - public String toString() { - return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]"; - } - } - - record InitializationEntry() implements Entry {} - - - record FinishEntry(TileArchiveMetadata metadata) implements Entry {} - } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java index cf5da59b9f..13b3aaea69 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java @@ -158,15 +158,17 @@ private static class ProtoTileArchiveWriter implements TileWriter { @Override public void write(TileEncodingResult encodingResult) { final TileCoord coord = encodingResult.coord(); - final StreamArchiveProto.TileEntry tile = StreamArchiveProto.TileEntry.newBuilder() + final byte[] data = encodingResult.tileData(); + StreamArchiveProto.TileEntry.Builder tileBuilder = StreamArchiveProto.TileEntry.newBuilder() .setZ(coord.z()) .setX(coord.x()) - .setY(coord.y()) - .setEncodedData(ByteString.copyFrom(encodingResult.tileData())) - .build(); + .setY(coord.y()); + if (data != null) { + tileBuilder = tileBuilder.setEncodedData(ByteString.copyFrom(encodingResult.tileData())); + } final StreamArchiveProto.Entry entry = StreamArchiveProto.Entry.newBuilder() - .setTile(tile) + .setTile(tileBuilder.build()) .build(); try { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java index 4efe6257de..6b44b21a64 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java @@ -2,8 +2,10 @@ import java.io.Closeable; import java.util.Iterator; +import java.util.NoSuchElementException; import java.util.Spliterators; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -56,4 +58,42 @@ public O next() { } }; } + + default CloseableIterator filter(Predicate predicate) { + final var parent = this; + return new CloseableIterator<>() { + private T nextValue; + + @Override + public void close() { + parent.close(); + } + + @Override + public boolean hasNext() { + if (nextValue != null) { + return true; + } + while (parent.hasNext()) { + final T parentNext = parent.next(); + if (predicate.test(parentNext)) { + nextValue = parentNext; + break; + } + } + return nextValue != null; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final T returnValue = nextValue; + nextValue = null; + return returnValue; + } + }; + } + } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java index 3e34120895..56884ed97a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java @@ -3,22 +3,42 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -public class Gzip { +public final class Gzip { - public static byte[] gzip(byte[] in) throws IOException { + private Gzip() {} + + @SuppressWarnings("java:S1168") // null in, null out + public static byte[] gzip(byte[] in) { + if (in == null) { + return null; + } var bos = new ByteArrayOutputStream(in.length); try (var gzipOS = new GZIPOutputStream(bos)) { gzipOS.write(in); + } catch (IOException e) { + throw new UncheckedIOException(e); } return bos.toByteArray(); } - public static byte[] gunzip(byte[] zipped) throws IOException { + @SuppressWarnings("java:S1168") // null in, null out + public static byte[] gunzip(byte[] zipped) { + if (zipped == null) { + return null; + } try (var is = new GZIPInputStream(new ByteArrayInputStream(zipped))) { return is.readAllBytes(); + } catch (IOException e) { + throw new UncheckedIOException(e); } } + + public static boolean isZipped(byte[] in) { + return in != null && in.length > 2 && in[0] == (byte) GZIPInputStream.GZIP_MAGIC && + in[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8); + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java index 45bc476e24..a5b6966377 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java @@ -61,6 +61,9 @@ public static int fnv1a32(byte... data) { */ public static long fnv1a64(long initHash, byte... data) { long hash = initHash; + if (data == null) { + return hash; + } for (byte datum : data) { hash ^= (datum & 0xff); hash *= FNV1_PRIME_64; diff --git a/planetiler-core/src/main/proto/stream_archive_proto.proto b/planetiler-core/src/main/proto/stream_archive_proto.proto index 1df9f81ff0..374ab4537f 100644 --- a/planetiler-core/src/main/proto/stream_archive_proto.proto +++ b/planetiler-core/src/main/proto/stream_archive_proto.proto @@ -21,7 +21,7 @@ message InitializationEntry { } message FinishEntry { - Metadata metadata = 1; + optional Metadata metadata = 1; } message Metadata { @@ -32,10 +32,10 @@ message Metadata { string version = 4; string type = 5; string format = 6; - Envelope bounds = 7; - Coordinate center = 8; - int32 min_zoom = 9; - int32 max_zoom = 10; + optional Envelope bounds = 7; + optional Coordinate center = 8; + optional int32 min_zoom = 9; + optional int32 max_zoom = 10; repeated VectorLayer vector_layers = 11; map others = 12; TileCompression tile_compression = 13; @@ -51,15 +51,15 @@ message Envelope { message Coordinate { double x = 1; double y = 2; - double z = 3; + optional double z = 3; } message VectorLayer { string id = 1; map fields = 2; string description = 3; - int32 min_zoom = 4; - int32 max_zoom = 5; + optional int32 min_zoom = 4; + optional int32 max_zoom = 5; enum FieldType { FIELD_TYPE_UNSPECIFIED = 0; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 3e4239a241..ccf7a287a0 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -31,7 +31,10 @@ import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; import com.onthegomap.planetiler.stats.Stats; -import com.onthegomap.planetiler.stream.InMemoryStreamArchive; +import com.onthegomap.planetiler.stream.ReadableCsvArchive; +import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive; +import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive; +import com.onthegomap.planetiler.stream.StreamArchiveConfig; import com.onthegomap.planetiler.util.BuildInfo; import com.onthegomap.planetiler.util.Gzip; import com.onthegomap.planetiler.util.TileSizeStats; @@ -1970,11 +1973,12 @@ void testPlanetilerRunner(String args) throws Exception { final ReadableTileArchiveFactory readableTileArchiveFactory = switch (format) { case MBTILES -> Mbtiles::newReadOnlyDatabase; - case CSV -> p -> InMemoryStreamArchive.fromCsv(p, ","); - case TSV -> p -> InMemoryStreamArchive.fromCsv(p, "\t"); - case JSON -> InMemoryStreamArchive::fromJson; + case CSV, TSV -> + p -> ReadableCsvArchive.newReader(format, p, new StreamArchiveConfig(false, Arguments.of())); + case JSON -> p -> ReadableJsonStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of())); case PMTILES -> ReadablePmtiles::newReadFromFile; - case PROTO, PBF -> InMemoryStreamArchive::fromProtobuf; + case PROTO, PBF -> + p -> ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of())); case FILES -> p -> ReadableFilesArchive.newReader(p, Arguments.of()); }; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 8065ac99d6..675f80d6f8 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -749,7 +749,7 @@ public static void assertFeatureNear(Mbtiles db, String layer, Map arguments, + @TempDir Path tempDir) throws Exception { + + final Path archiveInPath = tempDir.resolve(archiveDataIn.contains("{") ? "in.json" : "in.csv"); + final Path archiveOutPath = tempDir.resolve(archiveDataOut.contains("{") ? "out.json" : "out.csv"); + final Path inMetadataPath = tempDir.resolve("metadata.json"); + + Files.writeString(archiveInPath, archiveDataIn); + Files.writeString(inMetadataPath, EXTERNAL_METADATA); + + arguments = new LinkedHashMap<>(arguments); + arguments.replace("in_metadata", inMetadataPath.toString()); + + final Arguments args = Arguments.of(Map.of( + "input", archiveInPath.toString(), + "output", archiveOutPath.toString() + )).orElse(Arguments.of(arguments)); + + new TileCopy(TileCopy.TileCopyConfig.fromArguments(args)).run(); + + if (archiveDataOut.contains("{")) { + final List expectedLines = Arrays.stream(archiveDataOut.split("\n")).toList(); + final List actualLines = Files.readAllLines(archiveOutPath); + assertEquals(expectedLines.size(), actualLines.size()); + for (int i = 0; i < expectedLines.size(); i++) { + TestUtils.assertSameJson(expectedLines.get(i), actualLines.get(i)); + } + } else { + assertEquals(archiveDataOut, Files.readString(archiveOutPath)); + } + } + + private static String compressBase64(String archiveIn) { + final Base64.Encoder encoder = Base64.getEncoder(); + for (int i = 0; i <= 1; i++) { + archiveIn = archiveIn.replace( + encoder.encodeToString(new byte[]{(byte) i}), + encoder.encodeToString(Gzip.gzip(new byte[]{(byte) i})) + ); + } + return archiveIn; + } + + private static String replaceBase64(String archiveIn, String replacement) { + final Base64.Encoder encoder = Base64.getEncoder(); + for (int i = 0; i <= 1; i++) { + archiveIn = archiveIn.replace( + encoder.encodeToString(new byte[]{(byte) i}), + replacement + ); + } + return archiveIn; + } + + private static class TestArgs implements ArgumentsProvider { + + @Override + public Stream provideArguments(ExtensionContext context) { + return Stream.of( + argsOf( + "json(w/o meta, compression:none) to csv(compression:none)", + ARCHIVE_0_JSON_BASE.formatted("null"), + ARCHIVE_0_CSV_COMPRESSION_NONE, + Map.of("out_tile_compression", "none") + ), + argsOf( + "json(w/o meta, compression:none) to csv(compression:gzip)", + ARCHIVE_0_JSON_BASE.formatted("null"), + compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE), + Map.of("out_tile_compression", "gzip") + ), + argsOf( + "json(w/o meta, compression:gzip) to csv(compression:none)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")), + ARCHIVE_0_CSV_COMPRESSION_NONE, + Map.of("out_tile_compression", "none") + ), + argsOf( + "json(w/o meta, compression:gzip) to csv(compression:gzip)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")), + compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE), + Map.of("out_tile_compression", "gzip") + ), + argsOf( + "json(w/o meta, compression:gzip) to csv(compression:gzip)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")), + compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE), + Map.of("out_tile_compression", "gzip") + ), + argsOf( + "json(w/ meta, compression:gzip) to csv(compression:none)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)), + ARCHIVE_0_CSV_COMPRESSION_NONE, + Map.of("out_tile_compression", "none") + ), + argsOf( + "json(w/ meta, compression:gzip) to json(w/ meta, compression:gzip)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)), + compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)), + Map.of() + ), + argsOf( + "csv to json - use fallback metadata", + ARCHIVE_0_CSV_COMPRESSION_NONE, + ARCHIVE_0_JSON_BASE.formatted( + "{\"name\":\"unknown\",\"format\":\"pbf\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"), + Map.of("out_tile_compression", "none") + ), + argsOf( + "csv to json - use external metadata", + ARCHIVE_0_CSV_COMPRESSION_NONE, + ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"none\"}"), + Map.of("out_tile_compression", "none", "in_metadata", "blub") + ), + argsOf( + "csv to json - null handling", + replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), + replaceBase64(ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"gzip\"}"), "null") + .replace(",\"encodedData\":\"null\"", ""), + Map.of("in_metadata", "blub") + ), + argsOf( + "json to csv - null handling", + replaceBase64(ARCHIVE_0_JSON_BASE.formatted("null"), "null") + .replace(",\"encodedData\":\"null\"", ""), + replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), + Map.of() + ), + argsOf( + "csv to csv - empty skipping on", + """ + 0,0,0, + 1,2,3,AQ== + """, + """ + 1,2,3,AQ== + """, + Map.of("skip_empty", "true", "out_tile_compression", "none") + ), + argsOf( + "csv to csv - empty skipping off", + """ + 0,0,0, + 1,2,3,AQ== + """, + """ + 0,0,0, + 1,2,3,AQ== + """, + Map.of("skip_empty", "false", "out_tile_compression", "none") + ) + ); + } + + private static org.junit.jupiter.params.provider.Arguments argsOf(String testName, String archiveDataIn, + String archiveDataOut, Map arguments) { + return org.junit.jupiter.params.provider.Arguments.of(testName, archiveDataIn, archiveDataOut, arguments); + } + } + +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java deleted file mode 100644 index 1d30afb100..0000000000 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.onthegomap.planetiler.stream; - -import com.onthegomap.planetiler.archive.ReadableTileArchive; -import com.onthegomap.planetiler.archive.TileArchiveMetadata; -import com.onthegomap.planetiler.archive.TileEncodingResult; -import com.onthegomap.planetiler.geo.TileCoord; -import com.onthegomap.planetiler.proto.StreamArchiveProto; -import com.onthegomap.planetiler.util.CloseableIterator; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Base64; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.OptionalLong; - -public class InMemoryStreamArchive implements ReadableTileArchive { - - private final List tileEncodings; - private final TileArchiveMetadata metadata; - - private InMemoryStreamArchive(List tileEncodings, TileArchiveMetadata metadata) { - this.tileEncodings = tileEncodings; - this.metadata = metadata; - } - - public static InMemoryStreamArchive fromCsv(Path p, String columnSepatator) throws IOException { - var base64Decoder = Base64.getDecoder(); - final List tileEncodings = new ArrayList<>(); - try (var reader = Files.newBufferedReader(p)) { - String line; - while ((line = reader.readLine()) != null) { - final String[] splits = line.split(columnSepatator); - final TileCoord tileCoord = TileCoord.ofXYZ(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]), - Integer.parseInt(splits[2])); - tileEncodings.add(new TileEncodingResult(tileCoord, base64Decoder.decode(splits[3]), OptionalLong.empty())); - } - } - return new InMemoryStreamArchive(tileEncodings, null); - } - - public static InMemoryStreamArchive fromProtobuf(Path p) throws IOException { - final List tileEncodings = new ArrayList<>(); - try (var in = Files.newInputStream(p)) { - StreamArchiveProto.Entry entry; - while ((entry = StreamArchiveProto.Entry.parseDelimitedFrom(in)) != null) { - if (entry.getEntryCase() == StreamArchiveProto.Entry.EntryCase.TILE) { - final StreamArchiveProto.TileEntry tileProto = entry.getTile(); - final TileCoord tileCoord = TileCoord.ofXYZ(tileProto.getX(), tileProto.getY(), tileProto.getZ()); - tileEncodings - .add(new TileEncodingResult(tileCoord, tileProto.getEncodedData().toByteArray(), OptionalLong.empty())); - } - } - } - return new InMemoryStreamArchive(tileEncodings, null /* could add once the format is finalized*/); - } - - public static InMemoryStreamArchive fromJson(Path p) throws IOException { - final List tileEncodings = new ArrayList<>(); - final TileArchiveMetadata[] metadata = new TileArchiveMetadata[]{null}; - try (var reader = Files.newBufferedReader(p)) { - WriteableJsonStreamArchive.jsonMapper - .readerFor(WriteableJsonStreamArchive.Entry.class) - .readValues(reader) - .forEachRemaining(entry -> { - if (entry instanceof WriteableJsonStreamArchive.TileEntry te) { - final TileCoord tileCoord = TileCoord.ofXYZ(te.x(), te.y(), te.z()); - tileEncodings.add(new TileEncodingResult(tileCoord, te.encodedData(), OptionalLong.empty())); - } else if (entry instanceof WriteableJsonStreamArchive.FinishEntry fe) { - metadata[0] = fe.metadata(); - } - }); - } - return new InMemoryStreamArchive(tileEncodings, Objects.requireNonNull(metadata[0])); - } - - @Override - public void close() throws IOException {} - - @Override - public byte[] getTile(int x, int y, int z) { - - final TileCoord coord = TileCoord.ofXYZ(x, y, z); - - return tileEncodings.stream() - .filter(ter -> ter.coord().equals(coord)).findFirst() - .map(TileEncodingResult::tileData) - .orElse(null); - } - - @Override - public CloseableIterator getAllTileCoords() { - - final Iterator it = tileEncodings.iterator(); - - return new CloseableIterator() { - @Override - public TileCoord next() { - return it.next().coord(); - } - - @Override - public boolean hasNext() { - return it.hasNext(); - } - - @Override - public void close() {} - }; - } - - @Override - public TileArchiveMetadata metadata() { - return metadata; - } - -} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java new file mode 100644 index 0000000000..31258995a7 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java @@ -0,0 +1,75 @@ +package com.onthegomap.planetiler.stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.geo.TileCoord; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.apache.commons.text.StringEscapeUtils; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class ReadableCsvStreamArchiveTest { + + + @ParameterizedTest + @CsvSource(delimiter = '$', textBlock = """ + ,$\\n$false$BASE64 + ,$\\r$false$BASE64 + ,$\\r\\n$false$BASE64 + ,$x$false$BASE64 + ;$\\n$false$BASE64 + {$\\n$false$BASE64 + ,${$false$BASE64 + ,$\\n$false$HEX + ,$\\n$true$BASE64 + """) + void testSimple(String columnSeparator, String lineSeparator, boolean pad, CsvBinaryEncoding encoding, + @TempDir Path tempDir) throws IOException { + + final Path csvFile = tempDir.resolve("in.csv"); + final String csv = + """ + 0,0,0,AA== + 1,2,3,AQ== + """ + .replace("\n", StringEscapeUtils.unescapeJava(lineSeparator)) + .replace(",", columnSeparator + (pad ? " " : "")) + .replace("AA==", encoding == CsvBinaryEncoding.BASE64 ? "AA==" : "00") + .replace("AQ==", encoding == CsvBinaryEncoding.BASE64 ? "AQ==" : "01"); + + Files.writeString(csvFile, csv); + final StreamArchiveConfig config = new StreamArchiveConfig( + false, + Arguments.of(Map.of( + StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, columnSeparator, + StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, lineSeparator, + StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, encoding.id() + )) + ); + + final List expectedTiles = List.of( + new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), + new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) + ); + + try (var reader = ReadableCsvArchive.newReader(TileArchiveConfig.Format.CSV, csvFile, config)) { + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertNull(reader.metadata()); + assertNull(reader.metadata()); + assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3))); + assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0)); + assertNull(reader.getTile(4, 5, 6)); + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java new file mode 100644 index 0000000000..ab728f8ec4 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java @@ -0,0 +1,48 @@ +package com.onthegomap.planetiler.stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.geo.TileCoord; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ReadableJsonStreamArchiveTest { + + @Test + void testSimple(@TempDir Path tempDir) throws IOException { + + final Path jsonFile = tempDir.resolve("in.json"); + final String json = """ + {"type":"initialization"} + {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="} + {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="} + {"type":"finish","metadata":%s} + """.formatted(TestUtils.MAX_METADATA_SERIALIZED); + + Files.writeString(jsonFile, json); + final StreamArchiveConfig config = new StreamArchiveConfig(false, Arguments.of()); + + final List expectedTiles = List.of( + new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), + new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) + ); + try (var reader = ReadableJsonStreamArchive.newReader(jsonFile, config)) { + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata()); + assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata()); + assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3))); + assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0)); + assertNull(reader.getTile(4, 5, 6)); + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java new file mode 100644 index 0000000000..7681f98eef --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java @@ -0,0 +1,60 @@ +package com.onthegomap.planetiler.stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.protobuf.ByteString; +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.proto.StreamArchiveProto; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class ReadableProtoStreamArchiveTest { + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testSimple(boolean maxMetaData, @TempDir Path tempDir) throws IOException { + + final StreamArchiveProto.Metadata metadataSerialized = maxMetaData ? + WriteableProtoStreamArchiveTest.maxMetadataSerialized : WriteableProtoStreamArchiveTest.minMetadataSerialized; + + final TileArchiveMetadata metadataDeserialized = maxMetaData ? + WriteableProtoStreamArchiveTest.maxMetadataDeserialized : WriteableProtoStreamArchiveTest.minMetadataDeserialized; + + + final Path p = tempDir.resolve("out.proto"); + try (var out = Files.newOutputStream(p)) { + StreamArchiveProto.Entry.newBuilder().setInitialization( + StreamArchiveProto.InitializationEntry.newBuilder() + ).build().writeDelimitedTo(out); + StreamArchiveProto.Entry.newBuilder().setTile( + StreamArchiveProto.TileEntry.newBuilder() + .setX(0).setY(0).setZ(0).setEncodedData(ByteString.copyFrom(new byte[]{0})) + ).build().writeDelimitedTo(out); + StreamArchiveProto.Entry.newBuilder().setTile( + StreamArchiveProto.TileEntry.newBuilder() + .setX(1).setY(2).setZ(3).setEncodedData(ByteString.copyFrom(new byte[]{1})) + ).build().writeDelimitedTo(out); + StreamArchiveProto.Entry.newBuilder().setFinish( + StreamArchiveProto.FinishEntry.newBuilder() + .setMetadata(metadataSerialized) + ).build().writeDelimitedTo(out); + } + final List expectedTiles = List.of( + new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), + new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) + ); + try (var reader = ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()))) { + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(metadataDeserialized, reader.metadata()); + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java index 48d05956b3..e69d08e55b 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java @@ -113,7 +113,7 @@ void testWriteToMultipleFiles(@TempDir Path tempDir) throws IOException { void testColumnSeparator(@TempDir Path tempDir) throws IOException { final StreamArchiveConfig config = - new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_COLUMN_SEPARATOR, "' '"))); + new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, "' '"))); final String expectedCsv = """ @@ -128,7 +128,7 @@ void testColumnSeparator(@TempDir Path tempDir) throws IOException { void testLineSeparator(@TempDir Path tempDir) throws IOException { final StreamArchiveConfig config = - new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_LINE_SEPARTATOR, "'\\r'"))); + new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, "'\\r'"))); final String expectedCsv = """ @@ -143,7 +143,7 @@ void testLineSeparator(@TempDir Path tempDir) throws IOException { void testHexEncoding(@TempDir Path tempDir) throws IOException { final StreamArchiveConfig config = - new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_BINARY_ENCODING, "hex"))); + new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, "hex"))); final String expectedCsv = """ diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java index 934a62bf15..846739acc4 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java @@ -28,8 +28,8 @@ class WriteableProtoStreamArchiveTest { - private static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null); - private static final TileArchiveMetadata maxMetadataIn = + static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null); + static final TileArchiveMetadata maxMetadataDeserialized = new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3), new Coordinate(1.3, 3.7, 1.0), 2, 3, TileArchiveMetadata.TileArchiveMetadataJson.create( @@ -45,7 +45,7 @@ class WriteableProtoStreamArchiveTest { ), Map.of("a", "b", "c", "d"), TileCompression.GZIP); - private static final StreamArchiveProto.Metadata maxMetadataOut = StreamArchiveProto.Metadata.newBuilder() + static final StreamArchiveProto.Metadata maxMetadataSerialized = StreamArchiveProto.Metadata.newBuilder() .setName("name").setDescription("description").setAttribution("attribution").setVersion("version") .setType("type").setFormat("format") .setBounds(StreamArchiveProto.Envelope.newBuilder().setMinX(0).setMaxX(1).setMinY(2).setMaxY(3).build()) @@ -64,10 +64,10 @@ class WriteableProtoStreamArchiveTest { .setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_GZIP) .build(); - private static final TileArchiveMetadata minMetadataIn = - new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, + static final TileArchiveMetadata minMetadataDeserialized = + new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, Map.of(), TileCompression.NONE); - private static final StreamArchiveProto.Metadata minMetadataOut = StreamArchiveProto.Metadata.newBuilder() + static final StreamArchiveProto.Metadata minMetadataSerialized = StreamArchiveProto.Metadata.newBuilder() .setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_NONE) .build(); @@ -83,12 +83,12 @@ void testWriteSingleFile(@TempDir Path tempDir) throws IOException { tileWriter.write(tile0); tileWriter.write(tile1); } - archive.finish(minMetadataIn); + archive.finish(minMetadataDeserialized); } try (InputStream in = Files.newInputStream(csvFile)) { assertEquals( - List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)), + List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataSerialized)), readAllEntries(in) ); } @@ -119,12 +119,12 @@ void testWriteToMultipleFiles(@TempDir Path tempDir) throws IOException { try (var tileWriter = archive.newTileWriter()) { tileWriter.write(tile4); } - archive.finish(maxMetadataIn); + archive.finish(maxMetadataDeserialized); } try (InputStream in = Files.newInputStream(csvFilePrimary)) { assertEquals( - List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)), + List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataSerialized)), readAllEntries(in) ); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java new file mode 100644 index 0000000000..33f16c9051 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java @@ -0,0 +1,36 @@ +package com.onthegomap.planetiler.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; + +class CloseableIteratorTest { + + @Test + void testFilter() { + assertEquals( + List.of(2, 4), + CloseableIterator.of(Stream.of(1, 2, 3, 4, 5, 6)) + .filter(i -> i == 2 || i == 4) + .stream() + .toList() + ); + assertEquals( + List.of(), + CloseableIterator.of(Stream.of(100, 99, 98)) + .filter(i -> i == 2 || i == 4) + .stream() + .toList() + ); + assertEquals( + List.of(), + CloseableIterator.of(Stream.of()) + .filter(i -> i == 2 || i == 4) + .stream() + .toList() + ); + } + +} diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java index df949ed305..a64e04218e 100644 --- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java +++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java @@ -2,6 +2,7 @@ import static java.util.Map.entry; +import com.onthegomap.planetiler.archive.TileCopy; import com.onthegomap.planetiler.benchmarks.LongLongMapBench; import com.onthegomap.planetiler.benchmarks.OpenMapTilesMapping; import com.onthegomap.planetiler.custommap.ConfiguredMapMain; @@ -63,7 +64,9 @@ public class Main { entry("verify-mbtiles", Verify::main), entry("verify-monaco", VerifyMonaco::main), entry("stats", TileSizeStats::main), - entry("top-osm-tiles", TopOsmTiles::main) + entry("top-osm-tiles", TopOsmTiles::main), + + entry("tile-copy", TileCopy::main) ); private static EntryPoint bundledSchema(String path) {