From 0c4454679ea6a6d4fc6b13c13751331adadad495 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Tue, 9 Jan 2024 20:49:15 -0500 Subject: [PATCH] compare archives utility --- .../{Compare.java => CompareArchives.java} | 171 ++++++++++++------ .../planetiler/util/CompareArchivesTest.java | 91 ++++++++++ .../java/com/onthegomap/planetiler/Main.java | 4 +- 3 files changed, 208 insertions(+), 58 deletions(-) rename planetiler-core/src/main/java/com/onthegomap/planetiler/util/{Compare.java => CompareArchives.java} (55%) create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Compare.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java similarity index 55% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/util/Compare.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java index a44a859ae8..9496ee96a2 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Compare.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java @@ -7,15 +7,14 @@ import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.stats.ProgressLoggers; -import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; @@ -23,44 +22,115 @@ import org.slf4j.LoggerFactory; import vector_tile.VectorTileProto; -public class Compare { +/** + * Compares the contents of two tile archives. + *

+ * To run: + * + * 

{@code
+ * java -jar planetiler.jar compare [options] {path/to/archive1} {path/to/archive2}
+ * }
+ */ +public class CompareArchives { - private static final Logger LOGGER = LoggerFactory.getLogger(Compare.class); - private static final Map diffTypes = new ConcurrentHashMap<>(); - private static final Map> diffsByLayer = new ConcurrentHashMap<>(); + private static final Logger LOGGER = LoggerFactory.getLogger(CompareArchives.class); + private final Map diffTypes = new ConcurrentHashMap<>(); + private final Map> diffsByLayer = new ConcurrentHashMap<>(); + private final TileArchiveConfig input1; + private final TileArchiveConfig input2; + private CompareArchives(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2) { + this.input1 = archiveConfig1; + this.input2 = archiveConfig2; + } + + public static Result compare(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2, + PlanetilerConfig config) { + return new CompareArchives(archiveConfig1, archiveConfig2).getResult(config); + } public static void main(String[] args) { - var arguments = Arguments.fromArgsOrConfigFile(args); + if (args.length < 2) { + System.err.println("Usage: compare [options] {path/to/archive1} {path/to/archive2}"); + System.exit(1); + } + // last 2 args are paths to the archives + String inputString1 = args[args.length - 2]; + String inputString2 = args[args.length - 1]; + var arguments = Arguments.fromArgsOrConfigFile(Arrays.copyOf(args, args.length - 2)); var config = PlanetilerConfig.from(arguments); - var stats = Stats.inMemory(); - var inputString1 = arguments.getString("input1", "input file 1"); - var inputString2 = arguments.getString("input2", "input file 2"); var input1 = TileArchiveConfig.from(inputString1); var input2 = TileArchiveConfig.from(inputString2); + + try { + var result = compare(input1, input2, config); + + var format = Format.defaultInstance(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Detailed diffs:"); + for (var entry : result.diffsByLayer.entrySet()) { + LOGGER.info(" \"{}\" layer", entry.getKey()); + for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { + LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue())); + } + } + for (var entry : result.diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { + LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue())); + } + LOGGER.info("Total tiles: {}", format.integer(result.total)); + LOGGER.info("Total diffs: {} ({} of all tiles)", format.integer(result.tileDiffs), + format.percent(result.tileDiffs * 1d / result.total)); + } + } catch (IllegalArgumentException e) { + LOGGER.error("Error comparing archives {}", e.getMessage()); + System.exit(1); + } + } + + private Result getResult(PlanetilerConfig config) { + if (!input1.format().equals(input2.format())) { + throw new IllegalArgumentException( + "input1 and input2 must have the same format, got " + input1.format() + " and " + + input2.format()); + } + final TileCompression compression; + try ( + var reader1 = TileArchives.newReader(input1, config); + var reader2 = TileArchives.newReader(input2, config); + ) { + var metadata1 = reader1.metadata(); + var metadata2 = reader2.metadata(); + if (!Objects.equals(metadata1, metadata2)) { + LOGGER.warn(""" + archive1 and archive2 have different metadata + archive1: {} + archive2: {} + """, reader1.metadata(), reader2.metadata()); + } + compression = metadata1 == null ? TileCompression.UNKNWON : metadata1.tileCompression(); + TileCompression compression2 = metadata2 == null ? TileCompression.UNKNWON : metadata2.tileCompression(); + if (compression != compression2) { + throw new IllegalArgumentException( + "input1 and input2 must have the same compression, got " + compression + " and " + + compression2); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + var order = input1.format().preferredOrder(); + var stats = config.arguments().getStats(); var total = new AtomicLong(0); var diffs = new AtomicLong(0); - CompletableFuture compression = new CompletableFuture<>(); record Diff(Tile a, Tile b) {} var pipeline = WorkerPipeline.start("tilestats", stats) .fromGenerator("enumerate", next -> { - var order = input1.format().preferredOrder(); - if (order != input2.format().preferredOrder()) { - throw new IllegalArgumentException( - "input1 and input2 must have the same preferred order, got " + order + " and " + - input2.format().preferredOrder()); - } try ( var reader1 = TileArchives.newReader(input1, config); var tiles1 = reader1.getAllTiles(); var reader2 = TileArchives.newReader(input2, config); var tiles2 = reader2.getAllTiles() ) { - if (!Objects.equals(reader1.metadata(), reader2.metadata())) { - LOGGER.warn("input1 and input2 have different metadata: {} and {}", reader1.metadata(), reader2.metadata()); - } - compression - .complete(reader1.metadata() == null ? TileCompression.UNKNWON : reader1.metadata().tileCompression()); Supplier supplier1 = () -> tiles1.hasNext() ? tiles1.next() : null; Supplier supplier2 = () -> tiles2.hasNext() ? tiles2.next() : null; var tile1 = supplier1.get(); @@ -90,22 +160,21 @@ record Diff(Tile a, Tile b) {} }) .addBuffer("diffs", 1_000) .sinkTo("process", config.featureProcessThreads(), prev -> { - var tileCompression = compression.join(); for (var diff : prev) { var a = diff.a(); var b = diff.b(); total.incrementAndGet(); if (a == null) { - recordTileDiff("tile 1 missing"); + recordTileDiff("archive 1 missing tile"); diffs.incrementAndGet(); } else if (b == null) { - recordTileDiff("tile 2 missing"); + recordTileDiff("archive 2 missing tile"); diffs.incrementAndGet(); } else if (!Arrays.equals(a.bytes(), b.bytes())) { recordTileDiff("different contents"); diffs.incrementAndGet(); - var proto1 = decode(a.bytes(), tileCompression); - var proto2 = decode(b.bytes(), tileCompression); + var proto1 = decode(a.bytes(), compression); + var proto2 = decode(b.bytes(), compression); compareTiles(proto1, proto2); } } @@ -117,34 +186,19 @@ record Diff(Tile a, Tile b) {} .newLine() .addProcessStats(); loggers.awaitAndLog(pipeline.done(), config.logInterval()); - - var format = Format.defaultInstance(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("Detailed diffs:"); - for (var entry : diffsByLayer.entrySet()) { - LOGGER.info(" \"{}\" layer", entry.getKey()); - for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { - LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue())); - } - } - for (var entry : diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { - LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue())); - } - LOGGER.info("Total tiles: {}", format.integer(total.get())); - LOGGER.info("Total diffs: {} ({})", format.integer(diffs.get()), format.percent(diffs.get() * 1d / total.get())); - } + return new Result(total.get(), diffs.get(), diffTypes, diffsByLayer); } - private static void compareTiles(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) { + private void compareTiles(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) { compareLayerNames(proto1, proto2); - for (int i = 0; i < proto1.getLayersCount(); i++) { + for (int i = 0; i < proto1.getLayersCount() && i < proto2.getLayersCount(); i++) { var layer1 = proto1.getLayers(i); var layer2 = proto2.getLayers(i); compareLayer(layer1, layer2); } } - private static void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) { + private void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) { String name = layer1.getName(); compareValues(name, "version", layer1.getVersion(), layer2.getVersion()); compareValues(name, "extent", layer1.getExtent(), layer2.getExtent()); @@ -164,7 +218,7 @@ private static void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTilePr } } - private static void compareFeature(String layer, VectorTileProto.Tile.Feature feature1, + private void compareFeature(String layer, VectorTileProto.Tile.Feature feature1, VectorTileProto.Tile.Feature feature2) { compareValues(layer, "feature id", feature1.getId(), feature2.getId()); compareValues(layer, "feature type", feature1.getType(), feature2.getType()); @@ -172,29 +226,29 @@ private static void compareFeature(String layer, VectorTileProto.Tile.Feature fe compareValues(layer, "feature tags", feature1.getTagsCount(), feature2.getTagsCount()); } - private static void compareLayerNames(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) { + private void compareLayerNames(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) { var layers1 = proto1.getLayersList().stream().map(d -> d.getName()).toList(); var layers2 = proto2.getLayersList().stream().map(d -> d.getName()).toList(); - compareListDetailed("layer names", layers1, layers2); + compareListDetailed("tile layers", layers1, layers2); } - private static boolean compareList(String layer, String name, List value1, List value2) { + private boolean compareList(String layer, String name, List value1, List value2) { return compareValues(layer, name + " unique values", Set.copyOf(value1), Set.copyOf(value2)) && compareValues(layer, name + " order", value1, value2); } - private static void compareListDetailed(String name, List value1, List value2) { + private void compareListDetailed(String name, List value1, List value2) { if (!Objects.equals(value1, value2)) { boolean missing = false; for (var layer : value1) { if (!value2.contains(layer)) { - recordTileDiff(name + " 2 missing: " + layer); + recordTileDiff(name + " 2 missing " + layer); missing = true; } } for (var layer : value2) { if (!value1.contains(layer)) { - recordTileDiff(name + " 1 missing: " + layer); + recordTileDiff(name + " 1 missing " + layer); missing = true; } } @@ -204,7 +258,7 @@ private static void compareListDetailed(String name, List value1, List } } - private static boolean compareValues(String layer, String name, T value1, T value2) { + private boolean compareValues(String layer, String name, T value1, T value2) { if (!Objects.equals(value1, value2)) { recordLayerDiff(layer, name); return false; @@ -212,7 +266,7 @@ private static boolean compareValues(String layer, String name, T value1, T return true; } - private static VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCompression) throws IOException { + private VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCompression) throws IOException { byte[] decompressed = switch (tileCompression) { case GZIP -> Gzip.gunzip(bytes); case NONE -> bytes; @@ -221,7 +275,7 @@ private static VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCom return VectorTileProto.Tile.parseFrom(decompressed); } - private static void recordLayerDiff(String layer, String issue) { + private void recordLayerDiff(String layer, String issue) { var layerDiffs = diffsByLayer.get(layer); if (layerDiffs == null) { layerDiffs = diffsByLayer.computeIfAbsent(layer, k -> new ConcurrentHashMap<>()); @@ -229,7 +283,10 @@ private static void recordLayerDiff(String layer, String issue) { layerDiffs.merge(issue, 1L, Long::sum); } - private static void recordTileDiff(String issue) { + private void recordTileDiff(String issue) { diffTypes.merge(issue, 1L, Long::sum); } + + public record Result(long total, long tileDiffs, Map diffTypes, + Map> diffsByLayer) {} } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java new file mode 100644 index 0000000000..53b3f185cb --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java @@ -0,0 +1,91 @@ +package com.onthegomap.planetiler.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.pmtiles.WriteablePmtiles; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.OptionalLong; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import vector_tile.VectorTileProto; + +class CompareArchivesTest { + @TempDir + Path path; + PlanetilerConfig config = PlanetilerConfig.defaults(); + byte[] tile1 = VectorTileProto.Tile.newBuilder().addLayers( + VectorTileProto.Tile.Layer.newBuilder() + .setVersion(2) + .setName("layer1") + .addKeys("key1") + .addValues(VectorTileProto.Tile.Value.newBuilder().setStringValue("value1")) + .addFeatures(VectorTileProto.Tile.Feature.newBuilder().setId(1))) + .build() + .toByteArray(); + + byte[] tile2 = VectorTileProto.Tile.newBuilder().addLayers( + VectorTileProto.Tile.Layer.newBuilder() + .setVersion(2) + .setName("layer1") + .addKeys("key1") + .addValues(VectorTileProto.Tile.Value.newBuilder().setStringValue("value2")) + .addFeatures(VectorTileProto.Tile.Feature.newBuilder().setId(2))) + .build() + .toByteArray(); + + @Test + void testCompareArchives() throws IOException { + var aPath = path.resolve("a.pmtiles"); + var bPath = path.resolve("b.pmtiles"); + try ( + var a = WriteablePmtiles.newWriteToFile(aPath); + var b = WriteablePmtiles.newWriteToFile(bPath); + ) { + a.initialize(); + b.initialize(); + try ( + var aWriter = a.newTileWriter(); + var bWriter = b.newTileWriter() + ) { + aWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(0), new byte[]{0xa, 0x2}, OptionalLong.empty())); + aWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(2), Gzip.gzip(tile1), OptionalLong.empty())); + aWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(4), new byte[]{0xa, 0x2}, OptionalLong.empty())); + bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(1), new byte[]{0xa, 0x2}, OptionalLong.empty())); + bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), Gzip.gzip(tile2), OptionalLong.empty())); + bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(3), new byte[]{0xa, 0x2}, OptionalLong.empty())); + bWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(4), new byte[]{0xa, 0x2}, OptionalLong.empty())); + } + a.finish(new TileArchiveMetadata(new Profile.NullProfile(), config)); + b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config)); + } + var result = CompareArchives.compare( + TileArchiveConfig.from(aPath.toString()), + TileArchiveConfig.from(bPath.toString()), + config + ); + assertEquals(new CompareArchives.Result( + 5, 4, Map.of( + "archive 2 missing tile", 1L, + "archive 1 missing tile", 2L, + "different contents", 1L + ), Map.of( + "layer1", Map.of( + "values list unique values", 1L, + "feature ids", 1L + ) + ) + ), result); + } +} diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java index df949ed305..becc425ba0 100644 --- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java +++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java @@ -11,6 +11,7 @@ import com.onthegomap.planetiler.examples.ToiletsOverlay; import com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi; import com.onthegomap.planetiler.mbtiles.Verify; +import com.onthegomap.planetiler.util.CompareArchives; import com.onthegomap.planetiler.util.TileSizeStats; import com.onthegomap.planetiler.util.TopOsmTiles; import java.util.Arrays; @@ -63,7 +64,8 @@ public class Main { entry("verify-mbtiles", Verify::main), entry("verify-monaco", VerifyMonaco::main), entry("stats", TileSizeStats::main), - entry("top-osm-tiles", TopOsmTiles::main) + entry("top-osm-tiles", TopOsmTiles::main), + entry("compare", CompareArchives::main) ); private static EntryPoint bundledSchema(String path) {