diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Compare.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java
similarity index 55%
rename from planetiler-core/src/main/java/com/onthegomap/planetiler/util/Compare.java
rename to planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java
index a44a859ae8..9496ee96a2 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Compare.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java
@@ -7,15 +7,14 @@
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.stats.ProgressLoggers;
-import com.onthegomap.planetiler.stats.Stats;
import com.onthegomap.planetiler.worker.WorkerPipeline;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
@@ -23,44 +22,115 @@
import org.slf4j.LoggerFactory;
import vector_tile.VectorTileProto;
-public class Compare {
+/**
+ * Compares the contents of two tile archives.
+ *
+ * To run:
+ *
+ *
{@code
+ * java -jar planetiler.jar compare [options] {path/to/archive1} {path/to/archive2}
+ * }
+ */
+public class CompareArchives {
- private static final Logger LOGGER = LoggerFactory.getLogger(Compare.class);
- private static final Map diffTypes = new ConcurrentHashMap<>();
- private static final Map> diffsByLayer = new ConcurrentHashMap<>();
+ private static final Logger LOGGER = LoggerFactory.getLogger(CompareArchives.class);
+ private final Map diffTypes = new ConcurrentHashMap<>();
+ private final Map> diffsByLayer = new ConcurrentHashMap<>();
+ private final TileArchiveConfig input1;
+ private final TileArchiveConfig input2;
+ private CompareArchives(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2) {
+ this.input1 = archiveConfig1;
+ this.input2 = archiveConfig2;
+ }
+
+ public static Result compare(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2,
+ PlanetilerConfig config) {
+ return new CompareArchives(archiveConfig1, archiveConfig2).getResult(config);
+ }
public static void main(String[] args) {
- var arguments = Arguments.fromArgsOrConfigFile(args);
+ if (args.length < 2) {
+ System.err.println("Usage: compare [options] {path/to/archive1} {path/to/archive2}");
+ System.exit(1);
+ }
+ // last 2 args are paths to the archives
+ String inputString1 = args[args.length - 2];
+ String inputString2 = args[args.length - 1];
+ var arguments = Arguments.fromArgsOrConfigFile(Arrays.copyOf(args, args.length - 2));
var config = PlanetilerConfig.from(arguments);
- var stats = Stats.inMemory();
- var inputString1 = arguments.getString("input1", "input file 1");
- var inputString2 = arguments.getString("input2", "input file 2");
var input1 = TileArchiveConfig.from(inputString1);
var input2 = TileArchiveConfig.from(inputString2);
+
+ try {
+ var result = compare(input1, input2, config);
+
+ var format = Format.defaultInstance();
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Detailed diffs:");
+ for (var entry : result.diffsByLayer.entrySet()) {
+ LOGGER.info(" \"{}\" layer", entry.getKey());
+ for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
+ LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue()));
+ }
+ }
+ for (var entry : result.diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
+ LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue()));
+ }
+ LOGGER.info("Total tiles: {}", format.integer(result.total));
+ LOGGER.info("Total diffs: {} ({} of all tiles)", format.integer(result.tileDiffs),
+ format.percent(result.tileDiffs * 1d / result.total));
+ }
+ } catch (IllegalArgumentException e) {
+ LOGGER.error("Error comparing archives {}", e.getMessage());
+ System.exit(1);
+ }
+ }
+
+ private Result getResult(PlanetilerConfig config) {
+ if (!input1.format().equals(input2.format())) {
+ throw new IllegalArgumentException(
+ "input1 and input2 must have the same format, got " + input1.format() + " and " +
+ input2.format());
+ }
+ final TileCompression compression;
+ try (
+ var reader1 = TileArchives.newReader(input1, config);
+ var reader2 = TileArchives.newReader(input2, config);
+ ) {
+ var metadata1 = reader1.metadata();
+ var metadata2 = reader2.metadata();
+ if (!Objects.equals(metadata1, metadata2)) {
+ LOGGER.warn("""
+ archive1 and archive2 have different metadata
+ archive1: {}
+ archive2: {}
+ """, reader1.metadata(), reader2.metadata());
+ }
+ compression = metadata1 == null ? TileCompression.UNKNWON : metadata1.tileCompression();
+ TileCompression compression2 = metadata2 == null ? TileCompression.UNKNWON : metadata2.tileCompression();
+ if (compression != compression2) {
+ throw new IllegalArgumentException(
+ "input1 and input2 must have the same compression, got " + compression + " and " +
+ compression2);
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ var order = input1.format().preferredOrder();
+ var stats = config.arguments().getStats();
var total = new AtomicLong(0);
var diffs = new AtomicLong(0);
- CompletableFuture compression = new CompletableFuture<>();
record Diff(Tile a, Tile b) {}
var pipeline = WorkerPipeline.start("tilestats", stats)
.fromGenerator("enumerate", next -> {
- var order = input1.format().preferredOrder();
- if (order != input2.format().preferredOrder()) {
- throw new IllegalArgumentException(
- "input1 and input2 must have the same preferred order, got " + order + " and " +
- input2.format().preferredOrder());
- }
try (
var reader1 = TileArchives.newReader(input1, config);
var tiles1 = reader1.getAllTiles();
var reader2 = TileArchives.newReader(input2, config);
var tiles2 = reader2.getAllTiles()
) {
- if (!Objects.equals(reader1.metadata(), reader2.metadata())) {
- LOGGER.warn("input1 and input2 have different metadata: {} and {}", reader1.metadata(), reader2.metadata());
- }
- compression
- .complete(reader1.metadata() == null ? TileCompression.UNKNWON : reader1.metadata().tileCompression());
Supplier supplier1 = () -> tiles1.hasNext() ? tiles1.next() : null;
Supplier supplier2 = () -> tiles2.hasNext() ? tiles2.next() : null;
var tile1 = supplier1.get();
@@ -90,22 +160,21 @@ record Diff(Tile a, Tile b) {}
})
.addBuffer("diffs", 1_000)
.sinkTo("process", config.featureProcessThreads(), prev -> {
- var tileCompression = compression.join();
for (var diff : prev) {
var a = diff.a();
var b = diff.b();
total.incrementAndGet();
if (a == null) {
- recordTileDiff("tile 1 missing");
+ recordTileDiff("archive 1 missing tile");
diffs.incrementAndGet();
} else if (b == null) {
- recordTileDiff("tile 2 missing");
+ recordTileDiff("archive 2 missing tile");
diffs.incrementAndGet();
} else if (!Arrays.equals(a.bytes(), b.bytes())) {
recordTileDiff("different contents");
diffs.incrementAndGet();
- var proto1 = decode(a.bytes(), tileCompression);
- var proto2 = decode(b.bytes(), tileCompression);
+ var proto1 = decode(a.bytes(), compression);
+ var proto2 = decode(b.bytes(), compression);
compareTiles(proto1, proto2);
}
}
@@ -117,34 +186,19 @@ record Diff(Tile a, Tile b) {}
.newLine()
.addProcessStats();
loggers.awaitAndLog(pipeline.done(), config.logInterval());
-
- var format = Format.defaultInstance();
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Detailed diffs:");
- for (var entry : diffsByLayer.entrySet()) {
- LOGGER.info(" \"{}\" layer", entry.getKey());
- for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
- LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue()));
- }
- }
- for (var entry : diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
- LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue()));
- }
- LOGGER.info("Total tiles: {}", format.integer(total.get()));
- LOGGER.info("Total diffs: {} ({})", format.integer(diffs.get()), format.percent(diffs.get() * 1d / total.get()));
- }
+ return new Result(total.get(), diffs.get(), diffTypes, diffsByLayer);
}
- private static void compareTiles(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
+ private void compareTiles(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
compareLayerNames(proto1, proto2);
- for (int i = 0; i < proto1.getLayersCount(); i++) {
+ for (int i = 0; i < proto1.getLayersCount() && i < proto2.getLayersCount(); i++) {
var layer1 = proto1.getLayers(i);
var layer2 = proto2.getLayers(i);
compareLayer(layer1, layer2);
}
}
- private static void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) {
+ private void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) {
String name = layer1.getName();
compareValues(name, "version", layer1.getVersion(), layer2.getVersion());
compareValues(name, "extent", layer1.getExtent(), layer2.getExtent());
@@ -164,7 +218,7 @@ private static void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTilePr
}
}
- private static void compareFeature(String layer, VectorTileProto.Tile.Feature feature1,
+ private void compareFeature(String layer, VectorTileProto.Tile.Feature feature1,
VectorTileProto.Tile.Feature feature2) {
compareValues(layer, "feature id", feature1.getId(), feature2.getId());
compareValues(layer, "feature type", feature1.getType(), feature2.getType());
@@ -172,29 +226,29 @@ private static void compareFeature(String layer, VectorTileProto.Tile.Feature fe
compareValues(layer, "feature tags", feature1.getTagsCount(), feature2.getTagsCount());
}
- private static void compareLayerNames(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
+ private void compareLayerNames(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
var layers1 = proto1.getLayersList().stream().map(d -> d.getName()).toList();
var layers2 = proto2.getLayersList().stream().map(d -> d.getName()).toList();
- compareListDetailed("layer names", layers1, layers2);
+ compareListDetailed("tile layers", layers1, layers2);
}
- private static boolean compareList(String layer, String name, List value1, List value2) {
+ private boolean compareList(String layer, String name, List value1, List value2) {
return compareValues(layer, name + " unique values", Set.copyOf(value1), Set.copyOf(value2)) &&
compareValues(layer, name + " order", value1, value2);
}
- private static void compareListDetailed(String name, List value1, List value2) {
+ private void compareListDetailed(String name, List value1, List value2) {
if (!Objects.equals(value1, value2)) {
boolean missing = false;
for (var layer : value1) {
if (!value2.contains(layer)) {
- recordTileDiff(name + " 2 missing: " + layer);
+ recordTileDiff(name + " 2 missing " + layer);
missing = true;
}
}
for (var layer : value2) {
if (!value1.contains(layer)) {
- recordTileDiff(name + " 1 missing: " + layer);
+ recordTileDiff(name + " 1 missing " + layer);
missing = true;
}
}
@@ -204,7 +258,7 @@ private static void compareListDetailed(String name, List value1, List
}
}
- private static boolean compareValues(String layer, String name, T value1, T value2) {
+ private boolean compareValues(String layer, String name, T value1, T value2) {
if (!Objects.equals(value1, value2)) {
recordLayerDiff(layer, name);
return false;
@@ -212,7 +266,7 @@ private static boolean compareValues(String layer, String name, T value1, T
return true;
}
- private static VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCompression) throws IOException {
+ private VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCompression) throws IOException {
byte[] decompressed = switch (tileCompression) {
case GZIP -> Gzip.gunzip(bytes);
case NONE -> bytes;
@@ -221,7 +275,7 @@ private static VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCom
return VectorTileProto.Tile.parseFrom(decompressed);
}
- private static void recordLayerDiff(String layer, String issue) {
+ private void recordLayerDiff(String layer, String issue) {
var layerDiffs = diffsByLayer.get(layer);
if (layerDiffs == null) {
layerDiffs = diffsByLayer.computeIfAbsent(layer, k -> new ConcurrentHashMap<>());
@@ -229,7 +283,10 @@ private static void recordLayerDiff(String layer, String issue) {
layerDiffs.merge(issue, 1L, Long::sum);
}
- private static void recordTileDiff(String issue) {
+ private void recordTileDiff(String issue) {
diffTypes.merge(issue, 1L, Long::sum);
}
+
+ public record Result(long total, long tileDiffs, Map diffTypes,
+ Map> diffsByLayer) {}
}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java
new file mode 100644
index 0000000000..53b3f185cb
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java
@@ -0,0 +1,91 @@
+package com.onthegomap.planetiler.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.archive.TileArchiveConfig;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileEncodingResult;
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.geo.TileOrder;
+import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.OptionalLong;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import vector_tile.VectorTileProto;
+
+class CompareArchivesTest {
+ @TempDir
+ Path path;
+ PlanetilerConfig config = PlanetilerConfig.defaults();
+ byte[] tile1 = VectorTileProto.Tile.newBuilder().addLayers(
+ VectorTileProto.Tile.Layer.newBuilder()
+ .setVersion(2)
+ .setName("layer1")
+ .addKeys("key1")
+ .addValues(VectorTileProto.Tile.Value.newBuilder().setStringValue("value1"))
+ .addFeatures(VectorTileProto.Tile.Feature.newBuilder().setId(1)))
+ .build()
+ .toByteArray();
+
+ byte[] tile2 = VectorTileProto.Tile.newBuilder().addLayers(
+ VectorTileProto.Tile.Layer.newBuilder()
+ .setVersion(2)
+ .setName("layer1")
+ .addKeys("key1")
+ .addValues(VectorTileProto.Tile.Value.newBuilder().setStringValue("value2"))
+ .addFeatures(VectorTileProto.Tile.Feature.newBuilder().setId(2)))
+ .build()
+ .toByteArray();
+
+ @Test
+ void testCompareArchives() throws IOException {
+ var aPath = path.resolve("a.pmtiles");
+ var bPath = path.resolve("b.pmtiles");
+ try (
+ var a = WriteablePmtiles.newWriteToFile(aPath);
+ var b = WriteablePmtiles.newWriteToFile(bPath);
+ ) {
+ a.initialize();
+ b.initialize();
+ try (
+ var aWriter = a.newTileWriter();
+ var bWriter = b.newTileWriter()
+ ) {
+ aWriter
+ .write(new TileEncodingResult(TileOrder.HILBERT.decode(0), new byte[]{0xa, 0x2}, OptionalLong.empty()));
+ aWriter
+ .write(new TileEncodingResult(TileOrder.HILBERT.decode(2), Gzip.gzip(tile1), OptionalLong.empty()));
+ aWriter
+ .write(new TileEncodingResult(TileOrder.HILBERT.decode(4), new byte[]{0xa, 0x2}, OptionalLong.empty()));
+ bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(1), new byte[]{0xa, 0x2}, OptionalLong.empty()));
+ bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), Gzip.gzip(tile2), OptionalLong.empty()));
+ bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(3), new byte[]{0xa, 0x2}, OptionalLong.empty()));
+ bWriter
+ .write(new TileEncodingResult(TileOrder.HILBERT.decode(4), new byte[]{0xa, 0x2}, OptionalLong.empty()));
+ }
+ a.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
+ b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
+ }
+ var result = CompareArchives.compare(
+ TileArchiveConfig.from(aPath.toString()),
+ TileArchiveConfig.from(bPath.toString()),
+ config
+ );
+ assertEquals(new CompareArchives.Result(
+ 5, 4, Map.of(
+ "archive 2 missing tile", 1L,
+ "archive 1 missing tile", 2L,
+ "different contents", 1L
+ ), Map.of(
+ "layer1", Map.of(
+ "values list unique values", 1L,
+ "feature ids", 1L
+ )
+ )
+ ), result);
+ }
+}
diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
index df949ed305..becc425ba0 100644
--- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
+++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
@@ -11,6 +11,7 @@
import com.onthegomap.planetiler.examples.ToiletsOverlay;
import com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi;
import com.onthegomap.planetiler.mbtiles.Verify;
+import com.onthegomap.planetiler.util.CompareArchives;
import com.onthegomap.planetiler.util.TileSizeStats;
import com.onthegomap.planetiler.util.TopOsmTiles;
import java.util.Arrays;
@@ -63,7 +64,8 @@ public class Main {
entry("verify-mbtiles", Verify::main),
entry("verify-monaco", VerifyMonaco::main),
entry("stats", TileSizeStats::main),
- entry("top-osm-tiles", TopOsmTiles::main)
+ entry("top-osm-tiles", TopOsmTiles::main),
+ entry("compare", CompareArchives::main)
);
private static EntryPoint bundledSchema(String path) {