Skip to content

Commit

Permalink
compare archives utility
Browse files Browse the repository at this point in the history
  • Loading branch information
@msbarry
msbarry committed Jan 10, 2024
1 parent cca0906 commit 0c44546
Show file tree
Hide file tree
Showing 3 changed files with 208 additions and 58 deletions.
171 changes: 114 additions & 57 deletions ...m/onthegomap/planetiler/util/Compare.java → ...omap/planetiler/util/CompareArchives.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,60 +7,130 @@
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.stats.ProgressLoggers;
import com.onthegomap.planetiler.stats.Stats;
import com.onthegomap.planetiler.worker.WorkerPipeline;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import vector_tile.VectorTileProto;

public class Compare {
/**
* Compares the contents of two tile archives.
* <p>
* To run:
*
* <pre>{@code
* java -jar planetiler.jar compare [options] {path/to/archive1} {path/to/archive2}
* }</pre>
*/
public class CompareArchives {

private static final Logger LOGGER = LoggerFactory.getLogger(Compare.class);
private static final Map<String, Long> diffTypes = new ConcurrentHashMap<>();
private static final Map<String, Map<String, Long>> diffsByLayer = new ConcurrentHashMap<>();
private static final Logger LOGGER = LoggerFactory.getLogger(CompareArchives.class);
private final Map<String, Long> diffTypes = new ConcurrentHashMap<>();
private final Map<String, Map<String, Long>> diffsByLayer = new ConcurrentHashMap<>();
private final TileArchiveConfig input1;
private final TileArchiveConfig input2;

private CompareArchives(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2) {
this.input1 = archiveConfig1;
this.input2 = archiveConfig2;
}

public static Result compare(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2,
PlanetilerConfig config) {
return new CompareArchives(archiveConfig1, archiveConfig2).getResult(config);
}

public static void main(String[] args) {
var arguments = Arguments.fromArgsOrConfigFile(args);
if (args.length < 2) {
System.err.println("Usage: compare [options] {path/to/archive1} {path/to/archive2}");
System.exit(1);
}
// last 2 args are paths to the archives
String inputString1 = args[args.length - 2];
String inputString2 = args[args.length - 1];
var arguments = Arguments.fromArgsOrConfigFile(Arrays.copyOf(args, args.length - 2));
var config = PlanetilerConfig.from(arguments);
var stats = Stats.inMemory();
var inputString1 = arguments.getString("input1", "input file 1");
var inputString2 = arguments.getString("input2", "input file 2");
var input1 = TileArchiveConfig.from(inputString1);
var input2 = TileArchiveConfig.from(inputString2);

try {
var result = compare(input1, input2, config);

var format = Format.defaultInstance();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Detailed diffs:");
for (var entry : result.diffsByLayer.entrySet()) {
LOGGER.info(" \"{}\" layer", entry.getKey());
for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue()));
}
}
for (var entry : result.diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue()));
}
LOGGER.info("Total tiles: {}", format.integer(result.total));
LOGGER.info("Total diffs: {} ({} of all tiles)", format.integer(result.tileDiffs),
format.percent(result.tileDiffs * 1d / result.total));
}
} catch (IllegalArgumentException e) {
LOGGER.error("Error comparing archives {}", e.getMessage());
System.exit(1);
}
}

private Result getResult(PlanetilerConfig config) {
if (!input1.format().equals(input2.format())) {
throw new IllegalArgumentException(
"input1 and input2 must have the same format, got " + input1.format() + " and " +
input2.format());
}
final TileCompression compression;
try (
var reader1 = TileArchives.newReader(input1, config);
var reader2 = TileArchives.newReader(input2, config);
) {
var metadata1 = reader1.metadata();
var metadata2 = reader2.metadata();
if (!Objects.equals(metadata1, metadata2)) {
LOGGER.warn("""
archive1 and archive2 have different metadata
archive1: {}
archive2: {}
""", reader1.metadata(), reader2.metadata());
}
compression = metadata1 == null ? TileCompression.UNKNWON : metadata1.tileCompression();
TileCompression compression2 = metadata2 == null ? TileCompression.UNKNWON : metadata2.tileCompression();
if (compression != compression2) {
throw new IllegalArgumentException(
"input1 and input2 must have the same compression, got " + compression + " and " +
compression2);
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}

var order = input1.format().preferredOrder();
var stats = config.arguments().getStats();
var total = new AtomicLong(0);
var diffs = new AtomicLong(0);
CompletableFuture<TileCompression> compression = new CompletableFuture<>();
record Diff(Tile a, Tile b) {}
var pipeline = WorkerPipeline.start("tilestats", stats)
.<Diff>fromGenerator("enumerate", next -> {
var order = input1.format().preferredOrder();
if (order != input2.format().preferredOrder()) {
throw new IllegalArgumentException(
"input1 and input2 must have the same preferred order, got " + order + " and " +
input2.format().preferredOrder());
}
try (
var reader1 = TileArchives.newReader(input1, config);
var tiles1 = reader1.getAllTiles();
var reader2 = TileArchives.newReader(input2, config);
var tiles2 = reader2.getAllTiles()
) {
if (!Objects.equals(reader1.metadata(), reader2.metadata())) {
LOGGER.warn("input1 and input2 have different metadata: {} and {}", reader1.metadata(), reader2.metadata());
}
compression
.complete(reader1.metadata() == null ? TileCompression.UNKNWON : reader1.metadata().tileCompression());
Supplier<Tile> supplier1 = () -> tiles1.hasNext() ? tiles1.next() : null;
Supplier<Tile> supplier2 = () -> tiles2.hasNext() ? tiles2.next() : null;
var tile1 = supplier1.get();
Expand Down Expand Up @@ -90,22 +160,21 @@ record Diff(Tile a, Tile b) {}
})
.addBuffer("diffs", 1_000)
.sinkTo("process", config.featureProcessThreads(), prev -> {
var tileCompression = compression.join();
for (var diff : prev) {
var a = diff.a();
var b = diff.b();
total.incrementAndGet();
if (a == null) {
recordTileDiff("tile 1 missing");
recordTileDiff("archive 1 missing tile");
diffs.incrementAndGet();
} else if (b == null) {
recordTileDiff("tile 2 missing");
recordTileDiff("archive 2 missing tile");
diffs.incrementAndGet();
} else if (!Arrays.equals(a.bytes(), b.bytes())) {
recordTileDiff("different contents");
diffs.incrementAndGet();
var proto1 = decode(a.bytes(), tileCompression);
var proto2 = decode(b.bytes(), tileCompression);
var proto1 = decode(a.bytes(), compression);
var proto2 = decode(b.bytes(), compression);
compareTiles(proto1, proto2);
}
}
Expand All @@ -117,34 +186,19 @@ record Diff(Tile a, Tile b) {}
.newLine()
.addProcessStats();
loggers.awaitAndLog(pipeline.done(), config.logInterval());

var format = Format.defaultInstance();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Detailed diffs:");
for (var entry : diffsByLayer.entrySet()) {
LOGGER.info(" \"{}\" layer", entry.getKey());
for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue()));
}
}
for (var entry : diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue()));
}
LOGGER.info("Total tiles: {}", format.integer(total.get()));
LOGGER.info("Total diffs: {} ({})", format.integer(diffs.get()), format.percent(diffs.get() * 1d / total.get()));
}
return new Result(total.get(), diffs.get(), diffTypes, diffsByLayer);
}

private static void compareTiles(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
private void compareTiles(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
compareLayerNames(proto1, proto2);
for (int i = 0; i < proto1.getLayersCount(); i++) {
for (int i = 0; i < proto1.getLayersCount() && i < proto2.getLayersCount(); i++) {
var layer1 = proto1.getLayers(i);
var layer2 = proto2.getLayers(i);
compareLayer(layer1, layer2);
}
}

private static void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) {
private void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) {
String name = layer1.getName();
compareValues(name, "version", layer1.getVersion(), layer2.getVersion());
compareValues(name, "extent", layer1.getExtent(), layer2.getExtent());
Expand All @@ -164,37 +218,37 @@ private static void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTilePr
}
}

private static void compareFeature(String layer, VectorTileProto.Tile.Feature feature1,
private void compareFeature(String layer, VectorTileProto.Tile.Feature feature1,
VectorTileProto.Tile.Feature feature2) {
compareValues(layer, "feature id", feature1.getId(), feature2.getId());
compareValues(layer, "feature type", feature1.getType(), feature2.getType());
compareValues(layer, "feature geometry", feature1.getGeometryList(), feature2.getGeometryList());
compareValues(layer, "feature tags", feature1.getTagsCount(), feature2.getTagsCount());
}

private static void compareLayerNames(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
private void compareLayerNames(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
var layers1 = proto1.getLayersList().stream().map(d -> d.getName()).toList();
var layers2 = proto2.getLayersList().stream().map(d -> d.getName()).toList();
compareListDetailed("layer names", layers1, layers2);
compareListDetailed("tile layers", layers1, layers2);
}

private static <T> boolean compareList(String layer, String name, List<T> value1, List<T> value2) {
private <T> boolean compareList(String layer, String name, List<T> value1, List<T> value2) {
return compareValues(layer, name + " unique values", Set.copyOf(value1), Set.copyOf(value2)) &&
compareValues(layer, name + " order", value1, value2);
}

private static <T> void compareListDetailed(String name, List<T> value1, List<T> value2) {
private <T> void compareListDetailed(String name, List<T> value1, List<T> value2) {
if (!Objects.equals(value1, value2)) {
boolean missing = false;
for (var layer : value1) {
if (!value2.contains(layer)) {
recordTileDiff(name + " 2 missing: " + layer);
recordTileDiff(name + " 2 missing " + layer);
missing = true;
}
}
for (var layer : value2) {
if (!value1.contains(layer)) {
recordTileDiff(name + " 1 missing: " + layer);
recordTileDiff(name + " 1 missing " + layer);
missing = true;
}
}
Expand All @@ -204,15 +258,15 @@ private static <T> void compareListDetailed(String name, List<T> value1, List<T>
}
}

private static <T> boolean compareValues(String layer, String name, T value1, T value2) {
private <T> boolean compareValues(String layer, String name, T value1, T value2) {
if (!Objects.equals(value1, value2)) {
recordLayerDiff(layer, name);
return false;
}
return true;
}

private static VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCompression) throws IOException {
private VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCompression) throws IOException {
byte[] decompressed = switch (tileCompression) {
case GZIP -> Gzip.gunzip(bytes);
case NONE -> bytes;
Expand All @@ -221,15 +275,18 @@ private static VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCom
return VectorTileProto.Tile.parseFrom(decompressed);
}

private static void recordLayerDiff(String layer, String issue) {
private void recordLayerDiff(String layer, String issue) {
var layerDiffs = diffsByLayer.get(layer);
if (layerDiffs == null) {
layerDiffs = diffsByLayer.computeIfAbsent(layer, k -> new ConcurrentHashMap<>());
}
layerDiffs.merge(issue, 1L, Long::sum);
}

private static void recordTileDiff(String issue) {
private void recordTileDiff(String issue) {
diffTypes.merge(issue, 1L, Long::sum);
}

public record Result(long total, long tileDiffs, Map<String, Long> diffTypes,
Map<String, Map<String, Long>> diffsByLayer) {}
}
91 changes: 91 additions & 0 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package com.onthegomap.planetiler.util;

import static org.junit.jupiter.api.Assertions.assertEquals;

import com.onthegomap.planetiler.Profile;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileEncodingResult;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.geo.TileOrder;
import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
import java.util.OptionalLong;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import vector_tile.VectorTileProto;

class CompareArchivesTest {
@TempDir
Path path;
PlanetilerConfig config = PlanetilerConfig.defaults();
byte[] tile1 = VectorTileProto.Tile.newBuilder().addLayers(
VectorTileProto.Tile.Layer.newBuilder()
.setVersion(2)
.setName("layer1")
.addKeys("key1")
.addValues(VectorTileProto.Tile.Value.newBuilder().setStringValue("value1"))
.addFeatures(VectorTileProto.Tile.Feature.newBuilder().setId(1)))
.build()
.toByteArray();

byte[] tile2 = VectorTileProto.Tile.newBuilder().addLayers(
VectorTileProto.Tile.Layer.newBuilder()
.setVersion(2)
.setName("layer1")
.addKeys("key1")
.addValues(VectorTileProto.Tile.Value.newBuilder().setStringValue("value2"))
.addFeatures(VectorTileProto.Tile.Feature.newBuilder().setId(2)))
.build()
.toByteArray();

@Test
void testCompareArchives() throws IOException {
var aPath = path.resolve("a.pmtiles");
var bPath = path.resolve("b.pmtiles");
try (
var a = WriteablePmtiles.newWriteToFile(aPath);
var b = WriteablePmtiles.newWriteToFile(bPath);
) {
a.initialize();
b.initialize();
try (
var aWriter = a.newTileWriter();
var bWriter = b.newTileWriter()
) {
aWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(0), new byte[]{0xa, 0x2}, OptionalLong.empty()));
aWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), Gzip.gzip(tile1), OptionalLong.empty()));
aWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(4), new byte[]{0xa, 0x2}, OptionalLong.empty()));
bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(1), new byte[]{0xa, 0x2}, OptionalLong.empty()));
bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), Gzip.gzip(tile2), OptionalLong.empty()));
bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(3), new byte[]{0xa, 0x2}, OptionalLong.empty()));
bWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(4), new byte[]{0xa, 0x2}, OptionalLong.empty()));
}
a.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
}
var result = CompareArchives.compare(
TileArchiveConfig.from(aPath.toString()),
TileArchiveConfig.from(bPath.toString()),
config
);
assertEquals(new CompareArchives.Result(
5, 4, Map.of(
"archive 2 missing tile", 1L,
"archive 1 missing tile", 2L,
"different contents", 1L
), Map.of(
"layer1", Map.of(
"values list unique values", 1L,
"feature ids", 1L
)
)
), result);
}
}
Loading

0 comments on commit 0c44546

Please sign in to comment.