Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check for deterministic output in CI #792

Merged
merged 5 commits into from
Jan 15, 2024
Merged

Check for deterministic output in CI #792

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
02758b0
fail on nondeterministic output
msbarry Jan 15, 2024
792a9ab
undo
msbarry Jan 15, 2024
c914500
fix test
msbarry Jan 15, 2024
4dced7b
fix ci
msbarry Jan 15, 2024
2532f0f
check bytes too
msbarry Jan 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 104 additions & 30 deletions planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,21 @@
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
import com.onthegomap.planetiler.stats.ProgressLoggers;
import com.onthegomap.planetiler.stats.Stats;
import com.onthegomap.planetiler.worker.Worker;
import com.onthegomap.planetiler.worker.WorkerPipeline;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import org.locationtech.jts.geom.Geometry;
Expand All @@ -45,6 +51,7 @@ public class CompareArchives {
private static final Logger LOGGER = LoggerFactory.getLogger(CompareArchives.class);
private final Map<String, Long> diffTypes = new ConcurrentHashMap<>();
private final Map<String, Map<String, Long>> diffsByLayer = new ConcurrentHashMap<>();
private final List<String> archiveDiffs = new CopyOnWriteArrayList<>();
private final TileArchiveConfig input1;
private final TileArchiveConfig input2;
private final boolean verbose;
Expand All @@ -55,6 +62,9 @@ private CompareArchives(TileArchiveConfig archiveConfig1, TileArchiveConfig arch
this.input2 = archiveConfig2;
}

/**
* @throws FatalComparisonFailure if a comparison failure is encountered that prevents comparing the whole archives.
*/
public static Result compare(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2,
PlanetilerConfig config, boolean verbose) {
return new CompareArchives(archiveConfig1, archiveConfig2, verbose).getResult(config);
Expand All @@ -70,6 +80,7 @@ public static void main(String[] args) {
String inputString2 = args[args.length - 1];
var arguments = Arguments.fromArgsOrConfigFile(Arrays.copyOf(args, args.length - 2));
var verbose = arguments.getBoolean("verbose", "log each tile diff", false);
var strict = arguments.getBoolean("strict", "set to false to only fail on tile diffs", true);
var config = PlanetilerConfig.from(arguments);
var input1 = TileArchiveConfig.from(inputString1);
var input2 = TileArchiveConfig.from(inputString2);
Expand All @@ -80,61 +91,55 @@ public static void main(String[] args) {
var format = Format.defaultInstance();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Detailed diffs:");
for (var entry : result.diffsByLayer.entrySet()) {
for (var entry : result.tileDiffsByLayer.entrySet()) {
LOGGER.info(" \"{}\" layer", entry.getKey());
for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue()));
}
}
for (var entry : result.diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
for (var entry : result.tileDiffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue()));
}
for (var diffType : result.archiveDiffs) {
LOGGER.info(" {}", diffType);
}
LOGGER.info("Total tiles: {}", format.integer(result.total));
LOGGER.info("Total diffs: {} ({} of all tiles)", format.integer(result.tileDiffs),
LOGGER.info("Tile diffs: {} ({} of all tiles)", format.integer(result.tileDiffs),
format.percent(result.tileDiffs * 1d / result.total));
}
} catch (IllegalArgumentException e) {
System.exit((result.tileDiffs > 0 || (strict && !result.archiveDiffs.isEmpty())) ? 1 : 0);
} catch (FatalComparisonFailure e) {
LOGGER.error("Error comparing archives {}", e.getMessage());
System.exit(1);
}
}

public static class FatalComparisonFailure extends IllegalArgumentException {
FatalComparisonFailure(String message) {
super(message);
}
}

private Result getResult(PlanetilerConfig config) {
final TileCompression compression2;
final TileCompression compression1;
if (!input1.format().equals(input2.format())) {
LOGGER.warn("archive1 and archive2 have different formats, got {} and {}", input1.format(), input2.format());
}
compareArchive("format", input1.format(), input2.format());
try (
var reader1 = TileArchives.newReader(input1, config);
var reader2 = TileArchives.newReader(input2, config);
) {
var metadata1 = reader1.metadata();
var metadata2 = reader2.metadata();
if (!Objects.equals(metadata1, metadata2)) {
LOGGER.warn("""
archive1 and archive2 have different metadata
archive1: {}
archive2: {}
""", reader1.metadata(), reader2.metadata());
}
compareArchive("metadata", metadata1, metadata2);
if (reader1 instanceof ReadablePmtiles pmt1 && reader2 instanceof ReadablePmtiles pmt2) {
var header1 = pmt1.getHeader();
var header2 = pmt2.getHeader();
if (!Objects.equals(header1, header2)) {
LOGGER.warn("""
archive1 and archive2 have different pmtiles headers
archive1: {}
archive2: {}
""", header1, header2);
}
compareArchive("pmtiles header", header1, header2);
}
compression1 = metadata1 == null ? TileCompression.UNKNOWN : metadata1.tileCompression();
compression2 = metadata2 == null ? TileCompression.UNKNOWN : metadata2.tileCompression();
if (compression1 != compression2) {
LOGGER.warn(
"input1 and input2 must have the same compression, got {} and {} - will compare decompressed tile contents instead",
compression1, compression2);
if (!compareArchive("tile compression", compression1, compression2)) {
LOGGER.warn("Will compare decompressed tile contents instead");
}
} catch (IOException e) {
throw new UncheckedIOException(e);
Expand All @@ -143,7 +148,7 @@ private Result getResult(PlanetilerConfig config) {
var order = input1.format().preferredOrder();
var order2 = input2.format().preferredOrder();
if (order != order2) {
throw new IllegalArgumentException(
throw new FatalComparisonFailure(
"Archive orders must be the same to compare, got " + order + " and " + order2);
}
var stats = config.arguments().getStats();
Expand Down Expand Up @@ -230,7 +235,73 @@ record Diff(Tile a, Tile b) {}
.newLine()
.addProcessStats();
loggers.awaitAndLog(pipeline.done(), config.logInterval());
return new Result(total.get(), diffs.get(), diffTypes, diffsByLayer);
if (archiveDiffs.isEmpty() && diffs.get() == 0) {
var path1 = input1.getLocalPath();
var path2 = input2.getLocalPath();
if (path1 != null && path2 != null && Files.isRegularFile(path1) && Files.isRegularFile(path2)) {
LOGGER.info("No diffs so far, comparing bytes in {} vs. {}", path1, path2);
compareFiles(path1, path2, config);
}
}
return new Result(total.get(), diffs.get(), archiveDiffs, diffTypes, diffsByLayer);
}

private void compareFiles(Path path1, Path path2, PlanetilerConfig config) {
long size = FileUtils.fileSize(path1);
if (compareArchive("archive size", size, FileUtils.fileSize(path2))) {
AtomicLong bytesRead = new AtomicLong(0);
var worker = new Worker("compare", Stats.inMemory(), 1, () -> {
byte[] bytes1 = new byte[8192];
byte[] bytes2 = new byte[8192];
long n = 0;
try (
var is1 = Files.newInputStream(path1, StandardOpenOption.READ);
var is2 = Files.newInputStream(path2, StandardOpenOption.READ)
) {
do {
int len = is1.read(bytes1);
int len2 = is2.read(bytes2, 0, len);
if (len2 != len) {
String message = "Expected to read %s bytes from %s but got %s".formatted(len, path2, len2);
archiveDiffs.add(message);
LOGGER.warn(message);
return;
}
int mismatch = Arrays.mismatch(bytes1, bytes2);
if (mismatch >= 0 && mismatch < len) {
archiveDiffs.add("mismatch at byte %s".formatted(mismatch + n));
LOGGER.warn("Archives mismatch ay byte {}", mismatch + n);
return;
}
n += len;
bytesRead.set(n);
} while (n < size);
}
LOGGER.info("No mismatches! Analyzed {} / {} bytes", n, size);
});

var logger = ProgressLoggers.create()
.addStorageRatePercentCounter("bytes", size, bytesRead::get, true)
.newLine()
.addThreadPoolStats("compare", worker)
.newLine()
.addProcessStats();

worker.awaitAndLog(logger, config.logInterval());
}
}

private <T> boolean compareArchive(String name, T a, T b) {
if (Objects.equals(a, b)) {
return true;
}
LOGGER.warn("""
archive1 and archive2 have different {}
archive1: {}
archive2: {}
""", name, a, b);
archiveDiffs.add(name);
return false;
}

private void compareTiles(TileCoord coord, VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
Expand Down Expand Up @@ -356,7 +427,7 @@ private byte[] decompress(byte[] bytes, TileCompression tileCompression) throws
return switch (tileCompression) {
case GZIP -> Gzip.gunzip(bytes);
case NONE -> bytes;
case UNKNOWN -> throw new IllegalArgumentException("Unknown compression");
case UNKNOWN -> throw new FatalComparisonFailure("Unknown compression");
};
}

Expand All @@ -383,7 +454,10 @@ private void recordTileDiff(TileCoord coord, String issue) {
}

public record Result(
long total, long tileDiffs, Map<String, Long> diffTypes,
Map<String, Map<String, Long>> diffsByLayer
long total,
long tileDiffs,
List<String> archiveDiffs,
Map<String, Long> tileDiffTypes,
Map<String, Map<String, Long>> tileDiffsByLayer
) {}
}
49 changes: 47 additions & 2 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -78,7 +79,9 @@ void testCompareArchives() throws IOException {
false
);
assertEquals(new CompareArchives.Result(
5, 4, Map.of(
5, 4, List.of(
"pmtiles header"
), Map.of(
"archive 2 missing tile", 1L,
"archive 1 missing tile", 2L,
"different contents", 1L
Expand Down Expand Up @@ -132,7 +135,11 @@ void testCompareArchivesDifferentCompression() throws IOException {
false
);
assertEquals(new CompareArchives.Result(
5, 4, Map.of(
5, 4, List.of(
"metadata",
"pmtiles header",
"tile compression"
), Map.of(
"archive 2 missing tile", 1L,
"archive 1 missing tile", 2L,
"different decompressed contents", 1L
Expand All @@ -144,4 +151,42 @@ void testCompareArchivesDifferentCompression() throws IOException {
)
), result);
}

@Test
void testCompareArchivesSame() throws IOException {
var aPath = path.resolve("a.pmtiles");
var bPath = path.resolve("b.pmtiles");
byte[] a1 = new byte[]{0xa, 0x2};
try (
var a = WriteablePmtiles.newWriteToFile(aPath);
var b = WriteablePmtiles.newWriteToFile(bPath);
) {
a.initialize();
b.initialize();
try (
var aWriter = a.newTileWriter();
var bWriter = b.newTileWriter()
) {
aWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty()));
aWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty()));
bWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty()));
bWriter
.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty()));
}
a.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
}
var result = CompareArchives.compare(
TileArchiveConfig.from(aPath.toString()),
TileArchiveConfig.from(bPath.toString()),
config,
false
);
assertEquals(new CompareArchives.Result(
2, 0, List.of(), Map.of(), Map.of()
), result);
}
}
27 changes: 15 additions & 12 deletions scripts/test-release.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,27 @@ fi

echo "Test java build"
echo "::group::OpenMapTiles monaco (java)"
rm -f data/out.mbtiles
java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/out.mbtiles
./scripts/check-monaco.sh data/out.mbtiles
rm -f data/out*.mbtiles
# vary threads to stress-test determinism check
java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles --threads=32
./scripts/check-monaco.sh data/jar-monaco.mbtiles
echo "::endgroup::"
echo "::group::Example (java)"
rm -f data/out.mbtiles
java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/out.mbtiles
./scripts/check-mbtiles.sh data/out.mbtiles
java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/jar-example.mbtiles
./scripts/check-mbtiles.sh data/jar-example.mbtiles
echo "::endgroup::"

echo "::endgroup::"
echo "::group::OpenMapTiles monaco (docker)"
rm -f data/out.mbtiles
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/out.mbtiles
./scripts/check-monaco.sh data/out.mbtiles
# vary threads to stress-test determinism check
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles --threads=4
./scripts/check-monaco.sh data/docker-monaco.mbtiles
echo "::endgroup::"
echo "::group::Example (docker)"
rm -f data/out.mbtiles
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/out.mbtiles
./scripts/check-mbtiles.sh data/out.mbtiles
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/docker-example.mbtiles
./scripts/check-mbtiles.sh data/docker-example.mbtiles
echo "::endgroup::"

echo "::group::Compare"
java -jar planetiler-dist/target/*with-deps.jar compare data/jar-monaco.mbtiles data/docker-monaco.mbtiles
echo "::endgroup::"
Loading