From 09aa6e6c188cbe91aa4049032d04f926953b460e Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Thu, 21 Sep 2023 05:55:19 -0400 Subject: [PATCH] clean up formatters --- .../util/TilesetSummaryStatistics.java | 168 +++++++++--------- .../util/TilesetSummaryStatisticsTest.java | 29 +++ 2 files changed, 110 insertions(+), 87 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TilesetSummaryStatistics.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TilesetSummaryStatistics.java index 095c24db27..4d906bf53b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TilesetSummaryStatistics.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TilesetSummaryStatistics.java @@ -50,64 +50,6 @@ private static String tileBiggestLayers(Format formatter, TileSummary tile) { .collect(Collectors.joining(", ")); } - private static String writeStatsTableRow( - Summary result, - String firstColumn, - Function formatter, - Function extractCells, - Number lastColumn - ) { - return writeStatsTableRow(result, firstColumn, extractCells.andThen(formatter), formatter.apply(lastColumn)); - } - - private static String writeStatsTableRow( - Summary result, - String firstColumn, - Function extractStat, - String lastColumn - ) { - StringBuilder builder = new StringBuilder(); - int minZoom = result.minZoomWithData(); - int maxZoom = result.maxZoomWithData(); - List layers = result.layers().stream() - .sorted(Comparator.comparingInt(result::minZoomWithData)) - .toList(); - int maxLayerLength = Math.max(9, layers.stream().mapToInt(String::length).max().orElse(0)); - String cellFormat = "%1$5s"; - String layerFormat = "%1$" + maxLayerLength + "s"; - - builder.append(layerFormat.formatted(firstColumn)); - for (int z = minZoom; z <= maxZoom; z++) { - builder.append(cellFormat.formatted(extractStat.apply(z))); - builder.append(' '); - } - builder.append(cellFormat.formatted(lastColumn)); - return builder.toString(); - } - - private static String writeStatsTable(Summary result, Function formatter, - Function extractStat) { - StringBuilder builder = new StringBuilder(); - List layers = result.layers().stream() - .sorted(Comparator.comparingInt(result::minZoomWithData)) - .toList(); - - // header: 0 1 2 3 4 ... 15 - builder.append(writeStatsTableRow(result, "", z -> "z" + z, "all")).append('\n'); - - // each row: layer - for (var layer : layers) { - builder.append(writeStatsTableRow( - result, - layer, - formatter, - z -> extractStat.apply(result.get(z, layer)), - extractStat.apply(result.get(layer)) - )).append('\n'); - } - return builder.toString().stripTrailing(); - } - /** Returns a combined {@link Summary} from each thread's {@link Updater}. */ public Summary summary() { return summaries.stream().reduce(new Summary(), Summary::mergeIn); @@ -120,23 +62,10 @@ public void printStats(String debugUrlPattern) { Summary result = summary(); var overallStats = result.get(); var formatter = Format.defaultInstance(); - var biggestTiles = overallStats.biggestTiles(); - LOGGER.debug("Biggest tiles (gzipped)\n{}", - IntStream.range(0, biggestTiles.size()) - .mapToObj(index -> { - var tile = biggestTiles.get(index); - return "%d. %d/%d/%d (%s) %s (%s)".formatted( - index + 1, - tile.coord.z(), - tile.coord.x(), - tile.coord.y(), - formatter.storage(tile.archivedSize), - tile.coord.getDebugUrl(debugUrlPattern), - tileBiggestLayers(formatter, tile) - ); - }).collect(Collectors.joining("\n")) - ); - var alreadyListed = biggestTiles.stream().map(TileSummary::coord).collect(Collectors.toSet()); + LOGGER.debug("Biggest tiles (gzipped)\n{}", overallStats.formatBiggestTiles(debugUrlPattern)); + var alreadyListed = overallStats.biggestTiles().stream() + .map(TileSummary::coord) + .collect(Collectors.toSet()); var otherTiles = result.layers().stream() .flatMap(layer -> result.get(layer).biggestTiles().stream().limit(1)) .filter(tile -> !alreadyListed.contains(tile.coord) && tile.archivedSize > WARN_BYTES) @@ -155,18 +84,18 @@ public void printStats(String debugUrlPattern) { } LOGGER.debug("Max tile sizes\n{}\n{}\n{}", - writeStatsTable(result, n -> { + result.formatTable(n -> { String string = " " + formatter.storage(n, true); return n.intValue() > ERROR_BYTES ? AnsiColors.red(string) : n.intValue() > WARN_BYTES ? AnsiColors.yellow(string) : string; }, Cell::maxSize), - writeStatsTableRow(result, "full tile", + result.formatRow("full tile", formatter::storage, z -> result.get(z).maxSize(), result.get().maxSize() ), - writeStatsTableRow(result, "gzipped", + result.formatRow("gzipped", formatter::storage, z -> result.get(z).maxArchivedSize(), result.get().maxArchivedSize() @@ -250,15 +179,7 @@ public double weightedAverageSize() { } private Cell mergeIn(Cell other) { - totalWeight += other.totalWeight; - weightedBytesSum += other.weightedBytesSum; - weightedArchivedBytesSum += other.weightedArchivedBytesSum; - archivedBytes.combine(other.archivedBytes); - bytes.combine(other.bytes); - for (var bigTile : other.topTiles) { - acceptBigTile(bigTile.coord, bigTile.archivedSize, bigTile.layers); - } - return this; + return mergeIn(other, 1); } private Cell mergeIn(Cell other, double weight) { @@ -285,6 +206,24 @@ private void acceptBigTile(TileCoord coord, int archivedBytes, List { + var tile = biggestTiles.get(index); + return "%d. %d/%d/%d (%s) %s (%s)".formatted( + index + 1, + tile.coord.z(), + tile.coord.x(), + tile.coord.y(), + formatter.storage(tile.archivedSize), + tile.coord.getDebugUrl(debugUrlPattern), + tileBiggestLayers(formatter, tile) + ); + }).collect(Collectors.joining("\n")); + } } /** Statistics for a tile and its layers. */ @@ -406,6 +345,61 @@ private Cell combineZooms(List byTile) { } return result; } + + String formatRow( + String firstColumn, + Function formatter, + Function extractCells, + Number lastColumn + ) { + return formatRow(firstColumn, extractCells.andThen(formatter), formatter.apply(lastColumn)); + } + + String formatRow( + String firstColumn, + Function extractStat, + String lastColumn + ) { + StringBuilder builder = new StringBuilder(); + int minZoom = minZoomWithData(); + int maxZoom = maxZoomWithData(); + List layers = layers().stream() + .sorted(Comparator.comparingInt(this::minZoomWithData)) + .toList(); + int maxLayerLength = Math.max(9, layers.stream().mapToInt(String::length).max().orElse(0)); + String cellFormat = "%1$5s"; + String layerFormat = "%1$" + maxLayerLength + "s"; + + builder.append(layerFormat.formatted(firstColumn)); + for (int z = minZoom; z <= maxZoom; z++) { + builder.append(cellFormat.formatted(extractStat.apply(z))); + builder.append(' '); + } + builder.append(cellFormat.formatted(lastColumn)); + return builder.toString(); + } + + String formatTable(Function formatter, + Function extractStat) { + StringBuilder builder = new StringBuilder(); + List layers = layers().stream() + .sorted(Comparator.comparingInt(this::minZoomWithData)) + .toList(); + + // header: 0 1 2 3 4 ... 15 + builder.append(formatRow("", z -> "z" + z, "all")).append('\n'); + + // each row: layer + for (var layer : layers) { + builder.append(formatRow( + layer, + formatter, + z -> extractStat.apply(get(z, layer)), + extractStat.apply(get(layer)) + )).append('\n'); + } + return builder.toString().stripTrailing(); + } } /** Thread local updater that accepts individual statistics for each tile. */ diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TilesetSummaryStatisticsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TilesetSummaryStatisticsTest.java index 0b23cfd9dd..f41b00b32a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TilesetSummaryStatisticsTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/TilesetSummaryStatisticsTest.java @@ -49,6 +49,23 @@ void aggregateTileStats() { assertEquals(7, summary.get().maxSize()); assertEquals(2, summary.get().numTiles()); + + updater1.recordTile(TileCoord.ofXYZ(0, 0, 2), 0, List.of( + new TileSizeStats.LayerStats("c", 10, 7, 8, 9, 10) + )); + assertEquals(""" + z1 z2 all + a 1 0 1 + b 6 0 6 + c 6 10 10 + """.stripTrailing(), tileStats.summary().formatTable(Number::toString, cell -> cell.maxSize())); + + assertEquals(""" + z1 z2 all + a 1 0 1 + b 2 0 2 + c 1 1 2 + """.stripTrailing(), tileStats.summary().formatTable(Number::toString, cell -> cell.numTiles())); } @Test @@ -98,6 +115,18 @@ void topLayerTiles() { summaries.stream().map(d -> d.withSize(d.coord().encoded() * 3)).limit(10).toList(), tileStats.summary().get("b").biggestTiles() ); + assertEquals(""" + 1. 2/3/1 (19) 2.5/33.25663/135 (b:57) + 2. 2/3/2 (18) 2.5/-33.25663/135 (b:54) + 3. 2/3/3 (17) 2.5/-75.78219/135 (b:51) + 4. 2/2/0 (16) 2.5/75.78219/45 (b:48) + 5. 2/2/1 (15) 2.5/33.25663/45 (b:45) + 6. 2/2/2 (14) 2.5/-33.25663/45 (b:42) + 7. 2/2/3 (13) 2.5/-75.78219/45 (b:39) + 8. 2/1/0 (12) 2.5/75.78219/-45 (b:36) + 9. 2/1/1 (11) 2.5/33.25663/-45 (b:33) + 10. 2/1/2 (10) 2.5/-33.25663/-45 (b:30) + """.trim(), tileStats.summary().get().formatBiggestTiles("{z}/{lat}/{lon}")); } @Test