Skip to content

Commit

Permalink
clean up formatters
Browse files Browse the repository at this point in the history
  • Loading branch information
@msbarry
msbarry committed Sep 21, 2023
1 parent 3b1dedf commit 09aa6e6
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 87 deletions.
168 changes: 81 additions & 87 deletions planetiler-core/src/main/java/com/onthegomap/planetiler/util/TilesetSummaryStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,64 +50,6 @@ private static String tileBiggestLayers(Format formatter, TileSummary tile) {
.collect(Collectors.joining(", "));
}

private static String writeStatsTableRow(
Summary result,
String firstColumn,
Function<Number, String> formatter,
Function<Integer, Number> extractCells,
Number lastColumn
) {
return writeStatsTableRow(result, firstColumn, extractCells.andThen(formatter), formatter.apply(lastColumn));
}

private static String writeStatsTableRow(
Summary result,
String firstColumn,
Function<Integer, String> extractStat,
String lastColumn
) {
StringBuilder builder = new StringBuilder();
int minZoom = result.minZoomWithData();
int maxZoom = result.maxZoomWithData();
List<String> layers = result.layers().stream()
.sorted(Comparator.comparingInt(result::minZoomWithData))
.toList();
int maxLayerLength = Math.max(9, layers.stream().mapToInt(String::length).max().orElse(0));
String cellFormat = "%1$5s";
String layerFormat = "%1$" + maxLayerLength + "s";

builder.append(layerFormat.formatted(firstColumn));
for (int z = minZoom; z <= maxZoom; z++) {
builder.append(cellFormat.formatted(extractStat.apply(z)));
builder.append(' ');
}
builder.append(cellFormat.formatted(lastColumn));
return builder.toString();
}

private static String writeStatsTable(Summary result, Function<Number, String> formatter,
Function<Cell, Number> extractStat) {
StringBuilder builder = new StringBuilder();
List<String> layers = result.layers().stream()
.sorted(Comparator.comparingInt(result::minZoomWithData))
.toList();

// header: 0 1 2 3 4 ... 15
builder.append(writeStatsTableRow(result, "", z -> "z" + z, "all")).append('\n');

// each row: layer
for (var layer : layers) {
builder.append(writeStatsTableRow(
result,
layer,
formatter,
z -> extractStat.apply(result.get(z, layer)),
extractStat.apply(result.get(layer))
)).append('\n');
}
return builder.toString().stripTrailing();
}

/** Returns a combined {@link Summary} from each thread's {@link Updater}. */
public Summary summary() {
return summaries.stream().reduce(new Summary(), Summary::mergeIn);
Expand All @@ -120,23 +62,10 @@ public void printStats(String debugUrlPattern) {
Summary result = summary();
var overallStats = result.get();
var formatter = Format.defaultInstance();
var biggestTiles = overallStats.biggestTiles();
LOGGER.debug("Biggest tiles (gzipped)\n{}",
IntStream.range(0, biggestTiles.size())
.mapToObj(index -> {
var tile = biggestTiles.get(index);
return "%d. %d/%d/%d (%s) %s (%s)".formatted(
index + 1,
tile.coord.z(),
tile.coord.x(),
tile.coord.y(),
formatter.storage(tile.archivedSize),
tile.coord.getDebugUrl(debugUrlPattern),
tileBiggestLayers(formatter, tile)
);
}).collect(Collectors.joining("\n"))
);
var alreadyListed = biggestTiles.stream().map(TileSummary::coord).collect(Collectors.toSet());
LOGGER.debug("Biggest tiles (gzipped)\n{}", overallStats.formatBiggestTiles(debugUrlPattern));
var alreadyListed = overallStats.biggestTiles().stream()
.map(TileSummary::coord)
.collect(Collectors.toSet());
var otherTiles = result.layers().stream()
.flatMap(layer -> result.get(layer).biggestTiles().stream().limit(1))
.filter(tile -> !alreadyListed.contains(tile.coord) && tile.archivedSize > WARN_BYTES)
Expand All @@ -155,18 +84,18 @@ public void printStats(String debugUrlPattern) {
}

LOGGER.debug("Max tile sizes\n{}\n{}\n{}",
writeStatsTable(result, n -> {
result.formatTable(n -> {
String string = " " + formatter.storage(n, true);
return n.intValue() > ERROR_BYTES ? AnsiColors.red(string) :
n.intValue() > WARN_BYTES ? AnsiColors.yellow(string) :
string;
}, Cell::maxSize),
writeStatsTableRow(result, "full tile",
result.formatRow("full tile",
formatter::storage,
z -> result.get(z).maxSize(),
result.get().maxSize()
),
writeStatsTableRow(result, "gzipped",
result.formatRow("gzipped",
formatter::storage,
z -> result.get(z).maxArchivedSize(),
result.get().maxArchivedSize()
Expand Down Expand Up @@ -250,15 +179,7 @@ public double weightedAverageSize() {
}

private Cell mergeIn(Cell other) {
totalWeight += other.totalWeight;
weightedBytesSum += other.weightedBytesSum;
weightedArchivedBytesSum += other.weightedArchivedBytesSum;
archivedBytes.combine(other.archivedBytes);
bytes.combine(other.bytes);
for (var bigTile : other.topTiles) {
acceptBigTile(bigTile.coord, bigTile.archivedSize, bigTile.layers);
}
return this;
return mergeIn(other, 1);
}

private Cell mergeIn(Cell other, double weight) {
Expand All @@ -285,6 +206,24 @@ private void acceptBigTile(TileCoord coord, int archivedBytes, List<TileSizeStat
}
}
}

String formatBiggestTiles(String debugUrlPattern) {
var biggestTiles = biggestTiles();
var formatter = Format.defaultInstance();
return IntStream.range(0, biggestTiles.size())
.mapToObj(index -> {
var tile = biggestTiles.get(index);
return "%d. %d/%d/%d (%s) %s (%s)".formatted(
index + 1,
tile.coord.z(),
tile.coord.x(),
tile.coord.y(),
formatter.storage(tile.archivedSize),
tile.coord.getDebugUrl(debugUrlPattern),
tileBiggestLayers(formatter, tile)
);
}).collect(Collectors.joining("\n"));
}
}

/** Statistics for a tile and its layers. */
Expand Down Expand Up @@ -406,6 +345,61 @@ private Cell combineZooms(List<Cell> byTile) {
}
return result;
}

String formatRow(
String firstColumn,
Function<Number, String> formatter,
Function<Integer, Number> extractCells,
Number lastColumn
) {
return formatRow(firstColumn, extractCells.andThen(formatter), formatter.apply(lastColumn));
}

String formatRow(
String firstColumn,
Function<Integer, String> extractStat,
String lastColumn
) {
StringBuilder builder = new StringBuilder();
int minZoom = minZoomWithData();
int maxZoom = maxZoomWithData();
List<String> layers = layers().stream()
.sorted(Comparator.comparingInt(this::minZoomWithData))
.toList();
int maxLayerLength = Math.max(9, layers.stream().mapToInt(String::length).max().orElse(0));
String cellFormat = "%1$5s";
String layerFormat = "%1$" + maxLayerLength + "s";

builder.append(layerFormat.formatted(firstColumn));
for (int z = minZoom; z <= maxZoom; z++) {
builder.append(cellFormat.formatted(extractStat.apply(z)));
builder.append(' ');
}
builder.append(cellFormat.formatted(lastColumn));
return builder.toString();
}

String formatTable(Function<Number, String> formatter,
Function<Cell, Number> extractStat) {
StringBuilder builder = new StringBuilder();
List<String> layers = layers().stream()
.sorted(Comparator.comparingInt(this::minZoomWithData))
.toList();

// header: 0 1 2 3 4 ... 15
builder.append(formatRow("", z -> "z" + z, "all")).append('\n');

// each row: layer
for (var layer : layers) {
builder.append(formatRow(
layer,
formatter,
z -> extractStat.apply(get(z, layer)),
extractStat.apply(get(layer))
)).append('\n');
}
return builder.toString().stripTrailing();
}
}

/** Thread local updater that accepts individual statistics for each tile. */
Expand Down
29 changes: 29 additions & 0 deletions ...tiler-core/src/test/java/com/onthegomap/planetiler/util/TilesetSummaryStatisticsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,23 @@ void aggregateTileStats() {

assertEquals(7, summary.get().maxSize());
assertEquals(2, summary.get().numTiles());

updater1.recordTile(TileCoord.ofXYZ(0, 0, 2), 0, List.of(
new TileSizeStats.LayerStats("c", 10, 7, 8, 9, 10)
));
assertEquals("""
z1 z2 all
a 1 0 1
b 6 0 6
c 6 10 10
""".stripTrailing(), tileStats.summary().formatTable(Number::toString, cell -> cell.maxSize()));

assertEquals("""
z1 z2 all
a 1 0 1
b 2 0 2
c 1 1 2
""".stripTrailing(), tileStats.summary().formatTable(Number::toString, cell -> cell.numTiles()));
}

@Test
Expand Down Expand Up @@ -98,6 +115,18 @@ void topLayerTiles() {
summaries.stream().map(d -> d.withSize(d.coord().encoded() * 3)).limit(10).toList(),
tileStats.summary().get("b").biggestTiles()
);
assertEquals("""
1. 2/3/1 (19) 2.5/33.25663/135 (b:57)
2. 2/3/2 (18) 2.5/-33.25663/135 (b:54)
3. 2/3/3 (17) 2.5/-75.78219/135 (b:51)
4. 2/2/0 (16) 2.5/75.78219/45 (b:48)
5. 2/2/1 (15) 2.5/33.25663/45 (b:45)
6. 2/2/2 (14) 2.5/-33.25663/45 (b:42)
7. 2/2/3 (13) 2.5/-75.78219/45 (b:39)
8. 2/1/0 (12) 2.5/75.78219/-45 (b:36)
9. 2/1/1 (11) 2.5/33.25663/-45 (b:33)
10. 2/1/2 (10) 2.5/-33.25663/-45 (b:30)
""".trim(), tileStats.summary().get().formatBiggestTiles("{z}/{lat}/{lon}"));
}

@Test
Expand Down

0 comments on commit 09aa6e6

Please sign in to comment.