Skip to content

Commit

Permalink
biggest tiles
Browse files Browse the repository at this point in the history
  • Loading branch information
@msbarry
msbarry committed Sep 16, 2023
1 parent 87ac5dd commit 42d8cf5
Show file tree
Hide file tree
Showing 2 changed files with 138 additions and 2 deletions.
90 changes: 88 additions & 2 deletions planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@
import java.util.List;
import java.util.LongSummaryStatistics;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -53,6 +55,7 @@ public class TileStats {
.withColumnSeparator('\t')
.withLineSeparator("\n");
public static final ObjectWriter WRITER = MAPPER.writer(SCHEMA);
private static final int TOP_N_TILES = 10;
private final List<Summary> summaries = new CopyOnWriteArrayList<>();

public TileStats() {
Expand Down Expand Up @@ -227,8 +230,40 @@ public void printStats() {
Summary result = summary();
var overallStats = result.get();
var formatter = Format.defaultInstance();
// TODO top 10 biggest tiles
// TODO other notably large tiles
var biggestTiles = overallStats.biggestTiles();
LOGGER.debug("Biggest tiles (gzipped):\n{}",
IntStream.range(0, biggestTiles.size())
.mapToObj(index -> {
var tile = biggestTiles.get(index);
return "%d. %d/%d/%d (%s) %s (%s)".formatted(
index + 1,
tile.coord.z(),
tile.coord.x(),
tile.coord.y(),
formatter.storage(tile.size),
tile.coord.getDebugUrl(),
tileBiggestLayers(formatter, tile)
);
}).collect(Collectors.joining("\n"))
);
var alreadyListed = biggestTiles.stream().map(TileSummary::coord).collect(Collectors.toSet());
var otherTiles = result.layers().stream()
.flatMap(layer -> result.get(layer).biggestTiles().stream().limit(1))
.filter(tile -> !alreadyListed.contains(tile.coord) && tile.size > 100_000)
.toList();
if (!otherTiles.isEmpty()) {
LOGGER.info("Other tiles with large layers:\n{}",
otherTiles.stream()
.map(tile -> "%d/%d/%d (%s) %s (%s)".formatted(
tile.coord.z(),
tile.coord.x(),
tile.coord.y(),
formatter.storage(tile.size),
tile.coord.getDebugUrl(),
tileBiggestLayers(formatter, tile)
)).collect(Collectors.joining("\n")));
}

LOGGER.debug("Max tile sizes:\n{}\n{}",
writeStatsTable(result, formatter::storage, SummaryCell::maxSize),
writeStatsRow(result, "gzipped",
Expand All @@ -245,6 +280,16 @@ public void printStats() {
}
}

private static String tileBiggestLayers(Format formatter, TileSummary tile) {
return tile.layers.stream()
.filter(
d -> d.layerBytes > Math.min(100_000,
tile.layers.stream().mapToInt(l -> l.layerBytes).max().orElse(0)))
.sorted(Comparator.comparingInt(d -> -d.layerBytes))
.map(d -> d.layer + ":" + formatter.storage(d.layerBytes))
.collect(Collectors.joining(", "));
}

private static String writeStatsRow(
Summary result,
String firstColumn,
Expand Down Expand Up @@ -391,6 +436,8 @@ public int minZoomWithData(String layer) {
public static class SummaryCell {
private final LongSummaryStatistics archivedBytes = new LongSummaryStatistics();
private final LongSummaryStatistics bytes = new LongSummaryStatistics();
private int bigTileCutoff = 0;
private final PriorityQueue<TileSummary> topTiles = new PriorityQueue<>();

SummaryCell(String layer) {}

Expand All @@ -411,12 +458,32 @@ public long numTiles() {
public SummaryCell merge(SummaryCell other) {
archivedBytes.combine(other.archivedBytes);
bytes.combine(other.bytes);
for (var bigTile : other.topTiles) {
acceptBigTile(bigTile.coord, bigTile.size, bigTile.layers);
}
return this;
}

private void acceptBigTile(TileCoord coord, int archivedBytes, List<LayerStats> layerStats) {
if (archivedBytes >= bigTileCutoff) {
topTiles.offer(new TileSummary(coord, archivedBytes, layerStats));
while (topTiles.size() > TOP_N_TILES) {
topTiles.poll();
var min = topTiles.peek();
if (min != null) {
bigTileCutoff = min.size();
}
}
}
}

public static SummaryCell combine(SummaryCell a, SummaryCell b) {
return new SummaryCell().merge(a).merge(b);
}

public List<TileSummary> biggestTiles() {
return topTiles.stream().sorted(Comparator.comparingLong(s -> -s.size)).toList();
}
}


Expand Down Expand Up @@ -467,6 +534,22 @@ public int compareTo(LayerStats o) {
}
}

record TileSummary(TileCoord coord, int size, List<LayerStats> layers) implements Comparable<TileSummary> {

@Override
public int compareTo(TileSummary o) {
int result = Integer.compare(size, o.size);
if (result == 0) {
result = Integer.compare(coord.encoded(), o.coord.encoded());
}
return result;
}

TileSummary withSize(int newSize) {
return new TileSummary(coord, newSize, layers);
}
}

public class Updater {
private final Summary summary = new Summary();

Expand All @@ -478,10 +561,13 @@ public void recordTile(TileCoord coord, int archivedBytes, List<LayerStats> laye
var tileStat = summary.byTile.get(coord.z());
var layerStat = summary.byLayer.get(coord.z());
tileStat.archivedBytes.accept(archivedBytes);
tileStat.acceptBigTile(coord, archivedBytes, layerStats);

int sum = 0;
for (var layer : layerStats) {
var cell = layerStat.computeIfAbsent(layer.layer, SummaryCell::new);
cell.bytes.accept(layer.layerBytes);
cell.acceptBigTile(coord, layer.layerBytes, layerStats);
sum += layer.layerBytes;
}
tileStat.bytes.accept(sum);
Expand Down
50 changes: 50 additions & 0 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/util/TileStatsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.onthegomap.planetiler.VectorTile;
import com.onthegomap.planetiler.geo.TileCoord;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -139,4 +140,53 @@ void aggregateTileStats() {
assertEquals(7, summary.get().maxSize());
assertEquals(2, summary.get().numTiles());
}

@Test
void topGzippedTiles() {
var tileStats = new TileStats();
var updater1 = tileStats.threadLocalUpdater();
var updater2 = tileStats.threadLocalUpdater();
for (int i = 0; i < 20; i++) {
(i % 2 == 0 ? updater1 : updater2).recordTile(TileCoord.decode(i), i, List.of());
}
assertEquals(
List.of(
new TileStats.TileSummary(TileCoord.decode(19), 19, List.of()),
new TileStats.TileSummary(TileCoord.decode(18), 18, List.of()),
new TileStats.TileSummary(TileCoord.decode(17), 17, List.of()),
new TileStats.TileSummary(TileCoord.decode(16), 16, List.of()),
new TileStats.TileSummary(TileCoord.decode(15), 15, List.of()),
new TileStats.TileSummary(TileCoord.decode(14), 14, List.of()),
new TileStats.TileSummary(TileCoord.decode(13), 13, List.of()),
new TileStats.TileSummary(TileCoord.decode(12), 12, List.of()),
new TileStats.TileSummary(TileCoord.decode(11), 11, List.of()),
new TileStats.TileSummary(TileCoord.decode(10), 10, List.of())
),
tileStats.summary().get().biggestTiles()
);
}

@Test
void topLayerTiles() {
var tileStats = new TileStats();
var updater1 = tileStats.threadLocalUpdater();
var updater2 = tileStats.threadLocalUpdater();
List<TileStats.TileSummary> summaries = new ArrayList<>();
for (int i = 0; i < 20; i++) {
var summary = new TileStats.TileSummary(TileCoord.decode(i), i, List.of(
new TileStats.LayerStats("a", i * 2, i, 0, 0, 0),
new TileStats.LayerStats("b", i * 3, i, 0, 0, 0)
));
summaries.add(0, summary);
(i % 2 == 0 ? updater1 : updater2).recordTile(summary.coord(), summary.size(), summary.layers());
}
assertEquals(
summaries.stream().map(d -> d.withSize(d.coord().encoded() * 2)).limit(10).toList(),
tileStats.summary().get("a").biggestTiles()
);
assertEquals(
summaries.stream().map(d -> d.withSize(d.coord().encoded() * 3)).limit(10).toList(),
tileStats.summary().get("b").biggestTiles()
);
}
}

0 comments on commit 42d8cf5

Please sign in to comment.