Skip to content

Commit

Permalink
add layer_geometries column
Browse files Browse the repository at this point in the history
  • Loading branch information
@msbarry
msbarry committed Sep 26, 2023
1 parent f77abb7 commit 916f7e8
Show file tree
Hide file tree
Showing 7 changed files with 103 additions and 25 deletions.
9 changes: 5 additions & 4 deletions layerstats/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ The output is a gzipped tsv with a row per layer on each tile and the following
| layer | layer name |
| layer_bytes | encoded size of this layer on this tile |
| layer_features | number of features in this layer |
| layer_geometries | number of geometries in features in this layer, including inside multipoint/multipolygons/multilinestring features |
| layer_attr_bytes | encoded size of the [attribute key/value pairs](https://github.com/mapbox/vector-tile-spec/tree/master/2.1#44-feature-attributes) in this layer |
| layer_attr_keys | number of distinct attribute keys in this layer on this tile |
| layer_attr_values | number of distinct attribute values in this layer on this tile |
Expand All @@ -42,10 +43,10 @@ Then get the biggest layers:
SELECT * FROM layerstats ORDER BY layer_bytes DESC LIMIT 2;
```

| z | x | y | hilbert | archived_tile_bytes | layer | layer_bytes | layer_features | layer_attr_bytes | layer_attr_keys | layer_attr_values |
|----|-------|------|-----------|---------------------|-------------|-------------|----------------|------------------|-----------------|-------------------|
| 14 | 13722 | 7013 | 305278258 | 1261474 | housenumber | 2412464 | 108384 | 30764 | 1 | 3021 |
| 14 | 13723 | 7014 | 305278256 | 1064044 | housenumber | 1848990 | 83038 | 26022 | 1 | 2542 |
| z | x | y | hilbert | archived_tile_bytes | layer | layer_bytes | layer_features | layer_geometries | layer_attr_bytes | layer_attr_keys | layer_attr_values |
|----|-------|------|-----------|---------------------|-------------|-------------|----------------|------------------|------------------|-----------------|-------------------|
| 14 | 13722 | 7013 | 305278258 | 1260526 | housenumber | 2412589 | 108390 | 108390 | 30764 | 1 | 3021 |
| 14 | 13723 | 7014 | 305278256 | 1059752 | housenumber | 1850041 | 83038 | 83038 | 26022 | 1 | 2542 |

To get a table of biggest layers by zoom:

Expand Down
32 changes: 30 additions & 2 deletions planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,13 +421,41 @@ public static VectorGeometryMerger newMerger(GeometryType geometryType) {
return new VectorGeometryMerger(geometryType);
}

public static int hilbertIndex(Geometry g) {
Coordinate coord = g.getCoordinate();
/**
* Returns the hilbert index of the zig-zag-encoded first point of {@code geometry}.
* <p>
* This can be useful for sorting geometries to minimize encoded vector tile geometry command size since smaller
* values take fewer bytes using protobuf varint encoding.
*/
public static int hilbertIndex(Geometry geometry) {
Coordinate coord = geometry.getCoordinate();
int x = zigZagEncode((int) Math.round(coord.x * 4096 / 256));
int y = zigZagEncode((int) Math.round(coord.y * 4096 / 256));
return Hilbert.hilbertXYToIndex(15, x, y);
}

/**
* Returns the number of internal geometries in this feature including points/lines/polygons inside multigeometries.
*/
public static int countGeometries(VectorTileProto.Tile.Feature feature) {
int result = 0;
int idx = 0;
int geomCount = feature.getGeometryCount();
while (idx < geomCount) {
int length = feature.getGeometry(idx);
int command = length & ((1 << 3) - 1);
length = length >> 3;
if (command == Command.MOVE_TO.value) {
result += length;
}
idx += 1;
if (command != Command.CLOSE_PATH.value) {
idx += length * 2;
}
}
return result;
}

/**
* Adds features in a layer to this tile.
*
Expand Down
10 changes: 10 additions & 0 deletions planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
import com.onthegomap.planetiler.VectorTile;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.archive.TileArchives;
Expand Down Expand Up @@ -175,6 +176,7 @@ public static List<String> formatOutputRows(TileCoord tileCoord, int archivedByt
layer.layer,
layer.layerBytes,
layer.layerFeatures,
layer.layerGeometries,
layer.layerAttrBytes,
layer.layerAttrKeys,
layer.layerAttrValues
Expand Down Expand Up @@ -220,10 +222,15 @@ public static List<LayerStats> computeTileStats(VectorTileProto.Tile proto) {
for (var value : layer.getValuesList()) {
attrSize += value.getSerializedSize();
}
int geomCount = 0;
for (var feature : layer.getFeaturesList()) {
geomCount += VectorTile.countGeometries(feature);
}
result.add(new LayerStats(
layer.getName(),
layer.getSerializedSize(),
layer.getFeaturesCount(),
geomCount,
attrSize,
layer.getKeysCount(),
layer.getValuesCount()
Expand All @@ -243,6 +250,7 @@ public static List<LayerStats> computeTileStats(VectorTileProto.Tile proto) {
"layer",
"layer_bytes",
"layer_features",
"layer_geometries",
"layer_attr_bytes",
"layer_attr_keys",
"layer_attr_values"
Expand All @@ -257,6 +265,7 @@ public record OutputRow(
String layer,
int layerBytes,
int layerFeatures,
int layerGeometries,
int layerAttrBytes,
int layerAttrKeys,
int layerAttrValues
Expand All @@ -267,6 +276,7 @@ public record LayerStats(
String layer,
int layerBytes,
int layerFeatures,
int layerGeometries,
int layerAttrBytes,
int layerAttrKeys,
int layerAttrValues
Expand Down
1 change: 1 addition & 0 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2028,6 +2028,7 @@ public void processFeature(SourceFeature source, FeatureCollector features) {
"layer",
"layer_bytes",
"layer_features",
"layer_geometries",
"layer_attr_bytes",
"layer_attr_keys",
"layer_attr_values"
Expand Down
38 changes: 38 additions & 0 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,44 @@ void testFilterMultiPointsLastOutsideBuffer() {
);
}

@TestFactory
Stream<DynamicTest> testCountInternalGeometries() {
record Case(int expected, Geometry geom) {}
return Stream.of(
new Case(1, newPoint(0, 0)),
new Case(2, newMultiPoint(newPoint(0, 0), newPoint(0, 1))),
new Case(3, newMultiPoint(newPoint(0, 0), newPoint(0, 1), newPoint(0, 2))),
new Case(1, newLineString(0, 0, 1, 1)),
new Case(2, newMultiLineString(
newLineString(0, 0, 1, 1),
newLineString(0, 0, 2, 2)
)),
new Case(3, newMultiLineString(
newLineString(0, 0, 1, 1),
newLineString(0, 0, 2, 2),
newLineString(0, 0, 2, 3)
)),
new Case(1, rectangle(0, 1)),
new Case(2, newMultiPolygon(
rectangle(0, 1),
rectangle(3, 4)
)),
new Case(3, newMultiPolygon(
rectangle(0, 1),
rectangle(3, 4),
rectangle(6, 8)
))
).map(test -> dynamicTest(test.toString(),
() -> {
var feature = new VectorTile.Feature(
"layer", 1, VectorTile.encodeGeometry(test.geom), Map.of()
);
var tile = new VectorTile()
.addLayerFeatures("layer", List.of(feature));
assertEquals(test.expected, VectorTile.countGeometries(tile.toProto().getLayers(0).getFeatures(0)));
}));
}

private static void assertArrayEquals(int[] a, int[] b) {
assertEquals(
IntStream.of(a).boxed().toList(),
Expand Down
10 changes: 5 additions & 5 deletions planetiler-core/src/test/java/com/onthegomap/planetiler/util/TileSizeStatsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ void computeStatsOneFeature() throws IOException {
var formatted = TileSizeStats.formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
assertEquals(
"""
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_attr_bytes layer_attr_keys layer_attr_values
3 1 2 34 999 layer 55 1 18 2 2
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_geometries layer_attr_bytes layer_attr_keys layer_attr_values
3 1 2 34 999 layer 55 1 1 18 2 2
"""
.trim(),
(TileSizeStats.headerRow() + String.join("", formatted)).trim());
Expand Down Expand Up @@ -89,9 +89,9 @@ void computeStats2Features() throws IOException {
var formatted = TileSizeStats.formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
assertEquals(
"""
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_attr_bytes layer_attr_keys layer_attr_values
3 1 2 34 999 a 72 2 20 2 3
3 1 2 34 999 b 19 1 0 0 0
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_geometries layer_attr_bytes layer_attr_keys layer_attr_values
3 1 2 34 999 a 72 2 2 20 2 3
3 1 2 34 999 b 19 1 1 0 0 0
"""
.trim(),
(TileSizeStats.headerRow() + String.join("", formatted)).trim());
Expand Down
28 changes: 14 additions & 14 deletions ...tiler-core/src/test/java/com/onthegomap/planetiler/util/TilesetSummaryStatisticsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ void aggregateTileStats() {
var updater1 = tileStats.threadLocalUpdater();
var updater2 = tileStats.threadLocalUpdater();
updater1.recordTile(TileCoord.ofXYZ(0, 0, 1), 123, List.of(
new TileSizeStats.LayerStats("a", 1, 2, 3, 4, 5),
new TileSizeStats.LayerStats("b", 6, 7, 8, 9, 10)
new TileSizeStats.LayerStats("a", 1, 2, 2, 3, 4, 5),
new TileSizeStats.LayerStats("b", 6, 7, 7, 8, 9, 10)
));
updater2.recordTile(TileCoord.ofXYZ(0, 1, 1), 345, List.of(
new TileSizeStats.LayerStats("b", 1, 2, 3, 4, 5),
new TileSizeStats.LayerStats("c", 6, 7, 8, 9, 10)
new TileSizeStats.LayerStats("b", 1, 2, 2, 3, 4, 5),
new TileSizeStats.LayerStats("c", 6, 7, 7, 8, 9, 10)
));
var summary = tileStats.summary();
assertEquals(Set.of("a", "b", "c"), Set.copyOf(summary.layers()));
Expand Down Expand Up @@ -51,7 +51,7 @@ void aggregateTileStats() {
assertEquals(2, summary.get().numTiles());

updater1.recordTile(TileCoord.ofXYZ(0, 0, 2), 0, List.of(
new TileSizeStats.LayerStats("c", 10, 7, 8, 9, 10)
new TileSizeStats.LayerStats("c", 10, 7, 7, 8, 9, 10)
));
assertEquals("""
z1 z2 all
Expand Down Expand Up @@ -101,8 +101,8 @@ void topLayerTiles() {
List<TilesetSummaryStatistics.TileSummary> summaries = new ArrayList<>();
for (int i = 0; i < 20; i++) {
var summary = new TilesetSummaryStatistics.TileSummary(TileCoord.decode(i), i, List.of(
new TileSizeStats.LayerStats("a", i * 2, i, 0, 0, 0),
new TileSizeStats.LayerStats("b", i * 3, i, 0, 0, 0)
new TileSizeStats.LayerStats("a", i * 2, i, i * 2, 0, 0, 0),
new TileSizeStats.LayerStats("b", i * 3, i, i * 2, 0, 0, 0)
));
summaries.add(0, summary);
(i % 2 == 0 ? updater1 : updater2).recordTile(summary.coord(), summary.archivedSize(), summary.layers());
Expand Down Expand Up @@ -140,21 +140,21 @@ void tileWeights() {
updater1.recordTile(
TileCoord.ofXYZ(0, 0, 0),
100,
List.of(new TileSizeStats.LayerStats("a", 10, 0, 0, 0, 0))
List.of(new TileSizeStats.LayerStats("a", 10, 0, 0, 0, 0, 0))
);
updater2.recordTile(
TileCoord.ofXYZ(0, 0, 1),
200,
List.of(
new TileSizeStats.LayerStats("a", 20, 0, 0, 0, 0),
new TileSizeStats.LayerStats("b", 30, 0, 0, 0, 0)
new TileSizeStats.LayerStats("a", 20, 0, 0, 0, 0, 0),
new TileSizeStats.LayerStats("b", 30, 0, 0, 0, 0, 0)
)
);
updater2.recordTile(
TileCoord.ofXYZ(0, 0, 2), // no stats
400,
List.of(
new TileSizeStats.LayerStats("c", 40, 0, 0, 0, 0)
new TileSizeStats.LayerStats("c", 40, 0, 0, 0, 0, 0)
)
);

Expand Down Expand Up @@ -189,14 +189,14 @@ void tileWeightsScaledByZoom() {
updater1.recordTile(
TileCoord.ofXYZ(0, 0, 0),
100,
List.of(new TileSizeStats.LayerStats("a", 10, 0, 0, 0, 0))
List.of(new TileSizeStats.LayerStats("a", 10, 0, 0, 0, 0, 0))
);
updater2.recordTile(
TileCoord.ofXYZ(0, 0, 1),
200,
List.of(
new TileSizeStats.LayerStats("a", 20, 0, 0, 0, 0),
new TileSizeStats.LayerStats("b", 30, 0, 0, 0, 0)
new TileSizeStats.LayerStats("a", 20, 0, 0, 0, 0, 0),
new TileSizeStats.LayerStats("b", 30, 0, 0, 0, 0, 0)
)
);

Expand Down

0 comments on commit 916f7e8

Please sign in to comment.