Compute layer attr stats from actual vector tile features (#752)

planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java

@@ -27,6 +27,7 @@
 import com.onthegomap.planetiler.geo.GeometryType;
 import com.onthegomap.planetiler.geo.MutableCoordinateSequence;
 import com.onthegomap.planetiler.util.Hilbert;
+import com.onthegomap.planetiler.util.LayerAttrStats;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -80,6 +81,7 @@ public class VectorTile {
   private static final int EXTENT = 4096;
   private static final double SIZE = 256d;
   private final Map<String, Layer> layers = new LinkedHashMap<>();
+  private LayerAttrStats.Updater.ForZoom layerStatsTracker = LayerAttrStats.Updater.ForZoom.NOOP;
 
   private static int[] getCommands(Geometry input, int scale) {
     var encoder = new CommandEncoder(scale);
@@ -467,12 +469,12 @@ public VectorTile addLayerFeatures(String layerName, List<Feature> features) {
     if (features.isEmpty()) {
       return this;
     }
-
     Layer layer = layers.get(layerName);
     if (layer == null) {
       layer = new Layer();
       layers.put(layerName, layer);
     }
+    var statsTracker = layerStatsTracker.forLayer(layerName);
 
     for (Feature inFeature : features) {
       if (inFeature != null && inFeature.geometry().commands().length > 0) {
@@ -481,8 +483,11 @@ public VectorTile addLayerFeatures(String layerName, List<Feature> features) {
         for (Map.Entry<String, ?> e : inFeature.attrs().entrySet()) {
           // skip attribute without value
           if (e.getValue() != null) {
-            outFeature.tags.add(layer.key(e.getKey()));
-            outFeature.tags.add(layer.value(e.getValue()));
+            String key = e.getKey();
+            Object value = e.getValue();
+            outFeature.tags.add(layer.key(key));
+            outFeature.tags.add(layer.value(value));
+            statsTracker.accept(key, value);
           }
         }
 
@@ -594,6 +599,15 @@ public boolean likelyToBeDuplicated() {
     return layers.values().stream().allMatch(v -> v.encodedFeatures.isEmpty()) || containsOnlyFillsOrEdges();
   }
 
+  /**
+   * Call back to {@code layerStats} as vector tile features are being encoded in
+   * {@link #addLayerFeatures(String, List)} to track attribute types present on features in each layer, for example to
+   * emit in tilejson metadata stats.
+   */
+  public void trackLayerStats(LayerAttrStats.Updater.ForZoom layerStats) {
+    this.layerStatsTracker = layerStats;
+  }
+
   enum Command {
     MOVE_TO(1),
     LINE_TO(2),

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java

@@ -15,6 +15,7 @@
 import com.onthegomap.planetiler.util.DiskBacked;
 import com.onthegomap.planetiler.util.Format;
 import com.onthegomap.planetiler.util.Hashing;
+import com.onthegomap.planetiler.util.LayerAttrStats;
 import com.onthegomap.planetiler.util.TileSizeStats;
 import com.onthegomap.planetiler.util.TileWeights;
 import com.onthegomap.planetiler.util.TilesetSummaryStatistics;
@@ -59,6 +60,7 @@ public class TileArchiveWriter {
   private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
   private final TileArchiveMetadata tileArchiveMetadata;
   private final TilesetSummaryStatistics tileStats;
+  private final LayerAttrStats layerAttrStats = new LayerAttrStats();
 
   private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, WriteableTileArchive archive,
     PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats) {
@@ -105,9 +107,7 @@ public static void writeOutput(FeatureGroup features, WriteableTileArchive outpu
       readWorker = reader.readWorker();
     }
 
-    TileArchiveWriter writer =
-      new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata.withLayerStats(features.layerStats()
-        .getTileStats()), stats);
+    TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats);
 
     var pipeline = WorkerPipeline.start("archive", stats);
 
@@ -260,6 +260,7 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
     boolean skipFilled = config.skipFilledTiles();
 
     var tileStatsUpdater = tileStats.threadLocalUpdater();
+    var layerAttrStatsUpdater = layerAttrStats.handlerForThread();
     for (TileBatch batch : prev) {
       List<TileEncodingResult> result = new ArrayList<>(batch.size());
       FeatureGroup.TileFeatures last = null;
@@ -277,7 +278,7 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
           layerStats = lastLayerStats;
           memoizedTiles.inc();
         } else {
-          VectorTile tile = tileFeatures.getVectorTile();
+          VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater);
           if (skipFilled && (lastIsFill = tile.containsOnlyFills())) {
             encoded = null;
             layerStats = null;
@@ -333,7 +334,7 @@ private void tileWriter(Iterable<TileBatch> tileBatches) throws ExecutionExcepti
     var f = NumberFormat.getNumberInstance(Locale.getDefault());
     f.setMaximumFractionDigits(5);
 
-    archive.initialize(tileArchiveMetadata);
+    archive.initialize();
     var order = archive.tileOrder();
 
     TileCoord lastTile = null;
@@ -371,7 +372,7 @@ private void tileWriter(Iterable<TileBatch> tileBatches) throws ExecutionExcepti
       LOGGER.info("Finished z{} in {}", currentZ, time.stop());
     }
 
-    archive.finish(tileArchiveMetadata);
+    archive.finish(tileArchiveMetadata.withLayerStats(layerAttrStats.getTileStats()));
   }
 
   @SuppressWarnings("java:S2629")

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java

@@ -30,7 +30,7 @@ public interface WriteableTileArchive extends Closeable {
    * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any
    * required state here.
    */
-  default void initialize(TileArchiveMetadata metadata) {}
+  default void initialize() {}
 
   /**
    * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java

@@ -27,7 +27,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Consumer;
 import java.util.function.Function;
 import javax.annotation.concurrent.NotThreadSafe;
 import org.msgpack.core.MessageBufferPacker;
@@ -59,7 +58,6 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
   private final CommonStringEncoder.AsByte commonLayerStrings = new CommonStringEncoder.AsByte();
   private final CommonStringEncoder commonValueStrings = new CommonStringEncoder(100_000);
   private final Stats stats;
-  private final LayerAttrStats layerStats = new LayerAttrStats();
   private final PlanetilerConfig config;
   private volatile boolean prepared = false;
   private final TileOrder tileOrder;
@@ -141,14 +139,6 @@ static byte encodeGeomTypeAndScale(VectorTile.VectorGeometry geometry) {
     return (byte) ((geometry.geomType().asByte() & 0xff) | (geometry.scale() << 3));
   }
 
-  /**
-   * Returns statistics about each layer written through {@link #newRenderedFeatureEncoder()} including min/max zoom,
-   * features on elements in that layer, and their types.
-   */
-  public LayerAttrStats layerStats() {
-    return layerStats;
-  }
-
   public long numFeaturesWritten() {
     return sorter.numFeaturesWritten();
   }
@@ -159,16 +149,13 @@ public RenderedFeatureEncoder newRenderedFeatureEncoder() {
       // This method gets called billions of times when generating the planet, so these optimizations make a big difference:
       // 1) Re-use the same buffer packer to avoid allocating and resizing new byte arrays for every feature.
       private final MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
-      // 2) Avoid a ThreadLocal lookup on every layer stats call by getting the handler for this thread once
-      private final Consumer<RenderedFeature> threadLocalLayerStats = layerStats.handlerForThread();
-      // 3) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values
+      // 2) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values
       // FeatureRenderer ensures that a separate VectorTileEncoder.Feature is used for each zoom level
       private VectorTile.Feature lastFeature = null;
       private byte[] lastEncodedValue = null;
 
       @Override
       public SortableFeature apply(RenderedFeature feature) {
-        threadLocalLayerStats.accept(feature);
         var group = feature.group().orElse(null);
         var thisFeature = feature.vectorTileFeature();
         byte[] encodedValue;
@@ -450,7 +437,14 @@ private VectorTile.Feature decodeVectorTileFeature(SortableFeature entry) {
     }
 
     public VectorTile getVectorTile() {
+      return getVectorTile(null);
+    }
+
+    public VectorTile getVectorTile(LayerAttrStats.Updater layerStats) {
       VectorTile tile = new VectorTile();
+      if (layerStats != null) {
+        tile.trackLayerStats(layerStats.forZoom(tileCoord.z()));
+      }
       List<VectorTile.Feature> items = new ArrayList<>(entries.size());
       String currentLayer = null;
       for (SortableFeature entry : entries) {

planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java

@@ -220,7 +220,7 @@ public TileOrder tileOrder() {
   }
 
   @Override
-  public void initialize(TileArchiveMetadata tileArchiveMetadata) {
+  public void initialize() {
     if (skipIndexCreation) {
       createTablesWithoutIndexes();
       if (LOGGER.isInfoEnabled()) {
@@ -230,12 +230,11 @@ public void initialize(TileArchiveMetadata tileArchiveMetadata) {
     } else {
       createTablesWithIndexes();
     }
-
-    metadataTable().set(tileArchiveMetadata);
   }
 
   @Override
   public void finish(TileArchiveMetadata tileArchiveMetadata) {
+    metadataTable().set(tileArchiveMetadata);
     if (vacuumAnalyze) {
       vacuumAnalyze();
     }

planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java

@@ -76,11 +76,11 @@ protected TileWriter newTileWriter(OutputStream outputStream) {
   }
 
   @Override
-  public void initialize(TileArchiveMetadata metadata) {
+  public void initialize() {
     if (writeTilesOnly) {
       return;
     }
-    writeEntryFlush(new InitializationEntry(metadata));
+    writeEntryFlush(new InitializationEntry());
   }
 
   @Override
@@ -204,7 +204,7 @@ public String toString() {
     }
   }
 
-  record InitializationEntry(TileArchiveMetadata metadata) implements Entry {}
+  record InitializationEntry() implements Entry {}
 
 
   record FinishEntry(TileArchiveMetadata metadata) implements Entry {}

planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java

@@ -50,14 +50,8 @@ protected TileWriter newTileWriter(OutputStream outputStream) {
   }
 
   @Override
-  public void initialize(TileArchiveMetadata metadata) {
-    writeEntry(
-      StreamArchiveProto.Entry.newBuilder()
-        .setInitialization(
-          StreamArchiveProto.InitializationEntry.newBuilder().setMetadata(toExportData(metadata)).build()
-        )
-        .build()
-    );
+  public void initialize() {
+    writeEntry(StreamArchiveProto.Entry.newBuilder().build());
   }
 
   @Override

planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java

@@ -3,15 +3,13 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
-import com.onthegomap.planetiler.render.RenderedFeature;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.TreeMap;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.function.Consumer;
 import javax.annotation.concurrent.NotThreadSafe;
 import javax.annotation.concurrent.ThreadSafe;
 
@@ -27,7 +25,7 @@
  * @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#content">MBtiles spec</a>
  */
 @ThreadSafe
-public class LayerAttrStats implements Consumer<RenderedFeature> {
+public class LayerAttrStats {
   /*
    * This utility is called for billions of features by multiple threads when processing the planet which can make
    * access to shared data structures a bottleneck.  So give each thread an individual ThreadLocalLayerStatsHandler to
@@ -63,6 +61,11 @@ public List<VectorLayer> getTileStats() {
       .toList();
   }
 
+  /** Shortcut for tests */
+  void accept(String layer, int zoom, String key, Object value) {
+    handlerForThread().forZoom(zoom).forLayer(layer).accept(key, value);
+  }
+
   public enum FieldType {
     @JsonProperty("Number")
     NUMBER,
@@ -114,7 +117,7 @@ public VectorLayer withMaxzoom(int newMaxzoom) {
 
   /** Accepts features from a single thread that will be combined across all threads in {@link #getTileStats()}. */
   @NotThreadSafe
-  private class ThreadLocalHandler implements Consumer<RenderedFeature> {
+  private class ThreadLocalHandler implements Updater {
 
     private final Map<String, StatsForLayer> layers = new TreeMap<>();
 
@@ -123,42 +126,50 @@ private class ThreadLocalHandler implements Consumer<RenderedFeature> {
     }
 
     @Override
-    public void accept(RenderedFeature feature) {
-      var vectorTileFeature = feature.vectorTileFeature();
-      var stats = layers.computeIfAbsent(vectorTileFeature.layer(), StatsForLayer::new);
-      stats.expandZoomRangeToInclude(feature.tile().z());
-      for (var entry : vectorTileFeature.attrs().entrySet()) {
-        String key = entry.getKey();
-        Object value = entry.getValue();
-
-        FieldType fieldType = null;
-        if (value instanceof Number) {
-          fieldType = FieldType.NUMBER;
-        } else if (value instanceof Boolean) {
-          fieldType = FieldType.BOOLEAN;
-        } else if (value != null) {
-          fieldType = FieldType.STRING;
-        }
-        if (fieldType != null) {
-          // widen different types to string
-          stats.fields.merge(key, fieldType, FieldType::merge);
-        }
-      }
+    public Updater.ForZoom forZoom(int zoom) {
+      return layer -> {
+        var stats = layers.computeIfAbsent(layer, StatsForLayer::new);
+        stats.expandZoomRangeToInclude(zoom);
+        return (key, value) -> {
+          FieldType fieldType = null;
+          if (value instanceof Number) {
+            fieldType = FieldType.NUMBER;
+          } else if (value instanceof Boolean) {
+            fieldType = FieldType.BOOLEAN;
+          } else if (value != null) {
+            fieldType = FieldType.STRING;
+          }
+          if (fieldType != null) {
+            // widen different types to string
+            stats.fields.merge(key, fieldType, FieldType::merge);
+          }
+        };
+      };
     }
   }
 
   /**
    * Returns a handler optimized for accepting features from a single thread.
-   * <p>
-   * Use this instead of {@link #accept(RenderedFeature)}
    */
-  public Consumer<RenderedFeature> handlerForThread() {
+  public Updater handlerForThread() {
     return layerStats.get();
   }
 
-  @Override
-  public void accept(RenderedFeature feature) {
-    handlerForThread().accept(feature);
+  public interface Updater {
+
+    ForZoom forZoom(int zoom);
+
+    interface ForZoom {
+
+      ForZoom NOOP = layer -> (key, value) -> {
+      };
+
+      ForLayer forLayer(String layer);
+
+      interface ForLayer {
+        void accept(String key, Object value);
+      }
+    }
   }
 
   private static class StatsForLayer {

planetiler-core/src/main/proto/stream_archive_proto.proto

@@ -1,4 +1,3 @@
-
 syntax = "proto3";
 
 package com.onthegomap.planetiler.proto;
@@ -19,7 +18,6 @@ message TileEntry {
 }
 
 message InitializationEntry {
-  Metadata metadata = 1;
 }
 
 message FinishEntry {