From aa28d25dab5d61717540e4e581ded5507c5ddc2a Mon Sep 17 00:00:00 2001
From: Mike Barry <msb5014@gmail.com>
Date: Tue, 30 Jan 2024 07:25:25 -0500
Subject: [PATCH] omit() feature collector api and --refresh-sources arg

---
 .../com/onthegomap/planetiler/FeatureCollector.java   |  6 ++++++
 .../java/com/onthegomap/planetiler/Planetiler.java    |  9 +++++++--
 .../onthegomap/planetiler/FeatureCollectorTest.java   | 11 +++++++++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java
index f1e76d8a27..a136bd8e33 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureCollector.java
@@ -797,6 +797,12 @@ public Feature setNumPointsAttr(String numPointsAttr) {
       return this;
     }
 
+    /** Omit this feature from the output */
+    public Feature omit() {
+      output.remove(this);
+      return this;
+    }
+
     @Override
     public String toString() {
       return "Feature{" +
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
index b2503d76e3..7ad554095c 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
@@ -83,6 +83,7 @@ public class Planetiler {
   private final Path multipolygonPath;
   private final Path featureDbPath;
   private final boolean downloadSources;
+  private final boolean refreshSources;
   private final boolean onlyDownloadSources;
   private final boolean parseNodeBounds;
   private Profile profile = null;
@@ -117,6 +118,8 @@ private Planetiler(Arguments arguments) {
     tmpDir = config.tmpDir();
     onlyDownloadSources = arguments.getBoolean("only_download", "download source data then exit", false);
     downloadSources = onlyDownloadSources || arguments.getBoolean("download", "download sources", false);
+    refreshSources =
+      arguments.getBoolean("refresh_sources", "download new version of source files if they have changed", false);
     fetchOsmTileStats =
       arguments.getBoolean("download_osm_tile_weights", "download OSM tile weights file", downloadSources);
     nodeDbPath = arguments.file("temp_nodes", "temp node db location", tmpDir.resolve("node.db"));
@@ -891,11 +894,13 @@ private RunnableThatThrows ifSourceUsed(String name, RunnableThatThrows task) {
 
   private Path getPath(String name, String type, Path defaultPath, String defaultUrl) {
     Path path = arguments.file(name + "_path", name + " " + type + " path", defaultPath);
+    boolean refresh =
+      arguments.getBoolean("refresh_" + name, "Download new version of " + name + " if changed", refreshSources);
     boolean freeAfterReading = arguments.getBoolean("free_" + name + "_after_read",
       "delete " + name + " input file after reading to make space for output (reduces peak disk usage)", false);
-    if (downloadSources) {
+    if (downloadSources || refresh) {
       String url = arguments.getString(name + "_url", name + " " + type + " url", defaultUrl);
-      if (!Files.exists(path) && url != null) {
+      if ((!Files.exists(path) || refresh) && url != null) {
         toDownload.add(new ToDownload(name, url, path));
       }
     }
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java
index 678a29fd8d..e200140258 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/FeatureCollectorTest.java
@@ -85,6 +85,17 @@ void testPoint() {
 
   }
 
+  @Test
+  void testOmit() {
+    var collector = factory.get(newReaderFeature(newPoint(0, 0), Map.of(
+      "key", "val"
+    )));
+    var point = collector.point("layername");
+    assertFeatures(14, List.of(Map.of("_layer", "layername")), collector);
+    point.omit();
+    assertFeatures(14, List.of(), collector);
+  }
+
   @Test
   void testAttrWithMinzoom() {
     var collector = factory.get(newReaderFeature(newPoint(0, 0), Map.of(