fix downloading tile weights

onthegomap · Sep 22, 2023 · aef7c0a · aef7c0a
1 parent f604cab
commit aef7c0a
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 11 deletions.
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -64,7 +64,6 @@ jobs:
         set -eo pipefail
         cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
         cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
-        find data
 
     - name: 'Store build info'
       run: |

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
@@ -724,9 +724,8 @@ public void run() throws Exception {
     if (!toDownload.isEmpty()) {
       download();
     }
-    LOGGER.error("fetchOsmTileStats={}", fetchOsmTileStats);
     if (fetchOsmTileStats) {
-      TopOsmTiles.downloadPrecomputed(config, stats);
+      TopOsmTiles.downloadPrecomputed(config);
     }
     ensureInputFilesExist();
 

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java
@@ -70,7 +70,7 @@ public static void main(String... args) {
     var stats = Stats.inMemory();
     var download = arguments.getBoolean("download_osm_tile_weights", "download OSM tile weights file", true);
     if (download && !Files.exists(config.tileWeights())) {
-      TopOsmTiles.downloadPrecomputed(config, stats);
+      TopOsmTiles.downloadPrecomputed(config);
     }
     var tileStats = new TilesetSummaryStatistics(TileWeights.readFromFile(config.tileWeights()));
     var inputString = arguments.getString("input", "input file");

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java
@@ -1,6 +1,9 @@
 package com.onthegomap.planetiler.util;
 
 import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static java.nio.file.StandardOpenOption.WRITE;
 
 import com.google.common.io.LineReader;
 import com.onthegomap.planetiler.config.Arguments;
@@ -45,7 +48,7 @@
  * </pre>
  * <p>
  * You can also fetch precomputed top-1m tile stats from summer 2023 using
- * {@link #downloadPrecomputed(PlanetilerConfig, Stats)}
+ * {@link #downloadPrecomputed(PlanetilerConfig)}
  */
 public class TopOsmTiles {
 
@@ -180,12 +183,27 @@ public static void main(String[] args) throws IOException {
    * Download precomputed top-1m tile stats from 90 days of openstreetmap.org tile logs to
    * {@link PlanetilerConfig#tileWeights()} path if they don't already exist.
    */
-  public static void downloadPrecomputed(PlanetilerConfig config, Stats stats) {
-    LOGGER.error("TopOsmTiles.downloadPrecomputed={} {}", config.tileWeights(), Files.exists(config.tileWeights()));
-    if (!Files.exists(config.tileWeights())) {
-      LOGGER.error("TopOsmTiles.downloadPrecomputed downloading");
-      Downloader.create(config, stats)
-        .downloadIfNecessary(new Downloader.ResourceToDownload("osm-tile-weights", DOWLOAD_URL, config.tileWeights()));
+  @SuppressWarnings("java:S2142")
+  public static void downloadPrecomputed(PlanetilerConfig config) {
+    var dest = config.tileWeights();
+    if (!Files.exists(dest)) {
+      var tmp = dest.resolveSibling(dest.getFileName() + ".tmp");
+      FileUtils.deleteOnExit(tmp);
+      try {
+        try (
+          var in = Downloader.openStream(DOWLOAD_URL, config);
+          var out = Files.newOutputStream(tmp, CREATE, TRUNCATE_EXISTING, WRITE)
+        ) {
+          LOGGER.info("Downloading pre-computed tile weights from {} to {}", DOWLOAD_URL, dest);
+          in.transferTo(out);
+        }
+        Files.move(tmp, dest);
+      } catch (IOException e) {
+        LOGGER.warn("Failed downloading pre-computed tile weights: {}", e.toString());
+        FileUtils.delete(dest);
+      } finally {
+        FileUtils.delete(tmp);
+      }
     }
   }
 }