From 5fc11bf8dfb2a04d888dfe423a48e853353bff68 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 22 Sep 2023 05:33:17 -0400 Subject: [PATCH 1/5] show data we have --- .github/workflows/performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 497f560a0f..4890ad118c 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -64,6 +64,7 @@ jobs: set -eo pipefail cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + find data - name: 'Store build info' run: | From be134b3c042c3bc46504d82efdbadac89e9fee69 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 22 Sep 2023 05:46:32 -0400 Subject: [PATCH 2/5] more logging --- .github/workflows/performance.yml | 1 + .../src/main/java/com/onthegomap/planetiler/Planetiler.java | 1 + .../main/java/com/onthegomap/planetiler/util/TopOsmTiles.java | 2 ++ 3 files changed, 4 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 4890ad118c..6f497a5963 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -62,6 +62,7 @@ jobs: - name: 'Download data' run: | set -eo pipefail + find data cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" find data diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index b36d417d6b..d857629ef8 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -724,6 +724,7 @@ public void run() throws Exception { if (!toDownload.isEmpty()) { download(); } + LOGGER.error("fetchOsmTileStats={}", fetchOsmTileStats); if (fetchOsmTileStats) { TopOsmTiles.downloadPrecomputed(config, stats); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java index 652235e4a5..7202f39e20 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java @@ -181,7 +181,9 @@ public static void main(String[] args) throws IOException { * {@link PlanetilerConfig#tileWeights()} path if they don't already exist. */ public static void downloadPrecomputed(PlanetilerConfig config, Stats stats) { + LOGGER.error("TopOsmTiles.downloadPrecomputed={} {}", config.tileWeights(), Files.exists(config.tileWeights())); if (!Files.exists(config.tileWeights())) { + LOGGER.error("TopOsmTiles.downloadPrecomputed downloading"); Downloader.create(config, stats) .downloadIfNecessary(new Downloader.ResourceToDownload("osm-tile-weights", DOWLOAD_URL, config.tileWeights())); } From f604cabb4aecb84ac671b58cf66a7520bd9e8e81 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 22 Sep 2023 05:50:32 -0400 Subject: [PATCH 3/5] fix --- .github/workflows/performance.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 6f497a5963..4890ad118c 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -62,7 +62,6 @@ jobs: - name: 'Download data' run: | set -eo pipefail - find data cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" find data From aef7c0ad542d58ee475ad7297b72f9f7beed014f Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 22 Sep 2023 06:14:00 -0400 Subject: [PATCH 4/5] fix downloading tile weights --- .github/workflows/performance.yml | 1 - .../com/onthegomap/planetiler/Planetiler.java | 3 +- .../planetiler/util/TileSizeStats.java | 2 +- .../planetiler/util/TopOsmTiles.java | 32 +++++++++++++++---- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 4890ad118c..497f560a0f 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -64,7 +64,6 @@ jobs: set -eo pipefail cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" - find data - name: 'Store build info' run: | diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index d857629ef8..5fe00af280 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -724,9 +724,8 @@ public void run() throws Exception { if (!toDownload.isEmpty()) { download(); } - LOGGER.error("fetchOsmTileStats={}", fetchOsmTileStats); if (fetchOsmTileStats) { - TopOsmTiles.downloadPrecomputed(config, stats); + TopOsmTiles.downloadPrecomputed(config); } ensureInputFilesExist(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java index 2b07a5bd07..f8674dccde 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java @@ -70,7 +70,7 @@ public static void main(String... args) { var stats = Stats.inMemory(); var download = arguments.getBoolean("download_osm_tile_weights", "download OSM tile weights file", true); if (download && !Files.exists(config.tileWeights())) { - TopOsmTiles.downloadPrecomputed(config, stats); + TopOsmTiles.downloadPrecomputed(config); } var tileStats = new TilesetSummaryStatistics(TileWeights.readFromFile(config.tileWeights())); var inputString = arguments.getString("input", "input file"); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java index 7202f39e20..e66e8cbcb5 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TopOsmTiles.java @@ -1,6 +1,9 @@ package com.onthegomap.planetiler.util; import static com.onthegomap.planetiler.util.Exceptions.throwFatalException; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; import com.google.common.io.LineReader; import com.onthegomap.planetiler.config.Arguments; @@ -45,7 +48,7 @@ * *

* You can also fetch precomputed top-1m tile stats from summer 2023 using - * {@link #downloadPrecomputed(PlanetilerConfig, Stats)} + * {@link #downloadPrecomputed(PlanetilerConfig)} */ public class TopOsmTiles { @@ -180,12 +183,27 @@ public static void main(String[] args) throws IOException { * Download precomputed top-1m tile stats from 90 days of openstreetmap.org tile logs to * {@link PlanetilerConfig#tileWeights()} path if they don't already exist. */ - public static void downloadPrecomputed(PlanetilerConfig config, Stats stats) { - LOGGER.error("TopOsmTiles.downloadPrecomputed={} {}", config.tileWeights(), Files.exists(config.tileWeights())); - if (!Files.exists(config.tileWeights())) { - LOGGER.error("TopOsmTiles.downloadPrecomputed downloading"); - Downloader.create(config, stats) - .downloadIfNecessary(new Downloader.ResourceToDownload("osm-tile-weights", DOWLOAD_URL, config.tileWeights())); + @SuppressWarnings("java:S2142") + public static void downloadPrecomputed(PlanetilerConfig config) { + var dest = config.tileWeights(); + if (!Files.exists(dest)) { + var tmp = dest.resolveSibling(dest.getFileName() + ".tmp"); + FileUtils.deleteOnExit(tmp); + try { + try ( + var in = Downloader.openStream(DOWLOAD_URL, config); + var out = Files.newOutputStream(tmp, CREATE, TRUNCATE_EXISTING, WRITE) + ) { + LOGGER.info("Downloading pre-computed tile weights from {} to {}", DOWLOAD_URL, dest); + in.transferTo(out); + } + Files.move(tmp, dest); + } catch (IOException e) { + LOGGER.warn("Failed downloading pre-computed tile weights: {}", e.toString()); + FileUtils.delete(dest); + } finally { + FileUtils.delete(tmp); + } } } } From ff4fc058bda044b25cec49aba3c91895b0d79413 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 22 Sep 2023 06:18:45 -0400 Subject: [PATCH 5/5] tweak args to not download during tests --- .../main/java/com/onthegomap/planetiler/util/TileSizeStats.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java index f8674dccde..9276ef7e4d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileSizeStats.java @@ -68,7 +68,7 @@ public static void main(String... args) { var arguments = Arguments.fromArgsOrConfigFile(args); var config = PlanetilerConfig.from(arguments); var stats = Stats.inMemory(); - var download = arguments.getBoolean("download_osm_tile_weights", "download OSM tile weights file", true); + var download = arguments.getBoolean("download_osm_tile_weights|download", "download OSM tile weights file", false); if (download && !Files.exists(config.tileWeights())) { TopOsmTiles.downloadPrecomputed(config); }