Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix downloading tile weights in CI #668

Merged
merged 5 commits into from
Sep 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ public void run() throws Exception {
download();
}
if (fetchOsmTileStats) {
TopOsmTiles.downloadPrecomputed(config, stats);
TopOsmTiles.downloadPrecomputed(config);
}
ensureInputFilesExist();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ public static void main(String... args) {
var arguments = Arguments.fromArgsOrConfigFile(args);
var config = PlanetilerConfig.from(arguments);
var stats = Stats.inMemory();
var download = arguments.getBoolean("download_osm_tile_weights", "download OSM tile weights file", true);
var download = arguments.getBoolean("download_osm_tile_weights|download", "download OSM tile weights file", false);
if (download && !Files.exists(config.tileWeights())) {
TopOsmTiles.downloadPrecomputed(config, stats);
TopOsmTiles.downloadPrecomputed(config);
}
var tileStats = new TilesetSummaryStatistics(TileWeights.readFromFile(config.tileWeights()));
var inputString = arguments.getString("input", "input file");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.onthegomap.planetiler.util;

import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
import static java.nio.file.StandardOpenOption.WRITE;

import com.google.common.io.LineReader;
import com.onthegomap.planetiler.config.Arguments;
Expand Down Expand Up @@ -45,7 +48,7 @@
* </pre>
* <p>
* You can also fetch precomputed top-1m tile stats from summer 2023 using
* {@link #downloadPrecomputed(PlanetilerConfig, Stats)}
* {@link #downloadPrecomputed(PlanetilerConfig)}
*/
public class TopOsmTiles {

Expand Down Expand Up @@ -180,10 +183,27 @@ public static void main(String[] args) throws IOException {
* Download precomputed top-1m tile stats from 90 days of openstreetmap.org tile logs to
* {@link PlanetilerConfig#tileWeights()} path if they don't already exist.
*/
public static void downloadPrecomputed(PlanetilerConfig config, Stats stats) {
if (!Files.exists(config.tileWeights())) {
Downloader.create(config, stats)
.downloadIfNecessary(new Downloader.ResourceToDownload("osm-tile-weights", DOWLOAD_URL, config.tileWeights()));
@SuppressWarnings("java:S2142")
public static void downloadPrecomputed(PlanetilerConfig config) {
var dest = config.tileWeights();
if (!Files.exists(dest)) {
var tmp = dest.resolveSibling(dest.getFileName() + ".tmp");
FileUtils.deleteOnExit(tmp);
try {
try (
var in = Downloader.openStream(DOWLOAD_URL, config);
var out = Files.newOutputStream(tmp, CREATE, TRUNCATE_EXISTING, WRITE)
) {
LOGGER.info("Downloading pre-computed tile weights from {} to {}", DOWLOAD_URL, dest);
in.transferTo(out);
}
Files.move(tmp, dest);
} catch (IOException e) {
LOGGER.warn("Failed downloading pre-computed tile weights: {}", e.toString());
FileUtils.delete(dest);
} finally {
FileUtils.delete(tmp);
}
}
}
}