diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 066afbc8ab..c505ce08eb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -41,6 +41,7 @@ public record PlanetilerConfig( String httpUserAgent, Duration httpTimeout, int httpRetries, + Duration httpRetryWait, long downloadChunkSizeMB, int downloadThreads, double downloadMaxBandwidth, @@ -166,6 +167,7 @@ public static PlanetilerConfig from(Arguments arguments) { "Planetiler downloader (https://github.com/onthegomap/planetiler)"), arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"), arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1), + arguments.getDuration("http_retry_wait", "How long to wait before retrying HTTP request", "5s"), arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100), arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1), Parse.bandwidth(arguments.getString("download_max_bandwidth", diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java index c3961b0098..398713b576 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java @@ -300,10 +300,15 @@ private LongObjectMap> queryWikidata(List qidsToFetch) .POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8)) .build(); - InputStream response = null; - for (int i = 0; i <= config.httpRetries() && response == null; i++) { + LongObjectMap> result = null; + for (int i = 0; i <= config.httpRetries() && result == null; i++) { try { - response = client.send(request); + var response = client.send(request); + if (response != null) { + try (var bis = new BufferedInputStream(response)) { + result = parseResults(bis); + } + } } catch (IOException e) { boolean lastTry = i == config.httpRetries(); if (!lastTry) { @@ -312,13 +317,12 @@ private LongObjectMap> queryWikidata(List qidsToFetch) LOGGER.error("sparql query failed, exhausted retries: " + e); throw e; } + Thread.sleep(config.httpRetryWait()); } } - if (response != null) { - try (var bis = new BufferedInputStream(response)) { - return parseResults(bis); - } + if (result != null) { + return result; } else { throw new IllegalStateException("No response or exception"); // should never happen }