From 1b0d5526a793e6fb37b62adb42e4dc266da8b1b3 Mon Sep 17 00:00:00 2001 From: Peter Hanecak Date: Thu, 15 Feb 2024 17:12:01 +0100 Subject: [PATCH 1/5] attempt to fix IOException/GOAWAY happening within parseResults() --- .../onthegomap/planetiler/util/Wikidata.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java index c3961b0098..65fa4f75c0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java @@ -300,10 +300,15 @@ private LongObjectMap> queryWikidata(List qidsToFetch) .POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8)) .build(); - InputStream response = null; - for (int i = 0; i <= config.httpRetries() && response == null; i++) { + LongObjectMap> result = null; + for (int i = 0; i <= config.httpRetries() && result == null; i++) { try { - response = client.send(request); + var response = client.send(request); + if (response != null) { + try (var bis = new BufferedInputStream(response)) { + result = parseResults(bis); + } + } } catch (IOException e) { boolean lastTry = i == config.httpRetries(); if (!lastTry) { @@ -315,10 +320,8 @@ private LongObjectMap> queryWikidata(List qidsToFetch) } } - if (response != null) { - try (var bis = new BufferedInputStream(response)) { - return parseResults(bis); - } + if (result != null) { + return result; } else { throw new IllegalStateException("No response or exception"); // should never happen } From 8377e61e82d915dbfc35e1be35e459f730a8f57f Mon Sep 17 00:00:00 2001 From: Peter Hanecak Date: Fri, 16 Feb 2024 10:33:43 +0100 Subject: [PATCH 2/5] wait 5s berfore retrying WikiData request after GOAWAY, configurable with http_retry_wait --- .../onthegomap/planetiler/config/PlanetilerConfig.java | 2 ++ .../java/com/onthegomap/planetiler/util/Wikidata.java | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 066afbc8ab..067eec17a4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -41,6 +41,7 @@ public record PlanetilerConfig( String httpUserAgent, Duration httpTimeout, int httpRetries, + long httpRetryWait, long downloadChunkSizeMB, int downloadThreads, double downloadMaxBandwidth, @@ -166,6 +167,7 @@ public static PlanetilerConfig from(Arguments arguments) { "Planetiler downloader (https://github.com/onthegomap/planetiler)"), arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"), arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1), + arguments.getLong("http_retry_wait", "How long (in milliseconds) to wait before retrying HTTP request", 5_000), arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100), arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1), Parse.bandwidth(arguments.getString("download_max_bandwidth", diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java index 65fa4f75c0..04901f3c8b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java @@ -317,6 +317,14 @@ private LongObjectMap> queryWikidata(List qidsToFetch) LOGGER.error("sparql query failed, exhausted retries: " + e); throw e; } + if (e.getMessage() != null && e.getMessage().contains("GOAWAY")) { + try { + LOGGER.debug("GOAWAY received, waiting {} ms to give server some room", config.httpRetryWait()); + Thread.sleep(config.httpRetryWait()); + } catch (InterruptedException e2) { + Thread.currentThread().interrupt(); + } + } } } From f8679b26b8704c8d8a290881f3accacb9e8a3d93 Mon Sep 17 00:00:00 2001 From: Peter Hanecak Date: Fri, 16 Feb 2024 13:29:38 +0100 Subject: [PATCH 3/5] use Duration() for http_retry_wait for nicer handling of units --- .../com/onthegomap/planetiler/config/PlanetilerConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 067eec17a4..5657a61702 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -41,7 +41,7 @@ public record PlanetilerConfig( String httpUserAgent, Duration httpTimeout, int httpRetries, - long httpRetryWait, + Duration httpRetryWait, long downloadChunkSizeMB, int downloadThreads, double downloadMaxBandwidth, @@ -167,7 +167,7 @@ public static PlanetilerConfig from(Arguments arguments) { "Planetiler downloader (https://github.com/onthegomap/planetiler)"), arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"), arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1), - arguments.getLong("http_retry_wait", "How long (in milliseconds) to wait before retrying HTTP request", 5_000), + arguments.getDuration("http_retry_wait", "How long (in milliseconds) to wait before retrying HTTP request", "5s"), arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100), arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1), Parse.bandwidth(arguments.getString("download_max_bandwidth", From 01dec42302b30b4d2bd71bf0b62e9cf3683bdb3b Mon Sep 17 00:00:00 2001 From: Peter Hanecak Date: Fri, 16 Feb 2024 13:30:29 +0100 Subject: [PATCH 4/5] do a sleep on all IOException (not just GOAWAY) + no need to catch InterruptedException --- .../java/com/onthegomap/planetiler/util/Wikidata.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java index 04901f3c8b..398713b576 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java @@ -317,14 +317,7 @@ private LongObjectMap> queryWikidata(List qidsToFetch) LOGGER.error("sparql query failed, exhausted retries: " + e); throw e; } - if (e.getMessage() != null && e.getMessage().contains("GOAWAY")) { - try { - LOGGER.debug("GOAWAY received, waiting {} ms to give server some room", config.httpRetryWait()); - Thread.sleep(config.httpRetryWait()); - } catch (InterruptedException e2) { - Thread.currentThread().interrupt(); - } - } + Thread.sleep(config.httpRetryWait()); } } From 7b788137623d796ab1867b0267d0db76c5c05182 Mon Sep 17 00:00:00 2001 From: Peter Hanecak Date: Fri, 16 Feb 2024 13:38:38 +0100 Subject: [PATCH 5/5] clean-up: description of http_retry_wait adjusted --- .../java/com/onthegomap/planetiler/config/PlanetilerConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 5657a61702..c505ce08eb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -167,7 +167,7 @@ public static PlanetilerConfig from(Arguments arguments) { "Planetiler downloader (https://github.com/onthegomap/planetiler)"), arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"), arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1), - arguments.getDuration("http_retry_wait", "How long (in milliseconds) to wait before retrying HTTP request", "5s"), + arguments.getDuration("http_retry_wait", "How long to wait before retrying HTTP request", "5s"), arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100), arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1), Parse.bandwidth(arguments.getString("download_max_bandwidth",