Attempt to fix IOException/GOAWAY happening within parseResults() (#818)

onthegomap · Feb 16, 2024 · d703b62 · d703b62
1 parent e9a2bcc
commit d703b62
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 7 deletions.
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
@@ -41,6 +41,7 @@ public record PlanetilerConfig(
   String httpUserAgent,
   Duration httpTimeout,
   int httpRetries,
+  Duration httpRetryWait,
   long downloadChunkSizeMB,
   int downloadThreads,
   double downloadMaxBandwidth,
@@ -166,6 +167,7 @@ public static PlanetilerConfig from(Arguments arguments) {
         "Planetiler downloader (https://github.com/onthegomap/planetiler)"),
       arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"),
       arguments.getInteger("http_retries", "Retries to use when downloading files over HTTP", 1),
+      arguments.getDuration("http_retry_wait", "How long to wait before retrying HTTP request", "5s"),
       arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
       arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
       Parse.bandwidth(arguments.getString("download_max_bandwidth",

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Wikidata.java
@@ -300,10 +300,15 @@ private LongObjectMap<Map<String, String>> queryWikidata(List<Long> qidsToFetch)
       .POST(HttpRequest.BodyPublishers.ofString(query, StandardCharsets.UTF_8))
       .build();
 
-    InputStream response = null;
-    for (int i = 0; i <= config.httpRetries() && response == null; i++) {
+    LongObjectMap<Map<String, String>> result = null;
+    for (int i = 0; i <= config.httpRetries() && result == null; i++) {
       try {
-        response = client.send(request);
+        var response = client.send(request);
+        if (response != null) {
+          try (var bis = new BufferedInputStream(response)) {
+            result = parseResults(bis);
+          }
+        }
       } catch (IOException e) {
         boolean lastTry = i == config.httpRetries();
         if (!lastTry) {
@@ -312,13 +317,12 @@ private LongObjectMap<Map<String, String>> queryWikidata(List<Long> qidsToFetch)
           LOGGER.error("sparql query failed, exhausted retries: " + e);
           throw e;
         }
+        Thread.sleep(config.httpRetryWait());
       }
     }
 
-    if (response != null) {
-      try (var bis = new BufferedInputStream(response)) {
-        return parseResults(bis);
-      }
+    if (result != null) {
+      return result;
     } else {
       throw new IllegalStateException("No response or exception"); // should never happen
     }