From dccbf788791706ec26b3577b2054173540405193 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 27 Oct 2023 07:14:07 -0400 Subject: [PATCH] http request builder HEAD, closeable executors, HashMap.newHashMap constructor --- .../com/onthegomap/planetiler/VectorTile.java | 2 +- .../planetiler/collection/FeatureGroup.java | 2 +- .../planetiler/reader/GeoPackageReader.java | 2 +- .../planetiler/reader/NaturalEarthReader.java | 2 +- .../planetiler/reader/ShapefileReader.java | 2 +- .../planetiler/reader/osm/PbfDecoder.java | 4 +- .../planetiler/stats/PrometheusStats.java | 2 +- .../planetiler/util/Downloader.java | 2 +- .../onthegomap/planetiler/worker/Worker.java | 50 +++++++++---------- .../render/FeatureRendererTest.java | 2 +- 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index 97964fb721..cc7ca9b844 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -376,7 +376,7 @@ public static List decode(byte[] encoded) { for (VectorTileProto.Tile.Feature feature : layer.getFeaturesList()) { int tagsCount = feature.getTagsCount(); - Map attrs = new HashMap<>(tagsCount / 2); + Map attrs = HashMap.newHashMap(tagsCount / 2); int tagIdx = 0; while (tagIdx < feature.getTagsCount()) { String key = keys.get(feature.getTags(tagIdx++)); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index be4162db80..058b3a4218 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -417,7 +417,7 @@ private VectorTile.Feature decodeVectorTileFeature(SortableFeature entry) { GeometryType geomType = decodeGeomType(geomTypeAndScale); int scale = decodeScale(geomTypeAndScale); int mapSize = unpacker.unpackMapHeader(); - Map attrs = new HashMap<>(mapSize); + Map attrs = HashMap.newHashMap(mapSize); for (int i = 0; i < mapSize; i++) { String key = commonValueStrings.decode(unpacker.unpackInt()); Value v = unpacker.unpackValue(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java index fff8b20cd8..91882acba5 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java @@ -153,7 +153,7 @@ public void readFeatures(Consumer next) throws Exception { Geometry latLonGeom = (transform.isIdentity()) ? featureGeom : JTS.transform(featureGeom, transform); FeatureColumns columns = feature.getColumns(); - SimpleFeature geom = SimpleFeature.create(latLonGeom, new HashMap<>(columns.columnCount()), + SimpleFeature geom = SimpleFeature.create(latLonGeom, HashMap.newHashMap(columns.columnCount()), sourceName, featureName, ++id); for (int i = 0; i < columns.columnCount(); ++i) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java index c8fe0a939b..14ae862a82 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java @@ -177,7 +177,7 @@ public void readFeatures(Consumer next) throws Exception { // create the feature and pass to next stage Geometry latLonGeometry = GeoUtils.WKB_READER.read(geometry); - SimpleFeature readerGeometry = SimpleFeature.create(latLonGeometry, new HashMap<>(column.length - 1), + SimpleFeature readerGeometry = SimpleFeature.create(latLonGeometry, HashMap.newHashMap(column.length - 1), sourceName, table, ++id); for (int c = 0; c < column.length; c++) { if (c != geometryColumn) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java index d48f6f44a0..4532356175 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java @@ -137,7 +137,7 @@ public void readFeatures(Consumer next) throws TransformException latLonGeometry = JTS.transform(source, transformToLatLon); } if (latLonGeometry != null) { - SimpleFeature geom = SimpleFeature.create(latLonGeometry, new HashMap<>(attributeNames.length), + SimpleFeature geom = SimpleFeature.create(latLonGeometry, HashMap.newHashMap(attributeNames.length), sourceName, layer, ++id); for (int i = 1; i < attributeNames.length; i++) { geom.setTag(attributeNames[i], feature.getAttribute(i)); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java index dd4b6976af..dc6c925552 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java @@ -133,7 +133,7 @@ public Iterator iterator() { private Map buildTags(int num, IntUnaryOperator key, IntUnaryOperator value) { if (num > 0) { - Map tags = new HashMap<>(num); + Map tags = HashMap.newHashMap(num); for (int i = 0; i < num; i++) { String k = fieldDecoder.decodeString(key.applyAsInt(i)); String v = fieldDecoder.decodeString(value.applyAsInt(i)); @@ -366,7 +366,7 @@ public OsmElement.Node next() { if (tags == null) { // divide by 2 as key&value, multiply by 2 because of the better approximation - tags = new HashMap<>(Math.max(3, 2 * (nodes.getKeysValsCount() / 2) / nodes.getKeysValsCount())); + tags = HashMap.newHashMap(Math.max(3, 2 * (nodes.getKeysValsCount() / 2) / nodes.getKeysValsCount())); } tags.put(fieldDecoder.decodeString(keyIndex), fieldDecoder.decodeString(valueIndex)); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java index c3b42a81a6..bdf37f0ffb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java @@ -213,7 +213,7 @@ public Map dataErrors() { @Override public void close() { - executor.shutdown(); + executor.close(); push(); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java index a75a707074..5b5dde1742 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java @@ -244,7 +244,7 @@ private CompletableFuture httpHeadFollowRedirects(String url, CompletableFuture httpHead(String url) { return client - .sendAsync(newHttpRequest(url).method("HEAD", HttpRequest.BodyPublishers.noBody()).build(), + .sendAsync(newHttpRequest(url).HEAD().build(), responseInfo -> { int status = responseInfo.statusCode(); Optional location = Optional.empty(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/Worker.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/Worker.java index aec95fe535..c1d78a0d6e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/Worker.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/worker/Worker.java @@ -54,33 +54,33 @@ public Worker(String prefix, Stats stats, int threads, RunnableThatThrows task) public Worker(String prefix, Stats stats, int threads, IntConsumerThatThrows task) { this.prefix = prefix; stats.gauge(prefix + "_threads", threads); - var es = Executors.newFixedThreadPool(threads, new NamedThreadFactory(prefix)); - String parentStage = LogUtil.getStage(); - List> results = new ArrayList<>(); - for (int i = 0; i < threads; i++) { - final int threadId = i; - results.add(CompletableFuture.runAsync(() -> { - LogUtil.setStage(parentStage, prefix); - String id = Thread.currentThread().getName(); - LOGGER.trace("Starting worker"); - try { - long start = System.nanoTime(); - task.accept(threadId); - stats.timers().finishedWorker(prefix, Duration.ofNanos(System.nanoTime() - start)); - } catch (Throwable e) { - System.err.println("Worker " + id + " died"); - // when one worker dies it may close resources causing others to die as well, so only log the first - if (firstWorkerDied.compareAndSet(false, true)) { - e.printStackTrace(); // NOSONAR + try (var es = Executors.newFixedThreadPool(threads, new NamedThreadFactory(prefix))) { + String parentStage = LogUtil.getStage(); + List> results = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + final int threadId = i; + results.add(CompletableFuture.runAsync(() -> { + LogUtil.setStage(parentStage, prefix); + String id = Thread.currentThread().getName(); + LOGGER.trace("Starting worker"); + try { + long start = System.nanoTime(); + task.accept(threadId); + stats.timers().finishedWorker(prefix, Duration.ofNanos(System.nanoTime() - start)); + } catch (Throwable e) { + System.err.println("Worker " + id + " died"); + // when one worker dies it may close resources causing others to die as well, so only log the first + if (firstWorkerDied.compareAndSet(false, true)) { + e.printStackTrace(); // NOSONAR + } + throwFatalException(e); + } finally { + LOGGER.trace("Finished worker"); } - throwFatalException(e); - } finally { - LOGGER.trace("Finished worker"); - } - }, es)); + }, es)); + } + done = joinFutures(results); } - es.shutdown(); - done = joinFutures(results); } /** diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java index fab955013a..549f7569ff 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/render/FeatureRendererTest.java @@ -47,7 +47,7 @@ class FeatureRendererTest { private FeatureCollector collector(Geometry worldGeom) { var latLonGeom = GeoUtils.worldToLatLonCoords(worldGeom); return new FeatureCollector.Factory(config, stats) - .get(SimpleFeature.create(latLonGeom, new HashMap<>(0), null, null, + .get(SimpleFeature.create(latLonGeom, HashMap.newHashMap(0), null, null, 1)); }