diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/collection/BenchmarkKWayMerge.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/collection/BenchmarkKWayMerge.java index 6ad42302fb..f8983efef0 100644 --- a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/collection/BenchmarkKWayMerge.java +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/collection/BenchmarkKWayMerge.java @@ -1,5 +1,6 @@ package com.onthegomap.planetiler.collection; +import java.nio.ByteBuffer; import java.time.Duration; import java.util.PriorityQueue; import java.util.Random; @@ -7,15 +8,16 @@ import java.util.stream.IntStream; /** - * Performance tests for {@link LongMinHeap} implementations. + * Performance tests for {@link SortableFeatureMinHeap} implementations. * * Times how long it takes to merge N sorted lists of random elements. */ public class BenchmarkKWayMerge { + public static void main(String[] args) { for (int i = 0; i < 4; i++) { System.err.println(); - testMinHeap("quaternary", LongMinHeap::newArrayHeap); + testMinHeap("quaternary", SortableFeatureMinHeap::newArrayHeap); System.err.println(String.join("\t", "priorityqueue", Long.toString(testPriorityQueue(10).toMillis()), @@ -25,7 +27,7 @@ public static void main(String[] args) { } } - private static void testMinHeap(String name, IntFunction constructor) { + private static void testMinHeap(String name, IntFunction constructor) { System.err.println(String.join("\t", name, Long.toString(testUpdates(10, constructor).toMillis()), @@ -36,20 +38,28 @@ private static void testMinHeap(String name, IntFunction constructo private static final Random random = new Random(); - private static long[][] getVals(int size) { + final static ByteBuffer byteBuffer = ByteBuffer.allocate(Long.BYTES); + + private static SortableFeature newVal(long i) { + byteBuffer.clear().putLong(i); + return new SortableFeature(i, byteBuffer.array()); + } + + private static SortableFeature[][] getVals(int size) { int num = 10_000_000; return IntStream.range(0, size) .mapToObj(i -> random .longs(0, 1_000_000_000) .limit(num / size) .sorted() - .toArray() - ).toArray(long[][]::new); + .mapToObj(BenchmarkKWayMerge::newVal) + .toArray(SortableFeature[]::new) + ).toArray(SortableFeature[][]::new); } - private static Duration testUpdates(int size, IntFunction heapFn) { + private static Duration testUpdates(int size, IntFunction heapFn) { int[] indexes = new int[size]; - long[][] vals = getVals(size); + SortableFeature[][] vals = getVals(size); var heap = heapFn.apply(size); for (int i = 0; i < size; i++) { heap.push(i, vals[i][indexes[i]++]); @@ -58,7 +68,7 @@ private static Duration testUpdates(int size, IntFunction heapFn) { while (!heap.isEmpty()) { int id = heap.peekId(); int index = indexes[id]++; - long[] valList = vals[id]; + SortableFeature[] valList = vals[id]; if (index < valList.length) { heap.updateHead(valList[index]); } else { @@ -68,52 +78,24 @@ private static Duration testUpdates(int size, IntFunction heapFn) { return Duration.ofNanos(System.nanoTime() - start); } - static class Item implements Comparable { - long value; - int id; - - @Override - public int compareTo(Item o) { - return Long.compare(value, o.value); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Item item = (Item) o; - - return value == item.value; - } - - @Override - public int hashCode() { - return (int) (value ^ (value >>> 32)); - } - } - private static Duration testPriorityQueue(int size) { - long[][] vals = getVals(size); + SortableFeature[][] vals = getVals(size); int[] indexes = new int[size]; - PriorityQueue heap = new PriorityQueue<>(); + PriorityQueue heap = new PriorityQueue<>(); for (int i = 0; i < size; i++) { - Item item = new Item(); - item.id = i; - item.value = vals[i][indexes[i]++]; + byteBuffer.clear().putLong(i); + SortableFeature temp = vals[i][indexes[i]++]; + SortableFeature item = new SortableFeature(temp.key(), byteBuffer.array()); heap.offer(item); } var start = System.nanoTime(); while (!heap.isEmpty()) { var item = heap.poll(); - int index = indexes[item.id]++; - long[] valList = vals[item.id]; + int id = (int) byteBuffer.clear().put(item.value()).rewind().getLong(); + int index = indexes[id]++; + SortableFeature[] valList = vals[id]; if (index < valList.length) { - item.value = valList[index]; + item = valList[index]; heap.offer(item); } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index df7b02549b..36bc00515d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -28,10 +28,10 @@ import com.onthegomap.planetiler.geo.MutableCoordinateSequence; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.concurrent.NotThreadSafe; @@ -366,8 +366,7 @@ public static List decode(byte[] encoded) { } for (VectorTileProto.Tile.Feature feature : layer.getFeaturesList()) { - int tagsCount = feature.getTagsCount(); - Map attrs = new HashMap<>(tagsCount / 2); + Map attrs = new TreeMap<>(); int tagIdx = 0; while (tagIdx < feature.getTagsCount()) { String key = keys.get(feature.getTags(tagIdx++)); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongMinHeap.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArraySortableFeatureMinHeap.java similarity index 74% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongMinHeap.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArraySortableFeatureMinHeap.java index 176ea023e1..b6cf5b1c47 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongMinHeap.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArraySortableFeatureMinHeap.java @@ -36,11 +36,12 @@ * * @see d-ary heap (wikipedia) */ -class ArrayLongMinHeap implements LongMinHeap { +class ArraySortableFeatureMinHeap implements SortableFeatureMinHeap { protected static final int NOT_PRESENT = -1; protected final int[] tree; protected final int[] positions; protected final long[] vals; + protected final SortableFeature[] sortableFeatures; protected final int max; protected int size; @@ -48,13 +49,14 @@ class ArrayLongMinHeap implements LongMinHeap { * @param elements the number of elements that can be stored in this heap. Currently the heap cannot be resized or * shrunk/trimmed after initial creation. elements-1 is the maximum id that can be stored in this heap */ - ArrayLongMinHeap(int elements) { + ArraySortableFeatureMinHeap(int elements) { // we use an offset of one to make the arithmetic a bit simpler/more efficient, the 0th elements are not used! tree = new int[elements + 1]; positions = new int[elements + 1]; Arrays.fill(positions, NOT_PRESENT); vals = new long[elements + 1]; vals[0] = Long.MIN_VALUE; + sortableFeatures = new SortableFeature[elements + 1]; this.max = elements; } @@ -77,7 +79,7 @@ public boolean isEmpty() { } @Override - public void push(int id, long value) { + public void push(int id, SortableFeature sf) { checkIdInRange(id); if (size == max) { throw new IllegalStateException("Cannot push anymore, the heap is already full. size: " + size); @@ -89,7 +91,8 @@ public void push(int id, long value) { size++; tree[size] = id; positions[id] = size; - vals[size] = value; + vals[size] = sf.key(); + sortableFeatures[size] = sf; percolateUp(size); } @@ -100,7 +103,7 @@ public boolean contains(int id) { } @Override - public void update(int id, long value) { + public void update(int id, SortableFeature sf) { checkIdInRange(id); int index = positions[id]; if (index < 0) { @@ -108,17 +111,31 @@ public void update(int id, long value) { "The heap does not contain: " + id + ". Use the contains method to check this before calling update"); } long prev = vals[index]; + long value = sf.key(); vals[index] = value; if (value > prev) { + sortableFeatures[index] = sf; percolateDown(index); } else if (value < prev) { + sortableFeatures[index] = sf; percolateUp(index); + } else { + byte[] bytes = sf.value(); + byte[] prevBytes = sortableFeatures[index].value(); + sortableFeatures[index] = sf; + int compareResult = Arrays.compare(bytes, prevBytes); + if (compareResult > 0) { + percolateDown(index); + } else { + percolateUp(index); + } } } @Override - public void updateHead(long value) { - vals[1] = value; + public void updateHead(SortableFeature sf) { + vals[1] = sf.key(); + sortableFeatures[1] = sf; percolateDown(1); } @@ -128,8 +145,8 @@ public int peekId() { } @Override - public long peekValue() { - return vals[1]; + public SortableFeature peekValue() { + return sortableFeatures[1]; } @Override @@ -137,6 +154,8 @@ public int poll() { int id = peekId(); tree[1] = tree[size]; vals[1] = vals[size]; + sortableFeatures[1] = sortableFeatures[size]; + sortableFeatures[size] = null; positions[tree[1]] = 1; positions[id] = NOT_PRESENT; size--; @@ -152,6 +171,26 @@ public void clear() { size = 0; } + private void switchSortableFeatures(int index1, int index2) { + final SortableFeature temp = sortableFeatures[index1]; + sortableFeatures[index1] = sortableFeatures[index2]; + sortableFeatures[index2] = temp; + } + + private byte[] getValue(SortableFeature sf) { + if (sf == null) { + return null; + } + return sf.value(); + } + + private boolean isLessThanParent(int index, int parent, long val, long parentValue) { + if (val == parentValue) { + return Arrays.compare(getValue(sortableFeatures[index]), getValue(sortableFeatures[parent])) < 0; + } + return val < parentValue; + } + private void percolateUp(int index) { assert index != 0; if (index == 1) { @@ -162,8 +201,9 @@ private void percolateUp(int index) { // the finish condition (index==0) is covered here automatically because we set vals[0]=-inf int parent; long parentValue; - while (val < (parentValue = vals[parent = parent(index)])) { + while (isLessThanParent(index, parent = parent(index), val, parentValue = vals[parent])) { vals[index] = parentValue; + switchSortableFeatures(index, parent); positions[tree[index] = tree[parent]] = index; index = parent; } @@ -208,10 +248,14 @@ private void percolateDown(int index) { } } } - if (minValue >= val) { + if (minValue > val) { + break; + } else if (minValue == val && + Arrays.compare(sortableFeatures[minChild].value(), sortableFeatures[index].value()) >= 0) { break; } vals[index] = minValue; + switchSortableFeatures(index, minChild); positions[tree[index] = tree[minChild]] = index; index = minChild; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java index 43e8c8e8a7..0a15778b5b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java @@ -254,7 +254,7 @@ public Iterator iterator(int shard, int shards) { } } - return LongMerger.mergeIterators(iterators); + return SortableFeatureMerger.mergeIterators(iterators); } public int chunks() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index 46892bf13b..da4c58d272 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -21,11 +21,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.Function; @@ -421,7 +421,7 @@ private VectorTile.Feature decodeVectorTileFeature(SortableFeature entry) { GeometryType geomType = decodeGeomType(geomTypeAndScale); int scale = decodeScale(geomTypeAndScale); int mapSize = unpacker.unpackMapHeader(); - Map attrs = new HashMap<>(mapSize); + Map attrs = new TreeMap<>(); for (int i = 0; i < mapSize; i++) { String key = commonValueStrings.decode(unpacker.unpackInt()); Value v = unpacker.unpackValue(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureSort.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureSort.java index b9fbeb3870..62dd890a3d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureSort.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureSort.java @@ -131,7 +131,7 @@ default ParallelIterator parallelIterator(Stats stats, int threads) { } } }); - return new ParallelIterator(reader, LongMerger.mergeSuppliers(queues)); + return new ParallelIterator(reader, SortableFeatureMerger.mergeSuppliers(queues)); } record ParallelIterator(Worker reader, @Override Iterator iterator) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/HasLongSortKey.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/HasLongSortKey.java index 8482d43556..54ccffaf55 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/HasLongSortKey.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/HasLongSortKey.java @@ -4,7 +4,7 @@ * An item with a {@code long key} that can be used for sorting/grouping. * * These items can be sorted or grouped by {@link FeatureSort}/{@link FeatureGroup} implementations. Sorted lists can - * also be merged using {@link LongMerger}. + * also be merged using {@link SortableFeatureMerger}. */ public interface HasLongSortKey { /** Value to sort/group items by. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeature.java index e5df6c484f..918096c0b8 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeature.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeature.java @@ -6,7 +6,11 @@ public record SortableFeature(@Override long key, byte[] value) implements Compa @Override public int compareTo(SortableFeature o) { - return Long.compare(key, o.key); + int result = Long.compare(key, o.key); + if (result == 0) { + result = Arrays.compare(value, o.value); + } + return result; } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongMerger.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeatureMerger.java similarity index 85% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongMerger.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeatureMerger.java index 16bcfa9dcf..a509af1c53 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongMerger.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeatureMerger.java @@ -9,19 +9,19 @@ /** * A utility for merging sorted lists of items with a {@code long} key to sort by. */ -public class LongMerger { +public class SortableFeatureMerger { // Has a general-purpose KWayMerge implementation using a min heap and specialized (faster) // TwoWayMerge/ThreeWayMerge implementations when a small number of lists are being merged. - private LongMerger() {} + private SortableFeatureMerger() {} /** Merges sorted items from {@link Supplier Suppliers} that return {@code null} when there are no items left. */ - public static Iterator mergeSuppliers(List> suppliers) { + public static Iterator mergeSuppliers(List> suppliers) { return mergeIterators(suppliers.stream().map(SupplierIterator::new).toList()); } /** Merges sorted iterators into a combined iterator over all the items. */ - public static Iterator mergeIterators(List> iterators) { + public static Iterator mergeIterators(List> iterators) { return switch (iterators.size()) { case 0 -> Collections.emptyIterator(); case 1 -> iterators.get(0); @@ -31,7 +31,7 @@ public static Iterator mergeIterators(List implements Iterator { + private static class TwoWayMerge implements Iterator { T a, b; long ak = Long.MAX_VALUE, bk = Long.MAX_VALUE; final Iterator inputA, inputB; @@ -82,7 +82,7 @@ public T next() { } } - private static class ThreeWayMerge implements Iterator { + private static class ThreeWayMerge implements Iterator { T a, b, c; long ak = Long.MAX_VALUE, bk = Long.MAX_VALUE, ck = Long.MAX_VALUE; final Iterator inputA, inputB, inputC; @@ -163,23 +163,23 @@ public T next() { } } - private static class KWayMerge implements Iterator { + private static class KWayMerge implements Iterator { private final T[] items; private final Iterator[] iterators; - private final LongMinHeap heap; + private final SortableFeatureMinHeap heap; @SuppressWarnings("unchecked") KWayMerge(List> inputIterators) { this.iterators = new Iterator[inputIterators.size()]; - this.items = (T[]) new HasLongSortKey[inputIterators.size()]; - this.heap = LongMinHeap.newArrayHeap(inputIterators.size()); + this.items = (T[]) new SortableFeature[inputIterators.size()]; + this.heap = SortableFeatureMinHeap.newArrayHeap(inputIterators.size()); int outIdx = 0; for (Iterator iter : inputIterators) { if (iter.hasNext()) { var item = iter.next(); items[outIdx] = item; iterators[outIdx] = iter; - heap.push(outIdx++, item.key()); + heap.push(outIdx++, item); } } } @@ -200,7 +200,7 @@ public T next() { if (iterator.hasNext()) { T next = iterator.next(); items[id] = next; - heap.updateHead(next.key()); + heap.updateHead(next); } else { items[id] = null; heap.poll(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongMinHeap.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeatureMinHeap.java similarity index 88% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongMinHeap.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeatureMinHeap.java index f29985e995..6bda36acfd 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/LongMinHeap.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeatureMinHeap.java @@ -25,14 +25,14 @@ * "https://github.com/graphhopper/graphhopper/blob/master/core/src/main/java/com/graphhopper/coll/MinHeapWithUpdate.java">GraphHopper * and modified to extract a common interface for subclass implementations. */ -public interface LongMinHeap { +public interface SortableFeatureMinHeap { /** * Returns a new min-heap where each element has 4 children backed by elements in an array. *

* This is slightly faster than a traditional binary min heap due to a shallower, more cache-friendly memory layout. */ - static LongMinHeap newArrayHeap(int elements) { - return new ArrayLongMinHeap(elements); + static SortableFeatureMinHeap newArrayHeap(int elements) { + return new ArraySortableFeatureMinHeap(elements); } int size(); @@ -44,7 +44,7 @@ static LongMinHeap newArrayHeap(int elements) { * push the same id twice (unless it was polled/removed before). To update the value of an id contained in the heap * use the {@link #update} method. */ - void push(int id, long value); + void push(int id, SortableFeature value); /** * @return true if the heap contains an element with the given id @@ -55,20 +55,20 @@ static LongMinHeap newArrayHeap(int elements) { * Updates the element with the given id. The complexity of this method is O(log(N)), just like push/poll. Its illegal * to update elements that are not contained in the heap. Use {@link #contains} to check the existence of an id. */ - void update(int id, long value); + void update(int id, SortableFeature value); /** * Updates the weight of the head element in the heap, pushing it down and bubbling up the new min element if * necessary. */ - void updateHead(long value); + void updateHead(SortableFeature value); /** * @return the id of the next element to be polled, i.e. the same as calling poll() without removing the element */ int peekId(); - long peekValue(); + SortableFeature peekValue(); /** * Extracts the element with minimum value from the heap diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Compare.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Compare.java index 964d231f06..b1f7ae6d52 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Compare.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Compare.java @@ -10,9 +10,9 @@ import com.onthegomap.planetiler.geo.TileCoord; import java.nio.file.Path; import java.sql.SQLException; -import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import org.locationtech.jts.geom.Geometry; import org.slf4j.Logger; @@ -129,7 +129,7 @@ private record VectorTileFeatureForCmp( ) { static VectorTileFeatureForCmp fromActualFeature(VectorTile.Feature f) { try { - var attrs = new HashMap<>(f.attrs()); + var attrs = new TreeMap<>(f.attrs()); attrs.remove("rank"); return new VectorTileFeatureForCmp(f.layer(), f.geometry().decode().norm(), attrs, f.group()); } catch (GeometryException e) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java index 2dd7ed60cb..a56591eeb0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/NaturalEarthReader.java @@ -147,11 +147,14 @@ public WorkerPipeline.SourceStep read() { ResultSet rs = statement.executeQuery("SELECT * FROM %s;".formatted(table)); String[] column = new String[rs.getMetaData().getColumnCount()]; int geometryColumn = -1; + int neIdColumn = -1; for (int c = 0; c < column.length; c++) { String name = rs.getMetaData().getColumnName(c + 1); column[c] = name; if ("GEOMETRY".equals(name)) { geometryColumn = c; + } else if ("ne_id".equals(name)) { + neIdColumn = c; } } if (geometryColumn >= 0) { @@ -161,10 +164,17 @@ public WorkerPipeline.SourceStep read() { continue; } + long neId; + if (neIdColumn >= 0) { + neId = rs.getLong(neIdColumn + 1); + } else { + neId = id++; + } + // create the feature and pass to next stage Geometry latLonGeometry = GeoUtils.WKB_READER.read(geometry); SimpleFeature readerGeometry = SimpleFeature.create(latLonGeometry, new HashMap<>(column.length - 1), - sourceName, table, id); + sourceName, table, neId); for (int c = 0; c < column.length; c++) { if (c != geometryColumn) { Object value = rs.getObject(c + 1); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java index d0a07685ef..2626a9dc9b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java @@ -12,11 +12,10 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; +import java.util.TreeMap; import java.util.function.Consumer; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; @@ -40,8 +39,6 @@ */ public class FeatureRenderer implements Consumer, Closeable { - // generate globally-unique IDs shared by all vector tile features representing the same source feature - private static final AtomicLong idGenerator = new AtomicLong(0); private static final Logger LOGGER = LoggerFactory.getLogger(FeatureRenderer.class); private static final VectorTile.VectorGeometry FILL = VectorTile.encodeGeometry(GeoUtils.JTS_FACTORY .createPolygon(GeoUtils.JTS_FACTORY.createLinearRing(new PackedCoordinateSequence.Double(new double[]{ @@ -95,7 +92,7 @@ private void renderGeometry(Geometry geom, FeatureCollector.Feature feature) { } private void renderPoint(FeatureCollector.Feature feature, Coordinate... origCoords) { - long id = idGenerator.incrementAndGet(); + long id = feature.getSourceId(); boolean hasLabelGrid = feature.hasLabelGrid(); Coordinate[] coords = new Coordinate[origCoords.length]; for (int i = 0; i < origCoords.length; i++) { @@ -173,7 +170,7 @@ private void renderPoint(FeatureCollector.Feature feature, MultiPoint points) { } private void renderLineOrPolygon(FeatureCollector.Feature feature, Geometry input) { - long id = idGenerator.incrementAndGet(); + long id = feature.getSourceId(); boolean area = input instanceof Polygonal; double worldLength = (area || input.getNumGeometries() > 1) ? 0 : input.getLength(); String numPointsAttr = feature.getNumPointsAttr(); @@ -201,7 +198,7 @@ private void renderLineOrPolygon(FeatureCollector.Feature feature, Geometry inpu Map attrs = feature.getAttrsAtZoom(sliced.zoomLevel()); if (numPointsAttr != null) { // if profile wants the original number of points that the simplified but untiled geometry started with - attrs = new HashMap<>(attrs); + attrs = new TreeMap<>(attrs); attrs.put(numPointsAttr, geom.getNumPoints()); } writeTileFeatures(z, id, feature, sliced, attrs); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongMinHeapTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/ArraySortableFeatureMinHeapTest.java similarity index 57% rename from planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongMinHeapTest.java rename to planetiler-core/src/test/java/com/onthegomap/planetiler/collection/ArraySortableFeatureMinHeapTest.java index 7c3d787f8e..08b32aa19f 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongMinHeapTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/ArraySortableFeatureMinHeapTest.java @@ -37,52 +37,60 @@ * and modified to use long instead of float values, use stable random seed for reproducibility, and to use new * implementations. */ -class LongMinHeapTest { +class ArraySortableFeatureMinHeapTest { - protected LongMinHeap heap; + protected SortableFeatureMinHeap heap; void create(int capacity) { - heap = LongMinHeap.newArrayHeap(capacity); + heap = SortableFeatureMinHeap.newArrayHeap(capacity); + } + + private SortableFeature newEntry(long i) { + return new SortableFeature(i, new byte[]{(byte) i, (byte) (1 + i)}); + } + + private SortableFeature newEntry(long i, byte[] v) { + return new SortableFeature(i, v); } @Test void outOfRange() { create(4); - assertThrows(IllegalArgumentException.class, () -> heap.push(4, 12L)); - assertThrows(IllegalArgumentException.class, () -> heap.push(-1, 12L)); + assertThrows(IllegalArgumentException.class, () -> heap.push(4, newEntry(12L))); + assertThrows(IllegalArgumentException.class, () -> heap.push(-1, newEntry(12L))); } @Test void tooManyElements() { create(3); - heap.push(1, 1L); - heap.push(2, 1L); - heap.push(0, 1L); + heap.push(1, newEntry(1L)); + heap.push(2, newEntry(1L)); + heap.push(0, newEntry(1L)); // pushing element 1 again is not allowed (but this is not checked explicitly). however pushing more elements // than 3 is already an error - assertThrows(IllegalStateException.class, () -> heap.push(1, 1L)); - assertThrows(IllegalStateException.class, () -> heap.push(2, 61L)); + assertThrows(IllegalStateException.class, () -> heap.push(1, newEntry(1L))); + assertThrows(IllegalStateException.class, () -> heap.push(2, newEntry(61L))); } @Test void duplicateElements() { create(5); - heap.push(1, 2L); - heap.push(0, 4L); - heap.push(2, 1L); + heap.push(1, newEntry(2L)); + heap.push(0, newEntry(4L)); + heap.push(2, newEntry(1L)); assertEquals(2, heap.poll()); // pushing 2 again is ok because it was polled before - heap.push(2, 6L); + heap.push(2, newEntry(6L)); // but now its not ok to push it again - assertThrows(IllegalStateException.class, () -> heap.push(2, 4L)); + assertThrows(IllegalStateException.class, () -> heap.push(2, newEntry(4L))); } @Test void testContains() { create(4); - heap.push(1, 1L); - heap.push(2, 7L); - heap.push(0, 5L); + heap.push(1, newEntry(1L)); + heap.push(2, newEntry(7L)); + heap.push(0, newEntry(5L)); assertFalse(heap.contains(3)); assertTrue(heap.contains(1)); assertEquals(1, heap.poll()); @@ -92,8 +100,8 @@ void testContains() { @Test void containsAfterClear() { create(4); - heap.push(1, 1L); - heap.push(2, 1L); + heap.push(1, newEntry(1L)); + heap.push(2, newEntry(1L)); assertEquals(2, heap.size()); heap.clear(); assertFalse(heap.contains(0)); @@ -107,9 +115,9 @@ void testSize() { create(10); assertEquals(0, heap.size()); assertTrue(heap.isEmpty()); - heap.push(9, 36L); - heap.push(5, 23L); - heap.push(3, 23L); + heap.push(9, newEntry(36L)); + heap.push(5, newEntry(23L)); + heap.push(3, newEntry(23L)); assertEquals(3, heap.size()); assertFalse(heap.isEmpty()); } @@ -118,17 +126,17 @@ void testSize() { void testClear() { create(5); assertTrue(heap.isEmpty()); - heap.push(3, 12L); - heap.push(4, 3L); + heap.push(3, newEntry(12L)); + heap.push(4, newEntry(3L)); assertEquals(2, heap.size()); heap.clear(); assertTrue(heap.isEmpty()); - heap.push(4, 63L); - heap.push(1, 21L); + heap.push(4, newEntry(63L)); + heap.push(1, newEntry(21L)); assertEquals(2, heap.size()); assertEquals(1, heap.peekId()); - assertEquals(21L, heap.peekValue()); + assertEquals(newEntry(21L), heap.peekValue()); assertEquals(1, heap.poll()); assertEquals(4, heap.poll()); assertTrue(heap.isEmpty()); @@ -138,11 +146,11 @@ void testClear() { void testPush() { create(5); - heap.push(4, 63L); - heap.push(1, 21L); + heap.push(4, newEntry(63L)); + heap.push(1, newEntry(21L)); assertEquals(2, heap.size()); assertEquals(1, heap.peekId()); - assertEquals(21L, heap.peekValue()); + assertEquals(newEntry(21L), heap.peekValue()); assertEquals(1, heap.poll()); assertEquals(4, heap.poll()); assertTrue(heap.isEmpty()); @@ -151,20 +159,20 @@ void testPush() { @Test void testPeek() { create(5); - heap.push(4, -16L); - heap.push(2, 13L); - heap.push(1, -51L); - heap.push(3, 4L); + heap.push(4, newEntry(-16L)); + heap.push(2, newEntry(13L)); + heap.push(1, newEntry(-51L)); + heap.push(3, newEntry(4L)); assertEquals(1, heap.peekId()); - assertEquals(-51L, heap.peekValue()); + assertEquals(newEntry(-51L), heap.peekValue()); } @Test void pushAndPoll() { create(10); - heap.push(9, 36L); - heap.push(5, 23L); - heap.push(3, 23L); + heap.push(9, newEntry(36L)); + heap.push(5, newEntry(23L)); + heap.push(3, newEntry(23L)); assertEquals(3, heap.size()); heap.poll(); assertEquals(2, heap.size()); @@ -176,11 +184,11 @@ void pushAndPoll() { @Test void pollSorted() { create(10); - heap.push(9, 36L); - heap.push(5, 21L); - heap.push(3, 23L); - heap.push(8, 57L); - heap.push(7, 22L); + heap.push(9, newEntry(36L)); + heap.push(5, newEntry(21L)); + heap.push(3, newEntry(23L)); + heap.push(8, newEntry(57L)); + heap.push(7, newEntry(22L)); IntArrayList polled = new IntArrayList(); while (!heap.isEmpty()) { polled.add(heap.poll()); @@ -194,19 +202,19 @@ void poll() { assertTrue(heap.isEmpty()); assertEquals(0, heap.size()); - heap.push(9, 36L); + heap.push(9, newEntry(36L)); assertFalse(heap.isEmpty()); assertEquals(1, heap.size()); - heap.push(5, 21L); + heap.push(5, newEntry(21L)); assertFalse(heap.isEmpty()); assertEquals(2, heap.size()); - heap.push(3, 23L); + heap.push(3, newEntry(23L)); assertFalse(heap.isEmpty()); assertEquals(3, heap.size()); - heap.push(8, 57L); + heap.push(8, newEntry(57L)); assertFalse(heap.isEmpty()); assertEquals(4, heap.size()); @@ -230,9 +238,9 @@ void poll() { @Test void clear() { create(10); - heap.push(9, 36L); - heap.push(5, 21L); - heap.push(3, 23L); + heap.push(9, newEntry(36L)); + heap.push(5, newEntry(21L)); + heap.push(3, newEntry(23L)); heap.clear(); assertTrue(heap.isEmpty()); assertEquals(0, heap.size()); @@ -242,7 +250,7 @@ void clear() { void poll100Ascending() { create(100); for (int i = 1; i < 100; i++) { - heap.push(i, i); + heap.push(i, newEntry(i)); } for (int i = 1; i < 100; i++) { assertEquals(i, heap.poll()); @@ -253,7 +261,7 @@ void poll100Ascending() { void poll100Descending() { create(100); for (int i = 99; i >= 1; i--) { - heap.push(i, i); + heap.push(i, newEntry(i)); } for (int i = 1; i < 100; i++) { assertEquals(i, heap.poll()); @@ -263,16 +271,61 @@ void poll100Descending() { @Test void update() { create(10); - heap.push(9, 36L); - heap.push(5, 21L); - heap.push(3, 23L); - heap.update(3, 1L); + heap.push(9, newEntry(36L)); + heap.push(5, newEntry(21L)); + heap.push(3, newEntry(23L)); + heap.update(3, newEntry(1L)); assertEquals(3, heap.peekId()); - heap.update(3, 100L); + heap.update(3, newEntry(100L)); assertEquals(5, heap.peekId()); - heap.update(9, -13L); + heap.update(9, newEntry(-13L)); assertEquals(9, heap.peekId()); - assertEquals(-13L, heap.peekValue()); + assertEquals(newEntry(-13L), heap.peekValue()); + IntArrayList polled = new IntArrayList(); + while (!heap.isEmpty()) { + polled.add(heap.poll()); + } + assertEquals(IntArrayList.from(9, 5, 3), polled); + } + + @Test + void updateWithEqualKeys() { + create(10); + heap.push(9, newEntry(36L)); + heap.push(5, newEntry(21L, new byte[]{(byte) 1, (byte) 2})); + heap.push(3, newEntry(23L)); + heap.update(3, newEntry(1L)); + assertEquals(3, heap.peekId()); + heap.update(3, newEntry(100L)); + assertEquals(5, heap.peekId()); + // until here same as update() test, now some "key collisions" + + // prepare for hitting entry id=5 with sortable feature which has same ID but different value + heap.update(9, newEntry(21L, new byte[]{(byte) 100, (byte) 200})); + assertEquals(5, heap.peekId()); + + // hit "percolate up", NOT replacing item id=5 + heap.update(9, newEntry(21L, new byte[]{(byte) 10, (byte) 20})); + assertEquals(5, heap.peekId()); + + // hit "percolate down" + heap.update(9, newEntry(21L, new byte[]{(byte) 20, (byte) 30})); + assertEquals(5, heap.peekId()); + + // hit "percolate up", still NOT replacing item id=5 + heap.update(9, newEntry(21L, new byte[]{(byte) 5, (byte) 10})); + assertEquals(5, heap.peekId()); + + // hit "percolate up" one last time, now replacing item id=5 + SortableFeature SF = newEntry(21L, new byte[]{(byte) 0, (byte) 0}); + heap.update(9, SF); + assertEquals(9, heap.peekId()); + assertEquals(SF, heap.peekValue()); + + // and from now on again same as update() test + heap.update(9, newEntry(-13L)); + assertEquals(9, heap.peekId()); + assertEquals(newEntry(-13L), heap.peekValue()); IntArrayList polled = new IntArrayList(); while (!heap.isEmpty()) { polled.add(heap.poll()); @@ -283,14 +336,14 @@ void update() { @Test void updateHead() { create(10); - heap.push(1, 1); - heap.push(2, 2); - heap.push(3, 3); - heap.push(4, 4); - heap.push(5, 5); - heap.updateHead(6); - heap.updateHead(7); - heap.updateHead(8); + heap.push(1, newEntry(1)); + heap.push(2, newEntry(2)); + heap.push(3, newEntry(3)); + heap.push(4, newEntry(4)); + heap.push(5, newEntry(5)); + heap.updateHead(newEntry(6)); + heap.updateHead(newEntry(7)); + heap.updateHead(newEntry(8)); IntArrayList polled = new IntArrayList(); while (!heap.isEmpty()) { @@ -303,21 +356,23 @@ void updateHead() { void randomPushsThenPolls() { Random rnd = new Random(0); int size = 1 + rnd.nextInt(100); - PriorityQueue pq = new PriorityQueue<>(size); + PriorityQueue pq = new PriorityQueue<>(size); create(size); IntSet set = new IntHashSet(); while (pq.size() < size) { int id = rnd.nextInt(size); if (!set.add(id)) continue; - long val = (long) (Long.MAX_VALUE * rnd.nextFloat()); - pq.add(new Entry(id, val)); - heap.push(id, val); + long key = (long) (Long.MAX_VALUE * rnd.nextFloat()); + byte[] value = new byte[]{(byte) id, (byte) (id + 1)}; + SortableFeature sf = new SortableFeature(key, value); + pq.add(sf); + heap.push(id, sf); } while (!pq.isEmpty()) { - Entry entry = pq.poll(); - assertEquals(entry.val, heap.peekValue()); - assertEquals(entry.id, heap.poll()); + SortableFeature entry = pq.poll(); + assertEquals(entry, heap.peekValue()); + assertEquals(entry.value()[0], (byte) heap.poll()); assertEquals(pq.size(), heap.size()); } } @@ -326,7 +381,7 @@ void randomPushsThenPolls() { void randomPushsAndPolls() { Random rnd = new Random(0); int size = 1 + rnd.nextInt(100); - PriorityQueue pq = new PriorityQueue<>(size); + PriorityQueue pq = new PriorityQueue<>(size); create(size); IntSet set = new IntHashSet(); int pushCount = 0; @@ -336,34 +391,21 @@ void randomPushsAndPolls() { int id = rnd.nextInt(size); if (!set.add(id)) continue; - long val = (long) (Long.MAX_VALUE * rnd.nextFloat()); - pq.add(new Entry(id, val)); - heap.push(id, val); + long key = (long) (Long.MAX_VALUE * rnd.nextFloat()); + byte[] value = new byte[]{(byte) id, (byte) (id + 1)}; + SortableFeature sf = new SortableFeature(key, value); + heap.push(id, sf); pushCount++; } else { - Entry entry = pq.poll(); + SortableFeature entry = pq.poll(); assert entry != null; - assertEquals(entry.val, heap.peekValue()); - assertEquals(entry.id, heap.poll()); + assertEquals(entry, heap.peekValue()); + int id = heap.poll(); + assertEquals(entry.value()[0], (byte) id); assertEquals(pq.size(), heap.size()); - set.removeAll(entry.id); + set.removeAll(id); } } assertTrue(pushCount > 0); } - - static class Entry implements Comparable { - int id; - long val; - - public Entry(int id, long val) { - this.id = id; - this.val = val; - } - - @Override - public int compareTo(Entry o) { - return Long.compare(val, o.val); - } - } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java index a3f49da4b6..245c85b70e 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java @@ -116,6 +116,8 @@ void testPutPoints() { put(2, "layer", Map.of("a", 1.5d, "b", "string"), newPoint(5, 6)); put(1, "layer", Map.of("a", 1, "b", 2L), newPoint(1, 2)); put(1, "layer2", Map.of("c", 3d, "d", true), newPoint(3, 4)); + // special case: it will have (as SortableFeature) same key as previous + put(1, "layer2", Map.of("c", 2d, "d", false), newPoint(5, 6)); sorter.sort(); assertEquals(new TreeMap<>(Map.of( 1, new TreeMap<>(Map.of( @@ -123,7 +125,8 @@ void testPutPoints() { new Feature(Map.of("a", 1L, "b", 2L), newPoint(1, 2)) ), "layer2", List.of( - new Feature(Map.of("c", 3d, "d", true), newPoint(3, 4)) + new Feature(Map.of("c", 3d, "d", true), newPoint(3, 4)), + new Feature(Map.of("c", 2d, "d", false), newPoint(5, 6)) ) )), 2, new TreeMap<>(Map.of( "layer", List.of( @@ -192,21 +195,30 @@ void testPutPointsWithSortKey() { void testLimitPoints() { int x = 5, y = 6; putWithGroup( - 1, "layer", Map.of("id", 3), newPoint(x, y), 2, 1, 2 + 1, "layer", Map.of("id", 3), newPoint(x, y), 2, 1, 3 ); putWithGroup( - 1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2 + 1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 3 ); putWithGroup( - 1, "layer", Map.of("id", 1), newPoint(1, 2), 0, 1, 2 + 1, "layer", Map.of("id", 1), newPoint(1, 2), 0, 1, 3 + ); + // special case: it will have (as SortableFeature) same key as id=1 + putWithGroup( + 1, "layer", Map.of("id", 4), newPoint(5, 6), 0, 1, 3 + ); + // special case: it will have (as SortableFeature) same key as id=2 + putWithGroup( + 1, "layer", Map.of("id", 5), newPoint(5, 6), 1, 1, 3 ); sorter.sort(); assertEquals(new TreeMap<>(Map.of( 1, new TreeMap<>(Map.of( "layer", List.of( - // id=3 omitted because past limit + // id=3 and id=5 omitted because past limit // sorted by sortKey ascending new Feature(Map.of("id", 1L), newPoint(1, 2)), + new Feature(Map.of("id", 4L), newPoint(5, 6)), new Feature(Map.of("id", 2L), newPoint(3, 4)) ) )))), getFeatures()); @@ -224,12 +236,22 @@ void testLimitPointsInDifferentGroups() { putWithGroup( 1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2 ); + // special case: it will have (as SortableFeature) same key as id=1 + putWithGroup( + 1, "layer", Map.of("id", 4), newPoint(1, 2), 2, 1, 2 + ); + // special case: it will have (as SortableFeature) same key as id=3 + putWithGroup( + 1, "layer", Map.of("id", 5), newPoint(x, y), 0, 2, 2 + ); sorter.sort(); assertEquals(new TreeMap<>(Map.of( 1, new TreeMap<>(Map.of( "layer", List.of( + // id=4 omitted because past limit (given same key but different/higher value in SortableFeature.value will be pushed down) // ordered by sort key new Feature(Map.of("id", 3L), newPoint(x, y)), + new Feature(Map.of("id", 5L), newPoint(x, y)), new Feature(Map.of("id", 2L), newPoint(3, 4)), new Feature(Map.of("id", 1L), newPoint(1, 2)) ) diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureSortTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureSortTest.java index 2e803efe92..f74477d69a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureSortTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureSortTest.java @@ -26,6 +26,9 @@ private SortableFeature newEntry(int i) { return new SortableFeature(Long.MIN_VALUE + i, new byte[]{(byte) i, (byte) (1 + i)}); } + private SortableFeature newEntry(int id, byte byteSeed) { + return new SortableFeature(Long.MIN_VALUE + id, new byte[]{byteSeed, (byte) (1 + byteSeed)}); + } private FeatureSort newSorter(int workers, int chunkSizeLimit, boolean gzip, boolean mmap) { return new ExternalMergeSort(tmpDir, workers, chunkSizeLimit, gzip, mmap, true, true, config, @@ -68,6 +71,35 @@ void testTwoItemsTwoChunks() { assertEquals(List.of(newEntry(1), newEntry(2)), sorter.toList()); } + @Test + void testFourItemsFourChunks() { + FeatureSort sorter = newSorter(1, 0, false, false); + var writer = sorter.writerForThread(); + writer.accept(newEntry(4)); + writer.accept(newEntry(3)); + writer.accept(newEntry(2)); + writer.accept(newEntry(1)); + sorter.sort(); + assertEquals(List.of(newEntry(1), newEntry(2), newEntry(3), newEntry(4)), sorter.toList()); + } + + @Test + void testFourItemsWithSameKeyDifferentBytesFourChunks() { + int ITEMS = 4; + SortableFeature[] sf = new SortableFeature[ITEMS]; + for (int i = 0; i < ITEMS; i++) { + sf[i] = newEntry(1, (byte) i); + } + + FeatureSort sorter = newSorter(1, 0, false, false); + var writer = sorter.writerForThread(); + for (int i = ITEMS - 1; i >= 0; i--) { + writer.accept(sf[i]); + } + sorter.sort(); + assertEquals(List.of(sf), sorter.toList()); + } + @Test void testTwoWorkers() { FeatureSort sorter = newSorter(2, 0, false, false); @@ -93,6 +125,25 @@ void testTwoWriters() { assertEquals(List.of(newEntry(1), newEntry(2), newEntry(3), newEntry(4)), sorter.toList()); } + @Test + void testTwoWritersItemsWithSameKeyDifferentBytes() { + int ITEMS = 8; + SortableFeature[] sf = new SortableFeature[ITEMS]; + for (int i = 0; i < ITEMS; i++) { + sf[i] = newEntry(1, (byte) i); + } + + FeatureSort sorter = newSorter(2, 0, false, false); + var writer1 = sorter.writerForThread(); + var writer2 = sorter.writerForThread(); + for (int i = ITEMS - 1; i >= 0; i -= 2) { + writer2.accept(sf[i]); + writer1.accept(sf[i - 1]); + } + sorter.sort(); + assertEquals(List.of(sf), sorter.toList()); + } + @Test void testMultipleWritersThatGetCombined() { FeatureSort sorter = newSorter(2, 2_000_000, false, false); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongMergerTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/SortableFeatureMergerTest.java similarity index 64% rename from planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongMergerTest.java rename to planetiler-core/src/test/java/com/onthegomap/planetiler/collection/SortableFeatureMergerTest.java index e0a8f89665..d4372f15fb 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/LongMergerTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/SortableFeatureMergerTest.java @@ -13,20 +13,23 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -class LongMergerTest { - record Item(long key) implements HasLongSortKey {} - record ItemList(List items) {} +class SortableFeatureMergerTest { + record ItemList(List items) {} + + private static SortableFeature newItem(long i) { + return new SortableFeature(i, new byte[]{(byte) i, (byte) (i + 1)}); + } private static ItemList list(long... items) { - return new ItemList(LongStream.of(items).mapToObj(Item::new).toList()); + return new ItemList(LongStream.of(items).mapToObj(i -> newItem(i)).toList()); } - private static List merge(ItemList... lists) { - List list = new ArrayList<>(); - var iter = LongMerger.mergeIterators(Stream.of(lists) + private static List merge(ItemList... lists) { + List list = new ArrayList<>(); + var iter = SortableFeatureMerger.mergeIterators(Stream.of(lists) .map(d -> d.items.iterator()) .toList()); - iter.forEachRemaining(item -> list.add(item.key)); + iter.forEachRemaining(item -> list.add(item)); assertThrows(NoSuchElementException.class, iter::next); return list; } @@ -38,10 +41,10 @@ void testMergeEmpty() { @Test void testMergeSupplier() { - List list = new ArrayList<>(); - var iter = LongMerger.mergeSuppliers(Stream.of(new ItemList[]{list(1, 2)}) + List list = new ArrayList<>(); + var iter = SortableFeatureMerger.mergeSuppliers(Stream.of(new ItemList[]{list(1, 2)}) .map(d -> d.items.iterator()) - .>map(d -> () -> { + .>map(d -> () -> { try { return d.next(); } catch (NoSuchElementException e) { @@ -49,16 +52,16 @@ void testMergeSupplier() { } }) .toList()); - iter.forEachRemaining(item -> list.add(item.key)); + iter.forEachRemaining(item -> list.add(item)); assertThrows(NoSuchElementException.class, iter::next); - assertEquals(List.of(1L, 2L), list); + assertEquals(List.of(newItem(1L), newItem(2L)), list); } @Test void testMerge1() { assertEquals(List.of(), merge(list())); - assertEquals(List.of(1L), merge(list(1))); - assertEquals(List.of(1L, 2L), merge(list(1, 2))); + assertEquals(List.of(newItem(1L)), merge(list(1))); + assertEquals(List.of(newItem(1L), newItem(2L)), merge(list(1, 2))); } @ParameterizedTest @@ -76,11 +79,11 @@ void testMerge2(String a, String b, String output) { var listA = list(parse(a)); var listB = list(parse(b)); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listA, listB) ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listB, listA) ); } @@ -102,32 +105,32 @@ void testMerge3(String a, String b, String c, String output) { var listB = list(parse(b)); var listC = list(parse(c)); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listA, listB, listC), "ABC" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listA, listC, listB), "ACB" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listB, listA, listC), "BAC" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listB, listC, listA), "BCA" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listC, listA, listB), "CAB" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listC, listB, listA), "CBA" ); @@ -152,22 +155,22 @@ void testMerge4(String a, String b, String c, String d, String output) { var listD = list(parse(d)); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listA, listB, listC, listD), "ABCD" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listB, listA, listC, listD), "BACD" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listB, listC, listA, listD), "BCAD" ); assertEquals( - LongStream.of(parse(output)).boxed().toList(), + LongStream.of(parse(output)).boxed().map(l -> newItem(l)).toList(), merge(listB, listC, listD, listA), "BCDA" ); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/SortableFeatureTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/SortableFeatureTest.java new file mode 100644 index 0000000000..4be2bfcc58 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/SortableFeatureTest.java @@ -0,0 +1,51 @@ +package com.onthegomap.planetiler.collection; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class SortableFeatureTest { + + @Test + void testDifferentKeyDifferentValue() { + final SortableFeature SF1 = new SortableFeature(0, new byte[]{0, 1}); + final SortableFeature SF2 = new SortableFeature(1, new byte[]{2, 3}); + assertNotEquals(SF1, SF2); + assertTrue(SF1.compareTo(SF2) < 0); + assertFalse(SF1.compareTo(SF2) == 0); + assertFalse(SF2.compareTo(SF1) < 0); + } + + @Test + void testDifferentKeySameValue() { + final SortableFeature SF1 = new SortableFeature(0, new byte[]{0, 1}); + final SortableFeature SF2 = new SortableFeature(1, new byte[]{0, 1}); + assertNotEquals(SF1, SF2); + assertTrue(SF1.compareTo(SF2) < 0); + assertFalse(SF1.compareTo(SF2) == 0); + assertFalse(SF2.compareTo(SF1) < 0); + } + + @Test + void testSameKeyDifferentValue() { + final SortableFeature SF1 = new SortableFeature(0, new byte[]{0, 1}); + final SortableFeature SF2 = new SortableFeature(0, new byte[]{2, 3}); + assertNotEquals(SF1, SF2); + assertTrue(SF1.compareTo(SF2) < 0); + assertFalse(SF1.compareTo(SF2) == 0); + assertFalse(SF2.compareTo(SF1) < 0); + } + + @Test + void testSameKeySameValue() { + final SortableFeature SF1 = new SortableFeature(0, new byte[]{0, 1}); + final SortableFeature SF2 = new SortableFeature(0, new byte[]{0, 1}); + assertEquals(SF1, SF2); + assertFalse(SF1.compareTo(SF2) < 0); + assertTrue(SF1.compareTo(SF2) == 0); + assertFalse(SF2.compareTo(SF1) < 0); + } +}