Merge branch 'main' into structured-tag-api

onthegomap · May 24, 2024 · d5f851c · d5f851c
2 parents bde9de7 + 1cd8c98
commit d5f851c
Show file tree

Hide file tree

Showing 9 changed files with 50 additions and 52 deletions.
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -57,10 +57,10 @@ jobs:
     # Skip spotless since that gets checked in a separate task
     - name: Build with mvnw (linux/mac)
       if: ${{ !contains(matrix.os, 'windows') }}
-      run: ./mvnw ${{matrix.args}} -Dspotless.check.skip --batch-mode -no-transfer-progress package verify jib:buildTar --file pom.xml
+      run: ./mvnw ${{matrix.args}} -Dspotless.check.skip --batch-mode -no-transfer-progress verify jib:buildTar --file pom.xml
     - name: Build with mvnw.cmd (windows)
       if: ${{ contains(matrix.os, 'windows') }}
-      run: mvnw.cmd ${{matrix.args}} -Dspotless.check.skip --batch-mode -no-transfer-progress package verify jib:buildTar --file pom.xml
+      run: mvnw.cmd ${{matrix.args}} -Dspotless.check.skip --batch-mode -no-transfer-progress verify jib:buildTar --file pom.xml
       shell: cmd
 
   examples:

diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml
@@ -47,7 +47,7 @@
     <dependency>
       <groupId>org.locationtech.jts</groupId>
       <artifactId>jts-core</artifactId>
-      <version>1.19.0</version>
+      <version>${jts.version}</version>
     </dependency>
     <dependency>
       <groupId>org.tukaani</groupId>

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/GeometryReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/GeometryReader.java
@@ -13,8 +13,9 @@
  * Decodes geometries from a parquet record based on the {@link GeoParquetMetadata} provided.
  */
 class GeometryReader {
+
   private final Map<String, FunctionThatThrows<Object, Geometry>> converters = new HashMap<>();
-  private final String geometryColumn;
+  final String geometryColumn;
 
   GeometryReader(GeoParquetMetadata geoparquet) {
     this.geometryColumn = geoparquet.primaryColumn();
@@ -37,7 +38,6 @@ class GeometryReader {
         case "point", "geoarrow.point" -> GeoArrow::point;
         default -> throw new IllegalArgumentException("Unhandled type: " + columnInfo.encoding());
       };
-
       converters.put(column, converter);
     }
   }
@@ -47,7 +47,10 @@ Geometry readPrimaryGeometry(WithTags tags) throws GeometryException {
   }
 
   Geometry readGeometry(WithTags tags, String column) throws GeometryException {
-    var value = tags.getTag(column);
+    return parseGeometry(tags.getTag(column), column);
+  }
+
+  Geometry parseGeometry(Object value, String column) throws GeometryException {
     var converter = converters.get(column);
     if (value == null) {
       throw new GeometryException("no_parquet_column", "Missing geometry column column " + column);

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetFeature.java
@@ -4,7 +4,6 @@
 import com.onthegomap.planetiler.geo.GeometryException;
 import com.onthegomap.planetiler.reader.SourceFeature;
 import com.onthegomap.planetiler.reader.Struct;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -19,25 +18,21 @@
 public class ParquetFeature extends SourceFeature {
 
   private final GeometryReader geometryParser;
-  private final Path filename;
+  private final Object rawGeometry;
   private Geometry latLon;
   private Geometry world;
   private Struct struct = null;
 
-  ParquetFeature(String source, String sourceLayer, Path filename, long id, GeometryReader geometryParser,
+  ParquetFeature(String source, String sourceLayer, long id, GeometryReader geometryParser,
     Map<String, Object> tags) {
     super(tags, source, sourceLayer, List.of(), id);
     this.geometryParser = geometryParser;
-    this.filename = filename;
-  }
-
-  public Path getFilename() {
-    return filename;
+    this.rawGeometry = tags.remove(geometryParser.geometryColumn);
   }
 
   @Override
   public Geometry latLonGeometry() throws GeometryException {
-    return latLon == null ? latLon = geometryParser.readPrimaryGeometry(this) : latLon;
+    return latLon == null ? latLon = geometryParser.parseGeometry(rawGeometry, geometryParser.geometryColumn) : latLon;
   }
 
   @Override

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFile.java
@@ -50,7 +50,7 @@ public class ParquetInputFile {
   private final String layer;
   private final long count;
   private final int blockCount;
-  private final GeometryReader geometryReader;
+  final GeometryReader geometryReader;
   private final Map<String, Object> extraFields;
   private Envelope postFilterBounds = null;
   private boolean outOfBounds = false;
@@ -177,7 +177,6 @@ public ParquetFeature next() {
               var feature = new ParquetFeature(
                 source,
                 layer,
-                path,
                 idGenerator != null ? idGenerator.applyAsLong(item) :
                   Hashing.fnv1a64(blockHash, ByteBuffer.allocate(8).putLong(i).array()),
                 geometryReader,

diff --git a/...etiler-core/src/main/java/com/onthegomap/planetiler/util/SeekableInMemoryByteChannel.java b/...etiler-core/src/main/java/com/onthegomap/planetiler/util/SeekableInMemoryByteChannel.java
@@ -24,6 +24,7 @@
 import java.nio.channels.SeekableByteChannel;
 import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.concurrent.NotThreadSafe;
 
 /**
  * A {@link SeekableByteChannel} implementation that wraps a byte[].
@@ -38,8 +39,8 @@
  * </p>
  *
  * @since 1.13
- * @NotThreadSafe
  */
+@NotThreadSafe
 public class SeekableInMemoryByteChannel implements SeekableByteChannel {
 
   private static final int NAIVE_RESIZE_LIMIT = Integer.MAX_VALUE >> 1;

diff --git a/...ler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFileTest.java b/...ler-core/src/test/java/com/onthegomap/planetiler/reader/parquet/ParquetInputFileTest.java
@@ -1,8 +1,6 @@
 package com.onthegomap.planetiler.reader.parquet;
 
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.*;
 import static org.junit.jupiter.api.DynamicTest.dynamicTest;
 
 import com.onthegomap.planetiler.TestUtils;
@@ -37,10 +35,12 @@ static List<Path> bostons() {
   void testReadBoston(Path path) {
     for (int i = 0; i < 3; i++) {
       Set<Object> ids = new HashSet<>();
-      for (var block : new ParquetInputFile("parquet", "layer", path)
-        .get()) {
+      var file = new ParquetInputFile("parquet", "layer", path);
+      for (var block : file.get()) {
         for (var item : block) {
           ids.add(item.getString("id"));
+          assertFalse(item.hasTag(file.geometryReader.geometryColumn));
+          assertNull(item.getTag(file.geometryReader.geometryColumn));
         }
       }
       assertEquals(3, ids.size(), "iter " + i);

diff --git a/planetiler-dist/pom.xml b/planetiler-dist/pom.xml
@@ -48,6 +48,19 @@
       <artifactId>planetiler-examples</artifactId>
       <version>${project.parent.version}</version>
     </dependency>
+    <!-- Add sources to the jar to improve IDE experience when using jar with deps -->
+    <dependency>
+      <groupId>com.onthegomap.planetiler</groupId>
+      <artifactId>planetiler-core</artifactId>
+      <version>${project.parent.version}</version>
+      <classifier>sources</classifier>
+    </dependency>
+    <dependency>
+      <groupId>org.locationtech.jts</groupId>
+      <artifactId>jts-core</artifactId>
+      <version>${jts.version}</version>
+      <classifier>sources</classifier>
+    </dependency>
   </dependencies>
 
   <build>

diff --git a/pom.xml b/pom.xml
@@ -22,6 +22,7 @@
     <maven.source.excludeResources>true</maven.source.excludeResources>
     <jackson.version>2.17.1</jackson.version>
     <junit.version>5.10.2</junit.version>
+    <jts.version>1.19.0</jts.version>
     <sonar.host.url>https://sonarcloud.io</sonar.host.url>
     <sonar.organization>onthegomap</sonar.organization>
     <sonar.projectKey>onthegomap_planetiler</sonar.projectKey>
@@ -262,6 +263,21 @@
           </compilerArgs>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <version>3.3.1</version>
+        <executions>
+          <execution>
+            <id>attach-sources</id>
+            <phase>package</phase>
+            <goals>
+              <goal>jar-no-fork</goal>
+              <goal>test-jar-no-fork</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
 
     <pluginManagement>
@@ -309,22 +325,6 @@
             </execution>
           </executions>
         </plugin>
-
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-source-plugin</artifactId>
-          <version>3.3.1</version>
-          <executions>
-            <execution>
-              <id>attach-sources</id>
-              <phase>package</phase>
-              <goals>
-                <goal>jar-no-fork</goal>
-                <goal>test-jar-no-fork</goal>
-              </goals>
-            </execution>
-          </executions>
-        </plugin>
         <plugin>
           <groupId>org.sonarsource.scanner.maven</groupId>
           <artifactId>sonar-maven-plugin</artifactId>
@@ -405,19 +405,6 @@
               </execution>
             </executions>
           </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-source-plugin</artifactId>
-            <executions>
-              <execution>
-                <id>attach-sources</id>
-                <goals>
-                  <goal>jar-no-fork</goal>
-                  <goal>test-jar-no-fork</goal>
-                </goals>
-              </execution>
-            </executions>
-          </plugin>
         </plugins>
       </build>
     </profile>