Add lz4 compression to planetiler

- lz4 compressed files can be generated with libosmium based tools like osmium or python scripts using pyosmium like pyosmium-up-to-date - lz4 compressed files are bigger (~15-20%) then zlib compressed files but much faster to produce and process - this patch is in use by us for several month without any problems
onthegomap · Nov 16, 2024 · ce8b2c1 · ce8b2c1
1 parent b59c63e
commit ce8b2c1
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 0 deletions.
diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml
@@ -181,6 +181,11 @@
       <artifactId>parquet-floor</artifactId>
       <version>1.48</version>
     </dependency>
+    <dependency>
+      <groupId>org.lz4</groupId>
+      <artifactId>lz4-java</artifactId>
+      <version>1.8.0</version>
+    </dependency>
 
   </dependencies>
 

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java
@@ -24,6 +24,10 @@
 import java.util.zip.Inflater;
 import org.locationtech.jts.geom.Envelope;
 
+import net.jpountz.lz4.LZ4Exception;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+
 /**
  * Converts PBF block data into decoded entities. This class was adapted from Osmosis to expose an iterator over blocks
  * to give more control over the parallelism.
@@ -74,6 +78,20 @@ private static byte[] readBlobContent(Fileformat.Blob blob) {
         throw new FileFormatException("PBF blob contains incomplete compressed data.");
       }
       inflater.end();
+    } else if (blob.hasLz4Data()) {
+      final int decompressedLength = blob.getRawSize();
+      LZ4Factory factory = LZ4Factory.fastestInstance();
+      LZ4FastDecompressor decompressor = factory.fastDecompressor();
+      blobData = new byte[decompressedLength];
+      try {
+        int decompressedLength2 = decompressor.decompress(blob.getLz4Data().toByteArray(), 0, blobData, 0, decompressedLength);
+        if (decompressedLength != decompressedLength) {
+          // System.out.println("uncompressed size mismatch: "+decompressedLength2+" != "+decompressedLength);
+          throw new FileFormatException("Unable to decompress PBF blob. uncompressed size mismatch");
+        }
+      } catch (LZ4Exception e) {
+        throw new FileFormatException("Unable to decompress PBF blob.", e);
+      }
     } else {
       throw new FileFormatException("PBF blob uses unsupported compression, only raw or zlib may be used.");
     }