From ce8b2c1d31af9e319351772114fd85b565d5ff06 Mon Sep 17 00:00:00 2001
From: Flynn Marquardt <flynn.marquardt@movisda.de>
Date: Sat, 16 Nov 2024 16:11:19 +0100
Subject: [PATCH] Add lz4 compression to planetiler - lz4 compressed files can
 be generated with libosmium based tools like osmium or python scripts using
 pyosmium like pyosmium-up-to-date - lz4 compressed files are bigger (~15-20%)
 then zlib compressed files but much faster to produce and process - this
 patch is in use by us for several month without any problems

---
 planetiler-core/pom.xml                        |  5 +++++
 .../planetiler/reader/osm/PbfDecoder.java      | 18 ++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml
index ed9403936f..cbfcadf738 100644
--- a/planetiler-core/pom.xml
+++ b/planetiler-core/pom.xml
@@ -181,6 +181,11 @@
       <artifactId>parquet-floor</artifactId>
       <version>1.48</version>
     </dependency>
+    <dependency>
+      <groupId>org.lz4</groupId>
+      <artifactId>lz4-java</artifactId>
+      <version>1.8.0</version>
+    </dependency>
 
   </dependencies>
 
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java
index dc6c925552..48203a4b6d 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java
@@ -24,6 +24,10 @@
 import java.util.zip.Inflater;
 import org.locationtech.jts.geom.Envelope;
 
+import net.jpountz.lz4.LZ4Exception;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+
 /**
  * Converts PBF block data into decoded entities. This class was adapted from Osmosis to expose an iterator over blocks
  * to give more control over the parallelism.
@@ -74,6 +78,20 @@ private static byte[] readBlobContent(Fileformat.Blob blob) {
         throw new FileFormatException("PBF blob contains incomplete compressed data.");
       }
       inflater.end();
+    } else if (blob.hasLz4Data()) {
+      final int decompressedLength = blob.getRawSize();
+      LZ4Factory factory = LZ4Factory.fastestInstance();
+      LZ4FastDecompressor decompressor = factory.fastDecompressor();
+      blobData = new byte[decompressedLength];
+      try {
+        int decompressedLength2 = decompressor.decompress(blob.getLz4Data().toByteArray(), 0, blobData, 0, decompressedLength);
+        if (decompressedLength != decompressedLength) {
+          // System.out.println("uncompressed size mismatch: "+decompressedLength2+" != "+decompressedLength);
+          throw new FileFormatException("Unable to decompress PBF blob. uncompressed size mismatch");
+        }
+      } catch (LZ4Exception e) {
+        throw new FileFormatException("Unable to decompress PBF blob.", e);
+      }
     } else {
       throw new FileFormatException("PBF blob uses unsupported compression, only raw or zlib may be used.");
     }