diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml index 3d0fee0d9f..89258c44df 100644 --- a/planetiler-core/pom.xml +++ b/planetiler-core/pom.xml @@ -181,6 +181,11 @@ parquet-floor 1.48 + + org.lz4 + lz4-java + 1.8.0 + diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java index dc6c925552..ddfad722e1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/PbfDecoder.java @@ -22,6 +22,9 @@ import java.util.function.IntUnaryOperator; import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import net.jpountz.lz4.LZ4Exception; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; import org.locationtech.jts.geom.Envelope; /** @@ -74,8 +77,24 @@ private static byte[] readBlobContent(Fileformat.Blob blob) { throw new FileFormatException("PBF blob contains incomplete compressed data."); } inflater.end(); + } else if (blob.hasLz4Data()) { + final int decompressedLength = blob.getRawSize(); + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4FastDecompressor decompressor = factory.fastDecompressor(); + blobData = new byte[decompressedLength]; + try { + int compressedBytesRead = + decompressor.decompress(blob.getLz4Data().toByteArray(), 0, blobData, 0, decompressedLength); + int compressedBytesExpected = blob.getLz4Data().size(); + if (compressedBytesRead != compressedBytesExpected) { + throw new FileFormatException("Unable to decompress PBF blob. read %d compressed bytes but expected %d" + .formatted(decompressedLength, compressedBytesExpected)); + } + } catch (LZ4Exception e) { + throw new FileFormatException("Unable to decompress PBF blob.", e); + } } else { - throw new FileFormatException("PBF blob uses unsupported compression, only raw or zlib may be used."); + throw new FileFormatException("PBF blob uses unsupported compression, only lz4, zlib, or raw may be used."); } return blobData; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 112190cebb..0be94acf28 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -2088,10 +2088,13 @@ private static TileCompression extractTileCompression(String args) { "--tile-compression=none", "--tile-compression=gzip", "--output-layerstats", - "--max-point-buffer=1" + "--max-point-buffer=1", + "--osm-test-path=monaco-latest.lz4.osm.pbf", }) void testPlanetilerRunner(String args) throws Exception { - Path originalOsm = TestUtils.pathToResource("monaco-latest.osm.pbf"); + var argParsed = Arguments.fromArgs(args.split(" ")); + Path originalOsm = + TestUtils.pathToResource(argParsed.getString("osm-test-path", "osm-test-path", "monaco-latest.osm.pbf")); Path tempOsm = tempDir.resolve("monaco-temp.osm.pbf"); final TileCompression tileCompression = extractTileCompression(args); diff --git a/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf b/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf new file mode 100644 index 0000000000..9d4e0a34f0 Binary files /dev/null and b/planetiler-core/src/test/resources/monaco-latest.lz4.osm.pbf differ