diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index e23b1926c5..d07afbea20 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -19,7 +19,6 @@ import com.onthegomap.planetiler.stats.ProcessInfo; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.stats.Timers; -import com.onthegomap.planetiler.stream.StreamArchiveUtils; import com.onthegomap.planetiler.util.AnsiColors; import com.onthegomap.planetiler.util.BuildInfo; import com.onthegomap.planetiler.util.ByteBufferUtil; @@ -683,15 +682,15 @@ public void run() throws Exception { throw new IllegalArgumentException(output.format() + " doesn't support concurrent writes"); } IntStream.range(1, config.tileWriteThreads()) - .mapToObj(index -> StreamArchiveUtils.constructIndexedPath(output.getLocalPath(), index)) + .mapToObj(output::getPathForMultiThreadedWriter) .forEach(p -> { if (!config.append() && (overwrite || config.force())) { FileUtils.delete(p); } - if (config.append() && !Files.exists(p)) { - throw new IllegalArgumentException("indexed file \"" + p + "\" must exist when appending"); - } else if (!config.append() && Files.exists(p)) { - throw new IllegalArgumentException("indexed file \"" + p + "\" must not exist when not appending"); + if (config.append() && !output.exists(p)) { + throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending"); + } else if (!config.append() && output.exists(p)) { + throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending"); } }); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java index 284adfb37c..e30b93581c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java @@ -3,7 +3,10 @@ import static com.onthegomap.planetiler.util.LanguageUtils.nullIfEmpty; import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.stream.StreamArchiveUtils; import com.onthegomap.planetiler.util.FileUtils; +import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URI; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -11,6 +14,7 @@ import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import java.util.stream.Stream; /** * Definition for a tileset, parsed from a URI-like string. @@ -147,7 +151,30 @@ public void delete() { * Returns {@code true} if the archive already exists, {@code false} otherwise. */ public boolean exists() { - return getLocalPath() != null && Files.exists(getLocalPath()); + return exists(getLocalPath()); + } + + /** + * @param p path to the archive + * @return {@code true} if the archive already exists, {@code false} otherwise. + */ + public boolean exists(Path p) { + if (p == null) { + return false; + } + if (format() != Format.FILES) { + return Files.exists(p); + } else { + if (!Files.exists(p)) { + return false; + } + // file-archive exists only if it has any contents + try (Stream paths = Files.list(p)) { + return paths.findAny().isPresent(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } /** @@ -165,6 +192,14 @@ public Arguments applyFallbacks(Arguments arguments) { return Arguments.of(options).orElse(arguments.withPrefix(format.id)); } + public Path getPathForMultiThreadedWriter(int index) { + return switch (format) { + case CSV, TSV, JSON, PROTO, PBF -> StreamArchiveUtils.constructIndexedPath(getLocalPath(), index); + case FILES -> getLocalPath(); + default -> throw new UnsupportedOperationException("not supported by " + format); + }; + } + public enum Format { MBTILES("mbtiles", false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */, @@ -179,7 +214,9 @@ public enum Format { /** identical to {@link Format#PROTO} */ PBF("pbf", true, true), - JSON("json", true, true); + JSON("json", true, true), + + FILES("files", true, true); private final String id; private final boolean supportsAppend; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java index 232308a244..244289d1a8 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java @@ -1,6 +1,8 @@ package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.files.ReadableFilesArchive; +import com.onthegomap.planetiler.files.WriteableFilesArchive; import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; import com.onthegomap.planetiler.pmtiles.WriteablePmtiles; @@ -56,6 +58,7 @@ public static WriteableTileArchive newWriter(TileArchiveConfig archive, Planetil new StreamArchiveConfig(config, options)); case JSON -> WriteableJsonStreamArchive.newWriteToFile(archive.getLocalPath(), new StreamArchiveConfig(config, options)); + case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath()); }; } @@ -73,6 +76,7 @@ public static ReadableTileArchive newReader(TileArchiveConfig archive, Planetile case CSV, TSV -> throw new UnsupportedOperationException("reading CSV is not supported"); case PROTO, PBF -> throw new UnsupportedOperationException("reading PROTO is not supported"); case JSON -> throw new UnsupportedOperationException("reading JSON is not supported"); + case FILES -> ReadableFilesArchive.newReader(archive.getLocalPath()); }; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java new file mode 100644 index 0000000000..9db497a301 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java @@ -0,0 +1,21 @@ +package com.onthegomap.planetiler.files; + +import com.onthegomap.planetiler.geo.TileCoord; +import java.nio.file.Path; +import java.nio.file.Paths; + +final class FilesArchiveUtils { + + static final String PBF_FILE_ENDING = ".pbf"; + + private FilesArchiveUtils() {} + + static Path relativePathFromTileCoord(TileCoord tc) { + return Paths.get(Integer.toString(tc.z()), Integer.toString(tc.x()), + tc.y() + PBF_FILE_ENDING); + } + + static Path absolutePathFromTileCoord(Path basePath, TileCoord tc) { + return basePath.resolve(relativePathFromTileCoord(tc)); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/ReadableFilesArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/ReadableFilesArchive.java new file mode 100644 index 0000000000..16702faee8 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/ReadableFilesArchive.java @@ -0,0 +1,99 @@ +package com.onthegomap.planetiler.files; + +import static com.onthegomap.planetiler.files.FilesArchiveUtils.PBF_FILE_ENDING; +import static com.onthegomap.planetiler.files.FilesArchiveUtils.absolutePathFromTileCoord; + +import com.google.common.base.Preconditions; +import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.stream.Stream; +import org.apache.commons.lang3.math.NumberUtils; + +public class ReadableFilesArchive implements ReadableTileArchive { + + private final Path basePath; + + private ReadableFilesArchive(Path basePath) { + this.basePath = basePath; + Preconditions.checkArgument( + Files.isDirectory(basePath), + "require \"" + basePath + "\" to be an existing directory" + ); + } + + public static ReadableFilesArchive newReader(Path basePath) { + return new ReadableFilesArchive(basePath); + } + + @Override + @SuppressWarnings("java:S1168") // returning null is in sync with other mbtiles/pmtiles implementation + public byte[] getTile(int x, int y, int z) { + final Path absolute = absolutePathFromTileCoord(basePath, TileCoord.ofXYZ(x, y, z)); + if (!Files.exists(absolute)) { + return null; + } + try { + return Files.readAllBytes(absolute); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public CloseableIterator getAllTileCoords() { + + try { + final Stream it = Files.find(basePath, 3, (p, a) -> a.isRegularFile()) + .map(this::mapFileToTileCoord) + .flatMap(Optional::stream); + return CloseableIterator.of(it); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public TileArchiveMetadata metadata() { + return null; + } + + @Override + public void close() { + // nothing to do here + } + + private Optional mapFileToTileCoord(Path path) { + final Path relative = basePath.relativize(path); + if (relative.getNameCount() != 3) { + return Optional.empty(); + } + final int z = NumberUtils.toInt(relative.getName(0).toString(), -1); + if (z < 0) { + return Optional.empty(); + } + final int x = NumberUtils.toInt(relative.getName(1).toString(), -1); + if (x < 0) { + return Optional.empty(); + } + final String yPbf = relative.getName(2).toString(); + int dotIdx = yPbf.indexOf('.'); + if (dotIdx < 1) { + return Optional.empty(); + } + final int y = NumberUtils.toInt(yPbf.substring(0, dotIdx), -1); + if (y < 0) { + return Optional.empty(); + } + if (!PBF_FILE_ENDING.equals(yPbf.substring(dotIdx))) { + return Optional.empty(); + } + return Optional.of(TileCoord.ofXYZ(x, y, z)); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java new file mode 100644 index 0000000000..27343d579c --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java @@ -0,0 +1,92 @@ +package com.onthegomap.planetiler.files; + +import com.google.common.base.Preconditions; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.archive.WriteableTileArchive; +import com.onthegomap.planetiler.geo.TileOrder; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class WriteableFilesArchive implements WriteableTileArchive { + + private final Path basePath; + + private WriteableFilesArchive(Path basePath) { + this.basePath = basePath; + if (!Files.exists(basePath)) { + mkdirs(basePath); + } + Preconditions.checkArgument( + Files.isDirectory(basePath), + "require \"" + basePath + "\" to be a directory" + ); + } + + public static WriteableFilesArchive newWriter(Path basePath) { + return new WriteableFilesArchive(basePath); + } + + @Override + public boolean deduplicates() { + return false; + } + + @Override + public TileOrder tileOrder() { + return TileOrder.TMS; + } + + @Override + public TileWriter newTileWriter() { + return new FilesWriter(basePath); + } + + @Override + public void close() throws IOException { + // nothing to do here + } + + private static void mkdirs(Path p) { + try { + Files.createDirectories(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static class FilesWriter implements TileWriter { + + private final Path basePath; + private Path lastCheckedFolder; + + FilesWriter(Path basePath) { + this.basePath = basePath; + this.lastCheckedFolder = basePath; + } + + @Override + public void write(TileEncodingResult encodingResult) { + + final Path file = FilesArchiveUtils.absolutePathFromTileCoord(basePath, encodingResult.coord()); + final Path folder = file.getParent(); + + // tiny optimization in order to avoid too many unnecessary "folder-exists-checks" (I/O) + if (!lastCheckedFolder.equals(folder) && !Files.exists(folder)) { + mkdirs(folder); + } + lastCheckedFolder = folder; + try { + Files.write(file, encodingResult.tileData()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void close() { + // nothing to do here + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index d1224780f3..e5f5714def 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -15,6 +15,7 @@ import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.files.ReadableFilesArchive; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; @@ -1942,6 +1943,7 @@ private static TileCompression extractTileCompression(String args) { "--output-format=proto", "--output-format=pbf", "--output-format=json", + "--output-format=files", "--tile-compression=none", "--tile-compression=gzip", "--output-layerstats", @@ -1953,7 +1955,18 @@ void testPlanetilerRunner(String args) throws Exception { final TileCompression tileCompression = extractTileCompression(args); final TileArchiveConfig.Format format = extractFormat(args); - final Path output = tempDir.resolve("output." + format.id()); + final String outputUri; + final Path outputPath; + switch (format) { + case FILES -> { + outputPath = tempDir.resolve("output"); + outputUri = outputPath.toString() + "?format=files"; + } + default -> { + outputPath = tempDir.resolve("output." + format.id()); + outputUri = outputPath.toString(); + } + } final ReadableTileArchiveFactory readableTileArchiveFactory = switch (format) { case MBTILES -> Mbtiles::newReadOnlyDatabase; @@ -1962,6 +1975,7 @@ void testPlanetilerRunner(String args) throws Exception { case JSON -> InMemoryStreamArchive::fromJson; case PMTILES -> ReadablePmtiles::newReadFromFile; case PROTO, PBF -> InMemoryStreamArchive::fromProtobuf; + case FILES -> ReadableFilesArchive::newReader; }; @@ -1983,7 +1997,7 @@ public void processFeature(SourceFeature source, FeatureCollector features) { .addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite")) .addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip")) .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null) - .setOutput(output) + .setOutput(outputUri) .run(); // make sure it got deleted after write @@ -1991,7 +2005,7 @@ public void processFeature(SourceFeature source, FeatureCollector features) { assertFalse(Files.exists(tempOsm)); } - try (var db = readableTileArchiveFactory.create(output)) { + try (var db = readableTileArchiveFactory.create(outputPath)) { int features = 0; var tileMap = TestUtils.getTileMap(db, tileCompression); for (var tile : tileMap.values()) { @@ -2022,7 +2036,7 @@ public void processFeature(SourceFeature source, FeatureCollector features) { } } - final Path layerstats = output.resolveSibling(output.getFileName().toString() + ".layerstats.tsv.gz"); + final Path layerstats = outputPath.resolveSibling(outputPath.getFileName().toString() + ".layerstats.tsv.gz"); if (args.contains("--output-layerstats")) { assertTrue(Files.exists(layerstats)); byte[] data = Files.readAllBytes(layerstats); @@ -2063,7 +2077,7 @@ public void processFeature(SourceFeature source, FeatureCollector features) { // ensure tilestats standalone executable produces same output var standaloneLayerstatsOutput = tempDir.resolve("layerstats2.tsv.gz"); - TileSizeStats.main("--input=" + output, "--output=" + standaloneLayerstatsOutput); + TileSizeStats.main("--input=" + outputPath, "--output=" + standaloneLayerstatsOutput); byte[] standaloneData = Files.readAllBytes(standaloneLayerstatsOutput); byte[] standaloneUncompressed = Gzip.gunzip(standaloneData); assertEquals( diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java index 78ff0f06b3..70db6f0d4a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java @@ -1,10 +1,19 @@ package com.onthegomap.planetiler.archive; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; class TileArchiveConfigTest { @@ -33,4 +42,50 @@ void testPmtiles() { assertEquals(TileArchiveConfig.Format.PMTILES, TileArchiveConfig.from("file:///output.mbtiles?format=pmtiles").format()); } + + @ParameterizedTest + @EnumSource(TileArchiveConfig.Format.class) + void testByFormatParam(TileArchiveConfig.Format format) { + final var config = TileArchiveConfig.from("output?format=" + format.id()); + assertEquals(format, config.format()); + assertEquals(TileArchiveConfig.Scheme.FILE, config.scheme()); + assertEquals(Path.of("output").toAbsolutePath(), config.getLocalPath()); + assertEquals(Map.of("format", format.id()), config.options()); + } + + @ParameterizedTest + @EnumSource(TileArchiveConfig.Format.class) + void testGetPathForMultiThreadedWriter(TileArchiveConfig.Format format) { + final var config = TileArchiveConfig.from("output?format=" + format.id()); + if (!format.supportsConcurrentWrites()) { + assertThrows(UnsupportedOperationException.class, () -> config.getPathForMultiThreadedWriter(0)); + assertThrows(UnsupportedOperationException.class, () -> config.getPathForMultiThreadedWriter(1)); + } else { + assertEquals(config.getLocalPath(), config.getPathForMultiThreadedWriter(0)); + final Path p = config.getPathForMultiThreadedWriter(1); + switch (format) { + case FILES -> assertEquals(p, config.getLocalPath()); + default -> assertEquals(config.getLocalPath().getParent().resolve(Paths.get("output1")), p); + } + } + } + + @Test + void testExistsForFilesArchive(@TempDir Path tempDir) throws IOException { + final Path out = tempDir.resolve("outdir"); + final var config = TileArchiveConfig.from(out + "?format=files"); + assertFalse(config.exists()); + Files.createDirectory(out); + assertFalse(config.exists()); + Files.createFile(out.resolve("1")); + } + + @Test + void testExistsForNonFilesArchive(@TempDir Path tempDir) throws IOException { + final Path mbtilesOut = tempDir.resolve("out.mbtiles"); + final var config = TileArchiveConfig.from(mbtilesOut.toString()); + assertFalse(config.exists()); + Files.createFile(mbtilesOut); + assertTrue(config.exists()); + } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java new file mode 100644 index 0000000000..38d3c1dab6 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java @@ -0,0 +1,89 @@ +package com.onthegomap.planetiler.files; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.geo.TileCoord; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ReadableFilesArchiveTest { + + @Test + void testRead(@TempDir Path tempDir) throws IOException { + + final Path outputPath = tempDir.resolve("tiles"); + + final List files = List.of( + outputPath.resolve(Paths.get("0", "0", "0.pbf")), + outputPath.resolve(Paths.get("1", "2", "3.pbf")), + // invalid + outputPath.resolve(Paths.get("9", "9")), + outputPath.resolve(Paths.get("9", "x")), + outputPath.resolve(Paths.get("9", "8", "9")), + outputPath.resolve(Paths.get("9", "8", "9.")), + outputPath.resolve(Paths.get("9", "8", "x.pbf")), + outputPath.resolve(Paths.get("9", "b", "1.pbf")), + outputPath.resolve(Paths.get("a", "8", "1.pbf")), + outputPath.resolve(Paths.get("9", "7.pbf")), + outputPath.resolve(Paths.get("8.pbf")) + ); + for (int i = 0; i < files.size(); i++) { + final Path file = files.get(i); + Files.createDirectories(file.getParent()); + Files.write(files.get(i), new byte[]{(byte) i}); + } + + try (var reader = ReadableFilesArchive.newReader(outputPath)) { + final List tiles = reader.getAllTiles().stream().sorted().toList(); + assertEquals( + List.of( + new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), + new Tile(TileCoord.ofXYZ(2, 3, 1), new byte[]{1}) + ), + tiles + ); + } + } + + @Test + void testGetTileNotExists(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + Files.createDirectories(outputPath); + try (var reader = ReadableFilesArchive.newReader(outputPath)) { + assertNull(reader.getTile(0, 0, 0)); + } + } + + @Test + void testFailsToReadTileFromDir(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + Files.createDirectories(outputPath.resolve(Paths.get("0", "0", "0.pbf"))); + try (var reader = ReadableFilesArchive.newReader(outputPath)) { + assertThrows(UncheckedIOException.class, () -> reader.getTile(0, 0, 0)); + } + } + + @Test + void testRequiresExistingPath(@TempDir Path tempDir) { + final Path outputPath = tempDir.resolve("tiles"); + assertThrows(IllegalArgumentException.class, () -> ReadableFilesArchive.newReader(outputPath)); + } + + @Test + void testHasNoMetaData(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + Files.createDirectories(outputPath); + try (var reader = ReadableFilesArchive.newReader(outputPath)) { + assertNull(reader.metadata()); + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java new file mode 100644 index 0000000000..d3462c2326 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java @@ -0,0 +1,109 @@ +package com.onthegomap.planetiler.files; + +import static org.junit.jupiter.api.Assertions.*; + +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.geo.TileOrder; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.OptionalLong; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class WriteableFilesArchiveTest { + + @Test + void testWrite(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + try (var archive = WriteableFilesArchive.newWriter(outputPath)) { + try (var tileWriter = archive.newTileWriter()) { + tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty())); + tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1))); + tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 3, 3), new byte[]{2}, OptionalLong.of(2))); + tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 3, 4), new byte[]{3}, OptionalLong.of(3))); + } + } + + try (Stream s = Files.find(outputPath, 100, (p, attrs) -> attrs.isRegularFile())) { + final List filesInDir = s.sorted().toList(); + assertEquals( + List.of( + Paths.get("0", "0", "0.pbf"), + Paths.get("3", "1", "2.pbf"), + Paths.get("3", "1", "3.pbf"), + Paths.get("4", "1", "3.pbf") + ), + filesInDir.stream().map(outputPath::relativize).toList() + ); + assertArrayEquals(new byte[]{0}, Files.readAllBytes(filesInDir.get(0))); + assertArrayEquals(new byte[]{1}, Files.readAllBytes(filesInDir.get(1))); + assertArrayEquals(new byte[]{2}, Files.readAllBytes(filesInDir.get(2))); + assertArrayEquals(new byte[]{3}, Files.readAllBytes(filesInDir.get(3))); + } + } + + @Test + void testCreatesPathIfNotExists(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + try (var archive = WriteableFilesArchive.newWriter(outputPath)) { + try (var writer = archive.newTileWriter()) { + writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty())); + } + } + assertTrue(Files.isRegularFile(outputPath.resolve(Paths.get("0", "0", "0.pbf")))); + } + + @Test + void testFailsIfBasePathIsNoDirectory(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + Files.createFile(outputPath); + assertThrows(IllegalArgumentException.class, () -> WriteableFilesArchive.newWriter(outputPath)); + } + + @Test + void testFailsIfTileExistsAsDir(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + final Path tileAsDirPath = outputPath.resolve(Paths.get("0", "0", "0.pbf")); + Files.createDirectories(tileAsDirPath); + try (var archive = WriteableFilesArchive.newWriter(outputPath)) { + try (var writer = archive.newTileWriter()) { + assertThrows( + UncheckedIOException.class, + () -> writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty())) + ); + } + } + } + + @Test + void testFailsIfDirExistsAsFile(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + Files.createDirectories(outputPath); + Files.createFile(outputPath.resolve("0")); + try (var archive = WriteableFilesArchive.newWriter(outputPath)) { + try (var writer = archive.newTileWriter()) { + assertThrows( + UncheckedIOException.class, + () -> writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty())) + ); + } + } + } + + @Test + void testSettings(@TempDir Path tempDir) throws IOException { + final Path outputPath = tempDir.resolve("tiles"); + Files.createDirectories(outputPath); + try (var archive = WriteableFilesArchive.newWriter(outputPath)) { + assertFalse(archive.deduplicates()); + assertEquals(TileOrder.TMS, archive.tileOrder()); + + } + } +}