simplify files archive usage

1. allow to pass tile scheme directly via output: --output=tiles/{x}/{y}/{z}.pbf
2. auto-encode { (%7B) and } (%7D) => no need to encode it the URI on CLI

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -718,7 +718,7 @@ public void run() throws Exception {
     // in case any temp files are left from a previous run...
     FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath);
     Files.createDirectories(tmpDir);
-    FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalPath());
+    FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalBasePath());
 
     if (!toDownload.isEmpty()) {
       download();

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java

@@ -3,6 +3,7 @@
 import static com.onthegomap.planetiler.util.LanguageUtils.nullIfEmpty;
 
 import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.files.FilesArchiveUtils;
 import com.onthegomap.planetiler.stream.StreamArchiveUtils;
 import com.onthegomap.planetiler.util.FileUtils;
 import java.io.IOException;
@@ -43,6 +44,12 @@ public record TileArchiveConfig(
   Map<String, String> options
 ) {
 
+  // be more generous and encode some characters for the users
+  private static final Map<String, String> URI_ENCODINGS = Map.of(
+    "{", "%7B",
+    "}", "%7D"
+  );
+
   private static TileArchiveConfig.Scheme getScheme(URI uri) {
     String scheme = uri.getScheme();
     if (scheme == null) {
@@ -81,21 +88,20 @@ private static Map<String, String> parseQuery(URI uri) {
 
   private static TileArchiveConfig.Format getFormat(URI uri) {
     String format = parseQuery(uri).get("format");
-    if (format == null && uri.getPath().endsWith("/")) {
-      return TileArchiveConfig.Format.FILES; // no format query param and ends with / => assume files - regardless of the extension
-    }
-    if (format == null) {
-      format = getExtension(uri);
+    for (var value : TileArchiveConfig.Format.values()) {
+      if (value.isQueryFormatSupported(format)) {
+        return value;
+      }
     }
-    if (format == null) {
-      return TileArchiveConfig.Format.FILES; // no extension => assume files
+    if (format != null) {
+      throw new IllegalArgumentException("Unsupported format " + format + " from " + uri);
     }
     for (var value : TileArchiveConfig.Format.values()) {
-      if (value.id().equals(format)) {
+      if (value.isUriSupported(uri)) {
         return value;
       }
     }
-    throw new IllegalArgumentException("Unsupported format " + format + " from " + uri);
+    throw new IllegalArgumentException("Unsupported format " + getExtension(uri) + " from " + uri);
   }
 
   /**
@@ -110,6 +116,10 @@ public static TileArchiveConfig from(String string) {
         string += "?" + parts[1];
       }
     }
+    for (Map.Entry<String, String> uriEncoding : URI_ENCODINGS.entrySet()) {
+      string = string.replace(uriEncoding.getKey(), uriEncoding.getValue());
+    }
+
     return from(URI.create(string));
   }
 
@@ -144,6 +154,17 @@ public Path getLocalPath() {
     return scheme == Scheme.FILE ? Path.of(URI.create(uri.toString().replaceAll("\\?.*$", ""))) : null;
   }
 
+  /**
+   * Returns the local <b>base</b> path for this archive, for which directories should be pre-created for.
+   */
+  public Path getLocalBasePath() {
+    Path p = getLocalPath();
+    if (format() == Format.FILES) {
+      p = FilesArchiveUtils.cleanBasePath(p);
+    }
+    return p;
+  }
+
 
   /**
    * Deletes the archive if possible.
@@ -158,7 +179,7 @@ public void delete() {
    * Returns {@code true} if the archive already exists, {@code false} otherwise.
    */
   public boolean exists() {
-    return exists(getLocalPath());
+    return exists(getLocalBasePath());
   }
 
   /**
@@ -213,6 +234,16 @@ public enum Format {
       false),
     PMTILES("pmtiles", false, false),
 
+    // should be before PBF in order to avoid collisions
+    FILES("files", true, true) {
+      @Override
+      boolean isUriSupported(URI uri) {
+        final String path = uri.getPath();
+        return path != null && (path.endsWith("/") || path.contains("{") /* template string */ ||
+          !path.contains(".") /* no extension => assume files */);
+      }
+    },
+
     CSV("csv", true, true),
     /** identical to {@link Format#CSV} - except for the column separator */
     TSV("tsv", true, true),
@@ -221,9 +252,7 @@ public enum Format {
     /** identical to {@link Format#PROTO} */
     PBF("pbf", true, true),
 
-    JSON("json", true, true),
-
-    FILES("files", true, true);
+    JSON("json", true, true);
 
     private final String id;
     private final boolean supportsAppend;
@@ -246,6 +275,15 @@ public boolean supportsAppend() {
     public boolean supportsConcurrentWrites() {
       return supportsConcurrentWrites;
     }
+
+    boolean isUriSupported(URI uri) {
+      final String path = uri.getPath();
+      return path != null && path.endsWith("." + id);
+    }
+
+    boolean isQueryFormatSupported(String queryFormat) {
+      return id.equals(queryFormat);
+    }
   }
 
   public enum Scheme {

planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java

@@ -9,7 +9,7 @@
 import java.nio.file.Paths;
 import java.util.Optional;
 
-final class FilesArchiveUtils {
+public final class FilesArchiveUtils {
 
   static final String OPTION_METADATA_PATH = "metadata_path";
   static final String OPTION_TILE_SCHEME = "tile_scheme";
@@ -33,14 +33,41 @@ static Optional<Path> metadataPath(Path basePath, Arguments options) {
     }
   }
 
-  static TileSchemeEncoding tilesSchemeEncoding(Arguments options, Path basePath) {
+  static TileSchemeEncoding tilesSchemeEncoding(Arguments options, Path basePath, String defaultTileScheme) {
     final String tileScheme = options.getString(
       OPTION_TILE_SCHEME,
       "the tile scheme (e.g. {z}/{x}/{y}.pbf, {x}/{y}/{z}.pbf)" +
         " - instead of {x}/{y} {xs}/{ys} can be used which splits the x/y into 2 directories each" +
         " which ensures <1000 files per directory",
-      Path.of(Z_TEMPLATE, X_TEMPLATE, Y_TEMPLATE + ".pbf").toString()
+      defaultTileScheme
     );
     return new TileSchemeEncoding(tileScheme, basePath);
   }
+
+  static BasePathWithTileSchemeEncoding basePathWithTileSchemeEncoding(Arguments options, Path basePath) {
+    final String basePathStr = basePath.toString();
+    final int curlyIndex = basePathStr.indexOf('{');
+    if (curlyIndex >= 0) {
+      final Path newBasePath = Paths.get(basePathStr.substring(0, curlyIndex));
+      return new BasePathWithTileSchemeEncoding(
+        newBasePath,
+        tilesSchemeEncoding(options, newBasePath, basePathStr.substring(curlyIndex))
+      );
+    } else {
+      return new BasePathWithTileSchemeEncoding(
+        basePath,
+        tilesSchemeEncoding(options, basePath, Path.of(Z_TEMPLATE, X_TEMPLATE, Y_TEMPLATE + ".pbf").toString()));
+    }
+  }
+
+  public static Path cleanBasePath(Path basePath) {
+    final String basePathStr = basePath.toString();
+    final int curlyIndex = basePathStr.indexOf('{');
+    if (curlyIndex >= 0) {
+      return Paths.get(basePathStr.substring(0, curlyIndex));
+    }
+    return basePath;
+  }
+
+  record BasePathWithTileSchemeEncoding(Path basePath, TileSchemeEncoding tileSchemeEncoding) {}
 }

planetiler-core/src/main/java/com/onthegomap/planetiler/files/ReadableFilesArchive.java

@@ -15,6 +15,8 @@
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Stream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Reads tiles from a folder structure (e.g. BASEPATH/{z}/{x}{y}.pbf). Counterpart to {@link WriteableFilesArchive}.
@@ -34,6 +36,8 @@
  */
 public class ReadableFilesArchive implements ReadableTileArchive {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(ReadableFilesArchive.class);
+
   private final Path basePath;
   private final Path metadataPath;
   private final Function<TileCoord, Path> tileSchemeEncoder;
@@ -42,13 +46,19 @@ public class ReadableFilesArchive implements ReadableTileArchive {
   private final int searchDepth;
 
   private ReadableFilesArchive(Path basePath, Arguments options) {
+
+    final var pathAndScheme = FilesArchiveUtils.basePathWithTileSchemeEncoding(options, basePath);
+    basePath = pathAndScheme.basePath();
+
+    LOGGER.atInfo().log(() -> "using " + pathAndScheme.basePath() + " as base files archive path");
+
     this.basePath = basePath;
     Preconditions.checkArgument(
       Files.isDirectory(basePath),
       "require \"" + basePath + "\" to be an existing directory"
     );
     this.metadataPath = FilesArchiveUtils.metadataPath(basePath, options).orElse(null);
-    final TileSchemeEncoding tileSchemeEncoding = FilesArchiveUtils.tilesSchemeEncoding(options, basePath);
+    final TileSchemeEncoding tileSchemeEncoding = pathAndScheme.tileSchemeEncoding();
     this.tileSchemeEncoder = tileSchemeEncoding.encoder();
     this.tileSchemeDecoder = tileSchemeEncoding.decoder();
     this.searchDepth = tileSchemeEncoding.searchDepth();

planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java

@@ -17,6 +17,8 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.function.Function;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Writes tiles as separate files. The default tile scheme is z/x/y.pbf.
@@ -31,17 +33,21 @@
  * suppress writing metadata.</dd>
  * </ul>
  *
- * Usage:
+ * Usages:
  *
  * <pre>
  * --output=/path/to/tiles/ --files_tile_scheme={z}/{x}/{y}.pbf --files_metadata_path=/some/other/path/metadata.json
+ * --output=/path/to/tiles/{z}/{x}/{y}.pbf
+ * --output=/path/to/tiles?format=files&tile_scheme={z}/{x}/{y}.pbf
  * </pre>
  *
  * @see ReadableFilesArchive
  * @see TileSchemeEncoding
  */
 public class WriteableFilesArchive implements WriteableTileArchive {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(WriteableFilesArchive.class);
+
   private final Counter.MultiThreadCounter bytesWritten = Counter.newMultiThreadCounter();
 
   private final Path basePath;
@@ -52,6 +58,12 @@ public class WriteableFilesArchive implements WriteableTileArchive {
   private final TileOrder tileOrder;
 
   private WriteableFilesArchive(Path basePath, Arguments options, boolean overwriteMetadata) {
+
+    final var pathAndScheme = FilesArchiveUtils.basePathWithTileSchemeEncoding(options, basePath);
+    basePath = pathAndScheme.basePath();
+
+    LOGGER.atInfo().log("using {} as base files archive path", basePath);
+
     this.basePath = createValidateDirectory(basePath);
     this.metadataPath = FilesArchiveUtils.metadataPath(basePath, options)
       .flatMap(p -> FilesArchiveUtils.metadataPath(p.getParent(), options))
@@ -63,7 +75,7 @@ private WriteableFilesArchive(Path basePath, Arguments options, boolean overwrit
         throw new IllegalArgumentException("require " + this.metadataPath + " to be a regular file");
       }
     }
-    final TileSchemeEncoding tileSchemeEncoding = FilesArchiveUtils.tilesSchemeEncoding(options, basePath);
+    final TileSchemeEncoding tileSchemeEncoding = pathAndScheme.tileSchemeEncoding();
     this.tileSchemeEncoder = tileSchemeEncoding.encoder();
     this.tileOrder = tileSchemeEncoding.preferredTileOrder();
   }

planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java

@@ -106,9 +106,32 @@ void testExistsForNonFilesArchive(@TempDir Path tempDir) throws IOException {
     "output.mbtiles/,FILES", // trailing slash => files - regardless of the extension
     "output/,FILES",
     "output.mbtiles/?format=proto,PROTO", // format query param has precedence
+    "tiles/{x}/{y}/{z}.pbf,FILES"
   })
   void testPathMapping(String path, TileArchiveConfig.Format format) {
     final var config = TileArchiveConfig.from(path);
     assertEquals(format, config.format());
   }
+
+  @ParameterizedTest
+  @CsvSource({
+    "/a/output.mbtiles,/a/output.mbtiles",
+    "/a/tiles/{x}/{y}/{z}.pbf,/a/tiles",
+    "/a/tiles/{x}/{y}/{z}.pbf?format=proto,/a/tiles/{x}/{y}/{z}.pbf"
+  })
+  void testLocalBasePath(String path, Path localBasePath) {
+    final var config = TileArchiveConfig.from(path);
+    assertEquals(localBasePath, config.getLocalBasePath());
+  }
+
+  @ParameterizedTest
+  @CsvSource({
+    "/a/output.mbtiles,/a/output.mbtiles",
+    "/a/tiles/{x}/{y}/{z}.pbf,/a/tiles/{x}/{y}/{z}.pbf",
+    "/a/tiles/{x}/{y}/{z}.pbf?format=proto,/a/tiles/{x}/{y}/{z}.pbf"
+  })
+  void testLocalPath(String path, Path localPath) {
+    final var config = TileArchiveConfig.from(path);
+    assertEquals(localPath, config.getLocalPath());
+  }
 }

planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java

@@ -1,6 +1,7 @@
 package com.onthegomap.planetiler.files;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -111,6 +112,26 @@ void testReadCustomScheme(String tileScheme, Path tileFile, @TempDir Path tempDi
     }
   }
 
+  @Test
+  void testTileSchemeFromBasePath(@TempDir Path tempDir) throws IOException {
+    final Path tilesDir = tempDir.resolve("tiles");
+    final Path basePath = tilesDir.resolve(Paths.get("{x}", "{y}", "{z}.pbf"));
+    final Path tileFile = tilesDir.resolve(Paths.get("1", "2", "3.pbf"));
+    Files.createDirectories(tileFile.getParent());
+    Files.write(tileFile, new byte[]{1});
+
+    final Path metadataFile = tilesDir.resolve("metadata.json");
+    Files.writeString(metadataFile, TestUtils.MAX_METADATA_SERIALIZED);
+
+    try (var archive = ReadableFilesArchive.newReader(basePath, Arguments.of())) {
+      assertEquals(
+        List.of(TileCoord.ofXYZ(1, 2, 3)),
+        archive.getAllTileCoords().stream().toList()
+      );
+      assertNotNull(archive.metadata());
+    }
+  }
+
   @Test
   void testHasNoMetaData(@TempDir Path tempDir) throws IOException {
     final Path tilesDir = tempDir.resolve("tiles");

planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java

@@ -84,6 +84,21 @@ void testWriteCustomScheme(String tileScheme, Path expectedFile, @TempDir Path t
     assertTrue(Files.exists(expectedFile));
   }
 
+  @Test
+  void testTileSchemeFromBasePath(@TempDir Path tempDir) throws IOException {
+    final Path tilesDir = tempDir.resolve("tiles");
+    final Path basePath = tilesDir.resolve(Paths.get("{x}", "{y}", "{z}.pbf"));
+    try (var archive = WriteableFilesArchive.newWriter(basePath, Arguments.of(), false)) {
+      try (var tileWriter = archive.newTileWriter()) {
+        tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.empty()));
+      }
+      archive.finish(TestUtils.MAX_METADATA_DESERIALIZED);
+    }
+
+    assertTrue(Files.exists(tilesDir.resolve(Paths.get("1", "2", "3.pbf"))));
+    assertTrue(Files.exists(tilesDir.resolve("metadata.json")));
+  }
+
   private void testMetadataWrite(Arguments options, Path archiveOutput, Path metadataTilesDir) throws IOException {
     try (var archive = WriteableFilesArchive.newWriter(archiveOutput, options, false)) {
       archive.initialize();