initial s3 support (draft)

onthegomap · Jan 7, 2024 · d8ae3f1 · d8ae3f1
1 parent c480b35
commit d8ae3f1
Show file tree

Hide file tree

Showing 16 changed files with 637 additions and 49 deletions.
diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml
@@ -154,6 +154,11 @@
       <artifactId>geopackage</artifactId>
       <version>${geopackage.version}</version>
     </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+      <version>2.22.12</version>
+    </dependency>
   </dependencies>
 
   <build>

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java
@@ -159,7 +159,7 @@ public Path getLocalPath() {
    */
   public Path getLocalBasePath() {
     Path p = getLocalPath();
-    if (format() == Format.FILES) {
+    if (p != null && format() == Format.FILES) {
       p = FilesArchiveUtils.cleanBasePath(p);
     }
     return p;
@@ -223,7 +223,7 @@ public Arguments applyFallbacks(Arguments arguments) {
   public Path getPathForMultiThreadedWriter(int index) {
     return switch (format) {
       case CSV, TSV, JSON, PROTO, PBF -> StreamArchiveUtils.constructIndexedPath(getLocalPath(), index);
-      case FILES -> getLocalPath();
+      case FILES, S3 -> getLocalPath();
       default -> throw new UnsupportedOperationException("not supported by " + format);
     };
   }
@@ -239,8 +239,10 @@ public enum Format {
       @Override
       boolean isUriSupported(URI uri) {
         final String path = uri.getPath();
-        return path != null && (path.endsWith("/") || path.contains("{") /* template string */ ||
-          !path.contains(".") /* no extension => assume files */);
+        final String scheme = uri.getScheme();
+        return Scheme.FILE.id().equals(scheme) && path != null &&
+          (path.endsWith("/") || path.contains("{") /* template string */ ||
+            !path.contains(".") /* no extension => assume files */);
       }
     },
 
@@ -252,7 +254,13 @@ boolean isUriSupported(URI uri) {
     /** identical to {@link Format#PROTO} */
     PBF("pbf", true, true),
 
-    JSON("json", true, true);
+    JSON("json", true, true),
+    S3("s3", true, true) {
+      @Override
+      boolean isUriSupported(URI uri) {
+        return uri.getScheme().equals(Scheme.S3.id());
+      }
+    };
 
     private final String id;
     private final boolean supportsAppend;
@@ -278,7 +286,8 @@ public boolean supportsConcurrentWrites() {
 
     boolean isUriSupported(URI uri) {
       final String path = uri.getPath();
-      return path != null && path.endsWith("." + id);
+      final String scheme = uri.getScheme();
+      return Scheme.FILE.id().equals(scheme) && path != null && path.endsWith("." + id);
     }
 
     boolean isQueryFormatSupported(String queryFormat) {
@@ -287,7 +296,8 @@ boolean isQueryFormatSupported(String queryFormat) {
   }
 
   public enum Scheme {
-    FILE("file");
+    FILE("file"),
+    S3("s3");
 
     private final String id;
 

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java
@@ -6,6 +6,8 @@
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
 import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
+import com.onthegomap.planetiler.s3.ReadableS3Archive;
+import com.onthegomap.planetiler.s3.WriteableS3Archive;
 import com.onthegomap.planetiler.stream.StreamArchiveConfig;
 import com.onthegomap.planetiler.stream.WriteableCsvArchive;
 import com.onthegomap.planetiler.stream.WriteableJsonStreamArchive;
@@ -59,6 +61,7 @@ public static WriteableTileArchive newWriter(TileArchiveConfig archive, Planetil
       case JSON -> WriteableJsonStreamArchive.newWriteToFile(archive.getLocalPath(),
         new StreamArchiveConfig(config, options));
       case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, config.force() || config.append());
+      case S3 -> WriteableS3Archive.newWriter(archive.uri(), options);
     };
   }
 
@@ -77,6 +80,7 @@ public static ReadableTileArchive newReader(TileArchiveConfig archive, Planetile
       case PROTO, PBF -> throw new UnsupportedOperationException("reading PROTO is not supported");
       case JSON -> throw new UnsupportedOperationException("reading JSON is not supported");
       case FILES -> ReadableFilesArchive.newReader(archive.getLocalPath(), options);
+      case S3 -> ReadableS3Archive.newReader(archive.uri(), options);
     };
   }
 

diff --git a/.../planetiler/files/TileSchemeEncoding.java → ...lanetiler/archive/TileSchemeEncoding.java b/.../planetiler/files/TileSchemeEncoding.java → ...lanetiler/archive/TileSchemeEncoding.java
@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.files;
+package com.onthegomap.planetiler.archive;
 
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.geo.TileOrder;
@@ -59,25 +59,35 @@
  */
 public class TileSchemeEncoding {
 
-  static final String X_TEMPLATE = "{x}";
-  static final String X_SAFE_TEMPLATE = "{xs}";
-  static final String Y_TEMPLATE = "{y}";
-  static final String Y_SAFE_TEMPLATE = "{ys}";
-  static final String Z_TEMPLATE = "{z}";
+  public static final String X_TEMPLATE = "{x}";
+  public static final String X_SAFE_TEMPLATE = "{xs}";
+  public static final String Y_TEMPLATE = "{y}";
+  public static final String Y_SAFE_TEMPLATE = "{ys}";
+  public static final String Z_TEMPLATE = "{z}";
+
+  public static final String ARGUMENT_DESCRIPTION = "the tile scheme (e.g. {z}/{x}/{y}.pbf, {x}/{y}/{z}.pbf)" +
+    " - instead of {x}/{y} {xs}/{ys} can be used which splits the x/y into 2 directories each" +
+    " which ensures <1000 files per directory";
 
   private final String tileScheme;
-  private final Path basePath;
+  private final String prefix;
+  private final String delimiter;
 
   /**
    * @param tileScheme the tile scheme to use e.g. {z}/{x}/{y}.pbf
    * @param basePath   the base path to append the generated relative tile path to
    */
   public TileSchemeEncoding(String tileScheme, Path basePath) {
+    this(checkTileSchemePathNotAbsolute(tileScheme), basePath.toAbsolutePath().toString(), File.separator);
+  }
+
+  public TileSchemeEncoding(String tileScheme, String prefix, String delimiter) {
     this.tileScheme = validate(tileScheme);
-    this.basePath = basePath;
+    this.prefix = prefix;
+    this.delimiter = delimiter;
   }
 
-  public Function<TileCoord, Path> encoder() {
+  public Function<TileCoord, String> encoder() {
     final boolean xSafe = tileScheme.contains(X_SAFE_TEMPLATE);
     final boolean ySafe = tileScheme.contains(Y_SAFE_TEMPLATE);
     return tileCoord -> {
@@ -86,25 +96,24 @@ public Function<TileCoord, Path> encoder() {
 
       if (xSafe) {
         final String colStr = String.format("%06d", tileCoord.x());
-        p = p.replace(X_SAFE_TEMPLATE, Paths.get(colStr.substring(0, 3), colStr.substring(3)).toString());
+        p = p.replace(X_SAFE_TEMPLATE, String.join(delimiter, colStr.substring(0, 3), colStr.substring(3)));
       } else {
         p = p.replace(X_TEMPLATE, Integer.toString(tileCoord.x()));
       }
 
       if (ySafe) {
         final String rowStr = String.format("%06d", tileCoord.y());
-        p = p.replace(Y_SAFE_TEMPLATE, Paths.get(rowStr.substring(0, 3), rowStr.substring(3)).toString());
+        p = p.replace(Y_SAFE_TEMPLATE, String.join(delimiter, rowStr.substring(0, 3), rowStr.substring(3)));
       } else {
         p = p.replace(Y_TEMPLATE, Integer.toString(tileCoord.y()));
       }
-
-      return basePath.resolve(Paths.get(p));
+      return appendToPrefix(p);
     };
   }
 
-  Function<Path, Optional<TileCoord>> decoder() {
+  public Function<String, Optional<TileCoord>> decoder() {
 
-    final String tmpPath = basePath.resolve(tileScheme).toAbsolutePath().toString();
+    final String tmpPath = appendToPrefix(tileScheme);
 
     @SuppressWarnings("java:S1075") final String escapedPathSeparator = "\\" + File.separator;
 
@@ -120,8 +129,8 @@ Function<Path, Optional<TileCoord>> decoder() {
     final boolean xSafe = tileScheme.contains(X_SAFE_TEMPLATE);
     final boolean ySafe = tileScheme.contains(Y_SAFE_TEMPLATE);
 
-    return path -> {
-      final Matcher m = pathPattern.matcher(path.toAbsolutePath().toString());
+    return key -> {
+      final Matcher m = pathPattern.matcher(key);
       if (!m.matches()) {
         return Optional.empty();
       }
@@ -133,19 +142,23 @@ Function<Path, Optional<TileCoord>> decoder() {
     };
   }
 
-  int searchDepth() {
-    return Paths.get(tileScheme).getNameCount() +
+  public int searchDepth() {
+    return tileScheme.split(Pattern.quote(delimiter)).length +
       StringUtils.countMatches(tileScheme, X_SAFE_TEMPLATE) +
       StringUtils.countMatches(tileScheme, Y_SAFE_TEMPLATE);
   }
 
-  TileOrder preferredTileOrder() {
+  public TileOrder preferredTileOrder() {
     // there's only TMS currently - but once there are more, this can be changed according to the scheme
     return TileOrder.TMS;
   }
 
+  private String appendToPrefix(String s) {
+    return "".equals(prefix) ? s : prefix + delimiter + s;
+  }
+
   private static String validate(String tileScheme) {
-    if (Paths.get(tileScheme).isAbsolute()) {
+    if (tileScheme.startsWith("/")) {
       throw new IllegalArgumentException("tile scheme is not allowed to be absolute");
     }
     if (StringUtils.countMatches(tileScheme, Z_TEMPLATE) != 1 ||
@@ -162,22 +175,30 @@ private static String validate(String tileScheme) {
     return tileScheme;
   }
 
+  private static String checkTileSchemePathNotAbsolute(String tileScheme) {
+    if (Paths.get(tileScheme).isAbsolute()) {
+      throw new IllegalArgumentException("tile scheme is not allowed to be absolute");
+    }
+    return tileScheme;
+  }
+
   @Override
   public boolean equals(Object o) {
     return this == o || (o instanceof TileSchemeEncoding that && Objects.equals(tileScheme, that.tileScheme) &&
-      Objects.equals(basePath, that.basePath));
+      Objects.equals(prefix, that.prefix) && Objects.equals(delimiter, that.delimiter));
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(tileScheme, basePath);
+    return Objects.hash(tileScheme, prefix, delimiter);
   }
 
   @Override
   public String toString() {
     return "TileSchemeEncoding[" +
       "tileScheme='" + tileScheme + '\'' +
-      ", basePath=" + basePath +
+      ", prefix=" + prefix +
+      ", delimiter=" + delimiter +
       ']';
   }
 }
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java
@@ -1,9 +1,10 @@
 package com.onthegomap.planetiler.files;
 
-import static com.onthegomap.planetiler.files.TileSchemeEncoding.X_TEMPLATE;
-import static com.onthegomap.planetiler.files.TileSchemeEncoding.Y_TEMPLATE;
-import static com.onthegomap.planetiler.files.TileSchemeEncoding.Z_TEMPLATE;
+import static com.onthegomap.planetiler.archive.TileSchemeEncoding.X_TEMPLATE;
+import static com.onthegomap.planetiler.archive.TileSchemeEncoding.Y_TEMPLATE;
+import static com.onthegomap.planetiler.archive.TileSchemeEncoding.Z_TEMPLATE;
 
+import com.onthegomap.planetiler.archive.TileSchemeEncoding;
 import com.onthegomap.planetiler.config.Arguments;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -35,13 +36,8 @@ static Optional<Path> metadataPath(Path basePath, Arguments options) {
   }
 
   static TileSchemeEncoding tilesSchemeEncoding(Arguments options, Path basePath, String defaultTileScheme) {
-    final String tileScheme = options.getString(
-      OPTION_TILE_SCHEME,
-      "the tile scheme (e.g. {z}/{x}/{y}.pbf, {x}/{y}/{z}.pbf)" +
-        " - instead of {x}/{y} {xs}/{ys} can be used which splits the x/y into 2 directories each" +
-        " which ensures <1000 files per directory",
-      defaultTileScheme
-    );
+    final String tileScheme =
+      options.getString(OPTION_TILE_SCHEME, TileSchemeEncoding.ARGUMENT_DESCRIPTION, defaultTileScheme);
     return new TileSchemeEncoding(tileScheme, basePath);
   }
 

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/ReadableFilesArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/ReadableFilesArchive.java
@@ -4,6 +4,7 @@
 import com.onthegomap.planetiler.archive.ReadableTileArchive;
 import com.onthegomap.planetiler.archive.TileArchiveMetadata;
 import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer;
+import com.onthegomap.planetiler.archive.TileSchemeEncoding;
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.util.CloseableIterator;
@@ -12,6 +13,7 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -59,8 +61,9 @@ private ReadableFilesArchive(Path basePath, Arguments options) {
     );
     this.metadataPath = FilesArchiveUtils.metadataPath(basePath, options).orElse(null);
     final TileSchemeEncoding tileSchemeEncoding = pathAndScheme.tileSchemeEncoding();
-    this.tileSchemeEncoder = tileSchemeEncoding.encoder();
-    this.tileSchemeDecoder = tileSchemeEncoding.decoder();
+    final Function<String, Optional<TileCoord>> tileSchemeDecoderStr = tileSchemeEncoding.decoder();
+    this.tileSchemeEncoder = tileSchemeEncoding.encoder().andThen(Paths::get);
+    this.tileSchemeDecoder = p -> tileSchemeDecoderStr.apply(p.toAbsolutePath().toString());
     this.searchDepth = tileSchemeEncoding.searchDepth();
   }
 

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java
@@ -4,6 +4,7 @@
 import com.onthegomap.planetiler.archive.TileArchiveMetadata;
 import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer;
 import com.onthegomap.planetiler.archive.TileEncodingResult;
+import com.onthegomap.planetiler.archive.TileSchemeEncoding;
 import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.geo.TileCoord;
@@ -16,6 +17,7 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.function.Function;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -76,7 +78,7 @@ private WriteableFilesArchive(Path basePath, Arguments options, boolean overwrit
       }
     }
     final TileSchemeEncoding tileSchemeEncoding = pathAndScheme.tileSchemeEncoding();
-    this.tileSchemeEncoder = tileSchemeEncoding.encoder();
+    this.tileSchemeEncoder = tileSchemeEncoding.encoder().andThen(Paths::get);
     this.tileOrder = tileSchemeEncoding.preferredTileOrder();
   }