Skip to content

Commit

Permalink
Archive compare (#780)
Browse files Browse the repository at this point in the history
  • Loading branch information
msbarry authored Jan 10, 2024
1 parent 076d2ac commit 96eae61
Show file tree
Hide file tree
Showing 9 changed files with 409 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.files.FilesArchiveUtils;
import com.onthegomap.planetiler.geo.TileOrder;
import com.onthegomap.planetiler.stream.StreamArchiveUtils;
import com.onthegomap.planetiler.util.FileUtils;
import java.io.IOException;
Expand Down Expand Up @@ -231,11 +232,11 @@ public Path getPathForMultiThreadedWriter(int index) {
public enum Format {
MBTILES("mbtiles",
false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */,
false),
PMTILES("pmtiles", false, false),
false, TileOrder.TMS),
PMTILES("pmtiles", false, false, TileOrder.HILBERT),

// should be before PBF in order to avoid collisions
FILES("files", true, true) {
FILES("files", true, true, TileOrder.TMS) {
@Override
boolean isUriSupported(URI uri) {
final String path = uri.getPath();
Expand All @@ -244,24 +245,30 @@ boolean isUriSupported(URI uri) {
}
},

CSV("csv", true, true),
CSV("csv", true, true, TileOrder.TMS),
/** identical to {@link Format#CSV} - except for the column separator */
TSV("tsv", true, true),
TSV("tsv", true, true, TileOrder.TMS),

PROTO("proto", true, true),
PROTO("proto", true, true, TileOrder.TMS),
/** identical to {@link Format#PROTO} */
PBF("pbf", true, true),
PBF("pbf", true, true, TileOrder.TMS),

JSON("json", true, true);
JSON("json", true, true, TileOrder.TMS);

private final String id;
private final boolean supportsAppend;
private final boolean supportsConcurrentWrites;
private final TileOrder order;

Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites) {
Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, TileOrder order) {
this.id = id;
this.supportsAppend = supportsAppend;
this.supportsConcurrentWrites = supportsConcurrentWrites;
this.order = order;
}

public TileOrder preferredOrder() {
return order;
}

public String id() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
bytes = switch (config.tileCompression()) {
case GZIP -> gzip(encoded);
case NONE -> encoded;
case UNKNWON -> throw new IllegalArgumentException("cannot compress \"UNKNOWN\"");
case UNKNOWN -> throw new IllegalArgumentException("cannot compress \"UNKNOWN\"");
};
layerStats = TileSizeStats.computeTileStats(proto);
if (encoded.length > config.tileWarningSizeBytes()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public enum TileCompression {
@JsonProperty("gzip")
GZIP("gzip"),
@JsonProperty("unknown")
UNKNWON("unknown");
UNKNOWN("unknown");

private final String id;

Expand All @@ -42,7 +42,7 @@ public static Optional<TileCompression> findById(String id) {
}

public static Set<TileCompression> availableValues() {
return Arrays.stream(TileCompression.values()).filter(tc -> tc != UNKNWON).collect(Collectors.toUnmodifiableSet());
return Arrays.stream(TileCompression.values()).filter(tc -> tc != UNKNOWN).collect(Collectors.toUnmodifiableSet());
}

public String id() {
Expand All @@ -52,7 +52,7 @@ public String id() {
static class Deserializer extends JsonDeserializer<TileCompression> {
@Override
public TileCompression deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
return findById(p.getValueAsString()).orElse(TileCompression.UNKNWON);
return findById(p.getValueAsString()).orElse(TileCompression.UNKNOWN);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public TileArchiveMetadata metadata() {
TileCompression tileCompression = switch (header.tileCompression()) {
case GZIP -> TileCompression.GZIP;
case NONE -> TileCompression.NONE;
case UNKNOWN -> TileCompression.UNKNWON;
case UNKNOWN -> TileCompression.UNKNOWN;
};

String format = switch (header.tileType()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ private static StreamArchiveProto.Metadata toExportData(TileArchiveMetadata meta
final StreamArchiveProto.TileCompression tileCompression = switch (metadata.tileCompression()) {
case GZIP -> StreamArchiveProto.TileCompression.TILE_COMPRESSION_GZIP;
case NONE -> StreamArchiveProto.TileCompression.TILE_COMPRESSION_NONE;
case UNKNWON -> throw new IllegalArgumentException("should not produce \"UNKNOWN\" compression");
case UNKNOWN -> throw new IllegalArgumentException("should not produce \"UNKNOWN\" compression");
};
metaDataBuilder.setTileCompression(tileCompression);
if (metadata.vectorLayers() != null) {
Expand Down
Loading

0 comments on commit 96eae61

Please sign in to comment.