From 2b10382e065404b0ba4aab3756a2e64850a33e08 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Fri, 23 Jun 2023 21:02:11 +0800 Subject: [PATCH] TileJSON support for pmtiles serve (#55) * refactor metadata endpoint * pmtiles serve implements TileJSON * add --public-hostname option, required for TileJSON to work --- main.go | 13 ++--- pmtiles/directory.go | 17 +++++++ pmtiles/server.go | 111 ++++++++++++++++++++++++++++++++++------- pmtiles/server_test.go | 3 ++ 4 files changed, 120 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index b8afa8e..8df2790 100644 --- a/main.go +++ b/main.go @@ -58,11 +58,12 @@ var cli struct { } `cmd:"" help:"Verifies that a local archive is valid."` Serve struct { - Path string `arg:"" help:"Local path or bucket prefix"` - Port int `default:8080` - Cors string `help:"Value of HTTP CORS header."` - CacheSize int `default:64 help:"Size of cache in Megabytes."` - Bucket string `help:"Remote bucket"` + Path string `arg:"" help:"Local path or bucket prefix"` + Port int `default:8080` + Cors string `help:"Value of HTTP CORS header."` + CacheSize int `default:64 help:"Size of cache in Megabytes."` + Bucket string `help:"Remote bucket"` + PublicHostname string `help:"Public hostname of tile endpoint e.g. https://example.com"` } `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."` Upload struct { @@ -96,7 +97,7 @@ func main() { logger.Fatalf("Failed to show database, %v", err) } case "serve ": - server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors) + server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicHostname) if err != nil { logger.Fatalf("Failed to create new server, %v", err) diff --git a/pmtiles/directory.go b/pmtiles/directory.go index 4d50842..3326f71 100644 --- a/pmtiles/directory.go +++ b/pmtiles/directory.go @@ -76,6 +76,23 @@ func headerContentType(header HeaderV3) (string, bool) { } } +func headerExt(header HeaderV3) string { + switch header.TileType { + case Mvt: + return ".mvt" + case Png: + return ".png" + case Jpeg: + return ".jpg" + case Webp: + return ".webp" + case Avif: + return ".avif" + default: + return "" + } +} + func headerContentEncoding(compression Compression) (string, bool) { switch compression { case Gzip: diff --git a/pmtiles/server.go b/pmtiles/server.go index 2316f4d..cc0e8da 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -2,8 +2,11 @@ package pmtiles import ( "bytes" + "compress/gzip" "container/list" "context" + "encoding/json" + "errors" "fmt" "gocloud.dev/blob" "io" @@ -61,14 +64,15 @@ type Response struct { // } type Server struct { - reqs chan Request - bucket *blob.Bucket - logger *log.Logger - cacheSize int - cors string + reqs chan Request + bucket *blob.Bucket + logger *log.Logger + cacheSize int + cors string + publicHostname string } -func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string) (*Server, error) { +func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicHostname string) (*Server, error) { if bucketURL == "" { if strings.HasPrefix(prefix, "/") { bucketURL = "file:///" @@ -92,11 +96,12 @@ func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize in } l := &Server{ - reqs: reqs, - bucket: bucket, - logger: logger, - cacheSize: cacheSize, - cors: cors, + reqs: reqs, + bucket: bucket, + logger: logger, + cacheSize: cacheSize, + cors: cors, + publicHostname: publicHostname, } return l, nil @@ -210,32 +215,90 @@ func (server *Server) Start() { }() } -func (server *Server) get_metadata(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) { +func (server *Server) get_header_metadata(ctx context.Context, name string) (error, bool, HeaderV3, []byte) { root_req := Request{key: CacheKey{name: name, offset: 0, length: 0}, value: make(chan CachedValue, 1)} server.reqs <- root_req root_value := <-root_req.value header := root_value.header if !root_value.ok { - return 404, http_headers, []byte("Archive not found") + return nil, false, HeaderV3{}, nil } r, err := server.bucket.NewRangeReader(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength), nil) if err != nil { - return 404, http_headers, []byte("Archive not found") + return nil, false, HeaderV3{}, nil } defer r.Close() - b, err := io.ReadAll(r) + + var metadata_bytes []byte + if header.InternalCompression == Gzip { + metadata_reader, _ := gzip.NewReader(r) + defer metadata_reader.Close() + metadata_bytes, err = io.ReadAll(metadata_reader) + } else if header.InternalCompression == NoCompression { + metadata_bytes, err = io.ReadAll(r) + } else { + return errors.New("Unknown compression"), true, HeaderV3{}, nil + } + + return nil, true, header, metadata_bytes +} + +func (server *Server) get_tilejson(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) { + err, found, header, metadata_bytes := server.get_header_metadata(ctx, name) + if err != nil { return 500, http_headers, []byte("I/O Error") } - if header_val, ok := headerContentEncoding(header.InternalCompression); ok { - http_headers["Content-Encoding"] = header_val + if !found { + return 404, http_headers, []byte("Archive not found") + } + + var metadata_map map[string]interface{} + json.Unmarshal(metadata_bytes, &metadata_map) + + tilejson := make(map[string]interface{}) + + if server.publicHostname == "" { + return 501, http_headers, []byte("PUBLIC_HOSTNAME must be set for TileJSON") } + http_headers["Content-Type"] = "application/json" + tilejson["tilejson"] = "3.0.0" + tilejson["scheme"] = "xyz" + tilejson["tiles"] = []string{server.publicHostname + "/" + name + "/{z}/{x}/{y}" + headerExt(header)} + tilejson["vector_layers"] = metadata_map["vector_layers"] + tilejson["attribution"] = metadata_map["attribution"] + tilejson["description"] = metadata_map["description"] + tilejson["name"] = metadata_map["name"] + tilejson["version"] = metadata_map["version"] + + E7 := 10000000.0 + tilejson["bounds"] = []float64{float64(header.MinLonE7) / E7, float64(header.MinLatE7) / E7, float64(header.MaxLonE7) / E7, float64(header.MaxLatE7) / E7} + tilejson["center"] = []interface{}{float64(header.CenterLonE7) / E7, float64(header.CenterLatE7) / E7, header.CenterZoom} + tilejson["minzoom"] = header.MinZoom + tilejson["maxzoom"] = header.MaxZoom + + tilejson_bytes, err := json.Marshal(tilejson) + + return 200, http_headers, tilejson_bytes +} - return 200, http_headers, b +func (server *Server) get_metadata(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) { + err, found, _, metadata_bytes := server.get_header_metadata(ctx, name) + + if err != nil { + return 500, http_headers, []byte("I/O Error") + } + + if !found { + return 404, http_headers, []byte("Archive not found") + } + + http_headers["Content-Type"] = "application/json" + return 200, http_headers, metadata_bytes } func (server *Server) get_tile(ctx context.Context, http_headers map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) { @@ -318,6 +381,7 @@ func (server *Server) get_tile(ctx context.Context, http_headers map[string]stri var tilePattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`) var metadataPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/metadata$`) +var tileJSONPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\.json$`) func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string) { if res := tilePattern.FindStringSubmatch(path); res != nil { @@ -331,6 +395,14 @@ func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string) return false, "", 0, 0, 0, "" } +func parse_tilejson_path(path string) (bool, string) { + if res := tileJSONPattern.FindStringSubmatch(path); res != nil { + name := res[1] + return true, name + } + return false, "" +} + func parse_metadata_path(path string) (bool, string) { if res := metadataPattern.FindStringSubmatch(path); res != nil { name := res[1] @@ -348,6 +420,9 @@ func (server *Server) Get(ctx context.Context, path string) (int, map[string]str if ok, key, z, x, y, ext := parse_tile_path(path); ok { return server.get_tile(ctx, http_headers, key, z, x, y, ext) } + if ok, key := parse_tilejson_path(path); ok { + return server.get_tilejson(ctx, http_headers, key) + } if ok, key := parse_metadata_path(path); ok { return server.get_metadata(ctx, http_headers, key) } diff --git a/pmtiles/server_test.go b/pmtiles/server_test.go index d0bcaf2..0f639a9 100644 --- a/pmtiles/server_test.go +++ b/pmtiles/server_test.go @@ -33,4 +33,7 @@ func TestRegex(t *testing.T) { ok, key = parse_metadata_path("/!-_.*'()/metadata") assert.True(t, ok) assert.Equal(t, key, "!-_.*'()") + ok, key = parse_tilejson_path("/!-_.*'().json") + assert.True(t, ok) + assert.Equal(t, key, "!-_.*'()") }