From a1ac8b4fbcdb610f56f20dbe25b1bc0940bfa5a5 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sun, 11 Feb 2024 14:31:49 -0500 Subject: [PATCH] always calculate tile hash --- caddy/pmtiles_proxy.go | 9 +------- main.go | 3 +-- pmtiles/server.go | 51 +++++++++++++++--------------------------- pmtiles/server_test.go | 45 ++----------------------------------- 4 files changed, 22 insertions(+), 86 deletions(-) diff --git a/caddy/pmtiles_proxy.go b/caddy/pmtiles_proxy.go index f17a792..fc5ed01 100644 --- a/caddy/pmtiles_proxy.go +++ b/caddy/pmtiles_proxy.go @@ -30,7 +30,6 @@ type Middleware struct { Bucket string `json:"bucket"` CacheSize int `json:"cache_size"` PublicURL string `json:"public_url"` - TileEtag bool `json:"tile_etag"` logger *zap.Logger server *pmtiles.Server } @@ -47,7 +46,7 @@ func (m *Middleware) Provision(ctx caddy.Context) error { m.logger = ctx.Logger() logger := log.New(io.Discard, "", log.Ldate) prefix := "." // serve only the root of the bucket for now, at the root route of Caddyfile - server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL, m.TileEtag) + server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL) if err != nil { return err } @@ -96,12 +95,6 @@ func (m *Middleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if !d.Args(&m.PublicURL) { return d.ArgErr() } - case "tile_etag": - var tileEtag string - if !d.Args(&tileEtag) { - return d.ArgErr() - } - m.TileEtag = tileEtag == "true" } } } diff --git a/main.go b/main.go index 826f0d9..e6e1a80 100644 --- a/main.go +++ b/main.go @@ -88,7 +88,6 @@ var cli struct { CacheSize int `default:"64" help:"Size of cache in Megabytes."` Bucket string `help:"Remote bucket"` PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles/"` - TileEtag bool `help:"Generate etag for each tile instead of using archive etag"` } `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."` Download struct { @@ -131,7 +130,7 @@ func main() { logger.Fatalf("Failed to show tile, %v", err) } case "serve ": - server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL, cli.Serve.TileEtag) + server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL) if err != nil { logger.Fatalf("Failed to create new server, %v", err) diff --git a/pmtiles/server.go b/pmtiles/server.go index 6e6da51..309df28 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -53,11 +53,10 @@ type Server struct { cacheSize int cors string publicURL string - tileEtag bool } // NewServer creates a new pmtiles HTTP server. -func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) { +func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { ctx := context.Background() @@ -73,11 +72,11 @@ func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize in return nil, err } - return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL, tileEtag) + return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL) } // NewServerWithBucket creates a new HTTP server for a gocloud Bucket. -func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) { +func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { reqs := make(chan request, 8) @@ -88,7 +87,6 @@ func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize cacheSize: cacheSize, cors: cors, publicURL: publicURL, - tileEtag: tileEtag, } return l, nil @@ -233,30 +231,30 @@ func (server *Server) Start() { }() } -func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, string, error) { - found, header, metadataBytes, purgeEtag, newEtag, err := server.getHeaderMetadataAttempt(ctx, name, "") +func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, error) { + found, header, metadataBytes, purgeEtag, err := server.getHeaderMetadataAttempt(ctx, name, "") if len(purgeEtag) > 0 { - found, header, metadataBytes, _, newEtag, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag) + found, header, metadataBytes, _, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag) } - return found, header, metadataBytes, newEtag, err + return found, header, metadataBytes, err } -func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, string, error) { +func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, error) { rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1), purgeEtag: purgeEtag} server.reqs <- rootReq rootValue := <-rootReq.value header := rootValue.header if !rootValue.ok { - return false, HeaderV3{}, nil, "", rootValue.etag, nil + return false, HeaderV3{}, nil, "", nil } r, _, err := server.bucket.NewRangeReaderEtag(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength), rootValue.etag) if isRefreshRequredError(err) { - return false, HeaderV3{}, nil, rootValue.etag, rootValue.etag, nil + return false, HeaderV3{}, nil, rootValue.etag, nil } if err != nil { - return false, HeaderV3{}, nil, "", rootValue.etag, nil + return false, HeaderV3{}, nil, "", nil } defer r.Close() @@ -268,14 +266,14 @@ func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeE } else if header.InternalCompression == NoCompression { metadataBytes, err = io.ReadAll(r) } else { - return true, HeaderV3{}, nil, "", "", errors.New("unknown compression") + return true, HeaderV3{}, nil, "", errors.New("unknown compression") } - return true, header, metadataBytes, "", rootValue.etag, nil + return true, header, metadataBytes, "", nil } func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { - found, header, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name) + found, header, metadataBytes, err := server.getHeaderMetadata(ctx, name) if err != nil { return 500, httpHeaders, []byte("I/O Error") @@ -298,18 +296,13 @@ func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]st } httpHeaders["Content-Type"] = "application/json" - - if server.tileEtag { - httpHeaders["Etag"] = etag - } else { - httpHeaders["Etag"] = generateEtag(tilejsonBytes) - } + httpHeaders["Etag"] = generateEtag(tilejsonBytes) return 200, httpHeaders, tilejsonBytes } func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { - found, _, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name) + found, _, metadataBytes, err := server.getHeaderMetadata(ctx, name) if err != nil { return 500, httpHeaders, []byte("I/O Error") @@ -320,11 +313,7 @@ func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]st } httpHeaders["Content-Type"] = "application/json" - if server.tileEtag { - httpHeaders["Etag"] = etag - } else { - httpHeaders["Etag"] = generateEtag(metadataBytes) - } + httpHeaders["Etag"] = generateEtag(metadataBytes) return 200, httpHeaders, metadataBytes } func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) { @@ -407,11 +396,7 @@ func (server *Server) getTileAttempt(ctx context.Context, httpHeaders map[string return 500, httpHeaders, []byte("I/O error"), "" } - if server.tileEtag { - httpHeaders["Etag"] = generateEtag(b) - } else { - httpHeaders["Etag"] = rootValue.etag - } + httpHeaders["Etag"] = generateEtag(b) if headerVal, ok := headerContentType(header); ok { httpHeaders["Content-Type"] = headerVal } diff --git a/pmtiles/server_test.go b/pmtiles/server_test.go index 388736e..191d362 100644 --- a/pmtiles/server_test.go +++ b/pmtiles/server_test.go @@ -108,12 +108,8 @@ func fakeArchive(t *testing.T, header HeaderV3, metadata map[string]interface{}, } func newServer(t *testing.T) (mockBucket, *Server) { - return newServerWithTileEtags(t, false) -} - -func newServerWithTileEtags(t *testing.T, tileEtags bool) (mockBucket, *Server) { bucket := mockBucket{make(map[string][]byte)} - server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com", tileEtags) + server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com") assert.Nil(t, err) server.Start() return bucket, server @@ -375,45 +371,8 @@ func TestInvalidateCacheOnMetadataRequest(t *testing.T) { }`, string(data)) } -func TestEtagResponsesFromArchive(t *testing.T) { - mockBucket, server := newServerWithTileEtags(t, false) - header := HeaderV3{ - TileType: Mvt, - } - mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ - {0, 0, 0}: {0, 1, 2, 3}, - {4, 1, 2}: {1, 2, 3}, - }, false) - - statusCode, headers000v1, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers412v1, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers311v1, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") - assert.Equal(t, 204, statusCode) - - mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ - {0, 0, 0}: {0, 1, 2, 3}, - {4, 1, 2}: {1, 2, 3, 4}, // different - }, false) - - statusCode, headers000v2, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers412v2, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers311v2, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") - assert.Equal(t, 204, statusCode) - - assert.Equal(t, headers000v1["Etag"], headers412v1["Etag"]) - assert.NotEqual(t, headers000v1["Etag"], headers000v2["Etag"]) - assert.Equal(t, headers000v2["Etag"], headers412v2["Etag"]) - - assert.Equal(t, "", headers311v1["Etag"]) - assert.Equal(t, "", headers311v2["Etag"]) -} - func TestEtagResponsesFromTile(t *testing.T) { - mockBucket, server := newServerWithTileEtags(t, true) + mockBucket, server := newServer(t) header := HeaderV3{ TileType: Mvt, }