Skip to content

Commit

Permalink
always calculate tile hash
Browse files Browse the repository at this point in the history
  • Loading branch information
@msbarry
msbarry committed Feb 11, 2024
1 parent 99f7fc9 commit a1ac8b4
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 86 deletions.
9 changes: 1 addition & 8 deletions caddy/pmtiles_proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ type Middleware struct {
Bucket string `json:"bucket"`
CacheSize int `json:"cache_size"`
PublicURL string `json:"public_url"`
TileEtag bool `json:"tile_etag"`
logger *zap.Logger
server *pmtiles.Server
}
Expand All @@ -47,7 +46,7 @@ func (m *Middleware) Provision(ctx caddy.Context) error {
m.logger = ctx.Logger()
logger := log.New(io.Discard, "", log.Ldate)
prefix := "." // serve only the root of the bucket for now, at the root route of Caddyfile
server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL, m.TileEtag)
server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL)
if err != nil {
return err
}
Expand Down Expand Up @@ -96,12 +95,6 @@ func (m *Middleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
if !d.Args(&m.PublicURL) {
return d.ArgErr()
}
case "tile_etag":
var tileEtag string
if !d.Args(&tileEtag) {
return d.ArgErr()
}
m.TileEtag = tileEtag == "true"
}
}
}
Expand Down
3 changes: 1 addition & 2 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ var cli struct {
CacheSize int `default:"64" help:"Size of cache in Megabytes."`
Bucket string `help:"Remote bucket"`
PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles/"`
TileEtag bool `help:"Generate etag for each tile instead of using archive etag"`
} `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."`

Download struct {
Expand Down Expand Up @@ -131,7 +130,7 @@ func main() {
logger.Fatalf("Failed to show tile, %v", err)
}
case "serve <path>":
server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL, cli.Serve.TileEtag)
server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL)

if err != nil {
logger.Fatalf("Failed to create new server, %v", err)
Expand Down
51 changes: 18 additions & 33 deletions pmtiles/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,10 @@ type Server struct {
cacheSize int
cors string
publicURL string
tileEtag bool
}

// NewServer creates a new pmtiles HTTP server.
func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) {
func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) {

ctx := context.Background()

Expand All @@ -73,11 +72,11 @@ func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize in
return nil, err
}

return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL, tileEtag)
return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL)
}

// NewServerWithBucket creates a new HTTP server for a gocloud Bucket.
func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) {
func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) {

reqs := make(chan request, 8)

Expand All @@ -88,7 +87,6 @@ func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize
cacheSize: cacheSize,
cors: cors,
publicURL: publicURL,
tileEtag: tileEtag,
}

return l, nil
Expand Down Expand Up @@ -233,30 +231,30 @@ func (server *Server) Start() {
}()
}

func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, string, error) {
found, header, metadataBytes, purgeEtag, newEtag, err := server.getHeaderMetadataAttempt(ctx, name, "")
func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, error) {
found, header, metadataBytes, purgeEtag, err := server.getHeaderMetadataAttempt(ctx, name, "")
if len(purgeEtag) > 0 {
found, header, metadataBytes, _, newEtag, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag)
found, header, metadataBytes, _, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag)
}
return found, header, metadataBytes, newEtag, err
return found, header, metadataBytes, err
}

func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, string, error) {
func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, error) {
rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1), purgeEtag: purgeEtag}
server.reqs <- rootReq
rootValue := <-rootReq.value
header := rootValue.header

if !rootValue.ok {
return false, HeaderV3{}, nil, "", rootValue.etag, nil
return false, HeaderV3{}, nil, "", nil
}

r, _, err := server.bucket.NewRangeReaderEtag(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength), rootValue.etag)
if isRefreshRequredError(err) {
return false, HeaderV3{}, nil, rootValue.etag, rootValue.etag, nil
return false, HeaderV3{}, nil, rootValue.etag, nil
}
if err != nil {
return false, HeaderV3{}, nil, "", rootValue.etag, nil
return false, HeaderV3{}, nil, "", nil
}
defer r.Close()

Expand All @@ -268,14 +266,14 @@ func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeE
} else if header.InternalCompression == NoCompression {
metadataBytes, err = io.ReadAll(r)
} else {
return true, HeaderV3{}, nil, "", "", errors.New("unknown compression")
return true, HeaderV3{}, nil, "", errors.New("unknown compression")
}

return true, header, metadataBytes, "", rootValue.etag, nil
return true, header, metadataBytes, "", nil
}

func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) {
found, header, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name)
found, header, metadataBytes, err := server.getHeaderMetadata(ctx, name)

if err != nil {
return 500, httpHeaders, []byte("I/O Error")
Expand All @@ -298,18 +296,13 @@ func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]st
}

httpHeaders["Content-Type"] = "application/json"

if server.tileEtag {
httpHeaders["Etag"] = etag
} else {
httpHeaders["Etag"] = generateEtag(tilejsonBytes)
}
httpHeaders["Etag"] = generateEtag(tilejsonBytes)

return 200, httpHeaders, tilejsonBytes
}

func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) {
found, _, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name)
found, _, metadataBytes, err := server.getHeaderMetadata(ctx, name)

if err != nil {
return 500, httpHeaders, []byte("I/O Error")
Expand All @@ -320,11 +313,7 @@ func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]st
}

httpHeaders["Content-Type"] = "application/json"
if server.tileEtag {
httpHeaders["Etag"] = etag
} else {
httpHeaders["Etag"] = generateEtag(metadataBytes)
}
httpHeaders["Etag"] = generateEtag(metadataBytes)
return 200, httpHeaders, metadataBytes
}
func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) {
Expand Down Expand Up @@ -407,11 +396,7 @@ func (server *Server) getTileAttempt(ctx context.Context, httpHeaders map[string
return 500, httpHeaders, []byte("I/O error"), ""
}

if server.tileEtag {
httpHeaders["Etag"] = generateEtag(b)
} else {
httpHeaders["Etag"] = rootValue.etag
}
httpHeaders["Etag"] = generateEtag(b)
if headerVal, ok := headerContentType(header); ok {
httpHeaders["Content-Type"] = headerVal
}
Expand Down
45 changes: 2 additions & 43 deletions pmtiles/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,8 @@ func fakeArchive(t *testing.T, header HeaderV3, metadata map[string]interface{},
}

func newServer(t *testing.T) (mockBucket, *Server) {
return newServerWithTileEtags(t, false)
}

func newServerWithTileEtags(t *testing.T, tileEtags bool) (mockBucket, *Server) {
bucket := mockBucket{make(map[string][]byte)}
server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com", tileEtags)
server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com")
assert.Nil(t, err)
server.Start()
return bucket, server
Expand Down Expand Up @@ -375,45 +371,8 @@ func TestInvalidateCacheOnMetadataRequest(t *testing.T) {
}`, string(data))
}

func TestEtagResponsesFromArchive(t *testing.T) {
mockBucket, server := newServerWithTileEtags(t, false)
header := HeaderV3{
TileType: Mvt,
}
mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{
{0, 0, 0}: {0, 1, 2, 3},
{4, 1, 2}: {1, 2, 3},
}, false)

statusCode, headers000v1, _ := server.Get(context.Background(), "/archive/0/0/0.mvt")
assert.Equal(t, 200, statusCode)
statusCode, headers412v1, _ := server.Get(context.Background(), "/archive/4/1/2.mvt")
assert.Equal(t, 200, statusCode)
statusCode, headers311v1, _ := server.Get(context.Background(), "/archive/3/1/1.mvt")
assert.Equal(t, 204, statusCode)

mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{
{0, 0, 0}: {0, 1, 2, 3},
{4, 1, 2}: {1, 2, 3, 4}, // different
}, false)

statusCode, headers000v2, _ := server.Get(context.Background(), "/archive/0/0/0.mvt")
assert.Equal(t, 200, statusCode)
statusCode, headers412v2, _ := server.Get(context.Background(), "/archive/4/1/2.mvt")
assert.Equal(t, 200, statusCode)
statusCode, headers311v2, _ := server.Get(context.Background(), "/archive/3/1/1.mvt")
assert.Equal(t, 204, statusCode)

assert.Equal(t, headers000v1["Etag"], headers412v1["Etag"])
assert.NotEqual(t, headers000v1["Etag"], headers000v2["Etag"])
assert.Equal(t, headers000v2["Etag"], headers412v2["Etag"])

assert.Equal(t, "", headers311v1["Etag"])
assert.Equal(t, "", headers311v2["Etag"])
}

func TestEtagResponsesFromTile(t *testing.T) {
mockBucket, server := newServerWithTileEtags(t, true)
mockBucket, server := newServer(t)
header := HeaderV3{
TileType: Mvt,
}
Expand Down

0 comments on commit a1ac8b4

Please sign in to comment.