From b373253de29ffcb3eca6bd9f59464adf3753a2d2 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 10 Feb 2024 14:11:54 -0500 Subject: [PATCH 1/9] progress --- main.go | 18 +++++++-- pmtiles/bucket.go | 8 +++- pmtiles/server.go | 18 +++++++-- pmtiles/server_test.go | 87 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 122 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 524b398..7f3c4bc 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "fmt" "log" "net/http" @@ -88,6 +89,7 @@ var cli struct { CacheSize int `default:"64" help:"Size of cache in Megabytes."` Bucket string `help:"Remote bucket"` PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles/"` + TileEtag bool `help:"Generate etag for each tile instead of using archive etag"` } `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."` Download struct { @@ -130,7 +132,7 @@ func main() { logger.Fatalf("Failed to show tile, %v", err) } case "serve ": - server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL) + server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL, cli.Serve.TileEtag) if err != nil { logger.Fatalf("Failed to create new server, %v", err) @@ -144,8 +146,18 @@ func main() { for k, v := range headers { w.Header().Set(k, v) } - w.WriteHeader(statusCode) - w.Write(body) + if statusCode == 200 { + // handle if-match, if-none-match request headers based on response etag + http.ServeContent( + w, r, + "", // name used to infer content-type, but we've already set that + time.UnixMilli(0), // ignore setting last-modified time and handling if-modified-since headers + bytes.NewReader(body), + ) + } else { + w.WriteHeader(statusCode) + w.Write(body) + } logger.Printf("served %d %s in %s", statusCode, r.URL.Path, time.Since(start)) }) diff --git a/pmtiles/bucket.go b/pmtiles/bucket.go index a8cc269..3f8110c 100644 --- a/pmtiles/bucket.go +++ b/pmtiles/bucket.go @@ -77,6 +77,11 @@ func (b FileBucket) NewRangeReader(ctx context.Context, key string, offset, leng return body, err } +func generateEtag(data []byte) string { + hash := md5.Sum([]byte(data)) + return fmt.Sprintf(`"%s"`, hex.EncodeToString(hash[:])) +} + func (b FileBucket) NewRangeReaderEtag(_ context.Context, key string, offset, length int64, etag string) (io.ReadCloser, string, error) { name := filepath.Join(b.path, key) file, err := os.Open(name) @@ -89,8 +94,7 @@ func (b FileBucket) NewRangeReaderEtag(_ context.Context, key string, offset, le return nil, "", err } modInfo := fmt.Sprintf("%d %d", info.ModTime().UnixNano(), info.Size()) - hash := md5.Sum([]byte(modInfo)) - newEtag := fmt.Sprintf(`"%s"`, hex.EncodeToString(hash[:])) + newEtag := generateEtag([]byte(modInfo)) if len(etag) > 0 && etag != newEtag { return nil, "", &RefreshRequiredError{} } diff --git a/pmtiles/server.go b/pmtiles/server.go index 74df12f..b5be9ae 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -51,10 +51,14 @@ type Server struct { cacheSize int cors string publicURL string + tileEtag bool } +var emptyData = make([]byte, 0) +var emptyEtag = generateEtag(emptyData) + // NewServer creates a new pmtiles HTTP server. -func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { +func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) { ctx := context.Background() @@ -70,11 +74,11 @@ func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize in return nil, err } - return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL) + return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL, tileEtag) } // NewServerWithBucket creates a new HTTP server for a gocloud Bucket. -func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { +func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) { reqs := make(chan request, 8) @@ -85,6 +89,7 @@ func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize cacheSize: cacheSize, cors: cors, publicURL: publicURL, + tileEtag: tileEtag, } return l, nil @@ -320,6 +325,7 @@ func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string } return status, headers, data } + func (server *Server) getTileAttempt(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string, purgeEtag string) (int, map[string]string, []byte, string) { rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1), purgeEtag: purgeEtag} server.reqs <- rootReq @@ -390,6 +396,12 @@ func (server *Server) getTileAttempt(ctx context.Context, httpHeaders map[string if err != nil { return 500, httpHeaders, []byte("I/O error"), "" } + + if server.tileEtag { + httpHeaders["Etag"] = generateEtag(b) + } else { + httpHeaders["Etag"] = rootValue.etag + } if headerVal, ok := headerContentType(header); ok { httpHeaders["Content-Type"] = headerVal } diff --git a/pmtiles/server_test.go b/pmtiles/server_test.go index b7063a7..388736e 100644 --- a/pmtiles/server_test.go +++ b/pmtiles/server_test.go @@ -108,8 +108,12 @@ func fakeArchive(t *testing.T, header HeaderV3, metadata map[string]interface{}, } func newServer(t *testing.T) (mockBucket, *Server) { + return newServerWithTileEtags(t, false) +} + +func newServerWithTileEtags(t *testing.T, tileEtags bool) (mockBucket, *Server) { bucket := mockBucket{make(map[string][]byte)} - server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com") + server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com", tileEtags) assert.Nil(t, err) server.Start() return bucket, server @@ -370,3 +374,84 @@ func TestInvalidateCacheOnMetadataRequest(t *testing.T) { "meta": "data2" }`, string(data)) } + +func TestEtagResponsesFromArchive(t *testing.T) { + mockBucket, server := newServerWithTileEtags(t, false) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1, 2, 3}, + {4, 1, 2}: {1, 2, 3}, + }, false) + + statusCode, headers000v1, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers412v1, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers311v1, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") + assert.Equal(t, 204, statusCode) + + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1, 2, 3}, + {4, 1, 2}: {1, 2, 3, 4}, // different + }, false) + + statusCode, headers000v2, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers412v2, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers311v2, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") + assert.Equal(t, 204, statusCode) + + assert.Equal(t, headers000v1["Etag"], headers412v1["Etag"]) + assert.NotEqual(t, headers000v1["Etag"], headers000v2["Etag"]) + assert.Equal(t, headers000v2["Etag"], headers412v2["Etag"]) + + assert.Equal(t, "", headers311v1["Etag"]) + assert.Equal(t, "", headers311v2["Etag"]) +} + +func TestEtagResponsesFromTile(t *testing.T) { + mockBucket, server := newServerWithTileEtags(t, true) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1, 2, 3}, + {4, 1, 2}: {1, 2, 3}, + }, false) + + statusCode, headers000v1, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers412v1, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers311v1, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") + assert.Equal(t, 204, statusCode) + + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1, 2, 3}, + {4, 1, 2}: {1, 2, 3, 4}, // different + }, false) + + statusCode, headers000v2, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers412v2, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") + assert.Equal(t, 200, statusCode) + statusCode, headers311v2, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") + assert.Equal(t, 204, statusCode) + + // 204's have no etag + assert.Equal(t, "", headers311v1["Etag"]) + assert.Equal(t, "", headers311v2["Etag"]) + + // 000 and 311 didn't change + assert.Equal(t, headers000v1["Etag"], headers000v2["Etag"]) + + // 412 did change + assert.NotEqual(t, headers412v1["Etag"], headers412v2["Etag"]) + + // all are different + assert.NotEqual(t, headers000v1["Etag"], headers311v1["Etag"]) + assert.NotEqual(t, headers000v1["Etag"], headers412v1["Etag"]) +} From 6f42f761246fd429a2152d22a91cbd4b907e8850 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 10 Feb 2024 14:19:40 -0500 Subject: [PATCH 2/9] include etag on responses --- pmtiles/server.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/pmtiles/server.go b/pmtiles/server.go index b5be9ae..7349a1a 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -234,30 +234,30 @@ func (server *Server) Start() { }() } -func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, error) { - found, header, metadataBytes, purgeEtag, err := server.getHeaderMetadataAttempt(ctx, name, "") +func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, string, error) { + found, header, metadataBytes, purgeEtag, newEtag, err := server.getHeaderMetadataAttempt(ctx, name, "") if len(purgeEtag) > 0 { - found, header, metadataBytes, _, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag) + found, header, metadataBytes, _, newEtag, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag) } - return found, header, metadataBytes, err + return found, header, metadataBytes, newEtag, err } -func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, error) { +func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, string, error) { rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1), purgeEtag: purgeEtag} server.reqs <- rootReq rootValue := <-rootReq.value header := rootValue.header if !rootValue.ok { - return false, HeaderV3{}, nil, "", nil + return false, HeaderV3{}, nil, "", rootValue.etag, nil } r, _, err := server.bucket.NewRangeReaderEtag(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength), rootValue.etag) if isRefreshRequredError(err) { - return false, HeaderV3{}, nil, rootValue.etag, nil + return false, HeaderV3{}, nil, rootValue.etag, rootValue.etag, nil } if err != nil { - return false, HeaderV3{}, nil, "", nil + return false, HeaderV3{}, nil, "", rootValue.etag, nil } defer r.Close() @@ -269,14 +269,14 @@ func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeE } else if header.InternalCompression == NoCompression { metadataBytes, err = io.ReadAll(r) } else { - return true, HeaderV3{}, nil, "", errors.New("unknown compression") + return true, HeaderV3{}, nil, "", "", errors.New("unknown compression") } - return true, header, metadataBytes, "", nil + return true, header, metadataBytes, "", rootValue.etag, nil } func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { - found, header, metadataBytes, err := server.getHeaderMetadata(ctx, name) + found, header, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name) if err != nil { return 500, httpHeaders, []byte("I/O Error") @@ -300,11 +300,17 @@ func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]st httpHeaders["Content-Type"] = "application/json" + if server.tileEtag { + httpHeaders["Etag"] = etag + } else { + httpHeaders["Etag"] = generateEtag(tilejsonBytes) + } + return 200, httpHeaders, tilejsonBytes } func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { - found, _, metadataBytes, err := server.getHeaderMetadata(ctx, name) + found, _, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name) if err != nil { return 500, httpHeaders, []byte("I/O Error") @@ -315,6 +321,11 @@ func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]st } httpHeaders["Content-Type"] = "application/json" + if server.tileEtag { + httpHeaders["Etag"] = etag + } else { + httpHeaders["Etag"] = generateEtag(metadataBytes) + } return 200, httpHeaders, metadataBytes } func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) { From bac8bf44dd4dea0b7d0437de314fa3ee80e7a661 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 10 Feb 2024 14:30:35 -0500 Subject: [PATCH 3/9] fix --- caddy/pmtiles_proxy.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/caddy/pmtiles_proxy.go b/caddy/pmtiles_proxy.go index 0c2cb6d..e237bc1 100644 --- a/caddy/pmtiles_proxy.go +++ b/caddy/pmtiles_proxy.go @@ -2,6 +2,12 @@ package caddy import ( "fmt" + "io" + "log" + "net/http" + "strconv" + "time" + "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" @@ -12,11 +18,6 @@ import ( _ "gocloud.dev/blob/fileblob" _ "gocloud.dev/blob/gcsblob" _ "gocloud.dev/blob/s3blob" - "io" - "log" - "net/http" - "strconv" - "time" ) func init() { @@ -29,6 +30,7 @@ type Middleware struct { Bucket string `json:"bucket"` CacheSize int `json:"cache_size"` PublicURL string `json:"public_url"` + TileEtag bool `json:"tile_etag"` logger *zap.Logger server *pmtiles.Server } @@ -45,7 +47,7 @@ func (m *Middleware) Provision(ctx caddy.Context) error { m.logger = ctx.Logger() logger := log.New(io.Discard, "", log.Ldate) prefix := "." // serve only the root of the bucket for now, at the root route of Caddyfile - server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL) + server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL, m.TileEtag) if err != nil { return err } From f8d2c372ed6ac43a41c012a7c8bfa44cd4df9c3c Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 10 Feb 2024 15:40:50 -0500 Subject: [PATCH 4/9] add servecontent to caddy --- caddy/pmtiles_proxy.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/caddy/pmtiles_proxy.go b/caddy/pmtiles_proxy.go index e237bc1..f7386ec 100644 --- a/caddy/pmtiles_proxy.go +++ b/caddy/pmtiles_proxy.go @@ -1,6 +1,7 @@ package caddy import ( + "bytes" "fmt" "io" "log" @@ -72,8 +73,12 @@ func (m Middleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddy for k, v := range headers { w.Header().Set(k, v) } - w.WriteHeader(statusCode) - w.Write(body) + if statusCode == 200 { + http.ServeContent(w, r, "", time.UnixMilli(0), bytes.NewReader(body)) + } else { + w.WriteHeader(statusCode) + w.Write(body) + } m.logger.Info("response", zap.Int("status", statusCode), zap.String("path", r.URL.Path), zap.Duration("duration", time.Since(start))) return next.ServeHTTP(w, r) From 40d3b2d1ad9d5de6fd2521209ca48dfccfee6f87 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 10 Feb 2024 16:01:12 -0500 Subject: [PATCH 5/9] parse config --- caddy/pmtiles_proxy.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/caddy/pmtiles_proxy.go b/caddy/pmtiles_proxy.go index f7386ec..ff10dcd 100644 --- a/caddy/pmtiles_proxy.go +++ b/caddy/pmtiles_proxy.go @@ -106,6 +106,12 @@ func (m *Middleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if !d.Args(&m.PublicURL) { return d.ArgErr() } + case "tile_etag": + var tileEtag string + if !d.Args(&tileEtag) { + return d.ArgErr() + } + m.TileEtag = tileEtag == "true" } } } From 68a0a74bccc6549092dc97e1ac2381bd7f5d256b Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sun, 11 Feb 2024 13:48:55 -0500 Subject: [PATCH 6/9] pull ServeHTTP into server.go --- caddy/pmtiles_proxy.go | 12 +----------- main.go | 18 +----------------- pmtiles/server.go | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/caddy/pmtiles_proxy.go b/caddy/pmtiles_proxy.go index ff10dcd..f17a792 100644 --- a/caddy/pmtiles_proxy.go +++ b/caddy/pmtiles_proxy.go @@ -1,7 +1,6 @@ package caddy import ( - "bytes" "fmt" "io" "log" @@ -69,16 +68,7 @@ func (m *Middleware) Validate() error { func (m Middleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { start := time.Now() - statusCode, headers, body := m.server.Get(r.Context(), r.URL.Path) - for k, v := range headers { - w.Header().Set(k, v) - } - if statusCode == 200 { - http.ServeContent(w, r, "", time.UnixMilli(0), bytes.NewReader(body)) - } else { - w.WriteHeader(statusCode) - w.Write(body) - } + statusCode := m.server.ServeHTTP(w, r) m.logger.Info("response", zap.Int("status", statusCode), zap.String("path", r.URL.Path), zap.Duration("duration", time.Since(start))) return next.ServeHTTP(w, r) diff --git a/main.go b/main.go index 7f3c4bc..826f0d9 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "fmt" "log" "net/http" @@ -142,22 +141,7 @@ func main() { http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { start := time.Now() - statusCode, headers, body := server.Get(r.Context(), r.URL.Path) - for k, v := range headers { - w.Header().Set(k, v) - } - if statusCode == 200 { - // handle if-match, if-none-match request headers based on response etag - http.ServeContent( - w, r, - "", // name used to infer content-type, but we've already set that - time.UnixMilli(0), // ignore setting last-modified time and handling if-modified-since headers - bytes.NewReader(body), - ) - } else { - w.WriteHeader(statusCode) - w.Write(body) - } + statusCode := server.ServeHTTP(w, r) logger.Printf("served %d %s in %s", statusCode, r.URL.Path, time.Since(start)) }) diff --git a/pmtiles/server.go b/pmtiles/server.go index 7349a1a..1a79fc5 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -9,8 +9,10 @@ import ( "errors" "io" "log" + "net/http" "regexp" "strconv" + "time" "github.com/prometheus/client_golang/prometheus" ) @@ -488,3 +490,24 @@ func (server *Server) Get(ctx context.Context, path string) (int, map[string]str return 404, httpHeaders, []byte("Path not found") } + +// Serve an HTTP response from the archive +func (server *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) int { + statusCode, headers, body := server.Get(r.Context(), r.URL.Path) + for k, v := range headers { + w.Header().Set(k, v) + } + if statusCode == 200 { + // handle if-match, if-none-match request headers based on response etag + http.ServeContent( + w, r, + "", // name used to infer content-type, but we've already set that + time.UnixMilli(0), // ignore setting last-modified time and handling if-modified-since headers + bytes.NewReader(body), + ) + } else { + w.WriteHeader(statusCode) + w.Write(body) + } + return statusCode +} From 99f7fc920ab394b5281e477eb0b2c2a7dce0286b Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sun, 11 Feb 2024 14:25:24 -0500 Subject: [PATCH 7/9] switch to xxhash --- pmtiles/bucket.go | 33 ++++++++++++++++++++++++++------- pmtiles/server.go | 3 --- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/pmtiles/bucket.go b/pmtiles/bucket.go index 3f8110c..e83c15d 100644 --- a/pmtiles/bucket.go +++ b/pmtiles/bucket.go @@ -3,7 +3,7 @@ package pmtiles import ( "bytes" "context" - "crypto/md5" + "encoding/binary" "encoding/hex" "errors" "fmt" @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/s3" + "github.com/cespare/xxhash/v2" "gocloud.dev/blob" ) @@ -55,8 +56,7 @@ func (m mockBucket) NewRangeReaderEtag(_ context.Context, key string, offset int return nil, "", fmt.Errorf("Not found %s", key) } - hash := md5.Sum(bs) - resultEtag := hex.EncodeToString(hash[:]) + resultEtag := generateEtag(bs) if len(etag) > 0 && resultEtag != etag { return nil, "", &RefreshRequiredError{} } @@ -77,9 +77,29 @@ func (b FileBucket) NewRangeReader(ctx context.Context, key string, offset, leng return body, err } +func uintToBytes(n uint64) []byte { + bs := make([]byte, 8) + binary.LittleEndian.PutUint64(bs, n) + return bs +} + +func hasherToEtag(hasher *xxhash.Digest) string { + sum := uintToBytes(hasher.Sum64()) + return fmt.Sprintf(`"%s"`, hex.EncodeToString(sum)) +} + func generateEtag(data []byte) string { - hash := md5.Sum([]byte(data)) - return fmt.Sprintf(`"%s"`, hex.EncodeToString(hash[:])) + hasher := xxhash.New() + hasher.Write(data) + return hasherToEtag(hasher) +} + +func generateEtagFromInts(ns ...int64) string { + hasher := xxhash.New() + for _, n := range ns { + hasher.Write(uintToBytes(uint64(n))) + } + return hasherToEtag(hasher) } func (b FileBucket) NewRangeReaderEtag(_ context.Context, key string, offset, length int64, etag string) (io.ReadCloser, string, error) { @@ -93,8 +113,7 @@ func (b FileBucket) NewRangeReaderEtag(_ context.Context, key string, offset, le if err != nil { return nil, "", err } - modInfo := fmt.Sprintf("%d %d", info.ModTime().UnixNano(), info.Size()) - newEtag := generateEtag([]byte(modInfo)) + newEtag := generateEtagFromInts(info.ModTime().UnixNano(), info.Size()) if len(etag) > 0 && etag != newEtag { return nil, "", &RefreshRequiredError{} } diff --git a/pmtiles/server.go b/pmtiles/server.go index 1a79fc5..6e6da51 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -56,9 +56,6 @@ type Server struct { tileEtag bool } -var emptyData = make([]byte, 0) -var emptyEtag = generateEtag(emptyData) - // NewServer creates a new pmtiles HTTP server. func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) { From a1ac8b4fbcdb610f56f20dbe25b1bc0940bfa5a5 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sun, 11 Feb 2024 14:31:49 -0500 Subject: [PATCH 8/9] always calculate tile hash --- caddy/pmtiles_proxy.go | 9 +------- main.go | 3 +-- pmtiles/server.go | 51 +++++++++++++++--------------------------- pmtiles/server_test.go | 45 ++----------------------------------- 4 files changed, 22 insertions(+), 86 deletions(-) diff --git a/caddy/pmtiles_proxy.go b/caddy/pmtiles_proxy.go index f17a792..fc5ed01 100644 --- a/caddy/pmtiles_proxy.go +++ b/caddy/pmtiles_proxy.go @@ -30,7 +30,6 @@ type Middleware struct { Bucket string `json:"bucket"` CacheSize int `json:"cache_size"` PublicURL string `json:"public_url"` - TileEtag bool `json:"tile_etag"` logger *zap.Logger server *pmtiles.Server } @@ -47,7 +46,7 @@ func (m *Middleware) Provision(ctx caddy.Context) error { m.logger = ctx.Logger() logger := log.New(io.Discard, "", log.Ldate) prefix := "." // serve only the root of the bucket for now, at the root route of Caddyfile - server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL, m.TileEtag) + server, err := pmtiles.NewServer(m.Bucket, prefix, logger, m.CacheSize, "", m.PublicURL) if err != nil { return err } @@ -96,12 +95,6 @@ func (m *Middleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if !d.Args(&m.PublicURL) { return d.ArgErr() } - case "tile_etag": - var tileEtag string - if !d.Args(&tileEtag) { - return d.ArgErr() - } - m.TileEtag = tileEtag == "true" } } } diff --git a/main.go b/main.go index 826f0d9..e6e1a80 100644 --- a/main.go +++ b/main.go @@ -88,7 +88,6 @@ var cli struct { CacheSize int `default:"64" help:"Size of cache in Megabytes."` Bucket string `help:"Remote bucket"` PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles/"` - TileEtag bool `help:"Generate etag for each tile instead of using archive etag"` } `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."` Download struct { @@ -131,7 +130,7 @@ func main() { logger.Fatalf("Failed to show tile, %v", err) } case "serve ": - server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL, cli.Serve.TileEtag) + server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicURL) if err != nil { logger.Fatalf("Failed to create new server, %v", err) diff --git a/pmtiles/server.go b/pmtiles/server.go index 6e6da51..309df28 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -53,11 +53,10 @@ type Server struct { cacheSize int cors string publicURL string - tileEtag bool } // NewServer creates a new pmtiles HTTP server. -func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) { +func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { ctx := context.Background() @@ -73,11 +72,11 @@ func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize in return nil, err } - return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL, tileEtag) + return NewServerWithBucket(bucket, prefix, logger, cacheSize, cors, publicURL) } // NewServerWithBucket creates a new HTTP server for a gocloud Bucket. -func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string, tileEtag bool) (*Server, error) { +func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize int, cors string, publicURL string) (*Server, error) { reqs := make(chan request, 8) @@ -88,7 +87,6 @@ func NewServerWithBucket(bucket Bucket, _ string, logger *log.Logger, cacheSize cacheSize: cacheSize, cors: cors, publicURL: publicURL, - tileEtag: tileEtag, } return l, nil @@ -233,30 +231,30 @@ func (server *Server) Start() { }() } -func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, string, error) { - found, header, metadataBytes, purgeEtag, newEtag, err := server.getHeaderMetadataAttempt(ctx, name, "") +func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, error) { + found, header, metadataBytes, purgeEtag, err := server.getHeaderMetadataAttempt(ctx, name, "") if len(purgeEtag) > 0 { - found, header, metadataBytes, _, newEtag, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag) + found, header, metadataBytes, _, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag) } - return found, header, metadataBytes, newEtag, err + return found, header, metadataBytes, err } -func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, string, error) { +func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, error) { rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1), purgeEtag: purgeEtag} server.reqs <- rootReq rootValue := <-rootReq.value header := rootValue.header if !rootValue.ok { - return false, HeaderV3{}, nil, "", rootValue.etag, nil + return false, HeaderV3{}, nil, "", nil } r, _, err := server.bucket.NewRangeReaderEtag(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength), rootValue.etag) if isRefreshRequredError(err) { - return false, HeaderV3{}, nil, rootValue.etag, rootValue.etag, nil + return false, HeaderV3{}, nil, rootValue.etag, nil } if err != nil { - return false, HeaderV3{}, nil, "", rootValue.etag, nil + return false, HeaderV3{}, nil, "", nil } defer r.Close() @@ -268,14 +266,14 @@ func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeE } else if header.InternalCompression == NoCompression { metadataBytes, err = io.ReadAll(r) } else { - return true, HeaderV3{}, nil, "", "", errors.New("unknown compression") + return true, HeaderV3{}, nil, "", errors.New("unknown compression") } - return true, header, metadataBytes, "", rootValue.etag, nil + return true, header, metadataBytes, "", nil } func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { - found, header, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name) + found, header, metadataBytes, err := server.getHeaderMetadata(ctx, name) if err != nil { return 500, httpHeaders, []byte("I/O Error") @@ -298,18 +296,13 @@ func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]st } httpHeaders["Content-Type"] = "application/json" - - if server.tileEtag { - httpHeaders["Etag"] = etag - } else { - httpHeaders["Etag"] = generateEtag(tilejsonBytes) - } + httpHeaders["Etag"] = generateEtag(tilejsonBytes) return 200, httpHeaders, tilejsonBytes } func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { - found, _, metadataBytes, etag, err := server.getHeaderMetadata(ctx, name) + found, _, metadataBytes, err := server.getHeaderMetadata(ctx, name) if err != nil { return 500, httpHeaders, []byte("I/O Error") @@ -320,11 +313,7 @@ func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]st } httpHeaders["Content-Type"] = "application/json" - if server.tileEtag { - httpHeaders["Etag"] = etag - } else { - httpHeaders["Etag"] = generateEtag(metadataBytes) - } + httpHeaders["Etag"] = generateEtag(metadataBytes) return 200, httpHeaders, metadataBytes } func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) { @@ -407,11 +396,7 @@ func (server *Server) getTileAttempt(ctx context.Context, httpHeaders map[string return 500, httpHeaders, []byte("I/O error"), "" } - if server.tileEtag { - httpHeaders["Etag"] = generateEtag(b) - } else { - httpHeaders["Etag"] = rootValue.etag - } + httpHeaders["Etag"] = generateEtag(b) if headerVal, ok := headerContentType(header); ok { httpHeaders["Content-Type"] = headerVal } diff --git a/pmtiles/server_test.go b/pmtiles/server_test.go index 388736e..191d362 100644 --- a/pmtiles/server_test.go +++ b/pmtiles/server_test.go @@ -108,12 +108,8 @@ func fakeArchive(t *testing.T, header HeaderV3, metadata map[string]interface{}, } func newServer(t *testing.T) (mockBucket, *Server) { - return newServerWithTileEtags(t, false) -} - -func newServerWithTileEtags(t *testing.T, tileEtags bool) (mockBucket, *Server) { bucket := mockBucket{make(map[string][]byte)} - server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com", tileEtags) + server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com") assert.Nil(t, err) server.Start() return bucket, server @@ -375,45 +371,8 @@ func TestInvalidateCacheOnMetadataRequest(t *testing.T) { }`, string(data)) } -func TestEtagResponsesFromArchive(t *testing.T) { - mockBucket, server := newServerWithTileEtags(t, false) - header := HeaderV3{ - TileType: Mvt, - } - mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ - {0, 0, 0}: {0, 1, 2, 3}, - {4, 1, 2}: {1, 2, 3}, - }, false) - - statusCode, headers000v1, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers412v1, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers311v1, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") - assert.Equal(t, 204, statusCode) - - mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ - {0, 0, 0}: {0, 1, 2, 3}, - {4, 1, 2}: {1, 2, 3, 4}, // different - }, false) - - statusCode, headers000v2, _ := server.Get(context.Background(), "/archive/0/0/0.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers412v2, _ := server.Get(context.Background(), "/archive/4/1/2.mvt") - assert.Equal(t, 200, statusCode) - statusCode, headers311v2, _ := server.Get(context.Background(), "/archive/3/1/1.mvt") - assert.Equal(t, 204, statusCode) - - assert.Equal(t, headers000v1["Etag"], headers412v1["Etag"]) - assert.NotEqual(t, headers000v1["Etag"], headers000v2["Etag"]) - assert.Equal(t, headers000v2["Etag"], headers412v2["Etag"]) - - assert.Equal(t, "", headers311v1["Etag"]) - assert.Equal(t, "", headers311v2["Etag"]) -} - func TestEtagResponsesFromTile(t *testing.T) { - mockBucket, server := newServerWithTileEtags(t, true) + mockBucket, server := newServer(t) header := HeaderV3{ TileType: Mvt, } From 625b8d2cc76992908139690fac0d3ec15cd7c109 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sun, 11 Feb 2024 14:46:13 -0500 Subject: [PATCH 9/9] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 379eef5..e5b256a 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/alecthomas/kong v0.8.0 github.com/aws/aws-sdk-go v1.45.12 github.com/caddyserver/caddy/v2 v2.7.5 + github.com/cespare/xxhash/v2 v2.2.0 github.com/dustin/go-humanize v1.0.1 github.com/paulmach/orb v0.10.0 github.com/prometheus/client_golang v1.18.0 @@ -62,7 +63,6 @@ require ( github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/caddyserver/certmagic v0.19.2 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect