diff --git a/go.mod b/go.mod index 8f14549..379eef5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/RoaringBitmap/roaring v1.5.0 github.com/alecthomas/kong v0.8.0 + github.com/aws/aws-sdk-go v1.45.12 github.com/caddyserver/caddy/v2 v2.7.5 github.com/dustin/go-humanize v1.0.1 github.com/paulmach/orb v0.10.0 @@ -38,7 +39,6 @@ require ( github.com/Microsoft/go-winio v0.6.0 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b // indirect - github.com/aws/aws-sdk-go v1.45.12 // indirect github.com/aws/aws-sdk-go-v2 v1.20.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 // indirect github.com/aws/aws-sdk-go-v2/config v1.18.32 // indirect diff --git a/go.sum b/go.sum index ee2a4ee..350acb0 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,7 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/kms v1.15.2 h1:lh6qra6oC4AyWe5fUUUBe/S27k12OHAleOOOw6KakdE= +cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= @@ -22,6 +23,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQ github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -51,9 +53,11 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= +github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA= github.com/alecthomas/kong v0.8.0 h1:ryDCzutfIqJPnNn0omnrgHLbAggDQM2VWHikE1xqK7s= github.com/alecthomas/kong v0.8.0/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= +github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -113,6 +117,7 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.21.1/go.mod h1:G8SbvL0rFk4WOJroU8tKB github.com/aws/smithy-go v1.14.0 h1:+X90sB94fizKjDmwb4vyl2cTTPXTE5E2G/1mjByb0io= github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -177,6 +182,7 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -207,6 +213,7 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -254,9 +261,11 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.15.1 h1:iTgVZor2x9okXtmTrqO8cg4uvqIeaBcWhXtruaWFMYQ= github.com/google/cel-go v0.15.1/go.mod h1:YzWEoI07MC/a/wj9in8GeVatqfypkldgBlwXh9bCwqY= github.com/google/certificate-transparency-go v1.1.6 h1:SW5K3sr7ptST/pIvNkSVWMiJqemRmkjJPPT0jzXdOOY= +github.com/google/certificate-transparency-go v1.1.6/go.mod h1:0OJjOsOk+wj6aYQgP7FU0ioQ0AJUmnWPFMqTjQeazPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -269,12 +278,18 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE= +github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= +github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= github.com/google/go-tpm-tools v0.4.1 h1:gYU6iwRo0tY3V6NDnS6m+XYog+b3g6YFhHQl3sYaUL4= +github.com/google/go-tpm-tools v0.4.1/go.mod h1:w03m0jynhTo7puXTYoyfpNOMqyQ9SB7sixnKWsS/1L0= github.com/google/go-tspi v0.3.0 h1:ADtq8RKfP+jrTyIWIZDIYcKOMecRqNJFOew2IT0Inus= +github.com/google/go-tspi v0.3.0/go.mod h1:xfMGI3G0PhxCdNVcYr1C4C+EizojDg/TXuX5by8CiHI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -325,6 +340,7 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -413,10 +429,12 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -424,6 +442,7 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -501,6 +520,7 @@ github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -519,6 +539,7 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU= +github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= @@ -570,8 +591,10 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -582,6 +605,7 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/jsonstore v1.1.0 h1:WZBDjgezFS34CHI+myb4s8GGpir3UMpy7vWoCeO0n6E= +github.com/schollz/jsonstore v1.1.0/go.mod h1:15c6+9guw8vDRyozGjN3FoILt0wpruJk9Pi66vjaZfg= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -595,12 +619,15 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slackhq/nebula v1.6.1 h1:/OCTR3abj0Sbf2nGoLUrdDXImrCv0ZVFpVPP5qa0DsM= github.com/slackhq/nebula v1.6.1/go.mod h1:UmkqnXe4O53QwToSl/gG7sM4BroQwAB7dd4hUaT6MlI= github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY= +github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262/go.mod h1:MyOHs9Po2fbM1LHej6sBUT8ozbxmMOFG+E+rx/GSGuc= github.com/smallstep/certificates v0.25.0 h1:WWihtjQ7SprnRxDV44mBp8t5SMsNO5EWsQaEwy1rgFg= github.com/smallstep/certificates v0.25.0/go.mod h1:thJmekMKUplKYip+la99Lk4IwQej/oVH/zS9PVMagEE= github.com/smallstep/go-attestation v0.4.4-0.20230627102604-cf579e53cbd2 h1:UIAS8DTWkeclraEGH2aiJPyNPu16VbT41w4JoBlyFfU= +github.com/smallstep/go-attestation v0.4.4-0.20230627102604-cf579e53cbd2/go.mod h1:vNAduivU014fubg6ewygkAvQC0IQVXqdc8vaGl/0er4= github.com/smallstep/nosql v0.6.0 h1:ur7ysI8s9st0cMXnTvB8tA3+x5Eifmkb6hl4uqNV5jc= github.com/smallstep/nosql v0.6.0/go.mod h1:jOXwLtockXORUPPZ2MCUcIkGR6w0cN1QGZniY9DITQA= github.com/smallstep/truststore v0.12.1 h1:guLUKkc1UlsXeS3t6BuVMa4leOOpdiv02PCRTiy1WdY= @@ -699,6 +726,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -934,12 +962,14 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/main.go b/main.go index c486c33..524b398 100644 --- a/main.go +++ b/main.go @@ -146,7 +146,7 @@ func main() { } w.WriteHeader(statusCode) w.Write(body) - logger.Printf("served %s in %s", r.URL.Path, time.Since(start)) + logger.Printf("served %d %s in %s", statusCode, r.URL.Path, time.Since(start)) }) logger.Printf("Serving %s %s on port %d and interface %s with Access-Control-Allow-Origin: %s\n", cli.Serve.Bucket, cli.Serve.Path, cli.Serve.Port, cli.Serve.Interface, cli.Serve.Cors) diff --git a/pmtiles/bucket.go b/pmtiles/bucket.go index ca27779..979730d 100644 --- a/pmtiles/bucket.go +++ b/pmtiles/bucket.go @@ -1,9 +1,12 @@ package pmtiles import ( + "bytes" "context" + "crypto/md5" + "encoding/hex" + "errors" "fmt" - "gocloud.dev/blob" "io" "net/http" "net/url" @@ -11,55 +14,146 @@ import ( "path" "path/filepath" "strings" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "gocloud.dev/blob" ) // Bucket is an abstration over a gocloud or plain HTTP bucket. type Bucket interface { Close() error NewRangeReader(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error) + NewRangeReaderEtag(ctx context.Context, key string, offset int64, length int64, etag string) (io.ReadCloser, string, error) +} + +// RefreshRequiredError is an error that indicates the etag has chanced on the remote file +type RefreshRequiredError struct { + StatusCode int +} + +func (m *RefreshRequiredError) Error() string { + return fmt.Sprintf("HTTP error indicates file has changed: %d", m.StatusCode) +} + +type mockBucket struct { + items map[string][]byte +} + +func (m mockBucket) Close() error { + return nil +} + +func (m mockBucket) NewRangeReader(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error) { + body, _, err := m.NewRangeReaderEtag(ctx, key, offset, length, "") + return body, err + +} +func (m mockBucket) NewRangeReaderEtag(_ context.Context, key string, offset int64, length int64, etag string) (io.ReadCloser, string, error) { + bs, ok := m.items[key] + if !ok { + return nil, "", fmt.Errorf("Not found %s", key) + } + + hash := md5.Sum(bs) + resultEtag := hex.EncodeToString(hash[:]) + if len(etag) > 0 && resultEtag != etag { + return nil, "", &RefreshRequiredError{} + } + if offset+length > int64(len(bs)) { + return nil, "", &RefreshRequiredError{416} + } + + return io.NopCloser(bytes.NewReader(bs[offset:(offset + length)])), resultEtag, nil +} + +// HTTPClient is an interface that lets you swap out the default client with a mock one in tests +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) } type HTTPBucket struct { baseURL string + client HTTPClient } -func (b HTTPBucket) NewRangeReader(_ context.Context, key string, offset, length int64) (io.ReadCloser, error) { +func (b HTTPBucket) NewRangeReader(ctx context.Context, key string, offset, length int64) (io.ReadCloser, error) { + body, _, err := b.NewRangeReaderEtag(ctx, key, offset, length, "") + return body, err +} + +func (b HTTPBucket) NewRangeReaderEtag(ctx context.Context, key string, offset, length int64, etag string) (io.ReadCloser, string, error) { reqURL := b.baseURL + "/" + key - req, err := http.NewRequest("GET", reqURL, nil) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { - return nil, err + return nil, "", err } req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) + if len(etag) > 0 { + req.Header.Set("If-Match", etag) + } - resp, err := http.DefaultClient.Do(req) + resp, err := b.client.Do(req) if err != nil { - return nil, err + return nil, "", err } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { resp.Body.Close() - return nil, fmt.Errorf("HTTP error: %d", resp.StatusCode) + if isRefreshRequredCode(resp.StatusCode) { + err = &RefreshRequiredError{resp.StatusCode} + } else { + err = fmt.Errorf("HTTP error: %d", resp.StatusCode) + } + return nil, "", err } - return resp.Body, nil + return resp.Body, resp.Header.Get("ETag"), nil } func (b HTTPBucket) Close() error { return nil } +func isRefreshRequredCode(code int) bool { + return code == http.StatusPreconditionFailed || code == http.StatusRequestedRangeNotSatisfiable +} + type BucketAdapter struct { Bucket *blob.Bucket } func (ba BucketAdapter) NewRangeReader(ctx context.Context, key string, offset, length int64) (io.ReadCloser, error) { - reader, err := ba.Bucket.NewRangeReader(ctx, key, offset, length, nil) + body, _, err := ba.NewRangeReaderEtag(ctx, key, offset, length, "") + return body, err +} + +func (ba BucketAdapter) NewRangeReaderEtag(ctx context.Context, key string, offset, length int64, etag string) (io.ReadCloser, string, error) { + reader, err := ba.Bucket.NewRangeReader(ctx, key, offset, length, &blob.ReaderOptions{ + BeforeRead: func(asFunc func(interface{}) bool) error { + var req *s3.GetObjectInput + if len(etag) > 0 && asFunc(&req) { + req.IfMatch = &etag + } + return nil + }, + }) if err != nil { - return nil, err + var resp awserr.RequestFailure + errors.As(err, &resp) + if resp != nil && isRefreshRequredCode(resp.StatusCode()) { + return nil, "", &RefreshRequiredError{resp.StatusCode()} + } + return nil, "", err + } + resultETag := "" + var resp s3.GetObjectOutput + if reader.As(&resp) { + resultETag = *resp.ETag } - return reader, nil + return reader, resultETag, nil } func (ba BucketAdapter) Close() error { @@ -101,7 +195,7 @@ func NormalizeBucketKey(bucket string, prefix string, key string) (string, strin func OpenBucket(ctx context.Context, bucketURL string, bucketPrefix string) (Bucket, error) { if strings.HasPrefix(bucketURL, "http") { - bucket := HTTPBucket{bucketURL} + bucket := HTTPBucket{bucketURL, http.DefaultClient} return bucket, nil } bucket, err := blob.OpenBucket(ctx, bucketURL) diff --git a/pmtiles/bucket_test.go b/pmtiles/bucket_test.go index 2039a8f..f65681b 100644 --- a/pmtiles/bucket_test.go +++ b/pmtiles/bucket_test.go @@ -1,10 +1,14 @@ package pmtiles import ( - "github.com/stretchr/testify/assert" + "context" + "io" + "net/http" "os" "strings" "testing" + + "github.com/stretchr/testify/assert" ) func TestNormalizeLocalFile(t *testing.T) { @@ -35,3 +39,78 @@ func TestNormalizePathPrefixServer(t *testing.T) { assert.True(t, strings.HasSuffix(bucket, "/foo")) assert.True(t, strings.HasPrefix(bucket, "file://")) } + +type ClientMock struct { + request *http.Request + response *http.Response +} + +func (c *ClientMock) Do(req *http.Request) (*http.Response, error) { + c.request = req + return c.response, nil +} + +func TestHttpBucketRequestNormal(t *testing.T) { + mock := ClientMock{} + header := http.Header{} + header.Add("ETag", "etag") + bucket := HTTPBucket{"http://tiles.example.com/tiles", &mock} + mock.response = &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader("abc")), + Header: header, + } + data, etag, err := bucket.NewRangeReaderEtag(context.Background(), "a/b/c", 100, 3, "") + assert.Equal(t, "", mock.request.Header.Get("If-Match")) + assert.Equal(t, "bytes=100-102", mock.request.Header.Get("Range")) + assert.Equal(t, "http://tiles.example.com/tiles/a/b/c", mock.request.URL.String()) + assert.Nil(t, err) + b, err := io.ReadAll(data) + assert.Nil(t, err) + assert.Equal(t, "abc", string(b)) + assert.Equal(t, "etag", etag) + assert.Nil(t, err) +} + +func TestHttpBucketRequestRequestEtag(t *testing.T) { + mock := ClientMock{} + header := http.Header{} + header.Add("ETag", "etag2") + bucket := HTTPBucket{"http://tiles.example.com/tiles", &mock} + mock.response = &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader("abc")), + Header: header, + } + data, etag, err := bucket.NewRangeReaderEtag(context.Background(), "a/b/c", 0, 3, "etag1") + assert.Equal(t, "etag1", mock.request.Header.Get("If-Match")) + assert.Nil(t, err) + b, err := io.ReadAll(data) + assert.Nil(t, err) + assert.Equal(t, "abc", string(b)) + assert.Equal(t, "etag2", etag) + assert.Nil(t, err) +} + +func TestHttpBucketRequestRequestEtagFailed(t *testing.T) { + mock := ClientMock{} + header := http.Header{} + header.Add("ETag", "etag2") + bucket := HTTPBucket{"http://tiles.example.com/tiles", &mock} + mock.response = &http.Response{ + StatusCode: 412, + Body: io.NopCloser(strings.NewReader("abc")), + Header: header, + } + _, _, err := bucket.NewRangeReaderEtag(context.Background(), "a/b/c", 0, 3, "etag1") + assert.Equal(t, "etag1", mock.request.Header.Get("If-Match")) + assert.True(t, isRefreshRequredError(err)) + + mock.response.StatusCode = 416 + _, _, err = bucket.NewRangeReaderEtag(context.Background(), "a/b/c", 0, 3, "etag1") + assert.True(t, isRefreshRequredError(err)) + + mock.response.StatusCode = 404 + _, _, err = bucket.NewRangeReaderEtag(context.Background(), "a/b/c", 0, 3, "etag1") + assert.False(t, isRefreshRequredError(err)) +} diff --git a/pmtiles/server.go b/pmtiles/server.go index 9496bb4..74df12f 100644 --- a/pmtiles/server.go +++ b/pmtiles/server.go @@ -17,13 +17,15 @@ import ( type cacheKey struct { name string + etag string offset uint64 // is 0 for header length uint64 // is 0 for header } type request struct { - key cacheKey - value chan cachedValue + key cacheKey + value chan cachedValue + purgeEtag string } type cachedValue struct { @@ -31,6 +33,7 @@ type cachedValue struct { directory []EntryV3 etag string ok bool + badEtag bool } type response struct { @@ -115,6 +118,20 @@ func (server *Server) Start() { for { select { case req := <-server.reqs: + if len(req.purgeEtag) > 0 { + if _, dup := inflight[req.key]; !dup { + server.logger.Printf("re-fetching directories for changed file %s", req.key.name) + } + for k, v := range cache { + resp := v.Value.(*response) + if k.name == req.key.name && (k.etag == req.purgeEtag || resp.value.etag == req.purgeEtag) { + evictList.Remove(v) + delete(cache, k) + totalSize -= resp.size + } + } + cacheSize.Set(float64(len(cache))) + } key := req.key if val, ok := cache[key]; ok { evictList.MoveToFront(val) @@ -136,11 +153,11 @@ func (server *Server) Start() { } server.logger.Printf("fetching %s %d-%d", key.name, offset, length) - r, err := server.bucket.NewRangeReader(ctx, key.name+".pmtiles", offset, length) + r, etag, err := server.bucket.NewRangeReaderEtag(ctx, key.name+".pmtiles", offset, length, key.etag) - // TODO: store away ETag if err != nil { ok = false + result.badEtag = isRefreshRequredError(err) resps <- response{key: key, value: result} server.logger.Printf("failed to fetch %s %d-%d, %v", key.name, key.offset, key.length, err) return @@ -163,20 +180,20 @@ func (server *Server) Start() { // populate the root first before header rootEntries := deserializeEntries(bytes.NewBuffer(b[header.RootOffset : header.RootOffset+header.RootLength])) - result2 := cachedValue{directory: rootEntries, ok: true} + result2 := cachedValue{directory: rootEntries, ok: true, etag: etag} rootKey := cacheKey{name: key.name, offset: header.RootOffset, length: header.RootLength} resps <- response{key: rootKey, value: result2, size: 24 * len(rootEntries), ok: true} - result = cachedValue{header: header, ok: true} + result = cachedValue{header: header, ok: true, etag: etag} resps <- response{key: key, value: result, size: 127, ok: true} } else { directory := deserializeEntries(bytes.NewBuffer(b)) - result = cachedValue{directory: directory, ok: true} + result = cachedValue{directory: directory, ok: true, etag: etag} resps <- response{key: key, value: result, size: 24 * len(directory), ok: true} } - server.logger.Printf("fetched %s %d-%d", key.name, key.offset, key.length) + server.logger.Printf("fetched %s %d-%d", key.name, key.offset, length) }() } case resp := <-resps: @@ -213,18 +230,29 @@ func (server *Server) Start() { } func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, HeaderV3, []byte, error) { - rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1)} + found, header, metadataBytes, purgeEtag, err := server.getHeaderMetadataAttempt(ctx, name, "") + if len(purgeEtag) > 0 { + found, header, metadataBytes, _, err = server.getHeaderMetadataAttempt(ctx, name, purgeEtag) + } + return found, header, metadataBytes, err +} + +func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeEtag string) (bool, HeaderV3, []byte, string, error) { + rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1), purgeEtag: purgeEtag} server.reqs <- rootReq rootValue := <-rootReq.value header := rootValue.header if !rootValue.ok { - return false, HeaderV3{}, nil, nil + return false, HeaderV3{}, nil, "", nil } - r, err := server.bucket.NewRangeReader(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength)) + r, _, err := server.bucket.NewRangeReaderEtag(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength), rootValue.etag) + if isRefreshRequredError(err) { + return false, HeaderV3{}, nil, rootValue.etag, nil + } if err != nil { - return false, HeaderV3{}, nil, nil + return false, HeaderV3{}, nil, "", nil } defer r.Close() @@ -236,10 +264,10 @@ func (server *Server) getHeaderMetadata(ctx context.Context, name string) (bool, } else if header.InternalCompression == NoCompression { metadataBytes, err = io.ReadAll(r) } else { - return true, HeaderV3{}, nil, errors.New("unknown compression") + return true, HeaderV3{}, nil, "", errors.New("unknown compression") } - return true, header, metadataBytes, nil + return true, header, metadataBytes, "", nil } func (server *Server) getTileJSON(ctx context.Context, httpHeaders map[string]string, name string) (int, map[string]string, []byte) { @@ -284,9 +312,16 @@ func (server *Server) getMetadata(ctx context.Context, httpHeaders map[string]st httpHeaders["Content-Type"] = "application/json" return 200, httpHeaders, metadataBytes } - func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) { - rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1)} + status, headers, data, purgeEtag := server.getTileAttempt(ctx, httpHeaders, name, z, x, y, ext, "") + if len(purgeEtag) > 0 { + // file has new etag, retry once force-purging the etag that is no longer value + status, headers, data, _ = server.getTileAttempt(ctx, httpHeaders, name, z, x, y, ext, purgeEtag) + } + return status, headers, data +} +func (server *Server) getTileAttempt(ctx context.Context, httpHeaders map[string]string, name string, z uint8, x uint32, y uint32, ext string, purgeEtag string) (int, map[string]string, []byte, string) { + rootReq := request{key: cacheKey{name: name, offset: 0, length: 0}, value: make(chan cachedValue, 1), purgeEtag: purgeEtag} server.reqs <- rootReq // https://golang.org/doc/faq#atomic_maps @@ -294,33 +329,33 @@ func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string header := rootValue.header if !rootValue.ok { - return 404, httpHeaders, []byte("Archive not found") + return 404, httpHeaders, []byte("Archive not found"), "" } if z < header.MinZoom || z > header.MaxZoom { - return 404, httpHeaders, []byte("Tile not found") + return 404, httpHeaders, []byte("Tile not found"), "" } switch header.TileType { case Mvt: if ext != "mvt" { - return 400, httpHeaders, []byte("path mismatch: archive is type MVT (.mvt)") + return 400, httpHeaders, []byte("path mismatch: archive is type MVT (.mvt)"), "" } case Png: if ext != "png" { - return 400, httpHeaders, []byte("path mismatch: archive is type PNG (.png)") + return 400, httpHeaders, []byte("path mismatch: archive is type PNG (.png)"), "" } case Jpeg: if ext != "jpg" { - return 400, httpHeaders, []byte("path mismatch: archive is type JPEG (.jpg)") + return 400, httpHeaders, []byte("path mismatch: archive is type JPEG (.jpg)"), "" } case Webp: if ext != "webp" { - return 400, httpHeaders, []byte("path mismatch: archive is type WebP (.webp)") + return 400, httpHeaders, []byte("path mismatch: archive is type WebP (.webp)"), "" } case Avif: if ext != "avif" { - return 400, httpHeaders, []byte("path mismatch: archive is type AVIF (.avif)") + return 400, httpHeaders, []byte("path mismatch: archive is type AVIF (.avif)"), "" } } @@ -328,9 +363,12 @@ func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string dirOffset, dirLen := header.RootOffset, header.RootLength for depth := 0; depth <= 3; depth++ { - dirReq := request{key: cacheKey{name: name, offset: dirOffset, length: dirLen}, value: make(chan cachedValue, 1)} + dirReq := request{key: cacheKey{name: name, offset: dirOffset, length: dirLen, etag: rootValue.etag}, value: make(chan cachedValue, 1)} server.reqs <- dirReq dirValue := <-dirReq.value + if dirValue.badEtag { + return 500, httpHeaders, []byte("I/O Error"), rootValue.etag + } directory := dirValue.directory entry, ok := findTile(directory, tileID) if !ok { @@ -338,16 +376,19 @@ func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string } if entry.RunLength > 0 { - r, err := server.bucket.NewRangeReader(ctx, name+".pmtiles", int64(header.TileDataOffset+entry.Offset), int64(entry.Length)) + r, _, err := server.bucket.NewRangeReaderEtag(ctx, name+".pmtiles", int64(header.TileDataOffset+entry.Offset), int64(entry.Length), rootValue.etag) + if isRefreshRequredError(err) { + return 500, httpHeaders, []byte("I/O Error"), rootValue.etag + } // possible we have the header/directory cached but the archive has disappeared if err != nil { server.logger.Printf("failed to fetch tile %s %d-%d", name, entry.Offset, entry.Length) - return 404, httpHeaders, []byte("archive not found") + return 404, httpHeaders, []byte("archive not found"), "" } defer r.Close() b, err := io.ReadAll(r) if err != nil { - return 500, httpHeaders, []byte("I/O error") + return 500, httpHeaders, []byte("I/O error"), "" } if headerVal, ok := headerContentType(header); ok { httpHeaders["Content-Type"] = headerVal @@ -355,12 +396,17 @@ func (server *Server) getTile(ctx context.Context, httpHeaders map[string]string if headerVal, ok := headerContentEncoding(header.TileCompression); ok { httpHeaders["Content-Encoding"] = headerVal } - return 200, httpHeaders, b + return 200, httpHeaders, b, "" } dirOffset = header.LeafDirectoryOffset + entry.Offset dirLen = uint64(entry.Length) } - return 204, httpHeaders, nil + return 204, httpHeaders, nil, "" +} + +func isRefreshRequredError(err error) bool { + _, ok := err.(*RefreshRequiredError) + return ok } var tilePattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`) diff --git a/pmtiles/server_test.go b/pmtiles/server_test.go index bcbee7c..b7063a7 100644 --- a/pmtiles/server_test.go +++ b/pmtiles/server_test.go @@ -1,8 +1,15 @@ package pmtiles import ( - "github.com/stretchr/testify/assert" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "log" + "sort" "testing" + + "github.com/stretchr/testify/assert" ) func TestRegex(t *testing.T) { @@ -37,3 +44,329 @@ func TestRegex(t *testing.T) { assert.True(t, ok) assert.Equal(t, key, "!-_.*'()") } + +func fakeArchive(t *testing.T, header HeaderV3, metadata map[string]interface{}, tiles map[Zxy][]byte, leaves bool) []byte { + byTileID := make(map[uint64][]byte) + keys := make([]uint64, 0, len(tiles)) + for zxy, bytes := range tiles { + header.MaxZoom = max(header.MaxZoom, zxy.Z) + id := ZxyToID(zxy.Z, zxy.X, zxy.Y) + byTileID[id] = bytes + keys = append(keys, id) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + resolver := newResolver(false, false) + tileDataBytes := make([]byte, 0) + for _, id := range keys { + tileBytes := byTileID[id] + resolver.AddTileIsNew(id, tileBytes) + tileDataBytes = append(tileDataBytes, tileBytes...) + } + + var metadataBytes []byte + { + metadataBytesUncompressed, err := json.Marshal(metadata) + assert.Nil(t, err) + var b bytes.Buffer + w, _ := gzip.NewWriterLevel(&b, gzip.BestCompression) + w.Write(metadataBytesUncompressed) + w.Close() + metadataBytes = b.Bytes() + } + var rootBytes []byte + var leavesBytes []byte + if leaves { + rootBytes, leavesBytes, _ = buildRootsLeaves(resolver.Entries, 1) + } else { + rootBytes = serializeEntries(resolver.Entries) + leavesBytes = make([]byte, 0) + } + + header.InternalCompression = Gzip + if header.TileType == Mvt { + header.TileCompression = Gzip + } + + header.RootOffset = HeaderV3LenBytes + header.RootLength = uint64(len(rootBytes)) + header.MetadataOffset = header.RootOffset + header.RootLength + header.MetadataLength = uint64(len(metadataBytes)) + header.LeafDirectoryOffset = header.MetadataOffset + header.MetadataLength + header.LeafDirectoryLength = uint64(len(leavesBytes)) + header.TileDataOffset = header.LeafDirectoryOffset + header.LeafDirectoryLength + header.TileDataLength = resolver.Offset + + archiveBytes := serializeHeader(header) + archiveBytes = append(archiveBytes, rootBytes...) + archiveBytes = append(archiveBytes, metadataBytes...) + archiveBytes = append(archiveBytes, leavesBytes...) + archiveBytes = append(archiveBytes, tileDataBytes...) + if len(archiveBytes) < 16384 { + archiveBytes = append(archiveBytes, make([]byte, 16384-len(archiveBytes))...) + } + return archiveBytes +} + +func newServer(t *testing.T) (mockBucket, *Server) { + bucket := mockBucket{make(map[string][]byte)} + server, err := NewServerWithBucket(bucket, "", log.Default(), 10, "", "tiles.example.com") + assert.Nil(t, err) + server.Start() + return bucket, server +} + +func TestMissingFileReturns404(t *testing.T) { + _, server := newServer(t) + statusCode, _, _ := server.Get(context.Background(), "/") + assert.Equal(t, 204, statusCode) + statusCode, _, _ = server.Get(context.Background(), "/archive.json") + assert.Equal(t, 404, statusCode) + statusCode, _, _ = server.Get(context.Background(), "/archive/metadata") + assert.Equal(t, 404, statusCode) + statusCode, _, _ = server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 404, statusCode) +} + +func TestMvtEmptyArchiveReads(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{}, false) + + statusCode, _, _ := server.Get(context.Background(), "/") + assert.Equal(t, 204, statusCode) + statusCode, _, data := server.Get(context.Background(), "/archive.json") + assert.JSONEq(t, `{ + "bounds": [0,0,0,0], + "center": [0,0,0], + "maxzoom": 0, + "minzoom": 0, + "scheme": "xyz", + "tilejson": "3.0.0", + "tiles": ["tiles.example.com/archive/{z}/{x}/{y}.mvt"], + "vector_layers": null + }`, string(data)) + assert.Equal(t, 200, statusCode) + statusCode, _, data = server.Get(context.Background(), "/archive/metadata") + assert.JSONEq(t, `{}`, string(data)) + assert.Equal(t, 200, statusCode) + statusCode, _, _ = server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 204, statusCode) +} + +func TestReadMetadata(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{ + "vector_layers": []map[string]string{{"id": "layer1"}}, + "attribution": "Attribution", + "description": "Description", + "name": "Name", + "version": "1.0", + }, map[Zxy][]byte{}, false) + + statusCode, _, _ := server.Get(context.Background(), "/") + assert.Equal(t, 204, statusCode) + statusCode, _, data := server.Get(context.Background(), "/archive.json") + assert.JSONEq(t, `{ + "attribution": "Attribution", + "description": "Description", + "name": "Name", + "version": "1.0", + "bounds": [0,0,0,0], + "center": [0,0,0], + "maxzoom": 0, + "minzoom": 0, + "scheme": "xyz", + "tilejson": "3.0.0", + "tiles": ["tiles.example.com/archive/{z}/{x}/{y}.mvt"], + "vector_layers": [ + {"id": "layer1"} + ] + }`, string(data)) + assert.Equal(t, 200, statusCode) + statusCode, _, data = server.Get(context.Background(), "/archive/metadata") + assert.JSONEq(t, `{ + "attribution": "Attribution", + "description": "Description", + "name": "Name", + "version": "1.0", + "vector_layers": [ + {"id": "layer1"} + ] + }`, string(data)) +} + +func TestReadTiles(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1, 2, 3}, + {4, 1, 2}: {1, 2, 3}, + }, false) + + statusCode, _, _ := server.Get(context.Background(), "/") + assert.Equal(t, 204, statusCode) + statusCode, _, _ = server.Get(context.Background(), "/archive.json") + assert.Equal(t, 200, statusCode) + statusCode, _, _ = server.Get(context.Background(), "/archive/metadata") + assert.Equal(t, 200, statusCode) + statusCode, _, data := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{0, 1, 2, 3}, data) + statusCode, _, data = server.Get(context.Background(), "/archive/4/1/2.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{1, 2, 3}, data) + statusCode, _, _ = server.Get(context.Background(), "/archive/3/1/2.mvt") + assert.Equal(t, 204, statusCode) +} + +func TestReadTilesFromLeaves(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1, 2, 3}, + {4, 1, 2}: {1, 2, 3}, + }, true) + + statusCode, _, data := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{0, 1, 2, 3}, data) + statusCode, _, data = server.Get(context.Background(), "/archive/4/1/2.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{1, 2, 3}, data) + statusCode, _, _ = server.Get(context.Background(), "/archive/3/1/2.mvt") + assert.Equal(t, 204, statusCode) +} + +func TestInvalidateCacheOnTileRequest(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1, 2, 3}, + }, false) + + statusCode, _, data := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{0, 1, 2, 3}, data) + + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {4, 5, 6, 7}, + }, false) + + statusCode, _, data = server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{4, 5, 6, 7}, data) +} + +func TestInvalidateCacheOnDirRequest(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1}, + {1, 1, 1}: {2, 3}, + }, true) + + // cache first leaf dir + statusCode, _, data := server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{0, 1}, data) + + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {4, 5}, + {1, 1, 1}: {6, 7}, + }, false) + + // get etag mismatch on second leaf dir request + statusCode, _, data = server.Get(context.Background(), "/archive/1/1/1.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{6, 7}, data) + statusCode, _, data = server.Get(context.Background(), "/archive/0/0/0.mvt") + assert.Equal(t, 200, statusCode) + assert.Equal(t, []byte{4, 5}, data) +} + +func TestInvalidateCacheOnTileJSONRequest(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1}, + {1, 1, 1}: {2, 3}, + }, false) + statusCode, _, data := server.Get(context.Background(), "/archive.json") + assert.Equal(t, 200, statusCode) + assert.JSONEq(t, `{ + "bounds": [0,0,0,0], + "center": [0,0,0], + "maxzoom": 1, + "minzoom": 0, + "scheme": "xyz", + "tilejson": "3.0.0", + "tiles": ["tiles.example.com/archive/{z}/{x}/{y}.mvt"], + "vector_layers": null + }`, string(data)) + + header = HeaderV3{ + TileType: Mvt, + CenterZoom: 4, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{}, map[Zxy][]byte{ + {0, 0, 0}: {0, 1}, + {1, 1, 1}: {2, 3}, + }, false) + statusCode, _, data = server.Get(context.Background(), "/archive.json") + assert.Equal(t, 200, statusCode) + assert.JSONEq(t, `{ + "bounds": [0,0,0,0], + "center": [0,0,4], + "maxzoom": 1, + "minzoom": 0, + "scheme": "xyz", + "tilejson": "3.0.0", + "tiles": ["tiles.example.com/archive/{z}/{x}/{y}.mvt"], + "vector_layers": null + }`, string(data)) +} + +func TestInvalidateCacheOnMetadataRequest(t *testing.T) { + mockBucket, server := newServer(t) + header := HeaderV3{ + TileType: Mvt, + } + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{ + "meta": "data", + }, map[Zxy][]byte{ + {0, 0, 0}: {0, 1}, + {1, 1, 1}: {2, 3}, + }, false) + statusCode, _, data := server.Get(context.Background(), "/archive/metadata") + assert.Equal(t, 200, statusCode) + assert.JSONEq(t, `{ + "meta": "data" + }`, string(data)) + + mockBucket.items["archive.pmtiles"] = fakeArchive(t, header, map[string]interface{}{ + "meta": "data2", + }, map[Zxy][]byte{ + {0, 0, 0}: {0, 1}, + {1, 1, 1}: {2, 3}, + }, false) + statusCode, _, data = server.Get(context.Background(), "/archive/metadata") + assert.Equal(t, 200, statusCode) + assert.JSONEq(t, `{ + "meta": "data2" + }`, string(data)) +}