From e5c7c060e336be7c0318dc416744fa9016658d0e Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 09:19:52 +0200 Subject: [PATCH 1/7] wip: feat: add t0109 --- fixtures/t0109-redirects.car | Bin 0 -> 69257 bytes go.mod | 15 +- go.sum | 9 +- tests/t0109_gateway_web_redirects_test.go | 240 ++++++++++++++++++++++ tooling/car/fixture.go | 32 +++ tooling/car/unixfs.go | 2 +- 6 files changed, 282 insertions(+), 16 deletions(-) create mode 100644 fixtures/t0109-redirects.car create mode 100644 tests/t0109_gateway_web_redirects_test.go diff --git a/fixtures/t0109-redirects.car b/fixtures/t0109-redirects.car new file mode 100644 index 0000000000000000000000000000000000000000..ba4da8ba535d3fd0fb32546fda7f855e3d1232dd GIT binary patch literal 69257 zcmeI#dsGzH9S3lCgRqPmZ~z@&br8iwTxWNAE{d88sDMf(Px{kx$Vt{&egdJ-$@3ZmVq zZ)uq>Lz9?fAR+hc=?Q#DWXr_80gv6nPfo9rttndc=!ZOyxf$)XC5c*HZ1nNaP}+8x zDLGcBWkP9B{`oXl+x#=;{V_X?yZZ+3i5S_jPTsg>z4M-H?b_D5t#(c{6=yP8`*l%D zb9|8U^sM%c+F$B>?tEcmKdNii^M=)`l4YlLj;i|Gn-`fU(RO-cn#Q2lGQpOC6bIbh zeR)aSko=sb73J@(lZGeWa$7a(MrHGsc^;))RRk?F>e3B*BN<4K#L}^mA6@PaIs71; zI~=kntg)jiA*lHLwb|duR{X<#^`*KKS802*$s{*ulH+ttW3~+?l}R>8-e8)AWSHMo zu2j~2pVU27<+*#c$CyhO2Gz}Iy3XUV`<`jmuByjqf+zapb ziJG}n7nfVy`7}+w>!g!HFH1s?knE!KR!!3nABCYlrbuPfO_nq;#+mM@NcsU?NJS2lmubuOu9!SdyU zuNFNk{O&jApvKL=E!~-zYfv_4scA}Y)ao)`rqD8eS<^cqDTR~6MoU-CxMJ`fU68Nu zFr^ONY`@F(_Qr!YHnhxS)LBR4$Ir1L)#1~mzU7NHbv8$`flm|hvmI$$gzK-x#l881b&5C7|s+E#{ zosx3asgRRdWI5|pQkM(O)}@7Xfm4PIe)Dtlk+$0%?8p5>$3Upj|e>wGRKW$Uk$;UP0nAqW$CBU zWR?qM%0FqXEg>0P92u?f>&yCKNg+{Os!@}s(Hn#tFUy1_^+NGK{M$&FQ_@J96|Dm~ zzbJ2aoYNiMFl>6@@Ca4QfzYsdezD&w>$ZisWTi%j2b64~ZBz=CifNJX>uT;BJnzB3 zHLv1PeB!ii4!*2y+0MNqltG#9uLkE{Gc{pNnlbVVubzgQ-Q*+~!(iauvol0*eM|kIaniYLo*kE>bb?@>k&giVv zg$FCbn#K>DeWEYb8a}*zzsuzv{U5u~{o-{7gGrulN;YVjFn$hVXjU{zuq3!~hV${- za_8axrC(JqtZfZ!yVgCD&RHw{)rp|AwPA1f+EkqRSNqVcSP(^%^5`#giWNFVG5v`Q zVjI{i8HANWWFYk}wt>B>Ua}lX^+vtH1}4C8AffMhF6T=J-F?r!?(M5jclGrBdi9Ir zcJ4vjyCuUnJo%T6aRfi=DKsk%ySz^WN1WJEeD=qdxW>1lYIaOMkWfT#S{$N?P|&@4 zR3M?p^VRtG9cJ%>K@mQNse#osXA3W`*!TO7BV0EbE?qi7uc>INrEQjyjxaO$0h}UT z5&Hprr)^M!Qb zo#(*=^5YASxsF(()2wK7svNmLpweyp`ruyG>B!HFie|-vsJF?y$Ox+Z6i|$e^hWeZ zEcB)%zS-t_+sy~LG3Tp>`HU%aNKEQ8Ai3}cQTJ#Gt~xzhZ-SFZlW{HFXL6@@F@%GMUm&WfK>ral>5Z%EB}r>s|J0{%ZI zD1OL`X;!Rj)I>65?^p8nK0FNgANmiV81Dr%EBeoc7hg{en`~Q)9!Et?JNfY5Mc>*3 zZcWcVHUx!jPKykc%O4(G*=vIGE&qd5W!s1cQhC`nWB}{3ZMbF0rbKdfF_A(bg@hJn z0;>S^fV}{_B1C|O&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ejG=zrG z5E?>5Xb26VAvA=B&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ejG=zrG z5E?>5Xb26VAvA=B&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ejG=zrG z5E?>5Xb26VAvA=B&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ejG=zrG z5E?>5Xb26VAvA=B&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ejG=zrG z5E?>5Xb26VAvA=B&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ejG=zrG z5E?>5Xb26VAvA=B&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ejG=zrG z5E?>5Xb26VAvA=B&=49zLud#Mp&>MchR_fiLPKZ>4WS`4goe-%8bU*82o0ej#jMM= F`6u^#eR%)? literal 0 HcmV?d00001 diff --git a/go.mod b/go.mod index d94b2411b..99c0ec14d 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,18 @@ require ( github.com/ipfs/boxo v0.8.0-rc1 github.com/ipfs/go-cid v0.4.0 github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d + github.com/ipld/go-ipld-prime v0.20.0 github.com/urfave/cli/v2 v2.25.0 ) require ( github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/ipfs/go-ipfs-files v0.3.0 // indirect + github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect - github.com/ipld/go-ipld-prime v0.20.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + go.uber.org/goleak v1.1.12 // indirect golang.org/x/sync v0.1.0 // indirect ) @@ -29,7 +30,7 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.2 // indirect - github.com/ipfs/go-blockservice v0.5.0 + github.com/ipfs/go-blockservice v0.5.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect @@ -38,12 +39,10 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipld-legacy v0.1.1 // indirect - github.com/ipfs/go-libipfs v0.6.0 // indirect github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-merkledag v0.10.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect - github.com/ipfs/go-unixfs v0.4.5 github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect @@ -52,9 +51,9 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.8.1 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multibase v0.1.1 + github.com/multiformats/go-multicodec v0.8.1 + github.com/multiformats/go-multihash v0.2.1 github.com/multiformats/go-varint v0.0.7 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect diff --git a/go.sum b/go.sum index 407c60aa9..2cfcb2400 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -81,9 +82,6 @@ github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNo github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= -github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= -github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= -github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= @@ -97,8 +95,6 @@ github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSg github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= -github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= -github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= @@ -109,8 +105,6 @@ github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2m github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= -github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= @@ -260,6 +254,7 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go new file mode 100644 index 000000000..5811b64fa --- /dev/null +++ b/tests/t0109_gateway_web_redirects_test.go @@ -0,0 +1,240 @@ +package tests + +import ( + "fmt" + "testing" + + "github.com/ipfs/gateway-conformance/tooling/car" + . "github.com/ipfs/gateway-conformance/tooling/check" + "github.com/ipfs/gateway-conformance/tooling/test" + . "github.com/ipfs/gateway-conformance/tooling/test" + mb "github.com/multiformats/go-multibase" +) + +func TestRedirectsFileSupport(t *testing.T) { + fixture := car.MustOpenUnixfsCar("t0109-redirects.car") + + // root := fixture.MustGetNode() + redirectDir := fixture.MustGetNode("examples") + + redirectDirCID, err := mb.Encode(mb.Base32, redirectDir.Cid().Bytes()) + if err != nil { + t.Fatal(err) + } + + redirectDirHostname := fmt.Sprintf("%s.ipfs.localhost:8080", redirectDirCID) + + // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/404.html | cut -d "/" -f3) + custom404 := fixture.MustGetNode("examples", "404.html") + // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3) + custom410 := fixture.MustGetNode("examples", "410.html") + // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3) + custom451 := fixture.MustGetNode("examples", "451.html") + + tests := SugarTests{ + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/redirect-one" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /one.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("http://%s/redirect-one", redirectDirHostname), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/one.html"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/301-redirect-one" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /one.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("http://%s/301-redirect-one", redirectDirHostname), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/one.html"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/302-redirect-two" > response && + // test_should_contain "302 Found" response && + // test_should_contain "Location: /two.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("http://%s/302-redirect-two", redirectDirHostname), + Response: Expect(). + Status(302). + Headers( + Header("Location").Equals("/two.html"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/200-index" > response && + // test_should_contain "my index" response && + // test_should_contain "200 OK" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("http://%s/200-index", redirectDirHostname), + Response: Expect(). + Status(200). + Body(Contains("my index")), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/posts/2022/01/01/hello-world" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /articles/2022/01/01/hello-world" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("http://%s/posts/2022/01/01/hello-world", redirectDirHostname), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/articles/2022/01/01/hello-world"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/splat/one.html" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /redirected-splat/one.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("http://%s/splat/one.html", redirectDirHostname), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/redirected-splat/one.html"), + ), + }, + // # ensure custom 4xx works and has the same cache headers as regular /ipfs/ path + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry" > response && + // test_should_contain "404 Not Found" response && + // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && + // test_should_contain "my 404" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file", + Request: Request(). + URL("http://%s/not-found/has-no-redirects-entry", redirectDirHostname), + Response: Expect(). + Status(404). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals("\"%s\"", custom404.Cid().String()), + ). + Body(Contains(string(custom404.RawData()))), + }, + // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3) + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file" ' + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry" > response && + // test_should_contain "410 Gone" response && + // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && + // test_should_contain "my 410" response + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file", + Request: Request(). + URL("http://%s/gone/has-no-redirects-entry", redirectDirHostname), + Response: Expect(). + Status(410). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals("\"%s\"", custom410.Cid().String()), + ). + Body(Contains(string(custom410.RawData()))), + }, + // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3) + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry" > response && + // test_should_contain "451 Unavailable For Legal Reasons" response && + // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && + // test_should_contain "my 451" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file", + Request: Request(). + URL("http://%s/unavail/has-no-redirects-entry", redirectDirHostname), + Response: Expect(). + Status(451). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals("\"%s\"", custom451.Cid().String()), + ). + Body(Contains(string(custom451.RawData()))), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/catch-all" > response && + // test_should_contain "200 OK" response && + // test_should_contain "my index" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file", + Request: Request(). + URL("http://%s/catch-all", redirectDirHostname), + Response: Expect(). + Status(200). + Body(Contains("my index")), + }, + // # This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain) + // test_expect_success "request for http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one returns generic 404 (no custom 404 from _redirects since no origin isolation)" ' + // + // curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one" > response && + // test_should_contain "404 Not Found" response && + // test_should_not_contain "my 404" response + // + // ' + { + // TODO: how to test this correctly? + Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", + Request: Request(). + URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), + Response: Expect(). + Status(404). + Body(Not(Contains("my 404"))), + }, + } + + test.Run(t, tests.Build()) +} diff --git a/tooling/car/fixture.go b/tooling/car/fixture.go index ef2e4c1bc..9d82907eb 100644 --- a/tooling/car/fixture.go +++ b/tooling/car/fixture.go @@ -3,6 +3,8 @@ package car import ( "time" + _ "github.com/ipfs/boxo/ipld/merkledag" + "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" "github.com/ipld/go-ipld-prime" @@ -11,6 +13,7 @@ import ( type FixtureNode struct { node format.Node + dsvc format.DAGService } // get cid method @@ -29,6 +32,35 @@ func (n *FixtureNode) Formatted(codecStr string) []byte { return FormatDagNode(node, codecStr) } + +// func (n *FixtureNode) ToFile() files.File { +// f, err := unixfile.NewUnixfsFile(context.Background(), n.dsvc, n.node) +// if err != nil { +// panic(err) +// } + +// r, ok := f.(files.File) + +// if !ok { +// panic("not a file") +// } + +// return r +// } + +// // read the file +// func (n *FixtureNode) ReadFile() string { +// f := n.ToFile() + +// buf := new(strings.Builder) +// _, err := io.Copy(buf, f) +// if err != nil { +// panic(err) +// } + +// return buf.String() +// } + func RandomCID() cid.Cid { now := time.Now().UTC() timeBytes := []byte(now.Format(time.RFC3339)) diff --git a/tooling/car/unixfs.go b/tooling/car/unixfs.go index 797889834..21af7ffe8 100644 --- a/tooling/car/unixfs.go +++ b/tooling/car/unixfs.go @@ -96,7 +96,7 @@ func (d *UnixfsDag) mustGetNode(names ...string) format.Node { } func (d *UnixfsDag) MustGetNode(names ...string) *FixtureNode { - return &FixtureNode{node: d.mustGetNode(names...)} + return &FixtureNode{node: d.mustGetNode(names...), dsvc: d.dsvc} } func (d *UnixfsDag) MustGetRoot() *FixtureNode { From d2c241afc01547a183d2d062185f8b0c6fd83dcc Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 10:25:11 +0200 Subject: [PATCH 2/7] fix: t109 & module deps --- go.mod | 9 +---- go.sum | 26 +----------- tests/t0109_gateway_web_redirects_test.go | 11 +++-- tooling/car/fixture.go | 49 ++++++++++++----------- tooling/car/merge.go | 2 +- 5 files changed, 36 insertions(+), 61 deletions(-) diff --git a/go.mod b/go.mod index 99c0ec14d..07b4cc2cd 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,8 @@ module github.com/ipfs/gateway-conformance go 1.19 require ( - github.com/ipfs/boxo v0.8.0-rc1 + github.com/ipfs/boxo v0.8.0-rc3 github.com/ipfs/go-cid v0.4.0 - github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d github.com/ipld/go-ipld-prime v0.20.0 github.com/urfave/cli/v2 v2.25.0 ) @@ -30,20 +29,14 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.2 // indirect - github.com/ipfs/go-blockservice v0.5.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect - github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect - github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect - github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipld-legacy v0.1.1 // indirect github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 // indirect - github.com/ipfs/go-merkledag v0.10.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect - github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/mattn/go-isatty v0.0.17 // indirect diff --git a/go.sum b/go.sum index 2cfcb2400..ae56defbd 100644 --- a/go.sum +++ b/go.sum @@ -34,7 +34,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -46,44 +45,31 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc1 h1:DL5SDbBNSS9ZNsF+UhoQ39d05/wgoJ2k/T+y7JeWRaw= -github.com/ipfs/boxo v0.8.0-rc1/go.mod h1:EgDiNox/+W/+ySwEotRrHlvdmrhbSAB4p22ELg+ZsCc= +github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc= +github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= -github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= -github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= -github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= -github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= -github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= -github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM= -github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= -github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= -github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= -github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= -github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= -github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= -github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= @@ -100,16 +86,10 @@ github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JP github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-merkledag v0.10.0 h1:IUQhj/kzTZfam4e+LnaEpoiZ9vZF6ldimVlby+6OXL4= -github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2mCwxkQ6GIj8= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo= -github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= -github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= -github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d h1:22g+x1tgWSXK34i25qjs+afr7basaneEkHaglBshd2g= -github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d/go.mod h1:SH2pi/NgfGBsV/CGBAQPxMfghIgwzbh5lQ2N+6dNRI8= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= @@ -131,7 +111,6 @@ github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y7 github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -335,7 +314,6 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index 5811b64fa..6d95fc38b 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -158,7 +158,7 @@ func TestRedirectsFileSupport(t *testing.T) { Header("Cache-Control").Equals("public, max-age=29030400, immutable"), Header("Etag").Equals("\"%s\"", custom404.Cid().String()), ). - Body(Contains(string(custom404.RawData()))), + Body(Contains(custom404.ReadFile())), }, // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3) // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file" ' @@ -178,7 +178,7 @@ func TestRedirectsFileSupport(t *testing.T) { Header("Cache-Control").Equals("public, max-age=29030400, immutable"), Header("Etag").Equals("\"%s\"", custom410.Cid().String()), ). - Body(Contains(string(custom410.RawData()))), + Body(Contains(custom410.ReadFile())), }, // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3) // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file" ' @@ -200,7 +200,7 @@ func TestRedirectsFileSupport(t *testing.T) { Header("Cache-Control").Equals("public, max-age=29030400, immutable"), Header("Etag").Equals("\"%s\"", custom451.Cid().String()), ). - Body(Contains(string(custom451.RawData()))), + Body(Contains(custom451.ReadFile())), }, // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file" ' // @@ -228,11 +228,14 @@ func TestRedirectsFileSupport(t *testing.T) { { // TODO: how to test this correctly? Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", + Hint: ` + We expect the request to fail with a 404 (do not use the _redirect), and that 404 should not contain the custom 404 body. + `, Request: Request(). URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), Response: Expect(). Status(404). - Body(Not(Contains("my 404"))), + Body(Not(Contains(custom404.ReadFile()))), }, } diff --git a/tooling/car/fixture.go b/tooling/car/fixture.go index 9d82907eb..182efc0e5 100644 --- a/tooling/car/fixture.go +++ b/tooling/car/fixture.go @@ -1,10 +1,15 @@ package car import ( + "context" + "io" + "strings" "time" + files "github.com/ipfs/boxo/files" _ "github.com/ipfs/boxo/ipld/merkledag" + unixfile "github.com/ipfs/boxo/ipld/unixfs/file" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" "github.com/ipld/go-ipld-prime" @@ -16,50 +21,46 @@ type FixtureNode struct { dsvc format.DAGService } -// get cid method func (n *FixtureNode) Cid() cid.Cid { return n.node.Cid() } -// get raw data method func (n *FixtureNode) RawData() []byte { return n.node.RawData() } -// get formated node (pass codec name as parameter) func (n *FixtureNode) Formatted(codecStr string) []byte { node := n.node.(ipld.Node) return FormatDagNode(node, codecStr) } -// func (n *FixtureNode) ToFile() files.File { -// f, err := unixfile.NewUnixfsFile(context.Background(), n.dsvc, n.node) -// if err != nil { -// panic(err) -// } +func (n *FixtureNode) ToFile() files.File { + f, err := unixfile.NewUnixfsFile(context.Background(), n.dsvc, n.node) + if err != nil { + panic(err) + } -// r, ok := f.(files.File) + r, ok := f.(files.File) -// if !ok { -// panic("not a file") -// } + if !ok { + panic("not a file") + } -// return r -// } + return r +} -// // read the file -// func (n *FixtureNode) ReadFile() string { -// f := n.ToFile() +func (n *FixtureNode) ReadFile() string { + f := n.ToFile() -// buf := new(strings.Builder) -// _, err := io.Copy(buf, f) -// if err != nil { -// panic(err) -// } + buf := new(strings.Builder) + _, err := io.Copy(buf, f) + if err != nil { + panic(err) + } -// return buf.String() -// } + return buf.String() +} func RandomCID() cid.Cid { now := time.Now().UTC() diff --git a/tooling/car/merge.go b/tooling/car/merge.go index 6c00f4595..1097a2f0b 100644 --- a/tooling/car/merge.go +++ b/tooling/car/merge.go @@ -4,8 +4,8 @@ import ( "context" "fmt" + "github.com/ipfs/boxo/ipld/car/v2/blockstore" "github.com/ipfs/go-cid" - "github.com/ipld/go-car/v2/blockstore" ) func Merge(inputPaths []string, outputPath string) error { From 17b02c85046bd369e2c64cdcbbfbc4abd2044861 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 10:29:10 +0200 Subject: [PATCH 3/7] feat: move Base32Cid in fixtures --- tests/t0109_gateway_web_redirects_test.go | 8 +--- tooling/car/fixture.go | 58 +++++++++++++---------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index 6d95fc38b..742c33a0c 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -8,19 +8,13 @@ import ( . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/test" . "github.com/ipfs/gateway-conformance/tooling/test" - mb "github.com/multiformats/go-multibase" ) func TestRedirectsFileSupport(t *testing.T) { fixture := car.MustOpenUnixfsCar("t0109-redirects.car") - // root := fixture.MustGetNode() redirectDir := fixture.MustGetNode("examples") - - redirectDirCID, err := mb.Encode(mb.Base32, redirectDir.Cid().Bytes()) - if err != nil { - t.Fatal(err) - } + redirectDirCID := redirectDir.Base32Cid() redirectDirHostname := fmt.Sprintf("%s.ipfs.localhost:8080", redirectDirCID) diff --git a/tooling/car/fixture.go b/tooling/car/fixture.go index 182efc0e5..c9f473b09 100644 --- a/tooling/car/fixture.go +++ b/tooling/car/fixture.go @@ -13,6 +13,7 @@ import ( "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" "github.com/ipld/go-ipld-prime" + mb "github.com/multiformats/go-multibase" "github.com/multiformats/go-multihash" ) @@ -25,6 +26,14 @@ func (n *FixtureNode) Cid() cid.Cid { return n.node.Cid() } +func (n *FixtureNode) Base32Cid() string { + redirectDirCID, err := mb.Encode(mb.Base32, n.Cid().Bytes()) + if err != nil { + panic(err) + } + return redirectDirCID +} + func (n *FixtureNode) RawData() []byte { return n.node.RawData() } @@ -34,44 +43,43 @@ func (n *FixtureNode) Formatted(codecStr string) []byte { return FormatDagNode(node, codecStr) } - func (n *FixtureNode) ToFile() files.File { - f, err := unixfile.NewUnixfsFile(context.Background(), n.dsvc, n.node) - if err != nil { - panic(err) - } + f, err := unixfile.NewUnixfsFile(context.Background(), n.dsvc, n.node) + if err != nil { + panic(err) + } - r, ok := f.(files.File) + r, ok := f.(files.File) - if !ok { - panic("not a file") - } + if !ok { + panic("not a file") + } - return r + return r } func (n *FixtureNode) ReadFile() string { - f := n.ToFile() + f := n.ToFile() - buf := new(strings.Builder) - _, err := io.Copy(buf, f) - if err != nil { - panic(err) - } + buf := new(strings.Builder) + _, err := io.Copy(buf, f) + if err != nil { + panic(err) + } - return buf.String() + return buf.String() } func RandomCID() cid.Cid { - now := time.Now().UTC() - timeBytes := []byte(now.Format(time.RFC3339)) + now := time.Now().UTC() + timeBytes := []byte(now.Format(time.RFC3339)) - mh, err := multihash.Sum(timeBytes, multihash.SHA2_256, -1) - if err != nil { - panic(err) - } + mh, err := multihash.Sum(timeBytes, multihash.SHA2_256, -1) + if err != nil { + panic(err) + } - c := cid.NewCidV1(cid.Raw, mh) + c := cid.NewCidV1(cid.Raw, mh) return c -} \ No newline at end of file +} From faee6d069e69925d0276684791fa981167d63339 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 10:34:22 +0200 Subject: [PATCH 4/7] feat: add invalid tests --- tests/t0109_gateway_web_redirects_test.go | 57 +++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index 742c33a0c..bba3c1aec 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -233,5 +233,62 @@ func TestRedirectsFileSupport(t *testing.T) { }, } + // # Invalid file, containing forced redirect + // INVALID_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/forced | cut -d "/" -f3) + invalidRedirectsDirCID := fixture.MustGetNode("forced").Base32Cid() + // INVALID_REDIRECTS_DIR_HOSTNAME="${INVALID_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" + invalidDirHostname := fmt.Sprintf("%s.ipfs.localhost:8080", invalidRedirectsDirCID) + // TOO_LARGE_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/too-large | cut -d "/" -f3) + tooLargeRedirectsDirCID := fixture.MustGetNode("too-large").Base32Cid() + // TOO_LARGE_REDIRECTS_DIR_HOSTNAME="${TOO_LARGE_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" + tooLargeDirHostname := fmt.Sprintf("%s.ipfs.localhost:8080", tooLargeRedirectsDirCID) + + tests = append(tests, SugarTests{ + // # if accessing a path that doesn't exist, read _redirects and fail parsing, and return error + // test_expect_success "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file" ' + // curl -sD - --resolve $INVALID_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$INVALID_REDIRECTS_DIR_HOSTNAME/not-found" > response && + // test_should_contain "500" response && + // test_should_contain "could not parse _redirects:" response && + // test_should_contain "forced redirects (or \"shadowing\") are not supported" response + // ' + { + Name: "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file", + Hint: `if accessing a path that doesn't exist, read _redirects and fail parsing, and return error`, + Request: Request(). + URL("http://%s/not-found", invalidDirHostname), + Response: Expect(). + Status(500). + Body( + And( + Contains("could not parse _redirects:"), + Contains("forced redirects (or \"shadowing\") are not supported"), + ), + ), + }, + // # if accessing a path that doesn't exist and _redirects file is too large, return error + // test_expect_success "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file" ' + // curl -sD - --resolve $TOO_LARGE_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found" > response && + // test_should_contain "500" response && + // test_should_contain "could not parse _redirects:" response && + // test_should_contain "redirects file size cannot exceed" response + // ' + { + Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file", + Hint: `if accessing a path that doesn't exist and _redirects file is too large, return error`, + Request: Request(). + URL("http://%s/not-found", tooLargeDirHostname), + Response: Expect(). + Status(500). + Body( + And( + Contains("could not parse _redirects:"), + Contains("redirects file size cannot exceed"), + ), + ), + }, + }...) + test.Run(t, tests.Build()) } + +// TODO: dnslink tests \ No newline at end of file From e842ed9b58bd64fa5578d551e0379a57160816a9 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 10:36:48 +0200 Subject: [PATCH 5/7] feat: subdomain dependency --- tests/t0109_gateway_web_redirects_test.go | 8 ++++++-- tests/t0114_gateway_subdomains_test.go | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index bba3c1aec..f5519344d 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" - "github.com/ipfs/gateway-conformance/tooling/test" + "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) @@ -288,7 +288,11 @@ func TestRedirectsFileSupport(t *testing.T) { }, }...) - test.Run(t, tests.Build()) + if specs.SubdomainGateway.IsEnabled() { + Run(t, tests.Build()) + } else { + t.Skip("subdomain gateway disabled") + } } // TODO: dnslink tests \ No newline at end of file diff --git a/tests/t0114_gateway_subdomains_test.go b/tests/t0114_gateway_subdomains_test.go index ff1fb7194..5000cb08d 100644 --- a/tests/t0114_gateway_subdomains_test.go +++ b/tests/t0114_gateway_subdomains_test.go @@ -7,7 +7,7 @@ import ( "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" - . "github.com/ipfs/gateway-conformance/tooling/specs" + "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) @@ -369,8 +369,10 @@ func TestGatewaySubdomains(t *testing.T) { )) } - if SubdomainGateway.IsEnabled() { + if specs.SubdomainGateway.IsEnabled() { Run(t, tests) + } else { + t.Skip("subdomain gateway disabled") } } From ec56daac7edcd3ac7dd162d09d6c74d564d2daa0 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 10:51:06 +0200 Subject: [PATCH 6/7] feat: use correct urls --- tests/t0109_gateway_web_redirects_test.go | 611 +++++++++++++--------- 1 file changed, 352 insertions(+), 259 deletions(-) diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index f5519344d..acb8ed651 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -2,6 +2,7 @@ package tests import ( "fmt" + "net/url" "testing" "github.com/ipfs/gateway-conformance/tooling/car" @@ -16,8 +17,6 @@ func TestRedirectsFileSupport(t *testing.T) { redirectDir := fixture.MustGetNode("examples") redirectDirCID := redirectDir.Base32Cid() - redirectDirHostname := fmt.Sprintf("%s.ipfs.localhost:8080", redirectDirCID) - // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/404.html | cut -d "/" -f3) custom404 := fixture.MustGetNode("examples", "404.html") // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3) @@ -25,274 +24,368 @@ func TestRedirectsFileSupport(t *testing.T) { // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3) custom451 := fixture.MustGetNode("examples", "451.html") - tests := SugarTests{ - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/redirect-one" > response && - // test_should_contain "301 Moved Permanently" response && - // test_should_contain "Location: /one.html" response - // - // ' - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", - Request: Request(). - DoNotFollowRedirects(). - URL("http://%s/redirect-one", redirectDirHostname), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/one.html"), - ), - }, - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/301-redirect-one" > response && - // test_should_contain "301 Moved Permanently" response && - // test_should_contain "Location: /one.html" response - // - // ' - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file", - Request: Request(). - DoNotFollowRedirects(). - URL("http://%s/301-redirect-one", redirectDirHostname), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/one.html"), - ), - }, - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/302-redirect-two" > response && - // test_should_contain "302 Found" response && - // test_should_contain "Location: /two.html" response - // - // ' + tests := SugarTests{} + + // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) + gatewayURLs := []string{ + SubdomainGatewayURL, + SubdomainLocalhostGatewayURL, + } + + for _, gatewayURL := range gatewayURLs { + u, err := url.Parse(gatewayURL) + if err != nil { + t.Fatal(err) + } + + redirectDirBaseURL := fmt.Sprintf("%s://%s.ipfs.%s", u.Scheme, redirectDirCID, u.Host) + + tests = append(tests, SugarTests{ + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/redirect-one" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /one.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("%s/redirect-one", redirectDirBaseURL), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/one.html"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/301-redirect-one" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /one.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("%s/301-redirect-one", redirectDirBaseURL), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/one.html"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/302-redirect-two" > response && + // test_should_contain "302 Found" response && + // test_should_contain "Location: /two.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("%s/302-redirect-two", redirectDirBaseURL), + Response: Expect(). + Status(302). + Headers( + Header("Location").Equals("/two.html"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/200-index" > response && + // test_should_contain "my index" response && + // test_should_contain "200 OK" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("%s/200-index", redirectDirBaseURL), + Response: Expect(). + Status(200). + Body(Contains("my index")), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/posts/2022/01/01/hello-world" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /articles/2022/01/01/hello-world" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("%s/posts/2022/01/01/hello-world", redirectDirBaseURL), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/articles/2022/01/01/hello-world"), + ), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/splat/one.html" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /redirected-splat/one.html" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file", + Request: Request(). + DoNotFollowRedirects(). + URL("%s/splat/one.html", redirectDirBaseURL), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/redirected-splat/one.html"), + ), + }, + // # ensure custom 4xx works and has the same cache headers as regular /ipfs/ path + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry" > response && + // test_should_contain "404 Not Found" response && + // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && + // test_should_contain "my 404" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file", + Request: Request(). + URL("%s/not-found/has-no-redirects-entry", redirectDirBaseURL), + Response: Expect(). + Status(404). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals("\"%s\"", custom404.Cid().String()), + ). + Body(Contains(custom404.ReadFile())), + }, + // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3) + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file" ' + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry" > response && + // test_should_contain "410 Gone" response && + // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && + // test_should_contain "my 410" response + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file", + Request: Request(). + URL("%s/gone/has-no-redirects-entry", redirectDirBaseURL), + Response: Expect(). + Status(410). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals("\"%s\"", custom410.Cid().String()), + ). + Body(Contains(custom410.ReadFile())), + }, + // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3) + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry" > response && + // test_should_contain "451 Unavailable For Legal Reasons" response && + // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && + // test_should_contain "my 451" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file", + Request: Request(). + URL("%s/unavail/has-no-redirects-entry", redirectDirBaseURL), + Response: Expect(). + Status(451). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals("\"%s\"", custom451.Cid().String()), + ). + Body(Contains(custom451.ReadFile())), + }, + // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file" ' + // + // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/catch-all" > response && + // test_should_contain "200 OK" response && + // test_should_contain "my index" response + // + // ' + { + Name: "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file", + Request: Request(). + URL("%s/catch-all", redirectDirBaseURL), + Response: Expect(). + Status(200). + Body(Contains("my index")), + }, + // # This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain) + // test_expect_success "request for http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one returns generic 404 (no custom 404 from _redirects since no origin isolation)" ' + // + // curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one" > response && + // test_should_contain "404 Not Found" response && + // test_should_not_contain "my 404" response + // + // ' + { + // TODO: how to test this correctly? + Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", + Hint: ` + We expect the request to fail with a 404 (do not use the _redirect), and that 404 should not contain the custom 404 body. + `, + Request: Request(). + URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), + Response: Expect(). + Status(404). + Body(Not(Contains(custom404.ReadFile()))), + }, + }...) + + // # Invalid file, containing forced redirect + // INVALID_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/forced | cut -d "/" -f3) + invalidRedirectsDirCID := fixture.MustGetNode("forced").Base32Cid() + // INVALID_REDIRECTS_DIR_HOSTNAME="${INVALID_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" + invalidDirBaseURL := fmt.Sprintf("%s://%s.ipfs.%s", u.Scheme, invalidRedirectsDirCID, u.Host) + // TOO_LARGE_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/too-large | cut -d "/" -f3) + tooLargeRedirectsDirCID := fixture.MustGetNode("too-large").Base32Cid() + // TOO_LARGE_REDIRECTS_DIR_HOSTNAME="${TOO_LARGE_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" + tooLargeDirBaseURL := fmt.Sprintf("%s://%s.ipfs.%s", u.Scheme, tooLargeRedirectsDirCID, u.Host) + + tests = append(tests, SugarTests{ + // # if accessing a path that doesn't exist, read _redirects and fail parsing, and return error + // test_expect_success "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file" ' + // curl -sD - --resolve $INVALID_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$INVALID_REDIRECTS_DIR_HOSTNAME/not-found" > response && + // test_should_contain "500" response && + // test_should_contain "could not parse _redirects:" response && + // test_should_contain "forced redirects (or \"shadowing\") are not supported" response + // ' + { + Name: "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file", + Hint: `if accessing a path that doesn't exist, read _redirects and fail parsing, and return error`, + Request: Request(). + URL("%s/not-found", invalidDirBaseURL), + Response: Expect(). + Status(500). + Body( + And( + Contains("could not parse _redirects:"), + Contains("forced redirects (or \"shadowing\") are not supported"), + ), + ), + }, + // # if accessing a path that doesn't exist and _redirects file is too large, return error + // test_expect_success "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file" ' + // curl -sD - --resolve $TOO_LARGE_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found" > response && + // test_should_contain "500" response && + // test_should_contain "could not parse _redirects:" response && + // test_should_contain "redirects file size cannot exceed" response + // ' + { + Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file", + Hint: `if accessing a path that doesn't exist and _redirects file is too large, return error`, + Request: Request(). + URL("%s/not-found", tooLargeDirBaseURL), + Response: Expect(). + Status(500). + Body( + And( + Contains("could not parse _redirects:"), + Contains("redirects file size cannot exceed"), + ), + ), + }, + }...) + } + + if specs.SubdomainGateway.IsEnabled() { + Run(t, unwrapTests(t, tests).Build()) + } else { + t.Skip("subdomain gateway disabled") + } +} + +// TODO: dnslink tests + +func unwrapTests(t *testing.T, tests SugarTests) SugarTests { + t.Helper() + + var out SugarTests + for _, test := range tests { + out = append(out, unwrapTestForGateway(t, test)...) + } + return out +} + +func unwrapTestForGateway(t *testing.T, test SugarTest) SugarTests { + t.Helper() + + baseURL := test.Request.GetURL() + req := test.Request + expected := test.Response + + u, err := url.Parse(baseURL) + if err != nil { + t.Fatal(err) + } + // Because you might be testing an IPFS node in CI, or on your local machine, the test are designed + // to test the subdomain behavior (querying http://{CID}.my-subdomain-gateway.io/) even if the node is + // actually living on http://127.0.0.1:8080 or somewhere else. + // + // The test knows two addresses: + // - GatewayURL: the URL we connect to, it might be "dweb.link", "127.0.0.1:8080", etc. + // - SubdomainGatewayURL: the URL we test for subdomain requests, it might be "dweb.link", "localhost", "example.com", etc. + + // host is the hostname of the gateway we are testing, it might be `localhost` or `example.com` + host := u.Host + + // raw url is the url but we replace the host with our local url, it might be `http://127.0.0.1/ipfs/something` + u.Host = GatewayHost + rawURL := u.String() + + return SugarTests{ { - Name: "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file", - Request: Request(). + Name: fmt.Sprintf("%s (direct HTTP)", test.Name), + Hint: fmt.Sprintf("%s\n%s", test.Hint, "direct HTTP request (hostname in URL, raw IP in Host header)"), + Request: req. + URL(rawURL). DoNotFollowRedirects(). - URL("http://%s/302-redirect-two", redirectDirHostname), - Response: Expect(). - Status(302). Headers( - Header("Location").Equals("/two.html"), + Header("Host", host), ), + Response: expected, }, - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/200-index" > response && - // test_should_contain "my index" response && - // test_should_contain "200 OK" response - // - // ' { - Name: "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file", - Request: Request(). - DoNotFollowRedirects(). - URL("http://%s/200-index", redirectDirHostname), - Response: Expect(). - Status(200). - Body(Contains("my index")), + Name: fmt.Sprintf("%s (HTTP proxy)", test.Name), + Hint: fmt.Sprintf("%s\n%s", test.Hint, "HTTP proxy (hostname is passed via URL)"), + Request: req. + URL(baseURL). + Proxy(GatewayURL). + DoNotFollowRedirects(), + Response: expected, }, - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/posts/2022/01/01/hello-world" > response && - // test_should_contain "301 Moved Permanently" response && - // test_should_contain "Location: /articles/2022/01/01/hello-world" response - // - // ' { - Name: "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file", - Request: Request(). + Name: fmt.Sprintf("%s (HTTP proxy tunneling via CONNECT)", test.Name), + Hint: fmt.Sprintf("%s\n%s", test.Hint, `HTTP proxy + In HTTP/1.x, the pseudo-method CONNECT, + can be used to convert an HTTP connection into a tunnel to a remote host + https://tools.ietf.org/html/rfc7231#section-4.3.6 + `), + Request: req. + URL(baseURL). + Proxy(GatewayURL). + WithProxyTunnel(). DoNotFollowRedirects(). - URL("http://%s/posts/2022/01/01/hello-world", redirectDirHostname), - Response: Expect(). - Status(301). Headers( - Header("Location").Equals("/articles/2022/01/01/hello-world"), + Header("Host", host), ), + Response: expected, }, - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/splat/one.html" > response && - // test_should_contain "301 Moved Permanently" response && - // test_should_contain "Location: /redirected-splat/one.html" response - // - // ' - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file", - Request: Request(). - DoNotFollowRedirects(). - URL("http://%s/splat/one.html", redirectDirHostname), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/redirected-splat/one.html"), - ), - }, - // # ensure custom 4xx works and has the same cache headers as regular /ipfs/ path - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry" > response && - // test_should_contain "404 Not Found" response && - // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && - // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && - // test_should_contain "my 404" response - // - // ' - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file", - Request: Request(). - URL("http://%s/not-found/has-no-redirects-entry", redirectDirHostname), - Response: Expect(). - Status(404). - Headers( - Header("Cache-Control").Equals("public, max-age=29030400, immutable"), - Header("Etag").Equals("\"%s\"", custom404.Cid().String()), - ). - Body(Contains(custom404.ReadFile())), - }, - // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3) - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file" ' - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry" > response && - // test_should_contain "410 Gone" response && - // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && - // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && - // test_should_contain "my 410" response - // ' - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file", - Request: Request(). - URL("http://%s/gone/has-no-redirects-entry", redirectDirHostname), - Response: Expect(). - Status(410). - Headers( - Header("Cache-Control").Equals("public, max-age=29030400, immutable"), - Header("Etag").Equals("\"%s\"", custom410.Cid().String()), - ). - Body(Contains(custom410.ReadFile())), - }, - // CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3) - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry" > response && - // test_should_contain "451 Unavailable For Legal Reasons" response && - // test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && - // test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && - // test_should_contain "my 451" response - // - // ' - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file", - Request: Request(). - URL("http://%s/unavail/has-no-redirects-entry", redirectDirHostname), - Response: Expect(). - Status(451). - Headers( - Header("Cache-Control").Equals("public, max-age=29030400, immutable"), - Header("Etag").Equals("\"%s\"", custom451.Cid().String()), - ). - Body(Contains(custom451.ReadFile())), - }, - // test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file" ' - // - // curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/catch-all" > response && - // test_should_contain "200 OK" response && - // test_should_contain "my index" response - // - // ' - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file", - Request: Request(). - URL("http://%s/catch-all", redirectDirHostname), - Response: Expect(). - Status(200). - Body(Contains("my index")), - }, - // # This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain) - // test_expect_success "request for http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one returns generic 404 (no custom 404 from _redirects since no origin isolation)" ' - // - // curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one" > response && - // test_should_contain "404 Not Found" response && - // test_should_not_contain "my 404" response - // - // ' - { - // TODO: how to test this correctly? - Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", - Hint: ` - We expect the request to fail with a 404 (do not use the _redirect), and that 404 should not contain the custom 404 body. - `, - Request: Request(). - URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), - Response: Expect(). - Status(404). - Body(Not(Contains(custom404.ReadFile()))), - }, - } - - // # Invalid file, containing forced redirect - // INVALID_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/forced | cut -d "/" -f3) - invalidRedirectsDirCID := fixture.MustGetNode("forced").Base32Cid() - // INVALID_REDIRECTS_DIR_HOSTNAME="${INVALID_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" - invalidDirHostname := fmt.Sprintf("%s.ipfs.localhost:8080", invalidRedirectsDirCID) - // TOO_LARGE_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/too-large | cut -d "/" -f3) - tooLargeRedirectsDirCID := fixture.MustGetNode("too-large").Base32Cid() - // TOO_LARGE_REDIRECTS_DIR_HOSTNAME="${TOO_LARGE_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" - tooLargeDirHostname := fmt.Sprintf("%s.ipfs.localhost:8080", tooLargeRedirectsDirCID) - - tests = append(tests, SugarTests{ - // # if accessing a path that doesn't exist, read _redirects and fail parsing, and return error - // test_expect_success "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file" ' - // curl -sD - --resolve $INVALID_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$INVALID_REDIRECTS_DIR_HOSTNAME/not-found" > response && - // test_should_contain "500" response && - // test_should_contain "could not parse _redirects:" response && - // test_should_contain "forced redirects (or \"shadowing\") are not supported" response - // ' - { - Name: "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file", - Hint: `if accessing a path that doesn't exist, read _redirects and fail parsing, and return error`, - Request: Request(). - URL("http://%s/not-found", invalidDirHostname), - Response: Expect(). - Status(500). - Body( - And( - Contains("could not parse _redirects:"), - Contains("forced redirects (or \"shadowing\") are not supported"), - ), - ), - }, - // # if accessing a path that doesn't exist and _redirects file is too large, return error - // test_expect_success "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file" ' - // curl -sD - --resolve $TOO_LARGE_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found" > response && - // test_should_contain "500" response && - // test_should_contain "could not parse _redirects:" response && - // test_should_contain "redirects file size cannot exceed" response - // ' - { - Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file", - Hint: `if accessing a path that doesn't exist and _redirects file is too large, return error`, - Request: Request(). - URL("http://%s/not-found", tooLargeDirHostname), - Response: Expect(). - Status(500). - Body( - And( - Contains("could not parse _redirects:"), - Contains("redirects file size cannot exceed"), - ), - ), - }, - }...) - - if specs.SubdomainGateway.IsEnabled() { - Run(t, tests.Build()) - } else { - t.Skip("subdomain gateway disabled") } } - -// TODO: dnslink tests \ No newline at end of file From dc30238e4913e90986aa346a50a50e95cbd75607 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 5 Apr 2023 10:59:56 +0200 Subject: [PATCH 7/7] fix: drop unecessary import --- tooling/car/fixture.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tooling/car/fixture.go b/tooling/car/fixture.go index c9f473b09..745a0e0d7 100644 --- a/tooling/car/fixture.go +++ b/tooling/car/fixture.go @@ -7,7 +7,6 @@ import ( "time" files "github.com/ipfs/boxo/files" - _ "github.com/ipfs/boxo/ipld/merkledag" unixfile "github.com/ipfs/boxo/ipld/unixfs/file" "github.com/ipfs/go-cid"