From 253640ef768952c6846bdd9cae1b3f6db3fd9ee8 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 18 Jan 2024 17:45:31 +0100 Subject: [PATCH 1/2] gateway: new headers middleware --- CHANGELOG.md | 3 + examples/gateway/common/handler.go | 11 ++- gateway/README.md | 10 +-- gateway/errors_test.go | 4 +- gateway/gateway.go | 80 --------------------- gateway/gateway_test.go | 8 +-- gateway/handler.go | 8 --- gateway/handler_codec_test.go | 1 - gateway/headers.go | 112 +++++++++++++++++++++++++++++ gateway/hostname.go | 17 +---- gateway/utilities_test.go | 6 +- 11 files changed, 133 insertions(+), 127 deletions(-) create mode 100644 gateway/headers.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 12ad5df7e..4b16d4865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,11 +18,14 @@ The following emojis are used to highlight certain changes: - `blockservice` now has `ContextWithSession` and `EmbedSessionInContext` functions, which allows to embed a session in a context. Future calls to `BlockGetter.GetBlock`, `BlockGetter.GetBlocks` and `NewSession` will use the session in the context. - `blockservice.NewWritethrough` deprecated function has been removed, instead you can do `blockservice.New(..., ..., WriteThrough())` like previously. +- `gateway`: a new header configuration middleware has been added to replace the existing header configuration, which can be used more generically. ### Changed ### Removed +- 🛠 `gateway`: the header configuration `Config.Headers` and `AddAccessControlHeaders` has been replaced by the new middleware provided by `NewHeaders`. + ### Security ## [v0.17.0] diff --git a/examples/gateway/common/handler.go b/examples/gateway/common/handler.go index d21f38b64..5c4469aba 100644 --- a/examples/gateway/common/handler.go +++ b/examples/gateway/common/handler.go @@ -12,10 +12,6 @@ import ( func NewHandler(gwAPI gateway.IPFSBackend) http.Handler { conf := gateway.Config{ - // Initialize the headers. For this example, we do not add any special headers, - // only the required ones via gateway.AddAccessControlHeaders. - Headers: map[string][]string{}, - // If you set DNSLink to point at the CID from CAR, you can load it! NoDNSLink: false, @@ -58,9 +54,6 @@ func NewHandler(gwAPI gateway.IPFSBackend) http.Handler { }, } - // Add required access control headers to the configuration. - gateway.AddAccessControlHeaders(conf.Headers) - // Creates a mux to serve the gateway paths. This is not strictly necessary // and gwHandler could be used directly. However, on the next step we also want // to add prometheus metrics, hence needing the mux. @@ -86,6 +79,10 @@ func NewHandler(gwAPI gateway.IPFSBackend) http.Handler { // http.ServeMux which does not support CONNECT by default. handler = withConnect(handler) + // Add headers middleware that applies any headers we define to all requests + // as well as a default CORS configuration. + handler = gateway.NewHeaders(nil).ApplyCors().Wrap(handler) + // Finally, wrap with the otelhttp handler. This will allow the tracing system // to work and for correct propagation of tracing headers. This step is optional // and only required if you want to use tracing. Note that OTel must be correctly diff --git a/gateway/README.md b/gateway/README.md index a434f9b36..60e2dbcfb 100644 --- a/gateway/README.md +++ b/gateway/README.md @@ -14,13 +14,7 @@ This example shows how you can start your own gateway, assuming you have an `IPF implementation. ```go -// Initialize your headers and apply the default headers. -headers := map[string][]string{} -gateway.AddAccessControlHeaders(headers) - -conf := gateway.Config{ - Headers: headers, -} +conf := gateway.Config{} // Initialize an IPFSBackend interface for both an online and offline versions. // The offline version should not make any network request for missing content. @@ -29,9 +23,11 @@ ipfsBackend := ... // Create http mux and setup path gateway handler. mux := http.NewServeMux() handler := gateway.NewHandler(conf, ipfsBackend) +handler = gateway.NewHeaders(nil).ApplyCors().Wrap(handler) mux.Handle("/ipfs/", handler) mux.Handle("/ipns/", handler) + // Start the server on :8080 and voilá! You have a basic IPFS gateway running // in http://localhost:8080. _ = http.ListenAndServe(":8080", mux) diff --git a/gateway/errors_test.go b/gateway/errors_test.go index cad7ae061..ca41b759b 100644 --- a/gateway/errors_test.go +++ b/gateway/errors_test.go @@ -43,7 +43,7 @@ func TestWebError(t *testing.T) { t.Parallel() // Create a handler to be able to test `webError`. - config := &Config{Headers: map[string][]string{}} + config := &Config{} t.Run("429 Too Many Requests", func(t *testing.T) { t.Parallel() @@ -113,7 +113,7 @@ func TestWebError(t *testing.T) { t.Run("Error is sent as plain text when 'Accept' header contains 'text/html' and config.DisableHTMLErrors is true", func(t *testing.T) { t.Parallel() - config := &Config{Headers: map[string][]string{}, DisableHTMLErrors: true} + config := &Config{DisableHTMLErrors: true} w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/blah", nil) r.Header.Set("Accept", "something/else, text/html") diff --git a/gateway/gateway.go b/gateway/gateway.go index aa0d59f43..be9501281 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "io" - "net/http" - "sort" "strconv" "strings" "time" @@ -20,11 +18,6 @@ import ( // Config is the configuration used when creating a new gateway handler. type Config struct { - // Headers is a map containing all the headers that should be sent by default - // in all requests. You can define custom headers, as well as add the recommended - // headers via AddAccessControlHeaders. - Headers map[string][]string - // DeserializedResponses configures this gateway to support returning data // in deserialized format. By default, the gateway will only support // trustless, verifiable [application/vnd.ipld.raw] and @@ -394,79 +387,6 @@ type WithContextHint interface { WrapContextForRequest(context.Context) context.Context } -// cleanHeaderSet is an helper function that cleans a set of headers by -// (1) canonicalizing, (2) de-duplicating and (3) sorting. -func cleanHeaderSet(headers []string) []string { - // Deduplicate and canonicalize. - m := make(map[string]struct{}, len(headers)) - for _, h := range headers { - m[http.CanonicalHeaderKey(h)] = struct{}{} - } - result := make([]string, 0, len(m)) - for k := range m { - result = append(result, k) - } - - // Sort - sort.Strings(result) - return result -} - -// AddAccessControlHeaders ensures safe default HTTP headers are used for -// controlling cross-origin requests. This function adds several values to the -// [Access-Control-Allow-Headers] and [Access-Control-Expose-Headers] entries -// to be exposed on GET and OPTIONS responses, including [CORS Preflight]. -// -// If the Access-Control-Allow-Origin entry is missing, a default value of '*' is -// added, indicating that browsers should allow requesting code from any -// origin to access the resource. -// -// If the Access-Control-Allow-Methods entry is missing a value, 'GET, HEAD, -// OPTIONS' is added, indicating that browsers may use them when issuing cross -// origin requests. -// -// [Access-Control-Allow-Headers]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Headers -// [Access-Control-Expose-Headers]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Expose-Headers -// [CORS Preflight]: https://developer.mozilla.org/en-US/docs/Glossary/Preflight_request -func AddAccessControlHeaders(headers map[string][]string) { - // Hard-coded headers. - const ACAHeadersName = "Access-Control-Allow-Headers" - const ACEHeadersName = "Access-Control-Expose-Headers" - const ACAOriginName = "Access-Control-Allow-Origin" - const ACAMethodsName = "Access-Control-Allow-Methods" - - if _, ok := headers[ACAOriginName]; !ok { - // Default to *all* - headers[ACAOriginName] = []string{"*"} - } - if _, ok := headers[ACAMethodsName]; !ok { - // Default to GET, HEAD, OPTIONS - headers[ACAMethodsName] = []string{ - http.MethodGet, - http.MethodHead, - http.MethodOptions, - } - } - - headers[ACAHeadersName] = cleanHeaderSet( - append([]string{ - "Content-Type", - "User-Agent", - "Range", - "X-Requested-With", - }, headers[ACAHeadersName]...)) - - headers[ACEHeadersName] = cleanHeaderSet( - append([]string{ - "Content-Length", - "Content-Range", - "X-Chunked-Output", - "X-Stream-Output", - "X-Ipfs-Path", - "X-Ipfs-Roots", - }, headers[ACEHeadersName]...)) -} - // RequestContextKey is a type representing a [context.Context] value key. type RequestContextKey string diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 53f19ca08..4785ec759 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -352,8 +352,7 @@ func TestHeaders(t *testing.T) { headers := map[string][]string{} headers[headerACAO] = []string{expectedACAO} - ts := newTestServerWithConfig(t, backend, Config{ - Headers: headers, + ts := newTestServerWithConfigAndHeaders(t, backend, Config{ PublicGateways: map[string]*PublicGateway{ "subgw.example.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -362,7 +361,7 @@ func TestHeaders(t *testing.T) { }, }, DeserializedResponses: true, - }) + }, headers) t.Logf("test server url: %s", ts.URL) testCORSPreflightRequest := func(t *testing.T, path, hostHeader string, requestOriginHeader string, code int) { @@ -532,7 +531,6 @@ func TestRedirects(t *testing.T) { backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, NoDNSLink: false, PublicGateways: map[string]*PublicGateway{ "example.com": { @@ -590,7 +588,6 @@ func TestDeserializedResponses(t *testing.T) { backend, root := newMockBackend(t, "fixtures.car") ts := newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, NoDNSLink: false, PublicGateways: map[string]*PublicGateway{ "trustless.com": { @@ -670,7 +667,6 @@ func TestDeserializedResponses(t *testing.T) { backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, NoDNSLink: false, PublicGateways: map[string]*PublicGateway{ "trustless.com": { diff --git a/gateway/handler.go b/gateway/handler.go index 38a242d00..a64d22971 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -179,7 +179,6 @@ func (i *handler) optionsHandler(w http.ResponseWriter, r *http.Request) { // OPTIONS is a noop request that is used by the browsers to check if server accepts // cross-site XMLHttpRequest, which is indicated by the presence of CORS headers: // https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS#Preflighted_requests - addCustomHeaders(w, i.config.Headers) // return all custom headers (including CORS ones, if set) } // addAllowHeader sets Allow header with supported HTTP methods @@ -264,7 +263,6 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat)) i.requestTypeMetric.WithLabelValues(contentPath.Namespace(), responseFormat).Inc() - addCustomHeaders(w, i.config.Headers) // ok, _now_ write user's headers. w.Header().Set("X-Ipfs-Path", contentPath.String()) // Fail fast if unsupported request type was sent to a Trustless Gateway. @@ -340,12 +338,6 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } } -func addCustomHeaders(w http.ResponseWriter, headers map[string][]string) { - for k, v := range headers { - w.Header()[http.CanonicalHeaderKey(k)] = v - } -} - // isDeserializedResponsePossible returns true if deserialized responses // are allowed on the specified hostname, or globally. Host-specific rules // override global config. diff --git a/gateway/handler_codec_test.go b/gateway/handler_codec_test.go index d22579027..127e0bc8c 100644 --- a/gateway/handler_codec_test.go +++ b/gateway/handler_codec_test.go @@ -16,7 +16,6 @@ func TestDagJsonCborPreview(t *testing.T) { backend, root := newMockBackend(t, "fixtures.car") ts := newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, NoDNSLink: false, PublicGateways: map[string]*PublicGateway{ "example.com": { diff --git a/gateway/headers.go b/gateway/headers.go new file mode 100644 index 000000000..e7c47a605 --- /dev/null +++ b/gateway/headers.go @@ -0,0 +1,112 @@ +package gateway + +import ( + "net/http" + "sort" +) + +// Headers is an HTTP middleware that sets the configured headers in all requests. +type Headers struct { + headers map[string][]string +} + +// NewHeaders creates a new [Headers] middleware that applies the given headers +// to all requests. If you call [Headers.ApplyCors], the default CORS configuration +// will also be applied. +func NewHeaders(headers map[string][]string) *Headers { + h := &Headers{ + headers: map[string][]string{}, + } + + for k, v := range headers { + h.headers[http.CanonicalHeaderKey(k)] = v + } + + return h +} + +// ApplyCors applies safe default HTTP headers for controlling cross-origin +// requests. This function adds several values to the [Access-Control-Allow-Headers] +// and [Access-Control-Expose-Headers] entries to be exposed on GET and OPTIONS +// responses, including [CORS Preflight]. +// +// If the Access-Control-Allow-Origin entry is missing, a default value of '*' is +// added, indicating that browsers should allow requesting code from any +// origin to access the resource. +// +// If the Access-Control-Allow-Methods entry is missing a value, 'GET, HEAD, +// OPTIONS' is added, indicating that browsers may use them when issuing cross +// origin requests. +// +// [Access-Control-Allow-Headers]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Headers +// [Access-Control-Expose-Headers]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Expose-Headers +// [CORS Preflight]: https://developer.mozilla.org/en-US/docs/Glossary/Preflight_request +func (h *Headers) ApplyCors() *Headers { + // Hard-coded headers. + const ACAHeadersName = "Access-Control-Allow-Headers" + const ACEHeadersName = "Access-Control-Expose-Headers" + const ACAOriginName = "Access-Control-Allow-Origin" + const ACAMethodsName = "Access-Control-Allow-Methods" + + if _, ok := h.headers[ACAOriginName]; !ok { + // Default to *all* + h.headers[ACAOriginName] = []string{"*"} + } + if _, ok := h.headers[ACAMethodsName]; !ok { + // Default to GET, HEAD, OPTIONS + h.headers[ACAMethodsName] = []string{ + http.MethodGet, + http.MethodHead, + http.MethodOptions, + } + } + + h.headers[ACAHeadersName] = cleanHeaderSet( + append([]string{ + "Content-Type", + "User-Agent", + "Range", + "X-Requested-With", + }, h.headers[ACAHeadersName]...)) + + h.headers[ACEHeadersName] = cleanHeaderSet( + append([]string{ + "Content-Length", + "Content-Range", + "X-Chunked-Output", + "X-Stream-Output", + "X-Ipfs-Path", + "X-Ipfs-Roots", + }, h.headers[ACEHeadersName]...)) + + return h +} + +// Wrap wraps the given [http.Handler] with the headers middleware. +func (h *Headers) Wrap(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for k, v := range h.headers { + w.Header()[k] = v + } + + next.ServeHTTP(w, r) + }) +} + +// cleanHeaderSet is an helper function that cleans a set of headers by +// (1) canonicalizing, (2) de-duplicating and (3) sorting. +func cleanHeaderSet(headers []string) []string { + // Deduplicate and canonicalize. + m := make(map[string]struct{}, len(headers)) + for _, h := range headers { + m[http.CanonicalHeaderKey(h)] = struct{}{} + } + result := make([]string, 0, len(m)) + for k := range m { + result = append(result, k) + } + + // Sort + sort.Strings(result) + return result +} diff --git a/gateway/hostname.go b/gateway/hostname.go index 6b485f0b4..665cf1663 100644 --- a/gateway/hostname.go +++ b/gateway/hostname.go @@ -68,7 +68,7 @@ func NewHostnameHandler(c Config, backend IPFSBackend, next http.Handler) http.H return } if newURL != "" { - httpRedirectWithHeaders(w, r, newURL, http.StatusMovedPermanently, c.Headers) + http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } @@ -131,7 +131,7 @@ func NewHostnameHandler(c Config, backend IPFSBackend, next http.Handler) http.H if newURL != "" { // Redirect to deterministic CID to ensure CID // always gets the same Origin on the web - httpRedirectWithHeaders(w, r, newURL, http.StatusMovedPermanently, c.Headers) + http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } @@ -146,7 +146,7 @@ func NewHostnameHandler(c Config, backend IPFSBackend, next http.Handler) http.H } if newURL != "" { // Redirect to CID fixed inside of toSubdomainURL() - httpRedirectWithHeaders(w, r, newURL, http.StatusMovedPermanently, c.Headers) + http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } @@ -625,14 +625,3 @@ func (gws *hostnameGateways) knownSubdomainDetails(hostname string) (gw *PublicG // no match return nil, "", "", "", false } - -// httpRedirectWithHeaders applies custom headers before returning a redirect -// response to ensure consistency during transition from path to subdomain -// contexts. -func httpRedirectWithHeaders(w http.ResponseWriter, r *http.Request, url string, code int, headers map[string][]string) { - // ensure things like CORS are applied to redirect responses - // (https://github.com/ipfs/kubo/issues/9983#issuecomment-1599673976) - addCustomHeaders(w, headers) - - http.Redirect(w, r, url, code) -} diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 85153f808..68db84041 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -232,19 +232,21 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys, fixturesFile string) (*h func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server { return newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, DeserializedResponses: true, }) } func newTestServerWithConfig(t *testing.T, backend IPFSBackend, config Config) *httptest.Server { - AddAccessControlHeaders(config.Headers) + return newTestServerWithConfigAndHeaders(t, backend, config, map[string][]string{}) +} +func newTestServerWithConfigAndHeaders(t *testing.T, backend IPFSBackend, config Config, headers map[string][]string) *httptest.Server { handler := NewHandler(config, backend) mux := http.NewServeMux() mux.Handle("/ipfs/", handler) mux.Handle("/ipns/", handler) handler = NewHostnameHandler(config, backend, mux) + handler = NewHeaders(headers).ApplyCors().Wrap(handler) ts := httptest.NewServer(handler) t.Cleanup(func() { ts.Close() }) From 19948a24d21da5f047ce0ec821fff172ce8bb311 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 24 Jan 2024 10:21:37 +0100 Subject: [PATCH 2/2] Update gateway/headers.go Co-authored-by: Marcin Rataj --- gateway/headers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/headers.go b/gateway/headers.go index e7c47a605..66ad5d43a 100644 --- a/gateway/headers.go +++ b/gateway/headers.go @@ -12,7 +12,7 @@ type Headers struct { // NewHeaders creates a new [Headers] middleware that applies the given headers // to all requests. If you call [Headers.ApplyCors], the default CORS configuration -// will also be applied. +// will also be applied, if any of the CORS headers is missing. func NewHeaders(headers map[string][]string) *Headers { h := &Headers{ headers: map[string][]string{},