gateway: add Content-Location for non-default response formats

CHANGELOG.md

@@ -19,6 +19,8 @@ The following emojis are used to highlight certain changes:
 * ✨ `gateway` has new backend possibilities:
   * `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
   * `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`.
+* `gateway` now sets the [`Content-Location`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Location) header for requests with non-default content format, as a result of content negotiation. This allows generic and misconfigured HTTP caches to store Deserialized, CAR and Block responses separately, under distinct cache keys.
+* `gateway` now supports `car-dups`, `car-order` and `car-version` as query parameters in addition to the `application/vnd.ipld.car` parameters sent via `Accept` header. The parameters in the `Accept` header have always priority, but including them in URL simplifies HTTP caching and allows use in `Content-Location` header on CAR responses to maximize interoperability with wide array of HTTP caches.
 
 ### Changed
 

gateway/gateway.go

@@ -405,6 +405,11 @@ const (
 	// [Subdomain Gateway]: https://specs.ipfs.tech/http-gateways/subdomain-gateway/
 	SubdomainHostnameKey RequestContextKey = "subdomain-hostname"
 
-	// ContentPathKey is the key for the original [http.Request] URL Path, as an [ipath.Path].
+	// OriginalPathKey is the key for the original [http.Request] [url.URL.Path],
+	// as a string. This is the original path of the request, before [NewHostnameHandler].
+	OriginalPathKey RequestContextKey = "original-path-key"
+
+	// ContentPathKey is the key for the content [path.Path] of the current request.
+	// This already accounts with changes made with [NewHostnameHandler].
 	ContentPathKey RequestContextKey = "content-path"
 )

gateway/gateway_test.go

@@ -417,6 +417,82 @@ func TestHeaders(t *testing.T) {
 			testCORSPreflightRequest(t, "/", cid+".ipfs.subgw.example.com", "https://other.example.net", http.StatusOK)
 		})
 	})
+
+	t.Run("Content-Location is set when possible", func(t *testing.T) {
+		backend, root := newMockBackend(t, "fixtures.car")
+		backend.namesys["/ipns/dnslink-gateway.com"] = newMockNamesysItem(path.FromCid(root), 0)
+
+		ts := newTestServerWithConfig(t, backend, Config{
+			NoDNSLink: false,
+			PublicGateways: map[string]*PublicGateway{
+				"dnslink-gateway.com": {
+					Paths:                 []string{},
+					NoDNSLink:             false,
+					DeserializedResponses: true,
+				},
+				"subdomain-gateway.com": {
+					Paths:                 []string{"/ipfs", "/ipns"},
+					UseSubdomains:         true,
+					NoDNSLink:             true,
+					DeserializedResponses: true,
+				},
+			},
+			DeserializedResponses: true,
+		})
+
+		runTest := func(name, path, accept, host, expectedContentLocationHdr string) {
+			t.Run(name, func(t *testing.T) {
+				t.Parallel()
+
+				req := mustNewRequest(t, http.MethodGet, ts.URL+path, nil)
+
+				if accept != "" {
+					req.Header.Set("Accept", accept)
+				}
+
+				if host != "" {
+					req.Host = host
+				}
+
+				resp := mustDoWithoutRedirect(t, req)
+				defer resp.Body.Close()
+
+				body, err := io.ReadAll(resp.Body)
+				require.NoError(t, err)
+
+				require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+				require.Equal(t, expectedContentLocationHdr, resp.Header.Get("Content-Location"))
+			})
+		}
+
+		contentPath := path.FromCid(root).String() + "/empty-dir/"
+		subdomainGatewayHost := root.String() + ".ipfs.subdomain-gateway.com"
+		dnslinkGatewayHost := "dnslink-gateway.com"
+
+		runTest("Regular gateway with default format", contentPath, "", "", "")
+		runTest("Regular gateway with Accept: application/vnd.ipld.car;version=1;order=dfs;dups=n sets correct Content-Location", contentPath, "application/vnd.ipld.car;version=1;order=dfs;dups=n", "", contentPath+"?car-dups=n&car-order=dfs&car-version=1&format=car")
+		runTest("Regular gateway with ?dag-scope=entity&format=car", contentPath+"?dag-scope=entity&format=car", "", "", "")
+		runTest("Regular gateway preserves query parameters", contentPath+"?a=b&c=d", dagCborResponseFormat, "", contentPath+"?a=b&c=d&format=dag-cbor")
+		runTest("Subdomain gateway with default format", "/empty-dir/", "", subdomainGatewayHost, "")
+		runTest("DNSLink gateway with default format", "/empty-dir/", "", dnslinkGatewayHost, "")
+
+		for responseFormat, formatParam := range responseFormatToFormatParam {
+			if responseFormat == ipnsRecordResponseFormat {
+				continue
+			}
+
+			runTest("Regular gateway with Accept: "+responseFormat, contentPath, responseFormat, "", contentPath+"?format="+formatParam)
+			runTest("Regular gateway with ?format="+formatParam, contentPath+"?format="+formatParam, "", "", "")
+
+			runTest("Subdomain gateway with Accept: "+responseFormat, "/empty-dir/", responseFormat, subdomainGatewayHost, "/empty-dir/?format="+formatParam)
+			runTest("Subdomain gateway with ?format="+formatParam, "/empty-dir/?format="+formatParam, "", subdomainGatewayHost, "")
+
+			runTest("DNSLink gateway with Accept: "+responseFormat, "/empty-dir/", responseFormat, dnslinkGatewayHost, "/empty-dir/?format="+formatParam)
+			runTest("DNSLink gateway with ?format="+formatParam, "/empty-dir/?format="+formatParam, "", dnslinkGatewayHost, "")
+		}
+
+		runTest("Accept: application/vnd.ipld.car overrides ?format=raw in Content-Location", contentPath+"?format=raw", "application/vnd.ipld.car", "", contentPath+"?format=car")
+	})
 }
 
 func TestGoGetSupport(t *testing.T) {

gateway/handler.go

@@ -259,6 +259,8 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 		responseParams: formatParams,
 	}
 
+	addContentLocation(r, w, rq)
+
 	// IPNS Record response format can be handled now, since (1) it needs the
 	// non-resolved mutable path, and (2) has custom If-None-Match header handling
 	// due to custom ETag.
@@ -586,6 +588,27 @@ const (
 	ipnsRecordResponseFormat = "application/vnd.ipfs.ipns-record"
 )
 
+var (
+	formatParamToResponseFormat = map[string]string{
+		"raw":         rawResponseFormat,
+		"car":         carResponseFormat,
+		"tar":         tarResponseFormat,
+		"json":        jsonResponseFormat,
+		"cbor":        cborResponseFormat,
+		"dag-json":    dagJsonResponseFormat,
+		"dag-cbor":    dagCborResponseFormat,
+		"ipns-record": ipnsRecordResponseFormat,
+	}
+
+	responseFormatToFormatParam = map[string]string{}
+)
+
+func init() {
+	for k, v := range formatParamToResponseFormat {
+		responseFormatToFormatParam[v] = k
+	}
+}
+
 // return explicit response format if specified in request as query parameter or via Accept HTTP header
 func customResponseFormat(r *http.Request) (mediaType string, params map[string]string, err error) {
 	// First, inspect Accept header, as it may not only include content type, but also optional parameters.
@@ -615,23 +638,8 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 
 	// If no Accept header, translate query param to a content type, if present.
 	if formatParam := r.URL.Query().Get("format"); formatParam != "" {
-		switch formatParam {
-		case "raw":
-			return rawResponseFormat, nil, nil
-		case "car":
-			return carResponseFormat, nil, nil
-		case "tar":
-			return tarResponseFormat, nil, nil
-		case "json":
-			return jsonResponseFormat, nil, nil
-		case "cbor":
-			return cborResponseFormat, nil, nil
-		case "dag-json":
-			return dagJsonResponseFormat, nil, nil
-		case "dag-cbor":
-			return dagCborResponseFormat, nil, nil
-		case "ipns-record":
-			return ipnsRecordResponseFormat, nil, nil
+		if responseFormat, ok := formatParamToResponseFormat[formatParam]; ok {
+			return responseFormat, nil, nil
 		}
 	}
 
@@ -640,6 +648,46 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 	return "", nil, nil
 }
 
+// Add 'Content-Location' headers for non-default response formats. This allows
+// correct caching of such format requests when the format is passed via the
+// Accept header, for example.
+func addContentLocation(r *http.Request, w http.ResponseWriter, rq *requestData) {
+	// Skip Content-Location if no explicit format was requested
+	// via Accept HTTP header or ?format URL param
+	if rq.responseFormat == "" {
+		return
+	}
+
+	format := responseFormatToFormatParam[rq.responseFormat]
+
+	// Skip Content-Location if there is no conflict between
+	// 'format' in URL and value in 'Accept' header.
+	// If both are present and don't match, we continue and generate
+	// Content-Location to ensure value from Accept overrides 'format' from URL.
+	if urlFormat := r.URL.Query().Get("format"); urlFormat != "" && urlFormat == format {
+		return
+	}
+
+	path := r.URL.Path
+	if p, ok := r.Context().Value(OriginalPathKey).(string); ok {
+		path = p
+	}
+
+	// Copy all existing query parameters.
+	query := url.Values{}
+	for k, v := range r.URL.Query() {
+		query[k] = v
+	}
+	query.Set("format", format)
+
+	// Set response params as query elements.
+	for k, v := range rq.responseParams {
+		query.Set(format+"-"+k, v)
+	}
+
+	w.Header().Set("Content-Location", path+"?"+query.Encode())
+}
+
 // returns unquoted path with all special characters revealed as \u codes
 func debugStr(path string) string {
 	q := fmt.Sprintf("%+q", path)

gateway/handler_car.go

@@ -23,6 +23,9 @@ import (
 const (
 	carRangeBytesKey          = "entity-bytes"
 	carTerminalElementTypeKey = "dag-scope"
+	carVersionKey             = "car-version"
+	carDuplicatesKey          = "car-dups"
+	carOrderKey               = "car-order"
 )
 
 // serveCAR returns a CAR stream for specific DAG+selector
@@ -144,16 +147,31 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 
 	// application/vnd.ipld.car content type parameters from Accept header
 
+	// Get CAR version, duplicates and order from the query parameters and override
+	// with parameters from Accept header if they exist, since they have priority.
+	versionStr := queryParams.Get(carVersionKey)
+	duplicatesStr := queryParams.Get(carDuplicatesKey)
+	orderStr := queryParams.Get(carOrderKey)
+	if v, ok := contentTypeParams["version"]; ok {
+		versionStr = v
+	}
+	if v, ok := contentTypeParams["order"]; ok {
+		orderStr = v
+	}
+	if v, ok := contentTypeParams["dups"]; ok {
+		duplicatesStr = v
+	}
+
 	// version of CAR format
-	switch contentTypeParams["version"] {
+	switch versionStr {
 	case "": // noop, client does not care about version
 	case "1": // noop, we support this
 	default:
 		return CarParams{}, errors.New("unsupported application/vnd.ipld.car version: only version=1 is supported")
 	}
 
 	// optional order from IPIP-412
-	if order := DagOrder(contentTypeParams["order"]); order != DagOrderUnspecified {
+	if order := DagOrder(orderStr); order != DagOrderUnspecified {
 		switch order {
 		case DagOrderUnknown, DagOrderDFS:
 			params.Order = order
@@ -168,7 +186,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 	}
 
 	// optional dups from IPIP-412
-	dups, err := NewDuplicateBlocksPolicy(contentTypeParams["dups"])
+	dups, err := NewDuplicateBlocksPolicy(duplicatesStr)
 	if err != nil {
 		return CarParams{}, err
 	}

gateway/handler_car_test.go

@@ -2,6 +2,7 @@ package gateway
 
 import (
 	"net/http"
+	"net/url"
 	"testing"
 
 	"github.com/ipfs/boxo/path"
@@ -81,19 +82,22 @@ func TestCarParams(t *testing.T) {
 		// from the value read from Accept header
 		tests := []struct {
 			acceptHeader       string
+			params             url.Values
 			expectedOrder      DagOrder
 			expectedDuplicates DuplicateBlocksPolicy
 		}{
-			{"application/vnd.ipld.car; order=dfs; dups=y", DagOrderDFS, DuplicateBlocksIncluded},
-			{"application/vnd.ipld.car; order=unk; dups=n", DagOrderUnknown, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car; order=unk", DagOrderUnknown, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car; dups=y", DagOrderDFS, DuplicateBlocksIncluded},
-			{"application/vnd.ipld.car; dups=n", DagOrderDFS, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car", DagOrderDFS, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car;version=1;order=dfs;dups=y", DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car; order=dfs; dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car; order=unk; dups=n", nil, DagOrderUnknown, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car; order=unk", nil, DagOrderUnknown, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car; dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car; dups=n", nil, DagOrderDFS, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car", nil, DagOrderDFS, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car;version=1;order=dfs;dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car;version=1;order=dfs;dups=y", url.Values{"car-order": []string{"unk"}}, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car;version=1;dups=y", url.Values{"car-order": []string{"unk"}}, DagOrderUnknown, DuplicateBlocksIncluded},
 		}
 		for _, test := range tests {
-			r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil)
+			r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.params.Encode(), nil)
 			r.Header.Set("Accept", test.acceptHeader)
 
 			mediaType, formatParams, err := customResponseFormat(r)

gateway/hostname.go

@@ -28,6 +28,9 @@ func NewHostnameHandler(c Config, backend IPFSBackend, next http.Handler) http.H
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer panicHandler(w)
 
+		ctx := context.WithValue(r.Context(), OriginalPathKey, r.URL.Path)
+		r = r.WithContext(ctx)
+
 		// First check for protocol handler redirects.
 		if handleProtocolHandlerRedirect(w, r, &c) {
 			return