gateway: set Content-Location for non-default response formats

Author: hacdias

CHANGELOG.md

@@ -19,6 +19,7 @@ The following emojis are used to highlight certain changes:
 * ✨ `gateway` has new backend possibilities:
   * `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
   * `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`.
+* `gateway` now sets the [`Content-Location`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Location) header for requests with non-default content format, as a result of content negotiation.
 
 ### Changed
 

gateway/gateway.go

@@ -405,6 +405,11 @@ const (
 	// [Subdomain Gateway]: https://specs.ipfs.tech/http-gateways/subdomain-gateway/
 	SubdomainHostnameKey RequestContextKey = "subdomain-hostname"
 
-	// ContentPathKey is the key for the original [http.Request] URL Path, as an [ipath.Path].
+	// OriginalPathKey is the key for the original [http.Request] [url.URL.Path],
+	// as a string. This is the original path of the request, before [NewHostnameHandler].
+	OriginalPathKey RequestContextKey = "original-path-key"
+
+	// ContentPathKey is the key for the content [path.Path] of the current request.
+	// This already accounts with changes made with [NewHostnameHandler].
 	ContentPathKey RequestContextKey = "content-path"
 )

gateway/gateway_test.go

@@ -417,6 +417,79 @@ func TestHeaders(t *testing.T) {
 			testCORSPreflightRequest(t, "/", cid+".ipfs.subgw.example.com", "https://other.example.net", http.StatusOK)
 		})
 	})
+
+	t.Run("Content-Location is set when possible", func(t *testing.T) {
+		backend, root := newMockBackend(t, "fixtures.car")
+		backend.namesys["/ipns/dnslink-gateway.com"] = newMockNamesysItem(path.FromCid(root), 0)
+
+		ts := newTestServerWithConfig(t, backend, Config{
+			NoDNSLink: false,
+			PublicGateways: map[string]*PublicGateway{
+				"dnslink-gateway.com": {
+					Paths:                 []string{},
+					NoDNSLink:             false,
+					DeserializedResponses: true,
+				},
+				"subdomain-gateway.com": {
+					Paths:                 []string{"/ipfs", "/ipns"},
+					UseSubdomains:         true,
+					NoDNSLink:             true,
+					DeserializedResponses: true,
+				},
+			},
+			DeserializedResponses: true,
+		})
+
+		runTest := func(name, path, accept, host, expectedContentPath string) {
+			t.Run(name, func(t *testing.T) {
+				t.Parallel()
+
+				req := mustNewRequest(t, http.MethodGet, ts.URL+path, nil)
+
+				if accept != "" {
+					req.Header.Set("Accept", accept)
+				}
+
+				if host != "" {
+					req.Host = host
+				}
+
+				resp := mustDoWithoutRedirect(t, req)
+				defer resp.Body.Close()
+
+				body, err := io.ReadAll(resp.Body)
+				require.NoError(t, err)
+
+				require.Equal(t, http.StatusOK, resp.StatusCode, string(body))
+				require.Equal(t, expectedContentPath, resp.Header.Get("Content-Location"))
+			})
+		}
+
+		contentPath := path.FromCid(root).String() + "/empty-dir/"
+		subdomainGatewayHost := root.String() + ".ipfs.subdomain-gateway.com"
+		dnslinkGatewayHost := "dnslink-gateway.com"
+
+		runTest("Regular gateway with default format", contentPath, "", "", "")
+		runTest("Regular gateway with Accept: application/vnd.ipld.car has no Content-Location", contentPath, "application/vnd.ipld.car;version=1;order=dfs;dups=n", "", "")
+		runTest("Regular gateway with ?dag-scope=entity&format=car", contentPath+"?dag-scope=entity&format=car", "", "", contentPath+"?dag-scope=entity&format=car")
+		runTest("Subdomain gateway with default format", "/empty-dir/", "", subdomainGatewayHost, "")
+		runTest("DNSLink gateway with default format", "/empty-dir/", "", dnslinkGatewayHost, "")
+
+		for responseFormat, formatParam := range responseFormatToFormatParam {
+			if responseFormat == ipnsRecordResponseFormat {
+				continue
+			}
+
+			runTest("Regular gateway with Accept: "+responseFormat, contentPath, responseFormat, "", contentPath+"?format="+formatParam)
+			runTest("Regular gateway with ?format="+formatParam, contentPath+"?format="+formatParam, "", "", contentPath+"?format="+formatParam)
+
+			runTest("Subdomain gateway with Accept: "+responseFormat, "/empty-dir/", responseFormat, subdomainGatewayHost, "/empty-dir/?format="+formatParam)
+			runTest("Subdomain gateway with ?format="+formatParam, "/empty-dir/?format="+formatParam, "", subdomainGatewayHost, "/empty-dir/?format="+formatParam)
+
+			runTest("DNSLink gateway with Accept: "+responseFormat, "/empty-dir/", responseFormat, dnslinkGatewayHost, "/empty-dir/?format="+formatParam)
+			runTest("DNSLink gateway with ?format="+formatParam, "/empty-dir/?format="+formatParam, "", dnslinkGatewayHost, "/empty-dir/?format="+formatParam)
+		}
+	})
 }
 
 func TestGoGetSupport(t *testing.T) {

gateway/handler.go

@@ -259,6 +259,8 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 		responseParams: formatParams,
 	}
 
+	addContentLocation(r, w, rq)
+
 	// IPNS Record response format can be handled now, since (1) it needs the
 	// non-resolved mutable path, and (2) has custom If-None-Match header handling
 	// due to custom ETag.
@@ -586,6 +588,27 @@ const (
 	ipnsRecordResponseFormat = "application/vnd.ipfs.ipns-record"
 )
 
+var (
+	formatParamToResponseFormat = map[string]string{
+		"raw":         rawResponseFormat,
+		"car":         carResponseFormat,
+		"tar":         tarResponseFormat,
+		"json":        jsonResponseFormat,
+		"cbor":        cborResponseFormat,
+		"dag-json":    dagJsonResponseFormat,
+		"dag-cbor":    dagCborResponseFormat,
+		"ipns-record": ipnsRecordResponseFormat,
+	}
+
+	responseFormatToFormatParam = map[string]string{}
+)
+
+func init() {
+	for k, v := range formatParamToResponseFormat {
+		responseFormatToFormatParam[v] = k
+	}
+}
+
 // return explicit response format if specified in request as query parameter or via Accept HTTP header
 func customResponseFormat(r *http.Request) (mediaType string, params map[string]string, err error) {
 	// First, inspect Accept header, as it may not only include content type, but also optional parameters.
@@ -615,23 +638,8 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 
 	// If no Accept header, translate query param to a content type, if present.
 	if formatParam := r.URL.Query().Get("format"); formatParam != "" {
-		switch formatParam {
-		case "raw":
-			return rawResponseFormat, nil, nil
-		case "car":
-			return carResponseFormat, nil, nil
-		case "tar":
-			return tarResponseFormat, nil, nil
-		case "json":
-			return jsonResponseFormat, nil, nil
-		case "cbor":
-			return cborResponseFormat, nil, nil
-		case "dag-json":
-			return dagJsonResponseFormat, nil, nil
-		case "dag-cbor":
-			return dagCborResponseFormat, nil, nil
-		case "ipns-record":
-			return ipnsRecordResponseFormat, nil, nil
+		if responseFormat, ok := formatParamToResponseFormat[formatParam]; ok {
+			return responseFormat, nil, nil
 		}
 	}
 
@@ -640,6 +648,39 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 	return "", nil, nil
 }
 
+// Add 'Content-Location' headers for non-default response formats. This allows
+// correct caching of such format requests when the format is passed via the
+// Accept header, for example.
+func addContentLocation(r *http.Request, w http.ResponseWriter, rq *requestData) {
+	if rq.responseFormat == "" {
+		return
+	}
+
+	// Response format parameters, such as 'dups' and 'order' for CAR requests
+	// cannot be translated into the URL. Therefore, we cannot add a 'Content-Location'
+	// header.
+	if len(rq.responseParams) != 0 {
+		return
+	}
+
+	param := responseFormatToFormatParam[rq.responseFormat]
+	path := r.URL.Path
+	if p, ok := r.Context().Value(OriginalPathKey).(string); ok {
+		path = p
+	}
+
+	// Copy known query parameters.
+	query := url.Values{}
+	for _, param := range []string{carRangeBytesKey, carTerminalElementTypeKey, "filename", "download"} {
+		if v := r.URL.Query().Get(param); v != "" {
+			query.Set(param, v)
+		}
+	}
+	query.Set("format", param)
+
+	w.Header().Set("Content-Location", path+"?"+query.Encode())
+}
+
 // returns unquoted path with all special characters revealed as \u codes
 func debugStr(path string) string {
 	q := fmt.Sprintf("%+q", path)

gateway/hostname.go

@@ -28,6 +28,9 @@ func NewHostnameHandler(c Config, backend IPFSBackend, next http.Handler) http.H
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer panicHandler(w)
 
+		ctx := context.WithValue(r.Context(), OriginalPathKey, r.URL.Path)
+		r = r.WithContext(ctx)
+
 		// First check for protocol handler redirects.
 		if handleProtocolHandlerRedirect(w, r, &c) {
 			return