gateway: add support for car-* query parameters (#604)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

Author: hacdias

CHANGELOG.md

@@ -20,6 +20,7 @@ The following emojis are used to highlight certain changes:
   * `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
   * `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`.
 * `gateway` now sets the [`Content-Location`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Location) header for requests with non-default content format, as a result of content negotiation. This allows generic and misconfigured HTTP caches to store Deserialized, CAR and Block responses separately, under distinct cache keys.
+* `gateway` now supports `car-dups`, `car-order` and `car-version` as query parameters in addition to the `application/vnd.ipld.car` parameters sent via `Accept` header. The parameters in the `Accept` header have always priority, but including them in URL simplifies HTTP caching and allows use in `Content-Location` header on CAR responses to maximize interoperability with wide array of HTTP caches.
 
 ### Changed
 

gateway/gateway_test.go

@@ -470,7 +470,7 @@ func TestHeaders(t *testing.T) {
 		dnslinkGatewayHost := "dnslink-gateway.com"
 
 		runTest("Regular gateway with default format", contentPath, "", "", "")
-		runTest("Regular gateway with Accept: application/vnd.ipld.car has no Content-Location", contentPath, "application/vnd.ipld.car;version=1;order=dfs;dups=n", "", "")
+		runTest("Regular gateway with Accept: application/vnd.ipld.car;version=1;order=dfs;dups=n sets correct Content-Location", contentPath, "application/vnd.ipld.car;version=1;order=dfs;dups=n", "", contentPath+"?car-dups=n&car-order=dfs&car-version=1&format=car")
 		runTest("Regular gateway with ?dag-scope=entity&format=car", contentPath+"?dag-scope=entity&format=car", "", "", "")
 		runTest("Regular gateway preserves query parameters", contentPath+"?a=b&c=d", dagCborResponseFormat, "", contentPath+"?a=b&c=d&format=dag-cbor")
 		runTest("Subdomain gateway with default format", "/empty-dir/", "", subdomainGatewayHost, "")

gateway/handler.go

@@ -658,14 +658,7 @@ func addContentLocation(r *http.Request, w http.ResponseWriter, rq *requestData)
 		return
 	}
 
-	// Response format parameters, such as 'dups' and 'order' for CAR requests
-	// cannot be translated into the URL. Therefore, we cannot add a 'Content-Location'
-	// header.
-	if len(rq.responseParams) != 0 {
-		return
-	}
-
-	param := responseFormatToFormatParam[rq.responseFormat]
+	format := responseFormatToFormatParam[rq.responseFormat]
 	path := r.URL.Path
 	if p, ok := r.Context().Value(OriginalPathKey).(string); ok {
 		path = p
@@ -676,7 +669,12 @@ func addContentLocation(r *http.Request, w http.ResponseWriter, rq *requestData)
 	for k, v := range r.URL.Query() {
 		query[k] = v
 	}
-	query.Set("format", param)
+	query.Set("format", format)
+
+	// Set response params as query elements.
+	for k, v := range rq.responseParams {
+		query.Set(format+"-"+k, v)
+	}
 
 	w.Header().Set("Content-Location", path+"?"+query.Encode())
 }

gateway/handler_car.go

@@ -23,6 +23,9 @@ import (
 const (
 	carRangeBytesKey          = "entity-bytes"
 	carTerminalElementTypeKey = "dag-scope"
+	carVersionKey             = "car-version"
+	carDuplicatesKey          = "car-dups"
+	carOrderKey               = "car-order"
 )
 
 // serveCAR returns a CAR stream for specific DAG+selector
@@ -144,16 +147,31 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 
 	// application/vnd.ipld.car content type parameters from Accept header
 
+	// Get CAR version, duplicates and order from the query parameters and override
+	// with parameters from Accept header if they exist, since they have priority.
+	versionStr := queryParams.Get(carVersionKey)
+	duplicatesStr := queryParams.Get(carDuplicatesKey)
+	orderStr := queryParams.Get(carOrderKey)
+	if v, ok := contentTypeParams["version"]; ok {
+		versionStr = v
+	}
+	if v, ok := contentTypeParams["order"]; ok {
+		orderStr = v
+	}
+	if v, ok := contentTypeParams["dups"]; ok {
+		duplicatesStr = v
+	}
+
 	// version of CAR format
-	switch contentTypeParams["version"] {
+	switch versionStr {
 	case "": // noop, client does not care about version
 	case "1": // noop, we support this
 	default:
 		return CarParams{}, errors.New("unsupported application/vnd.ipld.car version: only version=1 is supported")
 	}
 
 	// optional order from IPIP-412
-	if order := DagOrder(contentTypeParams["order"]); order != DagOrderUnspecified {
+	if order := DagOrder(orderStr); order != DagOrderUnspecified {
 		switch order {
 		case DagOrderUnknown, DagOrderDFS:
 			params.Order = order
@@ -168,7 +186,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 	}
 
 	// optional dups from IPIP-412
-	dups, err := NewDuplicateBlocksPolicy(contentTypeParams["dups"])
+	dups, err := NewDuplicateBlocksPolicy(duplicatesStr)
 	if err != nil {
 		return CarParams{}, err
 	}

gateway/handler_car_test.go

@@ -2,6 +2,7 @@ package gateway
 
 import (
 	"net/http"
+	"net/url"
 	"testing"
 
 	"github.com/ipfs/boxo/path"
@@ -81,19 +82,22 @@ func TestCarParams(t *testing.T) {
 		// from the value read from Accept header
 		tests := []struct {
 			acceptHeader       string
+			params             url.Values
 			expectedOrder      DagOrder
 			expectedDuplicates DuplicateBlocksPolicy
 		}{
-			{"application/vnd.ipld.car; order=dfs; dups=y", DagOrderDFS, DuplicateBlocksIncluded},
-			{"application/vnd.ipld.car; order=unk; dups=n", DagOrderUnknown, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car; order=unk", DagOrderUnknown, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car; dups=y", DagOrderDFS, DuplicateBlocksIncluded},
-			{"application/vnd.ipld.car; dups=n", DagOrderDFS, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car", DagOrderDFS, DuplicateBlocksExcluded},
-			{"application/vnd.ipld.car;version=1;order=dfs;dups=y", DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car; order=dfs; dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car; order=unk; dups=n", nil, DagOrderUnknown, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car; order=unk", nil, DagOrderUnknown, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car; dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car; dups=n", nil, DagOrderDFS, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car", nil, DagOrderDFS, DuplicateBlocksExcluded},
+			{"application/vnd.ipld.car;version=1;order=dfs;dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car;version=1;order=dfs;dups=y", url.Values{"car-order": []string{"unk"}}, DagOrderDFS, DuplicateBlocksIncluded},
+			{"application/vnd.ipld.car;version=1;dups=y", url.Values{"car-order": []string{"unk"}}, DagOrderUnknown, DuplicateBlocksIncluded},
 		}
 		for _, test := range tests {
-			r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil)
+			r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.params.Encode(), nil)
 			r.Header.Set("Accept", test.acceptHeader)
 
 			mediaType, formatParams, err := customResponseFormat(r)