diff --git a/doc/release-notes/fix-url-signing-special-characters.md b/doc/release-notes/fix-url-signing-special-characters.md new file mode 100644 index 00000000000..d7a2853dfa2 --- /dev/null +++ b/doc/release-notes/fix-url-signing-special-characters.md @@ -0,0 +1,45 @@ +### Signed URLs work again for URLs with special characters + +Requesting a signed URL (e.g. via `/api/admin/requestSignedUrl`, used by external tools, the Globus +integration and third-party integrations such as the `rdm-integration` connector) was broken in 6.10 +for URLs whose query contained special characters — most notably persistent IDs such as +`doi:10.5072/FK2/ABC` (which contain `:` and `/`), as well as spaces, percent-encoded values and +non-ASCII characters. In 6.10 the signing step began re-encoding/normalizing the URL (for example +percent-encoding `:` and `/`) before computing the signature, while the request is validated against +the URL the caller actually presents back. The re-encoded signature no longer matched, so validation +failed with authentication / "signature does not match" errors. + +Signing no longer re-encodes the URL: it is signed exactly as provided, with only the reserved +signing parameters (`until`, `user`, `method`, `token`, `key`, `signed`) stripped out; the rest of +the URL is left untouched, character for character. + +**This restores the URL-signing behavior used before 6.10, so it is compatible with older versions +and with existing integrations.** Clients and connectors that build or consume signed URLs the way +they did before 6.10 keep working unchanged, signatures are computed the same way as before the +regression, and URLs containing special characters validate again. No client-side changes are +required. + +### A signing secret is now required for signed URLs + +Separately from the fix above, Dataverse no longer falls back to a weak signing key when +`dataverse.api.signing-secret` is unset. Previously, with no secret configured, signed URLs were +signed using only the user's API token (or, for a guest, a value derived from the public URL), which +is too weak to be a signing key. A non-empty `dataverse.api.signing-secret` is now required wherever +URLs are signed with a key based on a user's API token: + +- The endpoints that issue a signed URL on request - `/api/admin/requestSignedUrl` and the + guestbook-response file download (`POST /api/access/datafile/{id}`) - return an error instead of + issuing a weakly-signed URL. +- Signed callbacks and links built internally (external tool launches, Globus transfers, the + permission-history CSV links) are sent unsigned, with a warning logged, rather than weakly signed. + +Remote and Globus overlay stores are unaffected: they sign with their own per-store secret key, not +`dataverse.api.signing-secret`. + +**Upgrade note:** installations that rely on signed URLs - including the `rdm-integration` connector, +signed guestbook-response downloads, and external tools or Globus transfers that use signed callbacks - +must set `dataverse.api.signing-secret`. See the +[Configuration Guide](https://guides.dataverse.org/en/latest/installation/config.html#dataverse-api-signing-secret). +Treat the value like a password. Because the signing secret is part of the signing key, setting (or +later changing) it invalidates previously issued signed URLs: any existing signed URLs that have not +yet expired will stop working, and clients/integrations will need to request new ones. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 0670855791a..1dc2ad94f1c 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3349,9 +3349,10 @@ are time limited and only allow the action of the API call in the URL. See :ref: :ref:`api-native-signed-url` for more details. The key used to sign a URL is created from the API token of the creating user plus a signing-secret provided by an administrator. -**Using a signing-secret is highly recommended.** This setting defaults to an empty string. Using a non-empty -signing-secret makes it impossible for someone who knows an API token from forging signed URLs and provides extra security by -making the overall signing key longer. +**A non-empty signing-secret is required to request signed URLs through the API.** If it is not configured, the +``/api/admin/requestSignedUrl`` endpoint (see :ref:`api-native-signed-url`) returns an error instead of issuing a +weakly-signed URL. (The setting otherwise defaults to an empty string.) A non-empty signing-secret makes it impossible for +someone who only knows an API token to forge signed URLs, and provides extra security by making the overall signing key longer. **WARNING**: *Since the signing-secret is sensitive, you should treat it like a password.* diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b24bf0ed6f6..86aff92e523 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -60,6 +60,7 @@ services: -Ddataverse.pid.fake.label=FakeDOIProvider -Ddataverse.pid.fake.authority=10.5072 -Ddataverse.pid.fake.shoulder=FK2/ + -Ddataverse.api.signing-secret=dev-only-signing-secret-change-me -Ddataverse.cors.origin=* \ -Ddataverse.cors.methods=GET,POST,PUT,DELETE,OPTIONS \ -Ddataverse.cors.headers.allow=range,content-type,x-dataverse-key,accept \ diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index ee687305584..7317aca1717 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -22,7 +22,6 @@ import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.JsfHelper; @@ -640,9 +639,8 @@ public String getSignedUrlForRAHistoryCsv() { key = apiToken.getTokenString(); } } - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; - if(key.length() >= 36) { - return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key); + if (key != null && UrlSignerUtil.isSigningSecretConfigured()) { + return UrlSignerUtil.signUrlWithApiKey(fullApiPath, 10, userId, "GET", key); } } catch (Exception e) { logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index f5cd859e7ac..49be3631603 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -20,7 +20,6 @@ import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseDefaultContributorRoleCommand; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; @@ -736,9 +735,8 @@ public String getSignedUrlForRAHistoryCsv() { key = apiToken.getTokenString(); } } - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; - if(key.length() >= 36) { - return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key); + if (key != null && UrlSignerUtil.isSigningSecretConfigured()) { + return UrlSignerUtil.signUrlWithApiKey(fullApiPath, 10, userId, "GET", key); } } catch (Exception e) { logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index a2d7d3ed525..84c44ababd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -29,7 +29,6 @@ import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.mydata.Pager; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.*; import edu.harvard.iq.dataverse.util.json.JsonParseException; @@ -529,6 +528,12 @@ private Map getDatafilesMap(DataverseRequest req, String fileIds } private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, User user, String id, String gbrids) { + // Require a signing secret: without it the key is only the user's API token (or, for a guest, + // a guessable value derived from the URL), which is too weak. Mirrors Admin.getSignedUrl. + if (!UrlSignerUtil.isSigningSecretConfigured()) { + return error(INTERNAL_SERVER_ERROR, + "Requesting signed URLs requires a signing secret to be configured. Please set the dataverse.api.signing-secret JVM option."); + } // Create the signed URL String userIdentifier = null; String key = null; @@ -564,8 +569,7 @@ private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, U String baseUrlEncoded = builder.build().toString(); String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); baseUrl = baseUrl.replace(":persistentId", id); - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; - String signedUrl = UrlSignerUtil.signUrl(baseUrl, GUESTBOOK_RESPONSE_SIGNEDURL_TIMEOUT_MINUTES, userIdentifier, "GET", key); + String signedUrl = UrlSignerUtil.signUrlWithApiKey(baseUrl, GUESTBOOK_RESPONSE_SIGNEDURL_TIMEOUT_MINUTES, userIdentifier, "GET", key); return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 919fa7f67f9..feb87ba32ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -17,7 +17,6 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.auth.AuthRequired; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsValidationException; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; @@ -2453,7 +2452,13 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur if (superuser == null || !superuser.isSuperuser()) { return error(Response.Status.FORBIDDEN, "Requesting signed URLs is restricted to superusers."); } - + + // Require a signing secret: without it the key is only the user's API token, which is too weak. + if (!UrlSignerUtil.isSigningSecretConfigured()) { + return error(Response.Status.INTERNAL_SERVER_ERROR, + "Requesting signed URLs requires a signing secret to be configured. Please set the dataverse.api.signing-secret JVM option."); + } + String userId = urlInfo.getString("user"); String key=null; if (userId != null) { @@ -2475,14 +2480,13 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur if (key == null) { return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); } - key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; } - + String baseUrl = urlInfo.getString("url"); int timeout = urlInfo.getInt(URLTokenUtil.TIMEOUT, 10); String method = urlInfo.getString(URLTokenUtil.HTTP_METHOD, "GET"); - - String signedUrl = UrlSignerUtil.signUrl(baseUrl, timeout, userId, method, key); + + String signedUrl = UrlSignerUtil.signUrlWithApiKey(baseUrl, timeout, userId, method, key); return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index e7ae451cacf..4d0a3d49e74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -4,7 +4,6 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.users.ApiToken; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.URLTokenUtil; @@ -111,8 +110,12 @@ public String handleRequest(boolean preview) { + externalTool.getId(); } if (apiToken != null) { - callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(), HttpMethod.GET, - JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString()); + if (UrlSignerUtil.isSigningSecretConfigured()) { + callback = UrlSignerUtil.signUrlWithApiKey(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(), + HttpMethod.GET, apiToken.getTokenString()); + } else { + logger.warning("Cannot sign external tool callback: no signing secret configured (dataverse.api.signing-secret). Sending an unsigned callback."); + } } paramsString= "?callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback)); if (getLocaleCode() != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 789e0883a7c..9d43570a9d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -776,13 +776,14 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List 1) { - try { - URIBuilder uriBuilder = new URIBuilder(baseUrl); - List params = uriBuilder.getQueryParams(); - params.removeIf(pair -> reservedParameters.contains(pair.getName())); - uriBuilder.setParameters(params); - baseUrl = uriBuilder.build().toString(); - } catch (URISyntaxException e) { - logger.severe("Invalid URL for signing: " + baseUrl + " " + e.getMessage()); - } - } + // Strip reserved signing params that may already be in the base URL, using exact-string + // surgery rather than URIBuilder. The URL must be signed exactly as provided (the pre-6.10 + // behavior): validation reconstructs the signing string from the URL-decoded request, so + // re-encoding here (e.g. percent-encoding ':' and '/' in DOIs) would change the signed bytes + // and the signature would no longer match. + baseUrl = stripReservedParameters(baseUrl); boolean firstParam = !baseUrl.contains("?"); StringBuilder signedUrlBuilder = new StringBuilder(baseUrl); @@ -87,6 +79,63 @@ public static String signUrl(String baseUrl, Integer timeout, String user, Strin return signedUrl; } + /** + * Whether a non-empty API signing secret ({@code dataverse.api.signing-secret}) is configured. + * Every signed URL whose key is derived from a user's API token must be guarded by this: without + * the secret the signing key would be only the caller-supplied value (for a guest, even a value + * derived from the public URL), which is too weak. Callers either refuse the request or skip + * signing when this returns false, so a weakly-signed URL is never emitted. + */ + public static boolean isSigningSecretConfigured() { + return !JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("").isEmpty(); + } + + /** + * Signs a URL using the configured API signing secret prepended to the given per-user key + * (typically the user's API token). This is the single place that combines the server-side + * signing secret with a user key, so every API-token-based signed URL is produced the same way. + * + *

Stores that sign with their own per-store secret (the remote and Globus overlay stores) are + * the exception and must keep calling {@link #signUrl} directly with that secret. + * + * @throws IllegalStateException if no signing secret is configured - callers should normally + * guard with {@link #isSigningSecretConfigured()} first + */ + public static String signUrlWithApiKey(String baseUrl, Integer timeout, String user, String method, String apiKey) { + String secret = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse(""); + if (secret.isEmpty()) { + throw new IllegalStateException( + "Cannot sign a URL: no signing secret is configured. Please set the dataverse.api.signing-secret JVM option."); + } + return signUrl(baseUrl, timeout, user, method, secret + apiKey); + } + + /** + * Removes the reserved signing parameters from the query, preserving the exact bytes of the path + * and of every other parameter (unlike URIBuilder, which would re-encode and break the MAC). + */ + static String stripReservedParameters(String baseUrl) { + int queryStart = baseUrl.indexOf('?'); + if (queryStart < 0) { + return baseUrl; + } + String path = baseUrl.substring(0, queryStart); + String query = baseUrl.substring(queryStart + 1); + StringBuilder kept = new StringBuilder(); + for (String pair : query.split("&")) { + int equals = pair.indexOf('='); + String name = (equals < 0) ? pair : pair.substring(0, equals); + if (reservedParameters.contains(name)) { + continue; + } + if (kept.length() > 0) { + kept.append('&'); + } + kept.append(pair); + } + return kept.length() > 0 ? path + "?" + kept : path; + } + /** * This method will only return true if the URL and parameters except the * "token" are unchanged from the original/match the values sent to this method, diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java index 6fd7d2e1d8e..076894f4b10 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanismTest.java @@ -5,12 +5,17 @@ import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.util.UrlSignerUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import jakarta.ws.rs.container.ContainerRequestContext; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; + import static edu.harvard.iq.dataverse.api.auth.SignedUrlAuthMechanism.RESPONSE_MESSAGE_BAD_SIGNED_URL; import static org.junit.jupiter.api.Assertions.*; @@ -96,4 +101,86 @@ public void testFindUserFromRequest_SignedUrlTokenProvided_UserDoesNotExistForTh assertEquals(RESPONSE_MESSAGE_BAD_SIGNED_URL, wrappedUnauthorizedAuthErrorResponse.getMessage()); } + + // End-to-end validation through the REAL SignedUrlAuthMechanism (URLDecoder.decode + isValidUrl), + // which the isValidUrl-only tests in UrlSignerUtilTest do not exercise. No signing secret is + // configured here, so the signing key is just the API token. + + private void givenUserWithSigningKey(String key) { + AuthenticationServiceBean authStub = Mockito.mock(AuthenticationServiceBean.class); + Mockito.when(authStub.getAuthenticatedUser(TEST_SIGNED_URL_USER_ID)).thenReturn(testAuthenticatedUser); + ApiToken apiToken = Mockito.mock(ApiToken.class); + Mockito.when(apiToken.getTokenString()).thenReturn(key); + Mockito.when(authStub.findApiTokenByUser(testAuthenticatedUser)).thenReturn(apiToken); + sut.authSvc = authStub; + } + + @Test + public void testEndToEnd_tamperedSignedUrl_userNotAuthenticated() { + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + String base = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC"; + String signedUrl = UrlSignerUtil.signUrl(base, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + // Alter the signed portion of the URL after signing -> the signature must no longer validate. + String tampered = signedUrl.replace("FK2/ABC", "FK2/HACKED"); + + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, tampered); + + assertThrows(WrappedUnauthorizedAuthErrorResponse.class, () -> sut.findUserFromRequest(request)); + } + + // Runs the real rdm flow: un-escape, sign, request the original (encoded) URL + signature, then the + // server URL-decodes the request and checks it. Returns true iff the user authenticates. + private boolean validatesEndToEndAsRdmClient(String urlAsClientBuilds) { + givenUserWithSigningKey(TEST_SIGNED_URL_TOKEN); + String canonical = URLDecoder.decode(urlAsClientBuilds, StandardCharsets.UTF_8); + String signed = UrlSignerUtil.signUrl(canonical, 1000, TEST_SIGNED_URL_USER_ID, "GET", TEST_SIGNED_URL_TOKEN); + String requestUri = urlAsClientBuilds + signed.substring(canonical.length()); + ContainerRequestContext request = new SignedUrlContainerRequestTestFake(TEST_SIGNED_URL_TOKEN, TEST_SIGNED_URL_USER_ID, requestUri); + try { + return testAuthenticatedUser.equals(sut.findUserFromRequest(request)); + } catch (WrappedAuthErrorResponse e) { + return false; + } + } + + @Test + public void testEndToEnd_allRdmIntegrationUrls_authenticate() { + final String s = "https://demo.dataverse.org"; + final String pid = "doi:10.5072/FK2/ABC"; // raw, as most rdm paths send it + final String escPid = "doi%3A10.5072%2FFK2%2FABC"; // url.QueryEscape form (GetDatasetMetadata, GetDatasetUserPermissions) + + // Every URL shape rdm-integration signs - each must authenticate end to end. + List urls = List.of( + // raw persistentId in the query (GetNodeMap, CheckPermission, globus, writes, dataverse plugin) + s + "/api/v1/datasets/:persistentId/versions/:latest/files?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId?persistentId=" + pid, + s + "/api/v1/admin/permissions/:persistentId?persistentId=" + pid + "&unblock-key=UNBLOCK", + s + "/api/v1/datasets/:persistentId/requestGlobusUploadPaths?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/addGlobusFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/requestGlobusDownload?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/monitorGlobusDownload?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/globusDownloadParameters?persistentId=" + pid + "&downloadId=globus-task-123", + s + "/api/v1/datasets/:persistentId/add?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/addFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/replaceFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/deleteFiles?persistentId=" + pid, + s + "/api/v1/datasets/:persistentId/cleanStorage?persistentId=" + pid, + // url-escaped persistentId (GetDatasetMetadata, GetDatasetUserPermissions) + s + "/api/v1/datasets/:persistentId?persistentId=" + escPid + "&excludeFiles=true", + s + "/api/v1/datasets/:persistentId/userPermissions?persistentId=" + escPid, + // mydata/retrieve: url-escaped search term, '+' for spaces, repeated query params + s + "/api/v1/mydata/retrieve?selected_page=1&dvobject_types=Dataset" + + "&published_states=Published&published_states=Unpublished&published_states=Draft" + + "&role_ids=1&role_ids=6&mydata_search_term=text%3A%22hello+world%22", + // numeric-id / no-persistentId paths + s + "/api/v1/access/datafile/123/metadata/ddi", + s + "/api/v1/access/datafile/123", + s + "/api/v1/files/123", + s + "/api/v1/users/:me", + s + "/api/v1/datasets/42/versions/:latest?excludeFiles=true" + ); + for (String url : urls) { + assertTrue(validatesEndToEndAsRdmClient(url), "signed URL must authenticate end to end: " + url); + } + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java index df37f6723d3..6a1a440f713 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlContainerRequestTestFake.java @@ -6,7 +6,11 @@ public class SignedUrlContainerRequestTestFake extends ContainerRequestTestFake private final UriInfo uriInfo; public SignedUrlContainerRequestTestFake(String signedUrlToken, String signedUrlUserId) { - this.uriInfo = new SignedUrlUriInfoTestFake(signedUrlToken, signedUrlUserId); + this(signedUrlToken, signedUrlUserId, null); + } + + public SignedUrlContainerRequestTestFake(String signedUrlToken, String signedUrlUserId, String requestUriOverride) { + this.uriInfo = new SignedUrlUriInfoTestFake(signedUrlToken, signedUrlUserId, requestUriOverride); } @Override diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java index fa9da7fc8de..5edd31509f8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/doubles/SignedUrlUriInfoTestFake.java @@ -15,18 +15,29 @@ public class SignedUrlUriInfoTestFake extends UriInfoTestFake { private final String signedUrlToken; private final String signedUrlUserId; + private final String requestUriOverride; private static final String SIGNED_URL_BASE_URL = "http://localhost:8080/api/test1"; private static final Integer SIGNED_URL_TIMEOUT = 1000; public SignedUrlUriInfoTestFake(String signedUrlToken, String signedUrlUserId) { + this(signedUrlToken, signedUrlUserId, null); + } + + // Lets a test supply the exact request URI the server would see, to exercise the real + // URLDecoder.decode + isValidUrl validation path. + public SignedUrlUriInfoTestFake(String signedUrlToken, String signedUrlUserId, String requestUriOverride) { this.signedUrlToken = signedUrlToken; this.signedUrlUserId = signedUrlUserId; + this.requestUriOverride = requestUriOverride; } @Override public URI getRequestUri() { + if (requestUriOverride != null) { + return URI.create(requestUriOverride); + } return URI.create(UrlSignerUtil.signUrl(SIGNED_URL_BASE_URL, SIGNED_URL_TIMEOUT, signedUrlUserId, GET, signedUrlToken)); } diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index 639a7c542c4..6ed90780554 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -213,6 +213,7 @@ public void testGetToolUrlWithOptionalQueryParameters() { @Test @JvmSetting(key = JvmSettings.SITE_URL, value = "https://librascholar.org") + @JvmSetting(key = JvmSettings.API_SIGNING_SECRET, value = "test-only-signing-secret") public void testGetToolUrlWithAllowedApiCalls() { System.out.println("allowedApiCalls test"); Dataset ds = new Dataset(); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java index d92f8822e59..94b76a9678b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java @@ -8,6 +8,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -86,4 +87,68 @@ public void testSignAndValidateWithParams() { System.out.println(signedUrl3); assertTrue(signedUrl3.contains("&p2&")); // Show that this works with params that have no value } + + @Test + public void testStripReservedParametersPreservesSpecialCharacters() { + // The signature is a byte-exact MAC over the URL string, so removing the reserved signing + // params must not re-encode anything else: a DOI's ':' and '/' must survive unchanged. + String doiUrl = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&foo=bar"; + assertEquals(doiUrl, UrlSignerUtil.stripReservedParameters(doiUrl)); + + // Reserved params (here token/user/signed) are removed; everything else is left byte-for-byte. + String withReserved = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&token=spoofed&user=Mallory&signed=true&foo=bar"; + assertEquals("http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&foo=bar", + UrlSignerUtil.stripReservedParameters(withReserved)); + + // A URL with no query string is returned unchanged. + String noQuery = "http://localhost:8080/api/v1/datasets/:persistentId"; + assertEquals(noQuery, UrlSignerUtil.stripReservedParameters(noQuery)); + } + + @Test + public void testSignAndValidateSpecialCharacters() { + final int longTimeout = 1000; + final String user = "Alice"; + final String method = "GET"; + final String key = "abracadabara open sesame"; + + // DOIs (':' and '/'), pre-encoded values, spaces, unicode and embedded URLs must all sign + // byte-exact and be accepted by isValidUrl over those exact bytes (the signing primitive; + // end-to-end validation with the server's URLDecoder.decode is in SignedUrlAuthMechanismTest). + String[] baseUrls = new String[] { + "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123&foo=bar", + "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi%3A10.5072%2FFK2%2FABC123", + "http://localhost:8080/api/v1/search?q=hello%20world&persistentId=doi:10.1/2", + "http://localhost:8080/api/v1/search?q=hello world&pid=doi:10.1/2", + "http://localhost:8080/api/v1/search?q=café&name=測試", + "http://localhost:8080/api/v1/redirect?url=http%3A%2F%2Fexample.com%2Ff%3Fa%3D1%26b%3D2" + }; + for (String baseUrl : baseUrls) { + String signedUrl = UrlSignerUtil.signUrl(baseUrl, longTimeout, user, method, key); + // The base URL is preserved byte-for-byte in the signed URL (no re-encoding). + assertTrue(signedUrl.startsWith(baseUrl + "&"), + "base URL must be preserved byte-for-byte: " + signedUrl); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl, user, method, key), + "signed URL should validate when used verbatim: " + signedUrl); + } + } + + @Test + public void testSignedUrlIsByteExact() { + // Byte-exact contract: the signature is over the URL as provided, so a re-encoded variant + // must fail validation. This is the regression that URIBuilder normalization caused. + final int longTimeout = 1000; + final String user = "Alice"; + final String method = "GET"; + final String key = "abracadabara open sesame"; + + String baseUrl = "http://localhost:8080/api/v1/datasets/:persistentId?persistentId=doi:10.5072/FK2/ABC123"; + String signedUrl = UrlSignerUtil.signUrl(baseUrl, longTimeout, user, method, key); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl, user, method, key)); + + // Re-encoding ':' and '/' in the DOI changes the signed bytes, so it must be rejected. + String reEncoded = signedUrl.replace("doi:10.5072/FK2/ABC123", "doi%3A10.5072%2FFK2%2FABC123"); + assertFalse(UrlSignerUtil.isValidUrl(reEncoded, user, method, key), + "a re-encoded variant must not validate (byte-exact contract)"); + } }