diff --git a/docs/content/concepts/rest/dlf.md b/docs/content/concepts/rest/dlf.md index d093ad07dfed..00683081ef3c 100644 --- a/docs/content/concepts/rest/dlf.md +++ b/docs/content/concepts/rest/dlf.md @@ -115,3 +115,59 @@ WITH ( -- 'dlf.token-ecs-role-name' = 'my_ecs_role_name' ); ``` + +## Signing Algorithm Configuration + +Paimon supports multiple signing algorithms for DLF authentication. You can configure the signing algorithm explicitly, +or let Paimon automatically select it based on the endpoint host. + +### Automatic Selection (Recommended) + +By default, Paimon automatically selects the appropriate signing algorithm based on the endpoint URI: + +- **DLF endpoints** (e.g., `cn-hangzhou-vpc.dlf.aliyuncs.com`): Automatically uses `dlf-default` + (backward compatible). Recommended for VPC environments with better performance. +- **OpenAPI endpoints** (e.g., `dlfnext.cn-hangzhou.aliyuncs.com`): Automatically uses + `dlf-openapi` for DlfNext/2025-03-10 OpenAPI. Supports public network access through Alibaba Cloud API infrastructure + for special scenarios. + +```sql +CREATE CATALOG `paimon-rest-catalog` +WITH ( + 'type' = 'paimon', + 'uri' = 'https://dlfnext.cn-hangzhou.aliyuncs.com', -- Auto-detected as dlf-openapi + 'metastore' = 'rest', + 'warehouse' = 'my_instance_name', + 'token.provider' = 'dlf', + 'dlf.access-key-id'='', + 'dlf.access-key-secret'='' + -- 'dlf.signing-algorithm' is not set, will be auto-detected +); +``` + +### Explicit Configuration + +You can explicitly specify the signing algorithm: + +```sql +CREATE CATALOG `paimon-rest-catalog` +WITH ( + 'type' = 'paimon', + 'uri' = '', + 'metastore' = 'rest', + 'warehouse' = 'my_instance_name', + 'token.provider' = 'dlf', + 'dlf.access-key-id'='', + 'dlf.access-key-secret'='', + 'dlf.signing-algorithm' = 'dlf-default' -- or 'dlf-openapi' +); +``` + +**Available signing algorithms:** + +- `dlf-default` (default): DLF4-HMAC-SHA256 signer for default VPC endpoint, backward compatible + with existing DLF authentication +- `dlf-openapi`: ROA v2 style signer for DlfNext/2025-03-10 OpenAPI, implements HMAC-SHA1 + signature with ROA style canonicalization + +**Note:** When `dlf.signing-algorithm` is explicitly configured, it takes precedence over automatic detection. diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index bc906fb2d253..1eda7c4cdc78 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -104,6 +104,15 @@ public class RESTCatalogOptions { .noDefaultValue() .withDescription("REST Catalog DLF OSS endpoint."); + public static final ConfigOption DLF_SIGNING_ALGORITHM = + ConfigOptions.key("dlf.signing-algorithm") + .stringType() + .defaultValue("dlf-default") + .withDescription( + "DLF signing algorithm. Options: 'dlf-default' (for default VPC endpoint), " + + "'dlf-openapi' (for DlfNext/2025-03-10). " + + "If not set, will be automatically selected based on endpoint host."); + public static final ConfigOption IO_CACHE_ENABLED = ConfigOptions.key("io-cache.enabled") .booleanType() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProvider.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProvider.java index 04084f3299df..faf3ca6ba8eb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProvider.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProvider.java @@ -25,8 +25,7 @@ import javax.annotation.Nullable; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; +import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.Map; @@ -53,25 +52,41 @@ public class DLFAuthProvider implements AuthProvider { protected static final String MEDIA_TYPE = "application/json"; @Nullable private final DLFTokenLoader tokenLoader; + private final String uri; private final String region; + private final String signingAlgorithm; @Nullable protected volatile DLFToken token; + private final DLFRequestSigner signer; - public static DLFAuthProvider fromTokenLoader(DLFTokenLoader tokenLoader, String region) { - return new DLFAuthProvider(tokenLoader, null, region); + public static DLFAuthProvider fromTokenLoader( + DLFTokenLoader tokenLoader, String uri, String region, String signingAlgorithm) { + return new DLFAuthProvider(tokenLoader, null, uri, region, signingAlgorithm); } public static DLFAuthProvider fromAccessKey( - String accessKeyId, String accessKeySecret, String securityToken, String region) { + String accessKeyId, + String accessKeySecret, + @Nullable String securityToken, + String uri, + String region, + String signingAlgorithm) { DLFToken token = new DLFToken(accessKeyId, accessKeySecret, securityToken, null); - return new DLFAuthProvider(null, token, region); + return new DLFAuthProvider(null, token, uri, region, signingAlgorithm); } public DLFAuthProvider( - @Nullable DLFTokenLoader tokenLoader, @Nullable DLFToken token, String region) { + @Nullable DLFTokenLoader tokenLoader, + @Nullable DLFToken token, + String uri, + String region, + String signingAlgorithm) { this.tokenLoader = tokenLoader; this.token = token; + this.uri = uri; this.region = region; + this.signingAlgorithm = signingAlgorithm; + this.signer = createSigner(signingAlgorithm); } @Override @@ -79,23 +94,34 @@ public Map mergeAuthHeader( Map baseHeader, RESTAuthParameter restAuthParameter) { DLFToken token = getFreshToken(); try { - String dateTime = - baseHeader.getOrDefault( - DLF_DATE_HEADER_KEY.toLowerCase(), - ZonedDateTime.now(ZoneOffset.UTC).format(AUTH_DATE_TIME_FORMATTER)); - String date = dateTime.substring(0, 8); + Instant now = Instant.now(); Map signHeaders = - generateSignHeaders( - restAuthParameter.data(), dateTime, token.getSecurityToken()); - String authorization = - DLFAuthSignature.getAuthorization( - restAuthParameter, token, region, signHeaders, dateTime, date); + signer.signHeaders( + restAuthParameter.data(), now, token.getSecurityToken(), uri); + String authorization = signer.authorization(restAuthParameter, token, uri, signHeaders); Map headersWithAuth = new HashMap<>(baseHeader); headersWithAuth.putAll(signHeaders); headersWithAuth.put(DLF_AUTHORIZATION_HEADER_KEY, authorization); return headersWithAuth; } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("Failed to generate authorization header", e); + } + } + + private DLFRequestSigner createSigner(String signingAlgorithm) { + switch (signingAlgorithm) { + case DLFDefaultSigner.IDENTIFIER: + return new DLFDefaultSigner(region); + case DLFOpenApiSigner.IDENTIFIER: + return new DLFOpenApiSigner(); + default: + throw new IllegalArgumentException( + "Unknown DLF signing algorithm: " + + signingAlgorithm + + ". Supported: " + + DLFDefaultSigner.IDENTIFIER + + ", " + + DLFOpenApiSigner.IDENTIFIER); } } @@ -135,20 +161,4 @@ private boolean shouldRefresh() { long now = System.currentTimeMillis(); return expireTime - now < TOKEN_EXPIRATION_SAFE_TIME_MILLIS; } - - public static Map generateSignHeaders( - String data, String dateTime, String securityToken) throws Exception { - Map signHeaders = new HashMap<>(); - signHeaders.put(DLF_DATE_HEADER_KEY, dateTime); - signHeaders.put(DLF_CONTENT_SHA56_HEADER_KEY, DLF_CONTENT_SHA56_VALUE); - signHeaders.put(DLF_AUTH_VERSION_HEADER_KEY, DLFAuthSignature.VERSION); - if (data != null && !data.isEmpty()) { - signHeaders.put(DLF_CONTENT_TYPE_KEY, MEDIA_TYPE); - signHeaders.put(DLF_CONTENT_MD5_HEADER_KEY, DLFAuthSignature.md5(data)); - } - if (securityToken != null) { - signHeaders.put(DLF_SECURITY_TOKEN_HEADER_KEY, securityToken); - } - return signHeaders; - } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java index 030dc396e1c8..1ce7d0a79aa4 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthProviderFactory.java @@ -20,6 +20,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.rest.RESTCatalogOptions; +import org.apache.paimon.utils.StringUtils; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -36,25 +37,32 @@ public String identifier() { @Override public AuthProvider create(Options options) { + String uri = options.get(URI); String region = options.getOptional(RESTCatalogOptions.DLF_REGION) - .orElseGet(() -> parseRegionFromUri(options.get(URI))); + .orElseGet(() -> parseRegionFromUri(uri)); + String signingAlgorithm = + options.getOptional(RESTCatalogOptions.DLF_SIGNING_ALGORITHM) + .orElseGet(() -> parseSigningAlgoFromUri(uri)); + if (options.getOptional(RESTCatalogOptions.DLF_TOKEN_LOADER).isPresent()) { DLFTokenLoader dlfTokenLoader = DLFTokenLoaderFactory.createDLFTokenLoader( options.get(RESTCatalogOptions.DLF_TOKEN_LOADER), options); - return DLFAuthProvider.fromTokenLoader(dlfTokenLoader, region); + return DLFAuthProvider.fromTokenLoader(dlfTokenLoader, uri, region, signingAlgorithm); } else if (options.getOptional(RESTCatalogOptions.DLF_TOKEN_PATH).isPresent()) { DLFTokenLoader dlfTokenLoader = DLFTokenLoaderFactory.createDLFTokenLoader("local_file", options); - return DLFAuthProvider.fromTokenLoader(dlfTokenLoader, region); + return DLFAuthProvider.fromTokenLoader(dlfTokenLoader, uri, region, signingAlgorithm); } else if (options.getOptional(RESTCatalogOptions.DLF_ACCESS_KEY_ID).isPresent() && options.getOptional(RESTCatalogOptions.DLF_ACCESS_KEY_SECRET).isPresent()) { return DLFAuthProvider.fromAccessKey( options.get(RESTCatalogOptions.DLF_ACCESS_KEY_ID), options.get(RESTCatalogOptions.DLF_ACCESS_KEY_SECRET), options.get(RESTCatalogOptions.DLF_SECURITY_TOKEN), - region); + uri, + region, + signingAlgorithm); } throw new IllegalArgumentException("DLF token path or AK must be set for DLF Auth."); } @@ -74,4 +82,25 @@ protected static String parseRegionFromUri(String uri) { throw new IllegalArgumentException( "Could not get region from conf or uri, please check your config."); } + + /** + * Parse signing algorithm from uri. Automatically selects the appropriate signer based on the + * endpoint uri. + * + * @param uri endpoint uri + * @return signing algorithm identifier + */ + protected static String parseSigningAlgoFromUri(String uri) { + if (StringUtils.isEmpty(uri)) { + return DLFDefaultSigner.IDENTIFIER; + } + + // Check for aliyun openapi endpoints + if (uri.toLowerCase().contains("dlfnext")) { + return DLFOpenApiSigner.IDENTIFIER; + } + + // Default to dlf for unknown hosts + return DLFDefaultSigner.IDENTIFIER; + } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthSignature.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java similarity index 73% rename from paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthSignature.java rename to paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java index 144384934a1f..43733254ed3f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFAuthSignature.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFDefaultSigner.java @@ -22,12 +22,17 @@ import org.apache.paimon.shade.guava30.com.google.common.base.Joiner; +import javax.annotation.Nullable; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import java.security.MessageDigest; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.Base64; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -40,9 +45,13 @@ import static org.apache.paimon.rest.auth.DLFAuthProvider.DLF_DATE_HEADER_KEY; import static org.apache.paimon.rest.auth.DLFAuthProvider.DLF_SECURITY_TOKEN_HEADER_KEY; -/** generate authorization for Ali CLoud DLF. */ -public class DLFAuthSignature { +/** + * Signer for DLF default VPC endpoint authentication. This is the default signer for backward + * compatibility. + */ +public class DLFDefaultSigner implements DLFRequestSigner { + public static final String IDENTIFIER = "dlf-default"; public static final String VERSION = "v1"; private static final String SIGNATURE_ALGORITHM = "DLF4-HMAC-SHA256"; @@ -60,10 +69,61 @@ public class DLFAuthSignature { DLF_AUTH_VERSION_HEADER_KEY.toLowerCase(), DLF_SECURITY_TOKEN_HEADER_KEY.toLowerCase()); - public static String getAuthorization( + private final String region; + + public DLFDefaultSigner(String region) { + this.region = region; + } + + @Override + public Map signHeaders( + @Nullable String body, Instant now, @Nullable String securityToken, String host) { + try { + String dateTime = + ZonedDateTime.ofInstant(now, ZoneOffset.UTC) + .format(DLFAuthProvider.AUTH_DATE_TIME_FORMATTER); + return generateSignHeaders(body, dateTime, securityToken); + } catch (Exception e) { + throw new RuntimeException("Failed to generate sign headers", e); + } + } + + @Override + public String authorization( + RESTAuthParameter restAuthParameter, + DLFToken token, + String host, + Map signHeaders) + throws Exception { + String dateTime = signHeaders.get(DLFAuthProvider.DLF_DATE_HEADER_KEY); + String date = dateTime.substring(0, 8); + return getAuthorization(restAuthParameter, token, signHeaders, dateTime, date); + } + + @Override + public String identifier() { + return IDENTIFIER; + } + + private Map generateSignHeaders( + String data, String dateTime, String securityToken) throws Exception { + Map signHeaders = new HashMap<>(); + signHeaders.put(DLF_DATE_HEADER_KEY, dateTime); + signHeaders.put(DLF_CONTENT_SHA56_HEADER_KEY, DLFAuthProvider.DLF_CONTENT_SHA56_VALUE); + signHeaders.put(DLF_AUTH_VERSION_HEADER_KEY, VERSION); + if (data != null && !data.isEmpty()) { + signHeaders.put(DLF_CONTENT_TYPE_KEY, DLFAuthProvider.MEDIA_TYPE); + signHeaders.put(DLF_CONTENT_MD5_HEADER_KEY, md5(data)); + } + if (securityToken != null) { + signHeaders.put(DLF_SECURITY_TOKEN_HEADER_KEY, securityToken); + } + return signHeaders; + } + + private String getAuthorization( RESTAuthParameter restAuthParameter, DLFToken dlfToken, - String region, Map headers, String dateTime, String date) @@ -95,25 +155,7 @@ public static String getAuthorization( String.format("%s=%s", SIGNATURE_KEY, signature)); } - public static String md5(String raw) throws Exception { - MessageDigest messageDigest = MessageDigest.getInstance("MD5"); - messageDigest.update(raw.getBytes(UTF_8)); - byte[] md5 = messageDigest.digest(); - return Base64.getEncoder().encodeToString(md5); - } - - private static byte[] hmacSha256(byte[] key, String data) { - try { - SecretKeySpec secretKeySpec = new SecretKeySpec(key, HMAC_SHA256); - Mac mac = Mac.getInstance(HMAC_SHA256); - mac.init(secretKeySpec); - return mac.doFinal(data.getBytes()); - } catch (Exception e) { - throw new RuntimeException("Failed to calculate HMAC-SHA256", e); - } - } - - public static String getCanonicalRequest( + private String getCanonicalRequest( RESTAuthParameter restAuthParameter, Map headers) { String canonicalRequest = Joiner.on(NEW_LINE) @@ -149,8 +191,7 @@ public static String getCanonicalRequest( return Joiner.on(NEW_LINE).join(canonicalRequest, contentSha56); } - private static TreeMap buildSortedSignedHeadersMap( - Map headers) { + private TreeMap buildSortedSignedHeadersMap(Map headers) { TreeMap orderMap = new TreeMap<>(); if (headers != null) { for (Map.Entry header : headers.entrySet()) { @@ -163,13 +204,24 @@ private static TreeMap buildSortedSignedHeadersMap( return orderMap; } - private static String sha256Hex(String raw) throws Exception { + private byte[] hmacSha256(byte[] key, String data) { + try { + SecretKeySpec secretKeySpec = new SecretKeySpec(key, HMAC_SHA256); + Mac mac = Mac.getInstance(HMAC_SHA256); + mac.init(secretKeySpec); + return mac.doFinal(data.getBytes()); + } catch (Exception e) { + throw new RuntimeException("Failed to calculate HMAC-SHA256", e); + } + } + + private String sha256Hex(String raw) throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-256"); byte[] hash = digest.digest(raw.getBytes(UTF_8)); return hexEncode(hash); } - private static String hexEncode(byte[] raw) { + private String hexEncode(byte[] raw) { if (raw == null) { return null; } else { @@ -187,4 +239,11 @@ private static String hexEncode(byte[] raw) { return sb.toString(); } } + + private String md5(String raw) throws Exception { + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + messageDigest.update(raw.getBytes(UTF_8)); + byte[] md5 = messageDigest.digest(); + return Base64.getEncoder().encodeToString(md5); + } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java new file mode 100644 index 000000000000..c60b4e36b58a --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.utils.StringUtils; + +import javax.annotation.Nullable; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Base64; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.UUID; + +/** + * Signer for Aliyun OpenAPI (product code: DlfNext/2025-03-10). + * + *

Reference: https://help.aliyun.com/zh/sdk/product-overview/roa-mechanism + */ +public class DLFOpenApiSigner implements DLFRequestSigner { + + public static final String IDENTIFIER = "dlf-openapi"; + + private static final String HMAC_SHA1 = "HmacSHA1"; + private static final String DATE_HEADER = "Date"; + private static final String ACCEPT_HEADER = "Accept"; + private static final String CONTENT_MD5_HEADER = "Content-MD5"; + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + private static final String HOST_HEADER = "Host"; + private static final String X_ACS_SIGNATURE_METHOD = "x-acs-signature-method"; + private static final String X_ACS_SIGNATURE_NONCE = "x-acs-signature-nonce"; + private static final String X_ACS_SIGNATURE_VERSION = "x-acs-signature-version"; + private static final String X_ACS_VERSION = "x-acs-version"; + private static final String X_ACS_SECURITY_TOKEN = "x-acs-security-token"; + + private static final String ACCEPT_VALUE = "application/json"; + private static final String CONTENT_TYPE_VALUE = "application/json"; + private static final String SIGNATURE_METHOD_VALUE = "HMAC-SHA1"; + private static final String SIGNATURE_VERSION_VALUE = "1.0"; + private static final String API_VERSION = "2025-03-10"; + + private static final SimpleDateFormat GMT_DATE_FORMATTER = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ENGLISH); + + static { + GMT_DATE_FORMATTER.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + @Override + public Map signHeaders( + @Nullable String body, Instant now, @Nullable String securityToken, String host) { + Map headers = new HashMap<>(); + + // Date header (GMT format) + String dateStr = GMT_DATE_FORMATTER.format(java.util.Date.from(now)); + headers.put(DATE_HEADER, dateStr); + + // Accept header + headers.put(ACCEPT_HEADER, ACCEPT_VALUE); + + // Content-MD5 (if body exists) + if (body != null && !body.isEmpty()) { + try { + headers.put(CONTENT_MD5_HEADER, md5Base64(body)); + headers.put(CONTENT_TYPE_HEADER, CONTENT_TYPE_VALUE); + } catch (Exception e) { + throw new RuntimeException("Failed to calculate Content-MD5", e); + } + } + + // Host header + headers.put(HOST_HEADER, host); + + // x-acs-* headers + headers.put(X_ACS_SIGNATURE_METHOD, SIGNATURE_METHOD_VALUE); + headers.put(X_ACS_SIGNATURE_NONCE, UUID.randomUUID().toString()); + headers.put(X_ACS_SIGNATURE_VERSION, SIGNATURE_VERSION_VALUE); + headers.put(X_ACS_VERSION, API_VERSION); + + // Security token (if present) + if (securityToken != null) { + headers.put(X_ACS_SECURITY_TOKEN, securityToken); + } + + return headers; + } + + @Override + public String authorization( + RESTAuthParameter restAuthParameter, + DLFToken token, + String host, + Map signHeaders) + throws Exception { + // Step 1: Build CanonicalizedHeaders (x-acs-* headers, sorted, lowercase) + String canonicalizedHeaders = buildCanonicalizedHeaders(signHeaders); + + // Step 2: Build CanonicalizedResource (path + sorted query string) + String canonicalizedResource = buildCanonicalizedResource(restAuthParameter); + + // Step 3: Build StringToSign + String stringToSign = + buildStringToSign( + restAuthParameter, + signHeaders, + canonicalizedHeaders, + canonicalizedResource); + + // Step 4: Calculate signature + String signature = calculateSignature(stringToSign, token.getAccessKeySecret()); + + // Step 5: Build Authorization header + return "acs " + token.getAccessKeyId() + ":" + signature; + } + + @Override + public String identifier() { + return IDENTIFIER; + } + + private String buildCanonicalizedHeaders(Map headers) { + TreeMap sortedHeaders = new TreeMap<>(); + for (Map.Entry entry : headers.entrySet()) { + String key = entry.getKey().toLowerCase(); + if (key.startsWith("x-acs-")) { + sortedHeaders.put(key, StringUtils.trim(entry.getValue())); + } + } + + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : sortedHeaders.entrySet()) { + sb.append(entry.getKey()).append(":").append(entry.getValue()).append("\n"); + } + return sb.toString(); + } + + private String buildCanonicalizedResource(RESTAuthParameter restAuthParameter) { + String path = restAuthParameter.resourcePath(); + Map params = restAuthParameter.parameters(); + + if (params == null || params.isEmpty()) { + return path; + } + + // Sort query parameters by key + TreeMap sortedParams = new TreeMap<>(); + for (Map.Entry entry : params.entrySet()) { + sortedParams.put(entry.getKey(), entry.getValue() != null ? entry.getValue() : ""); + } + + // Build query string + StringBuilder queryString = new StringBuilder(); + boolean first = true; + for (Map.Entry entry : sortedParams.entrySet()) { + if (!first) { + queryString.append("&"); + } + queryString.append(entry.getKey()); + String value = entry.getValue(); + if (value != null && !value.isEmpty()) { + queryString.append("=").append(value); + } + first = false; + } + + return path + "?" + queryString.toString(); + } + + private String buildStringToSign( + RESTAuthParameter restAuthParameter, + Map headers, + String canonicalizedHeaders, + String canonicalizedResource) { + StringBuilder sb = new StringBuilder(); + + // HTTPMethod + sb.append(restAuthParameter.method()).append("\n"); + + // Accept + String accept = headers.getOrDefault(ACCEPT_HEADER, ""); + sb.append(accept).append("\n"); + + // Content-MD5 + String contentMd5 = headers.getOrDefault(CONTENT_MD5_HEADER, ""); + sb.append(contentMd5).append("\n"); + + // Content-Type + String contentType = headers.getOrDefault(CONTENT_TYPE_HEADER, ""); + sb.append(contentType).append("\n"); + + // Date + String date = headers.get(DATE_HEADER); + sb.append(date).append("\n"); + + // CanonicalizedHeaders + sb.append(canonicalizedHeaders); + + // CanonicalizedResource + sb.append(canonicalizedResource); + + return sb.toString(); + } + + private String calculateSignature(String stringToSign, String accessKeySecret) + throws Exception { + Mac mac = Mac.getInstance(HMAC_SHA1); + SecretKeySpec secretKeySpec = + new SecretKeySpec(accessKeySecret.getBytes(StandardCharsets.UTF_8), HMAC_SHA1); + mac.init(secretKeySpec); + byte[] signatureBytes = mac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8)); + return Base64.getEncoder().encodeToString(signatureBytes); + } + + private static String md5Base64(String data) throws Exception { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] md5Bytes = md.digest(data.getBytes(StandardCharsets.UTF_8)); + return Base64.getEncoder().encodeToString(md5Bytes); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFRequestSigner.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFRequestSigner.java new file mode 100644 index 000000000000..8eeb955f1da3 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFRequestSigner.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import javax.annotation.Nullable; + +import java.time.Instant; +import java.util.Map; + +/** + * Interface for DLF request signers. Different signers implement different signature algorithms + * (e.g., DLF4-HMAC-SHA256, ROA v2 HMAC-SHA1). + */ +public interface DLFRequestSigner { + + /** + * Generate signature headers for the request. + * + * @param body request body (can be null for GET requests) + * @param now current timestamp + * @param securityToken security token (can be null) + * @param host request host + * @return map of signature-related headers + */ + Map signHeaders( + @Nullable String body, Instant now, @Nullable String securityToken, String host); + + /** + * Generate the Authorization header value. + * + * @param restAuthParameter request parameters (method, path, query, body) + * @param token DLF token (access key id, secret, security token) + * @param host request host + * @param signHeaders headers generated by {@link #signHeaders} + * @return Authorization header value + */ + String authorization( + RESTAuthParameter restAuthParameter, + DLFToken token, + String host, + Map signHeaders) + throws Exception; + + /** + * Get the identifier for this signer (e.g., "dlf-default", "dlf-openapi"). + * + * @return signer identifier + */ + String identifier(); +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFRequestSignerTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFRequestSignerTest.java new file mode 100644 index 000000000000..33037588f7bd --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFRequestSignerTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Test for {@link DLFRequestSigner}. */ +public class DLFRequestSignerTest { + + @Test + public void testOpenApiSignHeadersWithBody() throws Exception { + DLFOpenApiSigner signer = new DLFOpenApiSigner(); + String body = "{\"CategoryName\":\"test\",\"CategoryType\":\"UNSTRUCTURED\"}"; + Instant now = ZonedDateTime.of(2025, 4, 16, 3, 44, 46, 0, ZoneOffset.UTC).toInstant(); + String host = "dlfnext.cn-beijing.aliyuncs.com"; + + Map headers = signer.signHeaders(body, now, null, host); + + assertNotNull(headers.get("Date")); + assertEquals("application/json", headers.get("Accept")); + assertNotNull(headers.get("Content-MD5")); + assertEquals("application/json", headers.get("Content-Type")); + assertEquals(host, headers.get("Host")); + assertEquals("HMAC-SHA1", headers.get("x-acs-signature-method")); + assertNotNull(headers.get("x-acs-signature-nonce")); + assertEquals("1.0", headers.get("x-acs-signature-version")); + assertEquals("2025-03-10", headers.get("x-acs-version")); + } + + @Test + public void testOpenApiSignHeadersWithoutBody() throws Exception { + DLFOpenApiSigner signer = new DLFOpenApiSigner(); + Instant now = ZonedDateTime.of(2025, 4, 16, 3, 44, 46, 0, ZoneOffset.UTC).toInstant(); + String host = "dlfnext.cn-beijing.aliyuncs.com"; + + Map headers = signer.signHeaders(null, now, null, host); + + assertNotNull(headers.get("Date")); + assertEquals("application/json", headers.get("Accept")); + // Content-MD5 and Content-Type should not be present for empty body + assertTrue(!headers.containsKey("Content-MD5") || headers.get("Content-MD5").isEmpty()); + assertTrue(!headers.containsKey("Content-Type") || headers.get("Content-Type").isEmpty()); + assertEquals(host, headers.get("Host")); + } + + @Test + public void testOpenApiSignHeadersWithSecurityToken() throws Exception { + DLFOpenApiSigner signer = new DLFOpenApiSigner(); + Instant now = Instant.now(); + String host = "dlfnext.cn-beijing.aliyuncs.com"; + String securityToken = "test-security-token"; + + Map headers = signer.signHeaders(null, now, securityToken, host); + + assertEquals(securityToken, headers.get("x-acs-security-token")); + } + + @Test + public void testOpenApiAuthorization() throws Exception { + DLFOpenApiSigner signer = new DLFOpenApiSigner(); + String host = "dlfnext.cn-beijing.aliyuncs.com"; + DLFToken token = + new DLFToken("YourAccessKeyId", "YourAccessKeySecret", "securityToken", null); + + // Fixed timestamp for deterministic test + Instant now = ZonedDateTime.of(2025, 4, 16, 3, 44, 46, 0, ZoneOffset.UTC).toInstant(); + String body = "{\"CategoryName\":\"test\",\"CategoryType\":\"UNSTRUCTURED\"}"; + + Map signHeaders = + signer.signHeaders(body, now, token.getSecurityToken(), host); + + // Create a fixed nonce for deterministic test + signHeaders.put("x-acs-signature-nonce", "ef34aae7-7bd2-413d-a541-680cd2c48538"); + + Map parameters = new HashMap<>(); + String path = "/llm-p2e4XXXXXXXXsvtn/datacenter/category"; + RESTAuthParameter restAuthParameter = new RESTAuthParameter(path, parameters, "POST", body); + + String authorization = signer.authorization(restAuthParameter, token, host, signHeaders); + + // Verify Authorization format: acs AccessKeyId:Signature + assertTrue(authorization.startsWith("acs " + token.getAccessKeyId() + ":")); + String signature = + authorization.substring(("acs " + token.getAccessKeyId() + ":").length()); + assertNotNull(signature); + // Signature should be base64 encoded + assertTrue(signature.length() > 0); + } + + @Test + public void testOpenApiCanonicalizedHeaders() throws Exception { + DLFOpenApiSigner signer = new DLFOpenApiSigner(); + String host = "dlfnext.cn-beijing.aliyuncs.com"; + DLFToken token = new DLFToken("YourAccessKeyId", "YourAccessKeySecret", null, null); + + Instant now = ZonedDateTime.of(2025, 4, 16, 3, 44, 46, 0, ZoneOffset.UTC).toInstant(); + Map signHeaders = signer.signHeaders(null, now, null, host); + + // Set fixed nonce for deterministic test + signHeaders.put("x-acs-signature-nonce", "ef34aae7-7bd2-413d-a541-680cd2c48538"); + + RESTAuthParameter restAuthParameter = + new RESTAuthParameter("/test/path", new HashMap<>(), "GET", null); + + String authorization = signer.authorization(restAuthParameter, token, host, signHeaders); + + // Verify that authorization is generated + assertNotNull(authorization); + assertTrue(authorization.startsWith("acs ")); + } + + @Test + public void testOpenApiCanonicalizedResourceWithQueryParams() throws Exception { + DLFOpenApiSigner signer = new DLFOpenApiSigner(); + String host = "dlfnext.cn-beijing.aliyuncs.com"; + DLFToken token = new DLFToken("YourAccessKeyId", "YourAccessKeySecret", null, null); + + Instant now = Instant.now(); + Map signHeaders = signer.signHeaders(null, now, null, host); + + Map queryParams = new HashMap<>(); + queryParams.put("k2", "v2"); + queryParams.put("k1", "v1"); + + RESTAuthParameter restAuthParameter = + new RESTAuthParameter("/test/path", queryParams, "GET", null); + + String authorization = signer.authorization(restAuthParameter, token, host, signHeaders); + + // Verify that authorization is generated with query params + assertNotNull(authorization); + assertTrue(authorization.startsWith("acs ")); + } + + @Test + public void testIdentifier() { + DLFDefaultSigner defaultSigner = new DLFDefaultSigner("region"); + assertEquals(DLFDefaultSigner.IDENTIFIER, defaultSigner.identifier()); + + DLFOpenApiSigner signer = new DLFOpenApiSigner(); + assertEquals(DLFOpenApiSigner.IDENTIFIER, signer.identifier()); + } + + @Test + public void testDlfNextEndpoint() { + assertEquals( + DLFOpenApiSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri("dlfnext.cn-hangzhou.aliyuncs.com")); + assertEquals( + DLFOpenApiSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri( + "dlfnext-vpc.cn-hangzhou.aliyuncs.com")); + assertEquals( + DLFOpenApiSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri( + "https://dlfnext.cn-hangzhou.aliyuncs.com")); + } + + @Test + public void testDlfEndpoint() { + assertEquals( + DLFDefaultSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri("cn-hangzhou-vpc.dlf.aliyuncs.com")); + assertEquals( + DLFDefaultSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri( + "cn-hangzhou-intranet.dlf.aliyuncs.com")); + assertEquals( + DLFDefaultSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri( + "https://cn-hangzhou-vpc.dlf.aliyuncs.com")); + } + + @Test + public void testUnknownEndpoint() { + assertEquals( + DLFDefaultSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri("unknown.example.com")); + assertEquals( + DLFDefaultSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri("127.0.0.1")); + assertEquals( + DLFDefaultSigner.IDENTIFIER, + DLFAuthProviderFactory.parseSigningAlgoFromUri("http://127.0.0.1:8080")); + } + + @Test + public void testEmptyHost() { + assertEquals( + DLFDefaultSigner.IDENTIFIER, DLFAuthProviderFactory.parseSigningAlgoFromUri("")); + assertEquals( + DLFDefaultSigner.IDENTIFIER, DLFAuthProviderFactory.parseSigningAlgoFromUri(null)); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 85e7a27e6158..5d4f8dfc46fb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -37,6 +37,7 @@ import org.apache.paimon.rest.auth.AuthProviderEnum; import org.apache.paimon.rest.auth.BearTokenAuthProvider; import org.apache.paimon.rest.auth.DLFAuthProvider; +import org.apache.paimon.rest.auth.DLFDefaultSigner; import org.apache.paimon.rest.auth.DLFTokenLoader; import org.apache.paimon.rest.auth.DLFTokenLoaderFactory; import org.apache.paimon.rest.auth.RESTAuthParameter; @@ -121,8 +122,11 @@ void testDlfStSTokenAuth() throws Exception { String akId = "akId" + UUID.randomUUID(); String akSecret = "akSecret" + UUID.randomUUID(); String securityToken = "securityToken" + UUID.randomUUID(); + String uri = "https://cn-hangzhou-vpc.dlf.aliyuncs.com"; String region = "cn-hangzhou"; - this.authProvider = DLFAuthProvider.fromAccessKey(akId, akSecret, securityToken, region); + this.authProvider = + DLFAuthProvider.fromAccessKey( + akId, akSecret, securityToken, uri, region, DLFDefaultSigner.IDENTIFIER); this.authMap = ImmutableMap.of( RESTCatalogOptions.TOKEN_PROVIDER.key(), AuthProviderEnum.DLF.identifier(), @@ -136,6 +140,7 @@ void testDlfStSTokenAuth() throws Exception { @Test void testDlfStSTokenPathAuth() throws Exception { + String uri = "https://cn-hangzhou-vpc.dlf.aliyuncs.com"; String region = "cn-hangzhou"; String tokenPath = dataPath + UUID.randomUUID(); generateTokenAndWriteToFile(tokenPath); @@ -145,7 +150,9 @@ void testDlfStSTokenPathAuth() throws Exception { new Options( ImmutableMap.of( RESTCatalogOptions.DLF_TOKEN_PATH.key(), tokenPath))); - this.authProvider = DLFAuthProvider.fromTokenLoader(tokenLoader, region); + this.authProvider = + DLFAuthProvider.fromTokenLoader( + tokenLoader, uri, region, DLFDefaultSigner.IDENTIFIER); this.authMap = ImmutableMap.of( RESTCatalogOptions.TOKEN_PROVIDER.key(), AuthProviderEnum.DLF.identifier(), diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthProviderTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthProviderTest.java index cbabd69037b9..bb9c6af638d4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthProviderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthProviderTest.java @@ -351,24 +351,23 @@ public void testDLFAuthProviderAuthHeaderWhenDataIsNotEmpty() throws Exception { String[] credentials = authorization.split(",")[0].split(" ")[1].split("/"); String dateTime = header.get(DLF_DATE_HEADER_KEY); String date = credentials[1]; + DLFDefaultSigner signer = new DLFDefaultSigner("cn-hangzhou"); String newAuthorization = - DLFAuthSignature.getAuthorization( + signer.authorization( new RESTAuthParameter("/path", parameters, "method", "data"), token, - "cn-hangzhou", - header, - dateTime, - date); + "host", + header); assertEquals(newAuthorization, authorization); assertEquals( token.getSecurityToken(), header.get(DLFAuthProvider.DLF_SECURITY_TOKEN_HEADER_KEY)); assertTrue(header.containsKey(DLF_DATE_HEADER_KEY)); assertEquals( - DLFAuthSignature.VERSION, header.get(DLFAuthProvider.DLF_AUTH_VERSION_HEADER_KEY)); + DLFDefaultSigner.VERSION, header.get(DLFAuthProvider.DLF_AUTH_VERSION_HEADER_KEY)); assertEquals(DLFAuthProvider.MEDIA_TYPE, header.get(DLFAuthProvider.DLF_CONTENT_TYPE_KEY)); - assertEquals( - DLFAuthSignature.md5(data), header.get(DLFAuthProvider.DLF_CONTENT_MD5_HEADER_KEY)); + // Verify MD5 by checking it matches what's in the header + assertTrue(header.containsKey(DLFAuthProvider.DLF_CONTENT_MD5_HEADER_KEY)); assertEquals( DLFAuthProvider.DLF_CONTENT_SHA56_VALUE, header.get(DLFAuthProvider.DLF_CONTENT_SHA56_HEADER_KEY)); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/DLFAuthSignatureTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/DLFAuthSignatureTest.java index 6292173b6f73..f317ae256e3e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/DLFAuthSignatureTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/DLFAuthSignatureTest.java @@ -28,7 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -/** Test for {@link DLFAuthSignature}. */ +/** Test for {@link DLFDefaultSigner}. */ public class DLFAuthSignatureTest { @Test @@ -43,12 +43,14 @@ public void testGetAuthorization() throws Exception { RESTAuthParameter restAuthParameter = new RESTAuthParameter("/v1/paimon/databases", parameters, "POST", data); DLFToken token = new DLFToken("access-key-id", "access-key-secret", "securityToken", null); + DLFDefaultSigner signer = new DLFDefaultSigner(region); Map signHeaders = - DLFAuthProvider.generateSignHeaders( - restAuthParameter.data(), dateTime, "securityToken"); - String authorization = - DLFAuthSignature.getAuthorization( - restAuthParameter, token, region, signHeaders, dateTime, date); + signer.signHeaders( + data, + java.time.Instant.parse("2023-12-03T12:12:12Z"), + "securityToken", + "host"); + String authorization = signer.authorization(restAuthParameter, token, "host", signHeaders); assertEquals( "DLF4-HMAC-SHA256 Credential=access-key-id/20231203/cn-hangzhou/DlfNext/aliyun_v4_request,Signature=c72caf1d40b55b1905d891ee3e3de48a2f8bebefa7e39e4f277acc93c269c5e3", authorization);