diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 91e68d7921a4..714a4eb612db 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -983,8 +983,8 @@ public enum ConfVars { " NOSASL: Raw transport" + " JWT: JSON Web Token authentication via JWT token. Only supported in Http/Https mode"), THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL("metastore.authentication.jwt.jwks.url", - "hive.metastore.authentication.jwt.jwks.url", "", "File URL from where URLBasedJWKSProvider " - + "in metastore server will try to load JWKS to match a JWT sent in HTTP request header. Used only when " + "hive.metastore.authentication.jwt.jwks.url", "", "File URL from where " + + "metastore server will try to load JWKS to match a JWT sent in HTTP request header. Used only when " + "Hive metastore server is running in JWT auth mode"), METASTORE_CUSTOM_AUTHENTICATION_CLASS("metastore.custom.authentication.class", "hive.metastore.custom.authentication.class", @@ -1873,8 +1873,56 @@ public enum ConfVars { " positive value will be used as-is." ), CATALOG_SERVLET_AUTH("metastore.catalog.servlet.auth", - "hive.metastore.catalog.servlet.auth", "jwt", new StringSetValidator("none", "simple", "jwt"), - "HMS Catalog servlet authentication method (none, simple, or jwt)." + "hive.metastore.catalog.servlet.auth", "jwt", new StringSetValidator("none", "simple", "jwt", "oauth2"), + "HMS Catalog servlet authentication method (none, simple, jwt, or oauth2)." + ), + CATALOG_SERVLET_AUTH_OAUTH2_ISSUER("metastore.catalog.servlet.auth.oauth2.issuer", + "hive.metastore.catalog.servlet.auth.oauth2.issuer", "", + "The authorization server's identifier, which is a URL. This is required when you use " + + "metastore.catalog.servlet.auth=oauth2" + ), + CATALOG_SERVLET_AUTH_OAUTH2_AUDIENCE("metastore.catalog.servlet.auth.oauth2.audience", + "hive.metastore.catalog.servlet.auth.oauth2.audience", "", + "The acceptable name in the audience(aud) claim. This is required when you use " + + "metastore.catalog.servlet.auth=oauth2" + ), + CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD("metastore.catalog.servlet.auth.oauth2.validation.method", + "hive.metastore.catalog.servlet.auth.oauth2.validation.method", "jwt", + new StringSetValidator("jwt", "introspection"), + "How to evaluate an access token. When your authorization server issues opaque tokens or you need " + + "to consider additional security requirements such as token revocations, use introspection." + ), + CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_ID("metastore.catalog.servlet.auth.oauth2.client.id", + "hive.metastore.catalog.servlet.auth.oauth2.client.id", "", + "The client ID to authenticate HMS, as a resource server, to the introspection endpoint. This is required to " + + "use metastore.catalog.servlet.auth.oauth2.validation.method=introspection." + ), + CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_SECRET("metastore.catalog.servlet.auth.oauth2.client.secret", + "hive.metastore.catalog.servlet.auth.oauth2.client.secret", "", + "The client secret to authenticate HMS, as a resource server, to the introspection endpoint. This is " + + "required to use metastore.catalog.servlet.auth.oauth2.validation.method=introspection." + ), + CATALOG_SERVLET_AUTH_OAUTH2_INTROSPECTION_CACHE_EXPIRY( + "metastore.catalog.servlet.auth.oauth2.introspection.cache.expiry", + "hive.metastore.catalog.servlet.auth.oauth2.introspection.cache.expiry", 60, TimeUnit.SECONDS, + "The expiry time of the token introspection cache. Set to 0 to disable caching." + ), + CATALOG_SERVLET_AUTH_OAUTH2_INTROSPECTION_CACHE_SIZE( + "metastore.catalog.servlet.auth.oauth2.introspection.cache.num", + "hive.metastore.catalog.servlet.auth.oauth2.introspection.cache.num", 1000L, + "The number of entries of the token introspection cache." + ), + CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_FIELD( + "metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.field", + "hive.metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.field", "sub", + "The claim name including a username. This is effective when you use RegexPrincipalMapper. For example, if " + + "you want to resolve a user name from the email claim, set this to email." + ), + CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_PATTERN( + "metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.pattern", + "hive.metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.pattern", "(.*)", + "The pattern to extract a user name. This is effective when you use RegexPrincipalMapper. For example, if " + + "you want to extract a user name from the local part of the email claim, set this to (.*)@example.com." ), ICEBERG_CATALOG_SERVLET_PATH("metastore.iceberg.catalog.servlet.path", "hive.metastore.iceberg.catalog.servlet.path", "iceberg", diff --git a/standalone-metastore/metastore-rest-catalog/pom.xml b/standalone-metastore/metastore-rest-catalog/pom.xml index edf41fdc17a7..dc63082c17ef 100644 --- a/standalone-metastore/metastore-rest-catalog/pom.xml +++ b/standalone-metastore/metastore-rest-catalog/pom.xml @@ -224,6 +224,16 @@ + + org.keycloak + keycloak-admin-client + test + + + org.testcontainers + testcontainers + test + diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java index c17ce318b9e7..7523ee55dca3 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java @@ -71,7 +71,6 @@ import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.iceberg.rest.responses.LoadViewResponse; -import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PropertyUtil; @@ -104,15 +103,6 @@ public class HMSCatalogAdapter implements RESTClient { .put(CommitStateUnknownException.class, 500) .buildOrThrow(); - private static final String URN_OAUTH_TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange"; - private static final String URN_OAUTH_ACCESS_TOKEN = "urn:ietf:params:oauth:token-type:access_token"; - private static final String GRANT_TYPE = "grant_type"; - private static final String CLIENT_CREDENTIALS = "client_credentials"; - private static final String BEARER = "Bearer"; - private static final String CLIENT_ID = "client_id"; - private static final String ACTOR_TOKEN = "actor_token"; - private static final String SUBJECT_TOKEN = "subject_token"; - private final Catalog catalog; private final SupportsNamespaces asNamespaceCatalog; private final ViewCatalog asViewCatalog; @@ -127,8 +117,6 @@ public HMSCatalogAdapter(Catalog catalog) { } enum Route { - TOKENS(HTTPMethod.POST, "v1/oauth/tokens", null), - SEPARATE_AUTH_TOKENS_URI(HTTPMethod.POST, "https://auth-server.com/token", null), CONFIG(HTTPMethod.GET, "v1/config", null), LIST_NAMESPACES(HTTPMethod.GET, ResourcePaths.V1_NAMESPACES, null), CREATE_NAMESPACE(HTTPMethod.POST, ResourcePaths.V1_NAMESPACES, CreateNamespaceRequest.class), @@ -240,35 +228,6 @@ private ConfigResponse config() { return castResponse(ConfigResponse.class, ConfigResponse.builder().withEndpoints(endpoints).build()); } - private OAuthTokenResponse tokens(Object body) { - @SuppressWarnings("unchecked") - Map request = (Map) castRequest(Map.class, body); - String grantType = request.get(GRANT_TYPE); - switch (grantType) { - case CLIENT_CREDENTIALS: - return OAuthTokenResponse.builder() - .withToken("client-credentials-token:sub=" + request.get(CLIENT_ID)) - .withIssuedTokenType(URN_OAUTH_ACCESS_TOKEN) - .withTokenType(BEARER) - .build(); - - case URN_OAUTH_TOKEN_EXCHANGE: - String actor = request.get(ACTOR_TOKEN); - String token = - String.format( - "token-exchange-token:sub=%s%s", - request.get(SUBJECT_TOKEN), actor != null ? ",act=" + actor : ""); - return OAuthTokenResponse.builder() - .withToken(token) - .withIssuedTokenType(URN_OAUTH_ACCESS_TOKEN) - .withTokenType(BEARER) - .build(); - - default: - throw new UnsupportedOperationException("Unsupported grant_type: " + grantType); - } - } - private ListNamespacesResponse listNamespaces(Map vars) { Namespace namespace; if (vars.containsKey("parent")) { @@ -469,9 +428,6 @@ private T handleRequest( counter.inc(); } switch (route) { - case TOKENS: - return (T) tokens(body); - case CONFIG: return (T) config(); diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java index 682e7c9e2649..4b085e9d34cf 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java @@ -18,6 +18,8 @@ */ package org.apache.iceberg.rest; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.TreeMap; import javax.servlet.http.HttpServlet; @@ -100,7 +102,9 @@ private Catalog createCatalog() { */ private HttpServlet createServlet(Catalog catalog) { String authType = MetastoreConf.getVar(configuration, ConfVars.CATALOG_SERVLET_AUTH); - ServletSecurity security = new ServletSecurity(AuthType.fromString(authType), configuration); + // Iceberg REST client uses "catalog" by default + List scopes = Collections.singletonList("catalog"); + ServletSecurity security = new ServletSecurity(AuthType.fromString(authType), configuration, req -> scopes); return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog))); } diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java index b391eb24f9e5..6140f40b2de5 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java @@ -20,8 +20,6 @@ package org.apache.iceberg.rest; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; import java.io.UncheckedIOException; import java.util.Map; import java.util.Optional; @@ -31,7 +29,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.io.CharStreams; import org.apache.iceberg.rest.HMSCatalogAdapter.Route; import org.apache.iceberg.rest.HTTPRequest.HTTPMethod; import org.apache.iceberg.rest.responses.ErrorResponse; @@ -152,10 +149,6 @@ static ServletRequestContext from(HttpServletRequest request) throws IOException if (route.requestClass() != null) { requestBody = RESTObjectMapper.mapper().readValue(request.getReader(), route.requestClass()); - } else if (route == Route.TOKENS) { - try (Reader reader = new InputStreamReader(request.getInputStream())) { - requestBody = RESTUtil.decodeFormData(CharStreams.toString(reader)); - } } Map queryParams = diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2Jwt.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2Jwt.java new file mode 100644 index 000000000000..c140bc0f44f2 --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2Jwt.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.rest; + +import java.util.Map; +import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; +import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +@Category(MetastoreCheckinTest.class) +class TestRESTCatalogOAuth2Jwt extends BaseRESTCatalogTests { + @RegisterExtension + private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION = + HiveRESTCatalogServerExtension.builder(AuthType.OAUTH2).build(); + + @Override + protected Map getDefaultClientConfiguration() { + return Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(), + "credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential() + ); + } + + @Test + void testWithAccessToken() { + Map properties = Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "token", REST_CATALOG_EXTENSION.getOAuth2AccessToken() + ); + Assertions.assertFalse(RCKUtils.initCatalogClient(properties).listNamespaces().isEmpty()); + } + + @Test + void testWithWrongCredential() { + Map properties = Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(), + "credential", "dummy:dummy" + ); + NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class, + () -> RCKUtils.initCatalogClient(properties)); + Assertions.assertEquals("Not authorized: invalid_client: Invalid client or Invalid client credentials", + error.getMessage()); + } + + @Test + void testWithWrongAccessToken() { + Map properties = Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "token", "invalid" + ); + NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class, + () -> RCKUtils.initCatalogClient(properties)); + Assertions.assertEquals("Not authorized: Authentication error: Invalid JWT serialization: Missing dot delimiter(s)", + error.getMessage()); + } +} diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2TokenIntrospection.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2TokenIntrospection.java new file mode 100644 index 000000000000..46a6e3649951 --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2TokenIntrospection.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.rest; + +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD; + +import java.util.Map; +import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; +import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +@Category(MetastoreCheckinTest.class) +class TestRESTCatalogOAuth2TokenIntrospection extends BaseRESTCatalogTests { + @RegisterExtension + private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION = + HiveRESTCatalogServerExtension.builder(AuthType.OAUTH2) + .configure(CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD.getVarname(), "introspection").build(); + + @Override + protected Map getDefaultClientConfiguration() { + return Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(), + "credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential() + ); + } + + @Test + void testWithAccessToken() { + Map properties = Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "token", REST_CATALOG_EXTENSION.getOAuth2AccessToken() + ); + Assertions.assertFalse(RCKUtils.initCatalogClient(properties).listNamespaces().isEmpty()); + } + + @Test + void testWithWrongCredential() { + Map properties = Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(), + "credential", "dummy:dummy" + ); + NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class, + () -> RCKUtils.initCatalogClient(properties)); + Assertions.assertEquals("Not authorized: invalid_client: Invalid client or Invalid client credentials", + error.getMessage()); + } + + @Test + void testWithWrongAccessToken() { + Map properties = Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "token", "invalid" + ); + NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class, + () -> RCKUtils.initCatalogClient(properties)); + Assertions.assertEquals("Not authorized: Authentication error: The token is not active", + error.getMessage()); + } +} diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogOAuth2Jwt.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogOAuth2Jwt.java new file mode 100644 index 000000000000..e5f794840291 --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogOAuth2Jwt.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.rest; + +import java.util.Map; +import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; +import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.extension.RegisterExtension; + +@Category(MetastoreCheckinTest.class) +class TestRESTViewCatalogOAuth2Jwt extends BaseRESTViewCatalogTests { + @RegisterExtension + private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION = + HiveRESTCatalogServerExtension.builder(AuthType.OAUTH2).build(); + + @Override + protected Map getDefaultClientConfiguration() throws Exception { + return Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(), + "credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential() + ); + } +} diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogOAuth2TokenIntrospection.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogOAuth2TokenIntrospection.java new file mode 100644 index 000000000000..b79851da43d4 --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogOAuth2TokenIntrospection.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.rest; + +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD; + +import java.util.Map; +import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; +import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.extension.RegisterExtension; + +@Category(MetastoreCheckinTest.class) +class TestRESTViewCatalogOAuth2TokenIntrospection extends BaseRESTViewCatalogTests { + @RegisterExtension + private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION = + HiveRESTCatalogServerExtension.builder(AuthType.OAUTH2) + .configure(CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD.getVarname(), "introspection").build(); + + @Override + protected Map getDefaultClientConfiguration() throws Exception { + return Map.of( + "uri", REST_CATALOG_EXTENSION.getRestEndpoint(), + "rest.auth.type", "oauth2", + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(), + "credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential() + ); + } +} diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/HiveRESTCatalogServerExtension.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/HiveRESTCatalogServerExtension.java index 5b80d2276671..38ee151b130c 100644 --- a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/HiveRESTCatalogServerExtension.java +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/HiveRESTCatalogServerExtension.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -41,9 +43,11 @@ public class HiveRESTCatalogServerExtension implements BeforeAllCallback, Before private final Configuration conf; private final JwksServer jwksServer; + private final OAuth2AuthorizationServer authorizationServer; private final RESTCatalogServer restCatalogServer; - private HiveRESTCatalogServerExtension(AuthType authType, Class schemaInfoClass) { + private HiveRESTCatalogServerExtension(AuthType authType, Class schemaInfoClass, + Map configurations) { this.conf = MetastoreConf.newMetastoreConf(); MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH, authType.name()); if (authType == AuthType.JWT) { @@ -54,12 +58,26 @@ private HiveRESTCatalogServerExtension(AuthType authType, Class metaStoreSchemaClass; + private final Map configurations = new HashMap<>(); private Builder(AuthType authType) { this.authType = authType; } - + public Builder addMetaStoreSchemaClassName(Class metaStoreSchemaClass) { this.metaStoreSchemaClass = metaStoreSchemaClass; return this; } + public Builder configure(String key, String value) { + configurations.put(key, value); + return this; + } + public HiveRESTCatalogServerExtension build() { - return new HiveRESTCatalogServerExtension(authType, metaStoreSchemaClass); + return new HiveRESTCatalogServerExtension(authType, metaStoreSchemaClass, configurations); } } diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/OAuth2AuthorizationServer.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/OAuth2AuthorizationServer.java new file mode 100644 index 000000000000..4f7731d2569d --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/OAuth2AuthorizationServer.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.rest.extension; + + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.keycloak.OAuth2Constants; +import org.keycloak.admin.client.Keycloak; +import org.keycloak.admin.client.KeycloakBuilder; +import org.keycloak.admin.client.resource.RealmResource; +import org.keycloak.representations.idm.ClientRepresentation; +import org.keycloak.representations.idm.ClientScopeRepresentation; +import org.keycloak.representations.idm.ProtocolMapperRepresentation; +import org.keycloak.representations.idm.RealmRepresentation; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +public class OAuth2AuthorizationServer { + private static final String REALM = "hive"; + static final String HMS_ID = "hive-metastore"; + static final String HMS_SECRET = "hive-metastore-secret"; + private static final String ICEBERG_CLIENT_ID = "iceberg-client"; + private static final String ICEBERG_CLIENT_SECRET = "iceberg-client-secret"; + + private GenericContainer container; + private Keycloak keycloak; + private String issuer; + private String tokenEndpoint; + private String accessToken; + + private static RealmResource createRealm(Keycloak keycloak) { + var realm = new RealmRepresentation(); + realm.setRealm(REALM); + realm.setEnabled(true); + keycloak.realms().create(realm); + return keycloak.realm(REALM); + } + + private static void createResourceServer(RealmResource realm) { + var resourceServer = new ClientRepresentation(); + resourceServer.setClientId(HMS_ID); + resourceServer.setSecret(HMS_SECRET); + resourceServer.setEnabled(true); + resourceServer.setProtocol("openid-connect"); + resourceServer.setPublicClient(false); + resourceServer.setServiceAccountsEnabled(true); + resourceServer.setAuthorizationServicesEnabled(true); + realm.clients().create(resourceServer).close(); + } + + private static void createScope(RealmResource realm) { + var scope = new ClientScopeRepresentation(); + scope.setName("catalog"); + scope.setProtocol("openid-connect"); + realm.clientScopes().create(scope).close(); + } + + private static ProtocolMapperRepresentation createAudience() { + var aud = new ProtocolMapperRepresentation(); + aud.setName("audience"); + aud.setProtocol("openid-connect"); + aud.setProtocolMapper("oidc-audience-mapper"); + aud.setConfig(Map.of( + "included.custom.audience", HMS_ID, + "access.token.claim", "true" + )); + return aud; + } + + private static ProtocolMapperRepresentation createEmailClaim() { + var mapper = new ProtocolMapperRepresentation(); + mapper.setName("email"); + mapper.setProtocol("openid-connect"); + mapper.setProtocolMapper("oidc-hardcoded-claim-mapper"); + mapper.setConfig(Map.of( + "claim.name", "email", + "claim.value", "iceberg-user@example.com", + "jsonType.label", "String", + "access.token.claim", "true" + )); + return mapper; + } + + private static void createClient(RealmResource realm, List scopes, + List protocolMappers) { + var client = new ClientRepresentation(); + client.setClientId(ICEBERG_CLIENT_ID); + client.setSecret(ICEBERG_CLIENT_SECRET); + client.setEnabled(true); + client.setProtocol("openid-connect"); + client.setPublicClient(false); + client.setServiceAccountsEnabled(true); + client.setOptionalClientScopes(scopes); + client.setAttributes(Collections.singletonMap("access.token.header.type.rfc9068", "true")); + client.setProtocolMappers(protocolMappers); + realm.clients().create(client).close(); + } + + private static String getAccessToken(String url, List scopes) { + try (var keycloak = KeycloakBuilder.builder() + .serverUrl(url) + .realm(REALM) + .clientId(ICEBERG_CLIENT_ID) + .clientSecret(ICEBERG_CLIENT_SECRET) + .scope(String.join(" ", scopes)) + .grantType(OAuth2Constants.CLIENT_CREDENTIALS) + .build()) { + return keycloak.tokenManager().getAccessTokenString(); + } + } + + void start() { + container = new GenericContainer<>(DockerImageName.parse("quay.io/keycloak/keycloak:26.3.4")) + .withEnv("KEYCLOAK_ADMIN", "admin") + .withEnv("KEYCLOAK_ADMIN_PASSWORD", "admin") + .withCommand("start-dev") + .withExposedPorts(8080) + .withStartupTimeout(Duration.ofMinutes(5)); + container.start(); + + var base = "http://%s:%d".formatted(container.getHost(), container.getMappedPort(8080)); + keycloak = Keycloak.getInstance(base, "master", "admin", "admin", "admin-cli"); + + var realm = createRealm(keycloak); + createResourceServer(realm); + issuer = "%s/realms/%s".formatted(base, REALM); + tokenEndpoint = "%s/protocol/openid-connect/token".formatted(issuer); + + createScope(realm); + var audience = createAudience(); + var email = createEmailClaim(); + createClient(realm, List.of("catalog"), List.of(audience, email)); + accessToken = getAccessToken(base, List.of("catalog")); + } + + void stop() { + if (container != null) { + container.stop(); + keycloak.close(); + } + } + + String getIssuer() { + return issuer; + } + + String getTokenEndpoint() { + return tokenEndpoint; + } + + String getClientCredential() { + return "%s:%s".formatted(ICEBERG_CLIENT_ID, ICEBERG_CLIENT_SECRET); + } + + String getAccessToken() { + return accessToken; + } +} diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/RESTCatalogServer.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/RESTCatalogServer.java index 7d2aac692db5..49d5ca7f5c1f 100644 --- a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/RESTCatalogServer.java +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/RESTCatalogServer.java @@ -44,7 +44,7 @@ private static int createMetastoreServerWithRESTCatalog(int restPort, Configurat return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf, true, false, false, false); } - + public void setSchemaInfoClass(Class schemaInfoClass) { this.schemaInfoClass = schemaInfoClass; } diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 08cff9453ab5..356130863af0 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -404,8 +404,8 @@ com.nimbusds - nimbus-jose-jwt - ${nimbus-jose-jwt.version} + oauth2-oidc-sdk + ${nimbus-oauth.version} org.pac4j @@ -470,6 +470,16 @@ mssqlserver test + + org.testcontainers + testcontainers + test + + + org.keycloak + keycloak-admin-client + test + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index 22985875f223..031571adcf53 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -17,10 +17,16 @@ package org.apache.hadoop.hive.metastore; +import static javax.ws.rs.core.HttpHeaders.WWW_AUTHENTICATE; + import com.google.common.base.Preconditions; +import java.util.List; +import java.util.function.Function; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; -import org.apache.hadoop.hive.metastore.auth.jwt.JWTValidator; +import org.apache.hadoop.hive.metastore.auth.jwt.SimpleJWTAuthenticator; +import org.apache.hadoop.hive.metastore.auth.oauth2.OAuth2Authenticator; +import org.apache.hadoop.hive.metastore.auth.oauth2.OAuth2AuthenticatorFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.security.SecurityUtil; @@ -79,7 +85,7 @@ */ public class ServletSecurity { public enum AuthType { - NONE, SIMPLE, JWT; + NONE, SIMPLE, JWT, OAUTH2; public static AuthType fromString(String type) { return AuthType.valueOf(type.toUpperCase()); @@ -91,12 +97,20 @@ public static AuthType fromString(String type) { private final boolean isSecurityEnabled; private final AuthType authType; private final Configuration conf; - private JWTValidator jwtValidator = null; + private final Function> scopeProvider; + private SimpleJWTAuthenticator jwtAuthenticator = null; + private OAuth2Authenticator oAuth2Authenticator = null; public ServletSecurity(AuthType authType, Configuration conf) { + this(authType, conf, null); + } + + public ServletSecurity(AuthType authType, Configuration conf, + Function> scopeProvider) { this.conf = conf; this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); this.authType = authType; + this.scopeProvider = scopeProvider; } /** @@ -104,9 +118,15 @@ public ServletSecurity(AuthType authType, Configuration conf) { * @throws ServletException if the jwt validator creation throws an exception */ public void init() throws ServletException { - if (authType == AuthType.JWT && jwtValidator == null) { + if (authType == AuthType.JWT && jwtAuthenticator == null) { + try { + jwtAuthenticator = SimpleJWTAuthenticator.create(this.conf); + } catch (Exception e) { + throw new ServletException("Failed to initialize ServletSecurity.", e); + } + } else if (authType == AuthType.OAUTH2 && oAuth2Authenticator == null) { try { - jwtValidator = new JWTValidator(this.conf); + oAuth2Authenticator = OAuth2AuthenticatorFactory.createAuthenticator(this.conf); } catch (Exception e) { throw new ServletException("Failed to initialize ServletSecurity.", e); } @@ -210,7 +230,7 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me String userFromHeader = extractUserName(request, response); // Temporary, and useless for now. Here only to allow this to work on an otherwise kerberized // server. - if (isSecurityEnabled || authType == AuthType.JWT) { + if (isSecurityEnabled || authType == AuthType.JWT || authType == AuthType.OAUTH2) { LOG.info("Creating proxy user for: {}", userFromHeader); clientUgi = UserGroupInformation.createProxyUser(userFromHeader, UserGroupInformation.getLoginUser()); } else { @@ -220,7 +240,8 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me clientUgi = UserGroupInformation.createRemoteUser(userFromHeader); } } catch (HttpAuthenticationException e) { - response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setStatus(e.getStatusCode()); + e.getWwwAuthenticateHeader().ifPresent(value -> response.setHeader(WWW_AUTHENTICATE, value)); response.getWriter().printf("Authentication error: %s", e.getMessage()); // Also log the error message on server side LOG.error("Authentication error: ", e); @@ -243,31 +264,38 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me private String extractUserName(HttpServletRequest request, HttpServletResponse response) throws HttpAuthenticationException { - if (authType == AuthType.SIMPLE) { + switch (authType) { + case NONE: + throw new IllegalArgumentException("This method should not be called when auth type is NONE"); + case SIMPLE: String userFromHeader = request.getHeader(X_USER); if (userFromHeader == null || userFromHeader.isEmpty()) { throw new HttpAuthenticationException("User header " + X_USER + " missing in request"); } return userFromHeader; + case JWT: + String signedJwt = extractBearerToken(request, response); + if (signedJwt == null) { + throw new HttpAuthenticationException("Couldn't find bearer token in the auth header in the request"); + } + String user; + try { + user = jwtAuthenticator.resolveUserName(signedJwt); + Preconditions.checkNotNull(user, "JWT needs to contain the user name as subject"); + Preconditions.checkState(!user.isEmpty(), "User name should not be empty in JWT"); + LOG.info("Successfully validated and extracted user name {} from JWT in Auth " + + "header in the request", user); + } catch (Exception e) { + throw new HttpAuthenticationException("Failed to validate JWT from Bearer token in " + + "Authentication header", e); + } + return user; + case OAUTH2: + String accessToken = extractBearerToken(request, response); + return oAuth2Authenticator.resolveUserName(accessToken, scopeProvider.apply(request)); + default: + throw new IllegalArgumentException("Unknown auth type: " + authType); } - // Unreachable in the case of NONE - Preconditions.checkState(authType == AuthType.JWT); - String signedJwt = extractBearerToken(request, response); - if (signedJwt == null) { - throw new HttpAuthenticationException("Couldn't find bearer token in the auth header in the request"); - } - String user; - try { - user = jwtValidator.validateJWTAndExtractUser(signedJwt); - Preconditions.checkNotNull(user, "JWT needs to contain the user name as subject"); - Preconditions.checkState(!user.isEmpty(), "User name should not be empty in JWT"); - LOG.info("Successfully validated and extracted user name {} from JWT in Auth " - + "header in the request", user); - } catch (Exception e) { - throw new HttpAuthenticationException("Failed to validate JWT from Bearer token in " - + "Authentication header", e); - } - return user; } /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java index dc0cc7c66f61..13559fc384db 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java @@ -14,6 +14,9 @@ package org.apache.hadoop.hive.metastore.auth; +import java.util.Optional; +import javax.servlet.http.HttpServletResponse; + /* Encapsulates any exceptions thrown by HiveMetastore server when authenticating http requests @@ -22,18 +25,14 @@ public class HttpAuthenticationException extends Exception { private static final long serialVersionUID = 0; - /** - * @param cause original exception - */ - public HttpAuthenticationException(Throwable cause) { - super(cause); - } + private final int statusCode; + private final String wwwAuthenticateHeader; /** * @param msg exception message */ public HttpAuthenticationException(String msg) { - super(msg); + this(msg, null); } /** @@ -41,7 +40,28 @@ public HttpAuthenticationException(String msg) { * @param cause original exception */ public HttpAuthenticationException(String msg, Throwable cause) { + this(msg, cause, HttpServletResponse.SC_UNAUTHORIZED); + } + + public HttpAuthenticationException(String msg, Throwable cause, int statusCode) { + this(msg, cause, statusCode, null); + } + + public HttpAuthenticationException(String msg, int statusCode, String wwwAuthenticateHeader) { + this(msg, null, statusCode, wwwAuthenticateHeader); + } + + public HttpAuthenticationException(String msg, Throwable cause, int statusCode, String wwwAuthenticateHeader) { super(msg, cause); + this.statusCode = statusCode; + this.wwwAuthenticateHeader = wwwAuthenticateHeader; } + public int getStatusCode() { + return statusCode; + } + + public Optional getWwwAuthenticateHeader() { + return Optional.ofNullable(wwwAuthenticateHeader); + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/JWTValidator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/JWTValidator.java index d95427c95e56..930700c7c4b9 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/JWTValidator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/JWTValidator.java @@ -20,23 +20,24 @@ import com.google.common.base.Preconditions; import com.nimbusds.jose.JOSEException; -import com.nimbusds.jose.JWSHeader; -import com.nimbusds.jose.JWSObject; -import com.nimbusds.jose.JWSVerifier; -import com.nimbusds.jose.crypto.factories.DefaultJWSVerifierFactory; -import com.nimbusds.jose.jwk.AsymmetricJWK; -import com.nimbusds.jose.jwk.JWK; +import com.nimbusds.jose.JOSEObjectType; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.JWSAlgorithm.Family; +import com.nimbusds.jose.jwk.source.JWKSource; +import com.nimbusds.jose.jwk.source.JWKSourceBuilder; +import com.nimbusds.jose.proc.BadJOSEException; +import com.nimbusds.jose.proc.DefaultJOSEObjectTypeVerifier; +import com.nimbusds.jose.proc.JWSVerificationKeySelector; +import com.nimbusds.jose.proc.SecurityContext; import com.nimbusds.jwt.JWTClaimsSet; -import com.nimbusds.jwt.SignedJWT; -import javax.security.sasl.AuthenticationException; -import org.apache.hadoop.conf.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.security.Key; +import com.nimbusds.jwt.proc.ConfigurableJWTProcessor; +import com.nimbusds.jwt.proc.DefaultJWTClaimsVerifier; +import com.nimbusds.jwt.proc.DefaultJWTProcessor; +import java.net.URL; import java.text.ParseException; -import java.util.Date; +import java.util.HashSet; +import java.util.Set; + import java.util.List; /** @@ -46,64 +47,35 @@ * This is cloned from JWTValidator in HS2 so as to NOT have any dependency on HS2 code. */ public class JWTValidator { - private static final Logger LOG = LoggerFactory.getLogger(JWTValidator.class.getName()); - private static final DefaultJWSVerifierFactory JWS_VERIFIER_FACTORY = new DefaultJWSVerifierFactory(); - private final URLBasedJWKSProvider jwksProvider; - public JWTValidator(Configuration conf) throws IOException, ParseException { - this.jwksProvider = new URLBasedJWKSProvider(conf); - } - - public String validateJWTAndExtractUser(String signedJwt) throws ParseException, AuthenticationException { - Preconditions.checkNotNull(jwksProvider); - Preconditions.checkNotNull(signedJwt, "No token found"); - final SignedJWT parsedJwt = SignedJWT.parse(signedJwt); - List matchedJWKS = jwksProvider.getJWKs(parsedJwt.getHeader()); - if (matchedJWKS.isEmpty()) { - throw new AuthenticationException("Failed to find matched JWKs with the JWT header: " + parsedJwt.getHeader()); - } + // Accept asymmetric cryptography based algorithms only + private static final Set ACCEPTABLE_ALGORITHMS = new HashSet<>(Family.SIGNATURE); - // verify signature - Exception lastException = null; - for (JWK matchedJWK : matchedJWKS) { - String keyID = matchedJWK.getKeyID() == null ? "null" : matchedJWK.getKeyID(); - try { - JWSVerifier verifier = getVerifier(parsedJwt.getHeader(), matchedJWK); - if (parsedJwt.verify(verifier)) { - LOG.debug("Verified JWT {} by JWK {}", parsedJwt.getPayload(), keyID); - break; - } - } catch (Exception e) { - lastException = e; - LOG.warn("Failed to verify JWT {} by JWK {}", parsedJwt.getPayload(), keyID, e); - } - } - // We use only the last seven characters to let a user can differentiate exceptions for different JWT - int startIndex = Math.max(0, signedJwt.length() - 7); - String lastSevenChars = signedJwt.substring(startIndex); - if (parsedJwt.getState() != JWSObject.State.VERIFIED) { - throw new AuthenticationException("Failed to verify the JWT signature (ends with " + lastSevenChars + ")", - lastException); - } + private final ConfigurableJWTProcessor jwtProcessor; - // verify claims - JWTClaimsSet claimsSet = parsedJwt.getJWTClaimsSet(); - Date expirationTime = claimsSet.getExpirationTime(); - if (expirationTime != null) { - Date now = new Date(); - if (now.after(expirationTime)) { - LOG.warn("Rejecting an expired JWT: {}", parsedJwt.getPayload()); - throw new AuthenticationException("JWT (ends with " + lastSevenChars + ") has been expired"); - } + public JWTValidator(Set acceptableTypes, List jwksURLs, String expectedIssuer, + String expectedAudience, Set requiredClaimNames) { + jwtProcessor = new DefaultJWTProcessor<>(); + jwtProcessor.setJWSTypeVerifier(new DefaultJOSEObjectTypeVerifier<>(acceptableTypes)); + Preconditions.checkArgument(!jwksURLs.isEmpty()); + final var keySelector = new JWSVerificationKeySelector<>(ACCEPTABLE_ALGORITHMS, getKeySource(jwksURLs)); + jwtProcessor.setJWSKeySelector(keySelector); + final var expectedClaimsBuilder = new JWTClaimsSet.Builder(); + if (expectedIssuer != null) { + expectedClaimsBuilder.issuer(expectedIssuer); } + jwtProcessor.setJWTClaimsSetVerifier(new DefaultJWTClaimsVerifier<>(expectedAudience, expectedClaimsBuilder.build(), + requiredClaimNames)); + } - // We assume the subject of claims is the query user - return claimsSet.getSubject(); + private JWKSource getKeySource(List jwkURLs) { + final var head = jwkURLs.getFirst(); + final var builder = JWKSourceBuilder.create(head).retrying(true); + final var tail = jwkURLs.subList(1, jwkURLs.size()); + return tail.isEmpty() ? builder.build() : builder.failover(getKeySource(tail)).build(); } - private static JWSVerifier getVerifier(JWSHeader header, JWK jwk) throws JOSEException { - Preconditions.checkArgument(jwk instanceof AsymmetricJWK, - "JWT signature verification with symmetric key is not allowed."); - Key key = ((AsymmetricJWK) jwk).toPublicKey(); - return JWS_VERIFIER_FACTORY.createJWSVerifier(header, key); + public JWTClaimsSet validateJWT(String signedJwt) throws BadJOSEException, ParseException, JOSEException { + Preconditions.checkNotNull(signedJwt, "No token found"); + return jwtProcessor.process(signedJwt, null); } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/SimpleJWTAuthenticator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/SimpleJWTAuthenticator.java new file mode 100644 index 000000000000..a6e85def82c3 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/SimpleJWTAuthenticator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.jwt; + +import com.google.common.collect.Sets; +import com.nimbusds.jose.JOSEException; +import com.nimbusds.jose.JOSEObjectType; +import com.nimbusds.jose.proc.BadJOSEException; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SimpleJWTAuthenticator { + private static final Logger LOG = LoggerFactory.getLogger(SimpleJWTAuthenticator.class.getName()); + private static final Set ACCEPTABLE_TYPES = Sets.newHashSet(null, JOSEObjectType.JWT); + + private final JWTValidator validator; + + public static SimpleJWTAuthenticator create(Configuration conf) throws IOException { + final var plainJwksURLs = MetastoreConf.getStringCollection(conf, + ConfVars.THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL); + if (plainJwksURLs.isEmpty()) { + throw new IOException("Invalid value of property: " + + ConfVars.THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL.getHiveName()); + } + final List jwksURLs = new ArrayList<>(plainJwksURLs.size()); + for (String url : plainJwksURLs) { + jwksURLs.add(URI.create(url).toURL()); + LOG.info("Loaded JWKS from {}", url); + } + final var validator = new JWTValidator(ACCEPTABLE_TYPES, jwksURLs, null, null, Collections.singleton("sub")); + return new SimpleJWTAuthenticator(validator); + } + + public SimpleJWTAuthenticator(JWTValidator validator) { + this.validator = validator; + } + + public String resolveUserName(String bearerToken) throws ParseException, BadJOSEException, JOSEException { + return validator.validateJWT(bearerToken).getSubject(); + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/URLBasedJWKSProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/URLBasedJWKSProvider.java deleted file mode 100644 index f13254d87831..000000000000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/jwt/URLBasedJWKSProvider.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore.auth.jwt; - -import com.nimbusds.jose.JWSHeader; -import com.nimbusds.jose.jwk.JWK; -import com.nimbusds.jose.jwk.JWKMatcher; -import com.nimbusds.jose.jwk.JWKSelector; -import com.nimbusds.jose.jwk.JWKSet; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.security.sasl.AuthenticationException; -import java.io.IOException; -import java.net.URL; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; - -/** - * Provides a way to get JWKS json. HiveMetastore will use this to verify the incoming JWTs. - * This is cloned from URLBasedJWKSProvider in HS2 so as to NOT have any dependency on HS2 code. - */ -public class URLBasedJWKSProvider { - - private static final Logger LOG = LoggerFactory.getLogger(URLBasedJWKSProvider.class.getName()); - private final Configuration conf; - private List jwkSets = new ArrayList<>(); - - public URLBasedJWKSProvider(Configuration conf) throws IOException, ParseException { - this.conf = conf; - loadJWKSets(); - } - - /** - * Fetches the JWKS and stores into memory. The JWKS are expected to be in the standard form as defined here - - * https://datatracker.ietf.org/doc/html/rfc7517#appendix-A. - */ - private void loadJWKSets() throws IOException, ParseException { - String jwksURL = MetastoreConf.getVar(conf, ConfVars.THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL); - if (jwksURL == null || jwksURL.isEmpty()) { - throw new IOException("Invalid value of property: " + - ConfVars.THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL.getHiveName()); - } - String[] jwksURLs = jwksURL.split(","); - for (String urlString : jwksURLs) { - URL url = new URL(urlString); - jwkSets.add(JWKSet.load(url)); - LOG.info("Loaded JWKS from " + urlString); - } - } - - /** - * Returns filtered JWKS by one or more criteria, such as kid, typ, alg. - */ - public List getJWKs(JWSHeader header) throws AuthenticationException { - JWKMatcher matcher = JWKMatcher.forJWSHeader(header); - if (matcher == null) { - throw new AuthenticationException("Unsupported algorithm: " + header.getAlgorithm()); - } - - List jwks = new ArrayList<>(); - JWKSelector selector = new JWKSelector(matcher); - for (JWKSet jwkSet : jwkSets) { - jwks.addAll(selector.select(jwkSet)); - } - return jwks; - } -} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/JWTAccessTokenAuthenticator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/JWTAccessTokenAuthenticator.java new file mode 100644 index 000000000000..0584aac2d606 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/JWTAccessTokenAuthenticator.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import com.nimbusds.jose.JOSEException; +import com.nimbusds.jose.JOSEObjectType; +import com.nimbusds.jose.proc.BadJOSEException; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.id.Issuer; +import com.nimbusds.oauth2.sdk.token.BearerTokenError; +import java.net.URL; +import java.text.ParseException; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; +import org.apache.hadoop.hive.metastore.auth.jwt.JWTValidator; + +/** + * RFC 9068 compliant JWT access token authenticator. + */ +public class JWTAccessTokenAuthenticator implements OAuth2Authenticator { + // RFC 9068 recommends using "at+jwt" to make sure the JWT is an access token, not an ID token or other tokens. + // However, as RFC 9068 is relatively new, some providers still use "JWT" by default. So, this might be a little too + // defensive. + private static final Set ACCEPTABLE_TYPES = Collections.singleton(new JOSEObjectType("at+jwt")); + + private final JWTValidator validator; + private final OAuth2PrincipalMapper principalMapper; + + public JWTAccessTokenAuthenticator(Issuer issuer, URL jwksURL, String audience, + OAuth2PrincipalMapper principalMapper) { + this.validator = new JWTValidator(ACCEPTABLE_TYPES, Collections.singletonList(jwksURL), + issuer.getValue(), audience, Collections.emptySet()); + this.principalMapper = principalMapper; + } + + @Override + public String resolveUserName(String bearerToken, List requiredScopes) throws HttpAuthenticationException { + OAuth2Authenticator.requireBearerToken(bearerToken); + final JWTClaimsSet claimSet; + try { + claimSet = validator.validateJWT(bearerToken); + } catch (ParseException | BadJOSEException e) { + final var error = BearerTokenError.INVALID_TOKEN; + throw new HttpAuthenticationException(e.getMessage(), e, error.getHTTPStatusCode(), + error.toWWWAuthenticateHeader()); + } catch (JOSEException e) { + throw new HttpAuthenticationException("Unexpectedly failed to validate JWT", e, 500); + } + final Scope scope; + try { + scope = Scope.parse(claimSet.getStringClaim("scope")); + } catch (ParseException e) { + final var error = BearerTokenError.INVALID_TOKEN; + throw new HttpAuthenticationException(e.getMessage(), e, error.getHTTPStatusCode(), + error.toWWWAuthenticateHeader()); + } + OAuth2Authenticator.requireScopes(scope, requiredScopes); + return principalMapper.getUserName(claimName -> getStringFromClaim(claimSet, claimName)); + } + + private static String getStringFromClaim(JWTClaimsSet claimSet, String claimName) throws HttpAuthenticationException { + try { + return claimSet.getStringClaim(claimName); + } catch (ParseException e) { + final var error = BearerTokenError.INVALID_TOKEN; + throw new HttpAuthenticationException(e.getMessage(), e, error.getHTTPStatusCode(), + error.toWWWAuthenticateHeader()); + } + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2Authenticator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2Authenticator.java new file mode 100644 index 000000000000..14a6c1c04f2c --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2Authenticator.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.token.BearerTokenError; +import java.util.List; +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; + +/** + * OAuth 2 based authenticator. + */ +public interface OAuth2Authenticator { + + /** + * Resolves the username from the given bearer token. + * + * @param bearerToken the bearer access token in the "Authorization" HTTP header + * @param requiredScopes the scopes required to access the resource + * @return the username + * @throws HttpAuthenticationException when it fails to resolve the bearer token + */ + String resolveUserName(String bearerToken, List requiredScopes) throws HttpAuthenticationException; + + static void requireBearerToken(String bearerToken) throws HttpAuthenticationException { + if (bearerToken == null) { + final var error = BearerTokenError.MISSING_TOKEN; + throw new HttpAuthenticationException("Missing bearer token", error.getHTTPStatusCode(), + error.toWWWAuthenticateHeader()); + } + } + + static void requireScopes(Scope tokenScope, List requiredScopes) throws HttpAuthenticationException { + if (tokenScope == null) { + final var error = BearerTokenError.INSUFFICIENT_SCOPE.setScope(Scope.parse(requiredScopes)); + throw new HttpAuthenticationException("This resource requires the following scopes: " + requiredScopes, + error.getHTTPStatusCode(), error.toWWWAuthenticateHeader()); + } + final var insufficient = requiredScopes.stream().filter(requiredScope -> !tokenScope.contains(requiredScope)) + .toList(); + if (!insufficient.isEmpty()) { + final var error = BearerTokenError.INSUFFICIENT_SCOPE.setScope(Scope.parse(requiredScopes)); + throw new HttpAuthenticationException("Insufficient scopes: " + insufficient, error.getHTTPStatusCode(), + error.toWWWAuthenticateHeader()); + } + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2AuthenticatorFactory.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2AuthenticatorFactory.java new file mode 100644 index 000000000000..7250cdf30532 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2AuthenticatorFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import com.nimbusds.oauth2.sdk.GeneralException; +import com.nimbusds.oauth2.sdk.as.AuthorizationServerMetadata; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.id.Issuer; +import java.io.IOException; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; + +/** + * A factory to create an {@link OAuth2Authenticator} instance based on the configuration. + */ +public final class OAuth2AuthenticatorFactory { + private OAuth2AuthenticatorFactory() { + throw new AssertionError(); + } + + public static OAuth2Authenticator createAuthenticator(Configuration conf) throws IOException { + final var issuer = Issuer.parse(MetastoreConf.getAsString(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_ISSUER)); + Objects.requireNonNull(issuer); + final var audience = MetastoreConf.getAsString(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_AUDIENCE); + final AuthorizationServerMetadata metadata; + try { + metadata = AuthorizationServerMetadata.resolve(issuer); + } catch (GeneralException e) { + throw new IOException("Failed to resolve the authorization server metadata. " + + "Please check %s/.well-known/oauth-authorization-server is available".formatted(issuer), e); + } + + final var claim = MetastoreConf.getAsString(conf, + ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_FIELD); + final var pattern = Pattern.compile(MetastoreConf.getAsString(conf, + ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_PATTERN)); + final var principalMapper = new RegexOAuth2PrincipalMapper(claim, pattern); + + final var validation = MetastoreConf.getAsString(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD); + switch (validation) { + case "jwt" -> { + if (metadata.getJWKSetURI() == null) { + throw new IllegalStateException(".well-known/oauth-authorization-server does not include jwks_uri"); + } + return new JWTAccessTokenAuthenticator(issuer, metadata.getJWKSetURI().toURL(), audience, principalMapper); + } + case "introspection" -> { + if (metadata.getIntrospectionEndpointURI() == null) { + throw new IllegalStateException( + ".well-known/oauth-authorization-server does not include introspection_endpoint"); + } + // RFC7662 does not specify any standard way to authenticate the HTTP client. At this moment, we use the client + // authentication, which is the most common one + final var clientId = MetastoreConf.getAsString(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_ID); + final var clientSecret = MetastoreConf.getPassword(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_SECRET); + final var credential = new ClientSecretBasic(new ClientID(clientId), new Secret(clientSecret)); + final var cacheExpiry = Duration.ofSeconds(MetastoreConf.getTimeVar(conf, + MetastoreConf.ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_INTROSPECTION_CACHE_EXPIRY, TimeUnit.SECONDS)); + final var cacheSize = MetastoreConf.getLongVar(conf, + ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_INTROSPECTION_CACHE_SIZE); + return new TokenIntrospectionAuthenticator(metadata.getIntrospectionEndpointURI(), new Audience(audience), + credential, principalMapper, cacheExpiry, cacheSize); + } + default -> throw new IllegalArgumentException("Illegal validation method: " + validation); + } + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2PrincipalMapper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2PrincipalMapper.java new file mode 100644 index 000000000000..25ce318360a1 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/OAuth2PrincipalMapper.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; + +/** + * Mapping from a claim set to a username. + */ +public interface OAuth2PrincipalMapper { + @FunctionalInterface + interface ClaimProvider { + String provide(String claimName) throws HttpAuthenticationException; + } + + /** + * Resolves the username from the provided claim set. + * + * @param rawValueProvider provides access to the raw claim values by name + * @return the username + * @throws HttpAuthenticationException when the username cannot be resolved + */ + String getUserName(ClaimProvider rawValueProvider) throws HttpAuthenticationException; +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/RegexOAuth2PrincipalMapper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/RegexOAuth2PrincipalMapper.java new file mode 100644 index 000000000000..79af4a7674da --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/RegexOAuth2PrincipalMapper.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import com.nimbusds.oauth2.sdk.token.BearerTokenError; +import java.util.regex.Pattern; +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; + +/** + * The regex-based principal mapper. + */ +public class RegexOAuth2PrincipalMapper implements OAuth2PrincipalMapper { + private final String claimName; + private final Pattern pattern; + + RegexOAuth2PrincipalMapper(String claimName, Pattern pattern) { + this.claimName = claimName; + this.pattern = pattern; + } + + @Override + public String getUserName(ClaimProvider rawValueProvider) throws HttpAuthenticationException { + final var rawValue = rawValueProvider.provide(claimName); + if (rawValue == null) { + final var error = BearerTokenError.INVALID_TOKEN; + throw new HttpAuthenticationException( + "Authentication error: Claim '%s' not found in token".formatted(claimName), + error.getHTTPStatusCode(), error.toWWWAuthenticateHeader()); + } + final var matcher = pattern.matcher(rawValue); + if (!matcher.find()) { + final var error = BearerTokenError.INVALID_TOKEN; + throw new HttpAuthenticationException( + "Authentication error: Claim '%s' does not match %s".formatted(claimName, pattern.pattern()), + error.getHTTPStatusCode(), error.toWWWAuthenticateHeader()); + } + if (matcher.groupCount() != 1) { + throw new IllegalStateException("Pattern must extract exactly one group, but %s picked up %d groups".formatted( + pattern.pattern(), matcher.groupCount())); + } + return matcher.group(1); + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/TokenIntrospectionAuthenticator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/TokenIntrospectionAuthenticator.java new file mode 100644 index 000000000000..f46910647367 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/oauth2/TokenIntrospectionAuthenticator.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.TokenIntrospectionRequest; +import com.nimbusds.oauth2.sdk.TokenIntrospectionResponse; +import com.nimbusds.oauth2.sdk.TokenIntrospectionSuccessResponse; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.oauth2.sdk.token.BearerTokenError; +import com.nimbusds.oauth2.sdk.token.TokenSchemeError; +import java.io.IOException; +import java.net.URI; +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; +import org.checkerframework.checker.index.qual.NonNegative; +import org.checkerframework.checker.nullness.qual.NonNull; + +/** + * RFC 7662 compliant OAuth2 token introspection based authenticator. This is must when access tokens are opaque, or + * this is helpful when you need further security requirements (e.g., to check token revocation). + */ +public class TokenIntrospectionAuthenticator implements OAuth2Authenticator { + private static final class UncheckedException extends RuntimeException { + private final HttpAuthenticationException underlying; + + private UncheckedException(HttpAuthenticationException cause) { + this.underlying = cause; + } + } + + private record TokenExpiry(Duration maxExpiration, Clock clock) implements + Expiry { + @Override + public long expireAfterCreate(@NonNull String key, @NonNull TokenIntrospectionSuccessResponse value, + long currentTime) { + final var expiresIn = Duration.between(value.getExpirationTime().toInstant(), clock.instant()); + return Math.min(expiresIn.toNanos(), maxExpiration.toNanos()); + } + + @Override + public long expireAfterUpdate(@NonNull String key, @NonNull TokenIntrospectionSuccessResponse value, + long currentTime, @NonNegative long currentDuration) { + final var expiresIn = Duration.between(value.getExpirationTime().toInstant(), clock.instant()); + return Math.min(expiresIn.toNanos(), maxExpiration.toNanos()); + } + + @Override + public long expireAfterRead(@NonNull String key, @NonNull TokenIntrospectionSuccessResponse value, long currentTime, + @NonNegative long currentDuration) { + return currentDuration; + } + } + + private final URI introspectionEndpoint; + private final Audience audience; + private final ClientAuthentication credential; + private final OAuth2PrincipalMapper principalMapper; + private final Cache cache; + + public TokenIntrospectionAuthenticator(URI introspectionEndpoint, Audience audience, ClientAuthentication credential, + OAuth2PrincipalMapper principalMapper, Duration maxCacheDuration, long cacheSize) { + this.introspectionEndpoint = introspectionEndpoint; + this.audience = audience; + this.credential = credential; + this.principalMapper = principalMapper; + this.cache = maxCacheDuration.isPositive() + ? Caffeine.newBuilder().maximumSize(cacheSize) + .expireAfter(new TokenExpiry(maxCacheDuration, Clock.systemUTC())).build() + : null; + } + + @Override + public String resolveUserName(String bearerToken, List requiredScopes) throws HttpAuthenticationException { + OAuth2Authenticator.requireBearerToken(bearerToken); + final var result = cache == null ? postIntrospection(bearerToken) : postIntrospectionWithCache(bearerToken); + OAuth2Authenticator.requireScopes(result.getScope(), requiredScopes); + return principalMapper.getUserName(result::getStringParameter); + } + + private TokenIntrospectionSuccessResponse postIntrospectionWithCache(String bearerToken) + throws HttpAuthenticationException { + try { + return cache.get(bearerToken, token -> { + try { + return postIntrospection(bearerToken); + } catch (HttpAuthenticationException e) { + throw new UncheckedException(e); + } + }); + } catch (UncheckedException e) { + throw e.underlying; + } + } + + private TokenIntrospectionSuccessResponse postIntrospection(String bearerToken) + throws HttpAuthenticationException { + final var request = new TokenIntrospectionRequest(introspectionEndpoint, credential, + new BearerAccessToken(bearerToken)); + final HTTPResponse httpResponse; + try { + httpResponse = request.toHTTPRequest().send(); + } catch (IOException e) { + throw new HttpAuthenticationException("The authorization server is unavailable", e, + HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + final TokenIntrospectionResponse response; + try { + response = TokenIntrospectionResponse.parse(httpResponse); + } catch (ParseException e) { + throw new HttpAuthenticationException("Received an invalid response from the authorization server", e, + HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + if (!response.indicatesSuccess()) { + final var error = response.toErrorResponse().getErrorObject(); + final var wwwAuthenticateHeader = error instanceof TokenSchemeError tokenSchemeError + ? tokenSchemeError.toWWWAuthenticateHeader() : null; + throw new HttpAuthenticationException("Failed to introspect the token", error.getHTTPStatusCode(), + wwwAuthenticateHeader); + } + final var result = response.toSuccessResponse(); + if (!result.isActive()) { + final var error = BearerTokenError.INVALID_TOKEN; + throw new HttpAuthenticationException("The token is not active", error.getHTTPStatusCode(), + error.toWWWAuthenticateHeader()); + } + if (result.getAudience() == null || !result.getAudience().contains(audience)) { + final var error = BearerTokenError.INVALID_TOKEN; + throw new HttpAuthenticationException("The aud is invalid: " + result.getAudience(), error.getHTTPStatusCode(), + error.toWWWAuthenticateHeader()); + } + // According RFC7662, an authorization server MUST validate the expiration, so this trusts the isActive flag + return result; + } +} diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java index bcecd7066cc2..3a83cd30ce6f 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java @@ -137,10 +137,9 @@ public void testValidJWT() throws Exception { @Test(expected = TTransportException.class) public void testExpiredJWT() throws Exception { String validJwtToken = generateJWT(USER_1, jwtAuthorizedKeyFile.toPath(), - TimeUnit.MILLISECONDS.toMillis(2)); + TimeUnit.MINUTES.toMillis(-2)); new EnvironmentVariables("HMS_JWT", validJwtToken).execute(() -> { - Thread.sleep(TimeUnit.MILLISECONDS.toMillis(3)); try (HiveMetaStoreClient client = new HiveMetaStoreClient(conf)) { String dbName = ("expired_jwt_" + TEST_DB_NAME_PREFIX + "_" + UUID.randomUUID()).toLowerCase(); Database createdDb = new Database(); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestJWTAccessTokenAuthenticator.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestJWTAccessTokenAuthenticator.java new file mode 100644 index 000000000000..253281ad68dc --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestJWTAccessTokenAuthenticator.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import com.nimbusds.jose.JOSEObjectType; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.JWSHeader; +import com.nimbusds.jose.crypto.RSASSASigner; +import com.nimbusds.jose.jwk.RSAKey; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.SignedJWT; +import com.nimbusds.oauth2.sdk.id.Issuer; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MetastoreUnitTest.class) +public class TestJWTAccessTokenAuthenticator { + private static final JOSEObjectType TYPE = new JOSEObjectType("at+jwt"); + private static final Issuer ISSUER = Issuer.parse("http://localhost:8080/auth"); + private static final String AUDIENCE = "http://localhost:8081/hive"; + private static final String USERNAME = "test-user"; + private static final List SCOPES = List.of("read", "update"); + private static final Date FUTURE_DATE = new Date(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1)); + private static final JWTClaimsSet CLAIMS_SET = new JWTClaimsSet.Builder().issuer(ISSUER.getValue()).audience(AUDIENCE) + .expirationTime(FUTURE_DATE).claim("email", USERNAME + "@example.com") + .claim("scope", String.join(" ", SCOPES)).build(); + private static final Path JWT_AUTHKEY; + private static final Path JWT_NOAUTHKEY; + private static final URL JWKS_URL = TestJWTAccessTokenAuthenticator.class.getClassLoader() + .getResource("auth/jwt/jwt-verification-jwks.json"); + private static final OAuth2PrincipalMapper PRINCIPAL_MAPPER = new RegexOAuth2PrincipalMapper("email", + Pattern.compile("(.*)@example.com")); + + static { + try { + JWT_AUTHKEY = Path.of(Objects.requireNonNull(TestJWTAccessTokenAuthenticator.class.getClassLoader() + .getResource("auth/jwt/jwt-authorized-key.json")).toURI()); + JWT_NOAUTHKEY = Path.of(Objects.requireNonNull(TestJWTAccessTokenAuthenticator.class.getClassLoader() + .getResource("auth/jwt/jwt-unauthorized-key.json")).toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + private static String generateJwt(JOSEObjectType type, JWTClaimsSet claimsSet, Path keyFile) { + try { + var rsaKeyPair = RSAKey.parse(Files.readString(keyFile)); + var signer = new RSASSASigner(rsaKeyPair); + var header = new JWSHeader.Builder(JWSAlgorithm.RS256).keyID(rsaKeyPair.getKeyID()).type(type).build(); + SignedJWT signedJWT = new SignedJWT(header, claimsSet); + signedJWT.sign(signer); + return signedJWT.serialize(); + } catch (Exception e) { + throw new AssertionError("Unexpectedly failed to generate JWT", e); + } + } + + @Test + public void testSuccess() throws HttpAuthenticationException { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var jwt = generateJwt(TYPE, CLAIMS_SET, JWT_AUTHKEY); + var actual = authenticator.resolveUserName(jwt, SCOPES); + Assert.assertEquals("test-user", actual); + } + + @Test + public void testExpired() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var pastDate = new Date(System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1)); + var claimSet = new JWTClaimsSet.Builder(CLAIMS_SET).expirationTime(pastDate).build(); + var jwt = generateJwt(TYPE, claimSet, JWT_AUTHKEY); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(jwt, SCOPES)); + Assert.assertEquals("Expired JWT", error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testNullToken() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(null, SCOPES)); + Assert.assertEquals("Missing bearer token", error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of("Bearer"), error.getWwwAuthenticateHeader()); + } + + @Test + public void testWrongJwtType() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var jwt = generateJwt(JOSEObjectType.JWT, CLAIMS_SET, JWT_AUTHKEY); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(jwt, SCOPES)); + Assert.assertEquals("JOSE header typ (type) JWT not allowed", error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testWrongJson() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName("invalid format", SCOPES)); + Assert.assertEquals("Invalid JWT serialization: Missing dot delimiter(s)", error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testWronglySignedJwt() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var jwt = generateJwt(TYPE, CLAIMS_SET, JWT_NOAUTHKEY); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(jwt, SCOPES)); + Assert.assertEquals("Signed JWT rejected: Another algorithm expected, or no matching key(s) found", + error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testWrongIssuer() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var claimSet = new JWTClaimsSet.Builder(CLAIMS_SET).issuer("http://localhost:8080/wrong").build(); + var jwt = generateJwt(TYPE, claimSet, JWT_AUTHKEY); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(jwt, SCOPES)); + Assert.assertEquals("JWT iss claim has value http://localhost:8080/wrong, must be http://localhost:8080/auth", + error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testWrongAudience() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var claimSet = new JWTClaimsSet.Builder(CLAIMS_SET).audience("http://localhost:8080/wrong").build(); + var jwt = generateJwt(TYPE, claimSet, JWT_AUTHKEY); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(jwt, SCOPES)); + Assert.assertEquals("JWT audience rejected: [http://localhost:8080/wrong]", + error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testMissingScope() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var claimSet = new JWTClaimsSet.Builder(CLAIMS_SET).claim("scope", null).build(); + var jwt = generateJwt(TYPE, claimSet, JWT_AUTHKEY); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(jwt, SCOPES)); + Assert.assertEquals("This resource requires the following scopes: [read, update]", + error.getMessage()); + Assert.assertEquals(403, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"insufficient_scope\", error_description=\"Insufficient scope\", scope=\"read update\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testInsufficientScope() { + var authenticator = new JWTAccessTokenAuthenticator(ISSUER, JWKS_URL, AUDIENCE, PRINCIPAL_MAPPER); + var claimSet = new JWTClaimsSet.Builder(CLAIMS_SET).claim("scope", "read delete").build(); + var jwt = generateJwt(TYPE, claimSet, JWT_AUTHKEY); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(jwt, SCOPES)); + Assert.assertEquals("Insufficient scopes: [update]", error.getMessage()); + Assert.assertEquals(403, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"insufficient_scope\", error_description=\"Insufficient scope\", scope=\"read update\""), + error.getWwwAuthenticateHeader()); + } +} diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestRegexOAuth2PrincipalMapper.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestRegexOAuth2PrincipalMapper.java new file mode 100644 index 000000000000..b58a007857b6 --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestRegexOAuth2PrincipalMapper.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import java.util.Map; +import java.util.regex.Pattern; +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MetastoreUnitTest.class) +public class TestRegexOAuth2PrincipalMapper { + private static final Map TEST_CLAIM_SET = Map.of( + "iss", "https://example.com", + "email", "alice@example.com", + "sub", "12345" + ); + + @Test + public void testSub() throws HttpAuthenticationException { + var mapper = new RegexOAuth2PrincipalMapper("sub", Pattern.compile("(.*)")); + Assert.assertEquals("12345", mapper.getUserName(TEST_CLAIM_SET::get)); + } + + @Test + public void testEmailLocalPart() throws HttpAuthenticationException { + var mapper = new RegexOAuth2PrincipalMapper("email", Pattern.compile("(.*)@example.com")); + Assert.assertEquals("alice", mapper.getUserName(TEST_CLAIM_SET::get)); + } + + @Test + public void testMissingClaim() { + var mapper = new RegexOAuth2PrincipalMapper("non-existent", Pattern.compile("(.*)")); + var error = Assert.assertThrows(HttpAuthenticationException.class, () -> mapper.getUserName(TEST_CLAIM_SET::get)); + Assert.assertEquals("Authentication error: Claim 'non-existent' not found in token", error.getMessage()); + } + + @Test + public void testNoMatching() { + var mapper = new RegexOAuth2PrincipalMapper("email", Pattern.compile("(.*)@another-domain")); + var error = Assert.assertThrows(HttpAuthenticationException.class, () -> mapper.getUserName(TEST_CLAIM_SET::get)); + Assert.assertEquals("Authentication error: Claim 'email' does not match (.*)@another-domain", error.getMessage()); + } + + @Test + public void testMultipleMatching() { + var mapper = new RegexOAuth2PrincipalMapper("email", Pattern.compile("(.*)@(.*)")); + var error = Assert.assertThrows(IllegalStateException.class, () -> mapper.getUserName(TEST_CLAIM_SET::get)); + Assert.assertEquals("Pattern must extract exactly one group, but (.*)@(.*) picked up 2 groups", error.getMessage()); + } +} diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestTokenIntrospectionAuthenticator.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestTokenIntrospectionAuthenticator.java new file mode 100644 index 000000000000..8e3652eeb7a8 --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/auth/oauth2/TestTokenIntrospectionAuthenticator.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.auth.oauth2; + +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.id.ClientID; +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; +import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException; +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.keycloak.OAuth2Constants; +import org.keycloak.admin.client.Keycloak; +import org.keycloak.admin.client.KeycloakBuilder; +import org.keycloak.admin.client.resource.RealmResource; +import org.keycloak.representations.idm.ClientRepresentation; +import org.keycloak.representations.idm.ClientScopeRepresentation; +import org.keycloak.representations.idm.ProtocolMapperRepresentation; +import org.keycloak.representations.idm.RealmRepresentation; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +@Category(MetastoreUnitTest.class) +public class TestTokenIntrospectionAuthenticator { + private static final String AUDIENCE = "http://localhost:8081/hive"; + private static final String USERNAME = "test-user"; + private static final List SCOPES = List.of("read", "update"); + private static final ClientSecretBasic RESOURCE_SERVER_CREDENTIAL = new ClientSecretBasic( + new ClientID("hive-resource-server-id"), new Secret("hive-resource-server-secret")); + private static final OAuth2PrincipalMapper PRINCIPAL_MAPPER = new RegexOAuth2PrincipalMapper("email", + Pattern.compile("(.*)@example.com")); + + private static GenericContainer container; + private static URI introspectionEndpoint; + private static String accessToken; + private static String accessTokenExpired; + private static String accessTokenWithWrongIssuer; + private static String accessTokenWithWrongAudience; + private static String accessTokenWithMissingScope; + private static String accessTokenWithInsufficientScope; + + private static RealmResource createRealm(Keycloak keycloak, String realmName) { + var realm = new RealmRepresentation(); + realm.setRealm(realmName); + realm.setEnabled(true); + keycloak.realms().create(realm); + return keycloak.realm(realmName); + } + + private static void createResourceServer(RealmResource realm) { + var resourceServer = new ClientRepresentation(); + resourceServer.setClientId(RESOURCE_SERVER_CREDENTIAL.getClientID().getValue()); + resourceServer.setSecret(RESOURCE_SERVER_CREDENTIAL.getClientSecret().getValue()); + resourceServer.setEnabled(true); + resourceServer.setProtocol("openid-connect"); + resourceServer.setPublicClient(false); + resourceServer.setServiceAccountsEnabled(true); + resourceServer.setAuthorizationServicesEnabled(true); + realm.clients().create(resourceServer).close(); + } + + private static void createScope(RealmResource realm, String name) { + var scope = new ClientScopeRepresentation(); + scope.setName(name); + scope.setProtocol("openid-connect"); + realm.clientScopes().create(scope).close(); + } + + private static ProtocolMapperRepresentation createAudience(String audience) { + var aud = new ProtocolMapperRepresentation(); + aud.setName("audience"); + aud.setProtocol("openid-connect"); + aud.setProtocolMapper("oidc-audience-mapper"); + aud.setConfig(Map.of( + "included.custom.audience", audience, + "access.token.claim", "true" + )); + return aud; + } + + private static ProtocolMapperRepresentation createEmailClaim() { + var mapper = new ProtocolMapperRepresentation(); + mapper.setName("email"); + mapper.setProtocol("openid-connect"); + mapper.setProtocolMapper("oidc-hardcoded-claim-mapper"); + mapper.setConfig(Map.of( + "claim.name", "email", + "claim.value", USERNAME + "@example.com", + "jsonType.label", "String", + "access.token.claim", "true" + )); + return mapper; + } + + private static void createClient(RealmResource realm, String clientId, String clientSecret, List scopes, + List protocolMappers) { + createClient(realm, clientId, clientSecret, scopes, protocolMappers, Collections.emptyMap()); + } + + private static void createClient(RealmResource realm, String clientId, String clientSecret, List scopes, + List protocolMappers, Map additionalAttributes) { + var client = new ClientRepresentation(); + client.setClientId(clientId); + client.setSecret(clientSecret); + client.setEnabled(true); + client.setProtocol("openid-connect"); + client.setPublicClient(false); + client.setServiceAccountsEnabled(true); + client.setOptionalClientScopes(scopes); + var attributes = new HashMap<>(additionalAttributes); + attributes.put("access.token.header.type.rfc9068", "true"); + client.setAttributes(attributes); + client.setProtocolMappers(protocolMappers); + realm.clients().create(client).close(); + } + + private static String getAccessToken(String url, String realm, String clientId, String clientSecret, + List scopes) { + try (var keycloak = KeycloakBuilder.builder() + .serverUrl(url) + .realm(realm) + .clientId(clientId) + .clientSecret(clientSecret) + .scope(scopes == null ? null : String.join(" ", scopes)) + .grantType(OAuth2Constants.CLIENT_CREDENTIALS) + .build()) { + return keycloak.tokenManager().getAccessTokenString(); + } + } + + @BeforeClass + public static void setup() throws URISyntaxException { + container = new GenericContainer<>(DockerImageName.parse("quay.io/keycloak/keycloak:26.3.4")) + .withEnv("KEYCLOAK_ADMIN", "admin") + .withEnv("KEYCLOAK_ADMIN_PASSWORD", "admin") + .withCommand("start-dev") + .withExposedPorts(8080) + .withStartupTimeout(Duration.ofMinutes(5)); + container.start(); + var base = "http://%s:%d".formatted(container.getHost(), container.getMappedPort(8080)); + var keycloak = Keycloak.getInstance(base, "master", "admin", "admin", "admin-cli"); + + var realmName = "hive"; + var realm = createRealm(keycloak, realmName); + var wrongRealmName = "hive-another"; + var wrongRealm = createRealm(keycloak, wrongRealmName); + introspectionEndpoint = new URI("%s/realms/hive/protocol/openid-connect/token/introspect".formatted(base)); + + createResourceServer(realm); + createResourceServer(wrongRealm); + + for (String scope : List.of("read", "update", "delete")) { + createScope(realm, scope); + createScope(wrongRealm, scope); + } + var audience = createAudience(AUDIENCE); + var wrongAudience = createAudience("http://localhost:8080/wrong"); + var email = createEmailClaim(); + + var clientId = "test-client-id"; + var clientSecret = "test-client-secret"; + createClient(realm, clientId, clientSecret, List.of("read", "update", "delete"), + List.of(audience, email)); + createClient(wrongRealm, clientId, clientSecret, List.of("read", "update", "delete"), + List.of(audience, email)); + var expiredClientId = "expired-client-id"; + var expiredClientSecret = "expired-client-secret"; + createClient(realm, expiredClientId, expiredClientSecret, List.of("read", "update", "delete"), + List.of(audience, email), Collections.singletonMap("access.token.lifespan", "1")); + var wrongAudienceClientId = "wrong-audience-id"; + var wrongAudienceClientSecret = "wrong-audience-secret"; + createClient(realm, wrongAudienceClientId, wrongAudienceClientSecret, + List.of("read", "update", "delete"), List.of(wrongAudience, email)); + + accessToken = getAccessToken(base, realmName, clientId, clientSecret, SCOPES); + accessTokenExpired = getAccessToken(base, realmName, expiredClientId, expiredClientSecret, SCOPES); + accessTokenWithWrongIssuer = getAccessToken(base, wrongRealmName, clientId, clientSecret, SCOPES); + accessTokenWithWrongAudience = getAccessToken(base, realmName, wrongAudienceClientId, wrongAudienceClientSecret, + SCOPES); + accessTokenWithMissingScope = getAccessToken(base, realmName, clientId, clientSecret, null); + accessTokenWithInsufficientScope = getAccessToken(base, realmName, clientId, clientSecret, + List.of("read", "delete")); + } + + @AfterClass + public static void teardown() { + if (container != null) { + container.stop(); + } + } + + @Test + public void testSuccess() throws HttpAuthenticationException { + var authenticator = new TokenIntrospectionAuthenticator(introspectionEndpoint, new Audience(AUDIENCE), + RESOURCE_SERVER_CREDENTIAL, PRINCIPAL_MAPPER, Duration.ofMinutes(1), 10); + var actual = authenticator.resolveUserName(accessToken, SCOPES); + Assert.assertEquals(USERNAME, actual); + } + + @Test + public void testExpired() { + var authenticator = new TokenIntrospectionAuthenticator(introspectionEndpoint, new Audience(AUDIENCE), + RESOURCE_SERVER_CREDENTIAL, PRINCIPAL_MAPPER, Duration.ofMinutes(0), 0); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(accessTokenExpired, SCOPES)); + Assert.assertEquals("The token is not active", error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + }); + } + + @Test + public void testNullToken() { + var authenticator = new TokenIntrospectionAuthenticator(introspectionEndpoint, new Audience(AUDIENCE), + RESOURCE_SERVER_CREDENTIAL, PRINCIPAL_MAPPER, Duration.ofMinutes(1), 10); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(null, SCOPES)); + Assert.assertEquals("Missing bearer token", error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of("Bearer"), error.getWwwAuthenticateHeader()); + } + + @Test + public void testWrongIssuer() { + var authenticator = new TokenIntrospectionAuthenticator(introspectionEndpoint, new Audience(AUDIENCE), + RESOURCE_SERVER_CREDENTIAL, PRINCIPAL_MAPPER, Duration.ofMinutes(1), 10); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(accessTokenWithWrongIssuer, SCOPES)); + Assert.assertEquals("The token is not active", error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testWrongAudience() { + var authenticator = new TokenIntrospectionAuthenticator(introspectionEndpoint, new Audience(AUDIENCE), + RESOURCE_SERVER_CREDENTIAL, PRINCIPAL_MAPPER, Duration.ofMinutes(1), 10); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(accessTokenWithWrongAudience, SCOPES)); + Assert.assertEquals("The aud is invalid: [http://localhost:8080/wrong]", + error.getMessage()); + Assert.assertEquals(401, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"invalid_token\", error_description=\"Invalid access token\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testMissingScope() { + var authenticator = new TokenIntrospectionAuthenticator(introspectionEndpoint, new Audience(AUDIENCE), + RESOURCE_SERVER_CREDENTIAL, PRINCIPAL_MAPPER, Duration.ofMinutes(1), 10); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(accessTokenWithMissingScope, SCOPES)); + Assert.assertEquals("This resource requires the following scopes: [read, update]", + error.getMessage()); + Assert.assertEquals(403, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"insufficient_scope\", error_description=\"Insufficient scope\", scope=\"read update\""), + error.getWwwAuthenticateHeader()); + } + + @Test + public void testInsufficientScope() { + var authenticator = new TokenIntrospectionAuthenticator(introspectionEndpoint, new Audience(AUDIENCE), + RESOURCE_SERVER_CREDENTIAL, PRINCIPAL_MAPPER, Duration.ofMinutes(1), 10); + var error = Assert.assertThrows(HttpAuthenticationException.class, + () -> authenticator.resolveUserName(accessTokenWithInsufficientScope, SCOPES)); + Assert.assertEquals("Insufficient scopes: [update]", error.getMessage()); + Assert.assertEquals(403, error.getStatusCode()); + Assert.assertEquals(Optional.of( + "Bearer error=\"insufficient_scope\", error_description=\"Insufficient scope\", scope=\"read update\""), + error.getWwwAuthenticateHeader()); + } +} diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index e23f779d0c1b..8a5e001a86cb 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -117,9 +117,10 @@ 4.4.13 4.5.13 4.5.8 - 10.4.2 + 11.28 9.4.57.v20241219 1.3.2 + 26.0.6 5.3.39 2.4.4 @@ -523,6 +524,11 @@ ${slf4j.version} test + + org.keycloak + keycloak-admin-client + ${keycloak.version} + org.testcontainers mariadb @@ -548,6 +554,11 @@ mssqlserver ${testcontainers.version} + + org.testcontainers + testcontainers + ${testcontainers.version} +