Skip to content

Commit e44cf34

Browse files
okuminokumin
authored
HIVE-29020: Support OAuth 2 in Iceberg REST Catalog (#6086)
* HIVE-29020: Support OAuth 2 in Iceberg REST Catalog * Apply some review comments * Rephrase some configurations * Normalize the var name of CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_FIELD
1 parent 96cf347 commit e44cf34

29 files changed

+1863
-255
lines changed

‎standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java‎

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -983,8 +983,8 @@ public enum ConfVars {
983983
" NOSASL: Raw transport" +
984984
" JWT: JSON Web Token authentication via JWT token. Only supported in Http/Https mode"),
985985
THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL("metastore.authentication.jwt.jwks.url",
986-
"hive.metastore.authentication.jwt.jwks.url", "", "File URL from where URLBasedJWKSProvider "
987-
+ "in metastore server will try to load JWKS to match a JWT sent in HTTP request header. Used only when "
986+
"hive.metastore.authentication.jwt.jwks.url", "", "File URL from where "
987+
+ "metastore server will try to load JWKS to match a JWT sent in HTTP request header. Used only when "
988988
+ "Hive metastore server is running in JWT auth mode"),
989989
METASTORE_CUSTOM_AUTHENTICATION_CLASS("metastore.custom.authentication.class",
990990
"hive.metastore.custom.authentication.class",
@@ -1873,8 +1873,56 @@ public enum ConfVars {
18731873
" positive value will be used as-is."
18741874
),
18751875
CATALOG_SERVLET_AUTH("metastore.catalog.servlet.auth",
1876-
"hive.metastore.catalog.servlet.auth", "jwt", new StringSetValidator("none", "simple", "jwt"),
1877-
"HMS Catalog servlet authentication method (none, simple, or jwt)."
1876+
"hive.metastore.catalog.servlet.auth", "jwt", new StringSetValidator("none", "simple", "jwt", "oauth2"),
1877+
"HMS Catalog servlet authentication method (none, simple, jwt, or oauth2)."
1878+
),
1879+
CATALOG_SERVLET_AUTH_OAUTH2_ISSUER("metastore.catalog.servlet.auth.oauth2.issuer",
1880+
"hive.metastore.catalog.servlet.auth.oauth2.issuer", "",
1881+
"The authorization server's identifier, which is a URL. This is required when you use " +
1882+
"metastore.catalog.servlet.auth=oauth2"
1883+
),
1884+
CATALOG_SERVLET_AUTH_OAUTH2_AUDIENCE("metastore.catalog.servlet.auth.oauth2.audience",
1885+
"hive.metastore.catalog.servlet.auth.oauth2.audience", "",
1886+
"The acceptable name in the audience(aud) claim. This is required when you use " +
1887+
"metastore.catalog.servlet.auth=oauth2"
1888+
),
1889+
CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD("metastore.catalog.servlet.auth.oauth2.validation.method",
1890+
"hive.metastore.catalog.servlet.auth.oauth2.validation.method", "jwt",
1891+
new StringSetValidator("jwt", "introspection"),
1892+
"How to evaluate an access token. When your authorization server issues opaque tokens or you need " +
1893+
"to consider additional security requirements such as token revocations, use introspection."
1894+
),
1895+
CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_ID("metastore.catalog.servlet.auth.oauth2.client.id",
1896+
"hive.metastore.catalog.servlet.auth.oauth2.client.id", "",
1897+
"The client ID to authenticate HMS, as a resource server, to the introspection endpoint. This is required to " +
1898+
"use metastore.catalog.servlet.auth.oauth2.validation.method=introspection."
1899+
),
1900+
CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_SECRET("metastore.catalog.servlet.auth.oauth2.client.secret",
1901+
"hive.metastore.catalog.servlet.auth.oauth2.client.secret", "",
1902+
"The client secret to authenticate HMS, as a resource server, to the introspection endpoint. This is " +
1903+
"required to use metastore.catalog.servlet.auth.oauth2.validation.method=introspection."
1904+
),
1905+
CATALOG_SERVLET_AUTH_OAUTH2_INTROSPECTION_CACHE_EXPIRY(
1906+
"metastore.catalog.servlet.auth.oauth2.introspection.cache.expiry",
1907+
"hive.metastore.catalog.servlet.auth.oauth2.introspection.cache.expiry", 60, TimeUnit.SECONDS,
1908+
"The expiry time of the token introspection cache. Set to 0 to disable caching."
1909+
),
1910+
CATALOG_SERVLET_AUTH_OAUTH2_INTROSPECTION_CACHE_SIZE(
1911+
"metastore.catalog.servlet.auth.oauth2.introspection.cache.num",
1912+
"hive.metastore.catalog.servlet.auth.oauth2.introspection.cache.num", 1000L,
1913+
"The number of entries of the token introspection cache."
1914+
),
1915+
CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_FIELD(
1916+
"metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.field",
1917+
"hive.metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.field", "sub",
1918+
"The claim name including a username. This is effective when you use RegexPrincipalMapper. For example, if " +
1919+
"you want to resolve a user name from the email claim, set this to email."
1920+
),
1921+
CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_PATTERN(
1922+
"metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.pattern",
1923+
"hive.metastore.catalog.servlet.auth.oauth2.principal.mapper.regex.username.pattern", "(.*)",
1924+
"The pattern to extract a user name. This is effective when you use RegexPrincipalMapper. For example, if " +
1925+
"you want to extract a user name from the local part of the email claim, set this to (.*)@example.com."
18781926
),
18791927
ICEBERG_CATALOG_SERVLET_PATH("metastore.iceberg.catalog.servlet.path",
18801928
"hive.metastore.iceberg.catalog.servlet.path", "iceberg",

‎standalone-metastore/metastore-rest-catalog/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,16 @@
224224
</exclusion>
225225
</exclusions>
226226
</dependency>
227+
<dependency>
228+
<groupId>org.keycloak</groupId>
229+
<artifactId>keycloak-admin-client</artifactId>
230+
<scope>test</scope>
231+
</dependency>
232+
<dependency>
233+
<groupId>org.testcontainers</groupId>
234+
<artifactId>testcontainers</artifactId>
235+
<scope>test</scope>
236+
</dependency>
227237
</dependencies>
228238
<build>
229239
<plugins>

‎standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@
7171
import org.apache.iceberg.rest.responses.ListTablesResponse;
7272
import org.apache.iceberg.rest.responses.LoadTableResponse;
7373
import org.apache.iceberg.rest.responses.LoadViewResponse;
74-
import org.apache.iceberg.rest.responses.OAuthTokenResponse;
7574
import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse;
7675
import org.apache.iceberg.util.Pair;
7776
import org.apache.iceberg.util.PropertyUtil;
@@ -104,15 +103,6 @@ public class HMSCatalogAdapter implements RESTClient {
104103
.put(CommitStateUnknownException.class, 500)
105104
.buildOrThrow();
106105

107-
private static final String URN_OAUTH_TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange";
108-
private static final String URN_OAUTH_ACCESS_TOKEN = "urn:ietf:params:oauth:token-type:access_token";
109-
private static final String GRANT_TYPE = "grant_type";
110-
private static final String CLIENT_CREDENTIALS = "client_credentials";
111-
private static final String BEARER = "Bearer";
112-
private static final String CLIENT_ID = "client_id";
113-
private static final String ACTOR_TOKEN = "actor_token";
114-
private static final String SUBJECT_TOKEN = "subject_token";
115-
116106
private final Catalog catalog;
117107
private final SupportsNamespaces asNamespaceCatalog;
118108
private final ViewCatalog asViewCatalog;
@@ -127,8 +117,6 @@ public HMSCatalogAdapter(Catalog catalog) {
127117
}
128118

129119
enum Route {
130-
TOKENS(HTTPMethod.POST, "v1/oauth/tokens", null),
131-
SEPARATE_AUTH_TOKENS_URI(HTTPMethod.POST, "https://auth-server.com/token", null),
132120
CONFIG(HTTPMethod.GET, "v1/config", null),
133121
LIST_NAMESPACES(HTTPMethod.GET, ResourcePaths.V1_NAMESPACES, null),
134122
CREATE_NAMESPACE(HTTPMethod.POST, ResourcePaths.V1_NAMESPACES, CreateNamespaceRequest.class),
@@ -240,35 +228,6 @@ private ConfigResponse config() {
240228
return castResponse(ConfigResponse.class, ConfigResponse.builder().withEndpoints(endpoints).build());
241229
}
242230

243-
private OAuthTokenResponse tokens(Object body) {
244-
@SuppressWarnings("unchecked")
245-
Map<String, String> request = (Map<String, String>) castRequest(Map.class, body);
246-
String grantType = request.get(GRANT_TYPE);
247-
switch (grantType) {
248-
case CLIENT_CREDENTIALS:
249-
return OAuthTokenResponse.builder()
250-
.withToken("client-credentials-token:sub=" + request.get(CLIENT_ID))
251-
.withIssuedTokenType(URN_OAUTH_ACCESS_TOKEN)
252-
.withTokenType(BEARER)
253-
.build();
254-
255-
case URN_OAUTH_TOKEN_EXCHANGE:
256-
String actor = request.get(ACTOR_TOKEN);
257-
String token =
258-
String.format(
259-
"token-exchange-token:sub=%s%s",
260-
request.get(SUBJECT_TOKEN), actor != null ? ",act=" + actor : "");
261-
return OAuthTokenResponse.builder()
262-
.withToken(token)
263-
.withIssuedTokenType(URN_OAUTH_ACCESS_TOKEN)
264-
.withTokenType(BEARER)
265-
.build();
266-
267-
default:
268-
throw new UnsupportedOperationException("Unsupported grant_type: " + grantType);
269-
}
270-
}
271-
272231
private ListNamespacesResponse listNamespaces(Map<String, String> vars) {
273232
Namespace namespace;
274233
if (vars.containsKey("parent")) {
@@ -469,9 +428,6 @@ private <T extends RESTResponse> T handleRequest(
469428
counter.inc();
470429
}
471430
switch (route) {
472-
case TOKENS:
473-
return (T) tokens(body);
474-
475431
case CONFIG:
476432
return (T) config();
477433

‎standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
*/
1919
package org.apache.iceberg.rest;
2020

21+
import java.util.Collections;
22+
import java.util.List;
2123
import java.util.Map;
2224
import java.util.TreeMap;
2325
import javax.servlet.http.HttpServlet;
@@ -100,7 +102,9 @@ private Catalog createCatalog() {
100102
*/
101103
private HttpServlet createServlet(Catalog catalog) {
102104
String authType = MetastoreConf.getVar(configuration, ConfVars.CATALOG_SERVLET_AUTH);
103-
ServletSecurity security = new ServletSecurity(AuthType.fromString(authType), configuration);
105+
// Iceberg REST client uses "catalog" by default
106+
List<String> scopes = Collections.singletonList("catalog");
107+
ServletSecurity security = new ServletSecurity(AuthType.fromString(authType), configuration, req -> scopes);
104108
return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog)));
105109
}
106110

‎standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020
package org.apache.iceberg.rest;
2121

2222
import java.io.IOException;
23-
import java.io.InputStreamReader;
24-
import java.io.Reader;
2523
import java.io.UncheckedIOException;
2624
import java.util.Map;
2725
import java.util.Optional;
@@ -31,7 +29,6 @@
3129
import javax.servlet.http.HttpServletRequest;
3230
import javax.servlet.http.HttpServletResponse;
3331
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
34-
import org.apache.iceberg.relocated.com.google.common.io.CharStreams;
3532
import org.apache.iceberg.rest.HMSCatalogAdapter.Route;
3633
import org.apache.iceberg.rest.HTTPRequest.HTTPMethod;
3734
import org.apache.iceberg.rest.responses.ErrorResponse;
@@ -152,10 +149,6 @@ static ServletRequestContext from(HttpServletRequest request) throws IOException
152149
if (route.requestClass() != null) {
153150
requestBody =
154151
RESTObjectMapper.mapper().readValue(request.getReader(), route.requestClass());
155-
} else if (route == Route.TOKENS) {
156-
try (Reader reader = new InputStreamReader(request.getInputStream())) {
157-
requestBody = RESTUtil.decodeFormData(CharStreams.toString(reader));
158-
}
159152
}
160153

161154
Map<String, String> queryParams =

‎standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2Jwt.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.iceberg.rest;
20+
21+
import java.util.Map;
22+
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
23+
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
24+
import org.apache.iceberg.exceptions.NotAuthorizedException;
25+
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
26+
import org.junit.experimental.categories.Category;
27+
import org.junit.jupiter.api.Assertions;
28+
import org.junit.jupiter.api.Test;
29+
import org.junit.jupiter.api.extension.RegisterExtension;
30+
31+
@Category(MetastoreCheckinTest.class)
32+
class TestRESTCatalogOAuth2Jwt extends BaseRESTCatalogTests {
33+
@RegisterExtension
34+
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
35+
HiveRESTCatalogServerExtension.builder(AuthType.OAUTH2).build();
36+
37+
@Override
38+
protected Map<String, String> getDefaultClientConfiguration() {
39+
return Map.of(
40+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
41+
"rest.auth.type", "oauth2",
42+
"oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(),
43+
"credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential()
44+
);
45+
}
46+
47+
@Test
48+
void testWithAccessToken() {
49+
Map<String, String> properties = Map.of(
50+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
51+
"rest.auth.type", "oauth2",
52+
"token", REST_CATALOG_EXTENSION.getOAuth2AccessToken()
53+
);
54+
Assertions.assertFalse(RCKUtils.initCatalogClient(properties).listNamespaces().isEmpty());
55+
}
56+
57+
@Test
58+
void testWithWrongCredential() {
59+
Map<String, String> properties = Map.of(
60+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
61+
"rest.auth.type", "oauth2",
62+
"oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(),
63+
"credential", "dummy:dummy"
64+
);
65+
NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class,
66+
() -> RCKUtils.initCatalogClient(properties));
67+
Assertions.assertEquals("Not authorized: invalid_client: Invalid client or Invalid client credentials",
68+
error.getMessage());
69+
}
70+
71+
@Test
72+
void testWithWrongAccessToken() {
73+
Map<String, String> properties = Map.of(
74+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
75+
"rest.auth.type", "oauth2",
76+
"token", "invalid"
77+
);
78+
NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class,
79+
() -> RCKUtils.initCatalogClient(properties));
80+
Assertions.assertEquals("Not authorized: Authentication error: Invalid JWT serialization: Missing dot delimiter(s)",
81+
error.getMessage());
82+
}
83+
}

‎standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogOAuth2TokenIntrospection.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.iceberg.rest;
20+
21+
import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD;
22+
23+
import java.util.Map;
24+
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
25+
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
26+
import org.apache.iceberg.exceptions.NotAuthorizedException;
27+
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
28+
import org.junit.experimental.categories.Category;
29+
import org.junit.jupiter.api.Assertions;
30+
import org.junit.jupiter.api.Test;
31+
import org.junit.jupiter.api.extension.RegisterExtension;
32+
33+
@Category(MetastoreCheckinTest.class)
34+
class TestRESTCatalogOAuth2TokenIntrospection extends BaseRESTCatalogTests {
35+
@RegisterExtension
36+
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
37+
HiveRESTCatalogServerExtension.builder(AuthType.OAUTH2)
38+
.configure(CATALOG_SERVLET_AUTH_OAUTH2_VALIDATION_METHOD.getVarname(), "introspection").build();
39+
40+
@Override
41+
protected Map<String, String> getDefaultClientConfiguration() {
42+
return Map.of(
43+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
44+
"rest.auth.type", "oauth2",
45+
"oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(),
46+
"credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential()
47+
);
48+
}
49+
50+
@Test
51+
void testWithAccessToken() {
52+
Map<String, String> properties = Map.of(
53+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
54+
"rest.auth.type", "oauth2",
55+
"token", REST_CATALOG_EXTENSION.getOAuth2AccessToken()
56+
);
57+
Assertions.assertFalse(RCKUtils.initCatalogClient(properties).listNamespaces().isEmpty());
58+
}
59+
60+
@Test
61+
void testWithWrongCredential() {
62+
Map<String, String> properties = Map.of(
63+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
64+
"rest.auth.type", "oauth2",
65+
"oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint(),
66+
"credential", "dummy:dummy"
67+
);
68+
NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class,
69+
() -> RCKUtils.initCatalogClient(properties));
70+
Assertions.assertEquals("Not authorized: invalid_client: Invalid client or Invalid client credentials",
71+
error.getMessage());
72+
}
73+
74+
@Test
75+
void testWithWrongAccessToken() {
76+
Map<String, String> properties = Map.of(
77+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint(),
78+
"rest.auth.type", "oauth2",
79+
"token", "invalid"
80+
);
81+
NotAuthorizedException error = Assertions.assertThrows(NotAuthorizedException.class,
82+
() -> RCKUtils.initCatalogClient(properties));
83+
Assertions.assertEquals("Not authorized: Authentication error: The token is not active",
84+
error.getMessage());
85+
}
86+
}

0 commit comments

Comments
 (0)