From b1c7adf54869ad0c25ae788ce8a03a3fd1d52b73 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 13 Aug 2025 14:06:40 +0200 Subject: [PATCH 1/3] Add integration tests with Keycloak --- gradle/libs.versions.toml | 2 + integration-tests/build.gradle.kts | 1 + .../polaris/service/it/env/CatalogApi.java | 26 +-- .../service/it/env/ClientCredentials.java | 12 +- .../service/it/env/ClientPrincipal.java | 12 +- .../service/it/env/GenericTableApi.java | 4 +- .../polaris/service/it/env/IcebergHelper.java | 7 +- .../it/env/IcebergTokenAccessManager.java | 43 ----- .../polaris/service/it/env/ManagementApi.java | 4 +- .../polaris/service/it/env/OAuth2Api.java | 65 +++++++ .../polaris/service/it/env/PolarisClient.java | 73 +++---- .../service/it/env/PolarisRestApi.java | 55 ++++++ .../polaris/service/it/env/PolicyApi.java | 2 +- .../polaris/service/it/env/RestApi.java | 18 +- .../service/it/ext/PolarisServerManager.java | 5 - .../ext/PolarisSparkIntegrationTestBase.java | 4 +- .../PolarisApplicationIntegrationTest.java | 13 +- ...larisManagementServiceIntegrationTest.java | 14 +- .../PolarisPolicyServiceIntegrationTest.java | 24 ++- .../PolarisRestCatalogIntegrationBase.java | 69 +++++-- ...PolarisRestCatalogViewIntegrationBase.java | 14 +- .../resources/application-test.properties | 5 +- runtime/service/build.gradle.kts | 4 + .../service/it/RestCatalogKeycloakFileIT.java | 81 ++++++++ .../service/it/RestCatalogMinIOSpecialIT.java | 9 +- runtime/test-common/build.gradle.kts | 11 ++ .../test/commons/keycloak/KeycloakAccess.java | 81 ++++++++ .../commons/keycloak/KeycloakContainer.java | 182 ++++++++++++++++++ .../keycloak/KeycloakLifecycleManager.java | 57 ++++++ .../commons/keycloak/KeycloakProfile.java | 29 ++- .../keycloak/Dockerfile-keycloak-version | 22 +++ 31 files changed, 747 insertions(+), 201 deletions(-) delete mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergTokenAccessManager.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/OAuth2Api.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisRestApi.java create mode 100644 runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java create mode 100644 runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java create mode 100644 runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java create mode 100644 runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakLifecycleManager.java rename integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisAccessManager.java => runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakProfile.java (50%) create mode 100644 runtime/test-common/src/main/resources/org/apache/polaris/test/commons/keycloak/Dockerfile-keycloak-version diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1697eafc52..4635dffb3a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -72,6 +72,7 @@ jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" } javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } junit-bom = { module = "org.junit:junit-bom", version = "5.13.4" } +keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version = "26.0.6" } logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" } micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.15.3" } microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" } @@ -94,6 +95,7 @@ spark35-sql-scala212 = { module = "org.apache.spark:spark-sql_2.12", version.ref swagger-annotations = { module = "io.swagger:swagger-annotations", version.ref = "swagger" } swagger-jaxrs = { module = "io.swagger:swagger-jaxrs", version.ref = "swagger" } testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version = "1.21.3" } +testcontainers-keycloak = { module = "com.github.dasniko:testcontainers-keycloak", version = "3.8.0" } threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" } [plugins] diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts index 6836ff89b1..fca66bc18b 100644 --- a/integration-tests/build.gradle.kts +++ b/integration-tests/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { implementation(project(":polaris-api-management-model")) implementation(project(":polaris-api-catalog-service")) + implementation(libs.jakarta.annotation.api) implementation(libs.jakarta.ws.rs.api) implementation(libs.guava) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java index 25a77ecbc0..eb400b1e66 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java @@ -25,7 +25,6 @@ import com.google.common.base.Joiner; import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.MultivaluedHashMap; import jakarta.ws.rs.core.Response; import java.net.URI; import java.util.ArrayList; @@ -42,38 +41,17 @@ import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; -import org.apache.iceberg.rest.responses.OAuthTokenResponse; /** * A simple, non-exhaustive set of helper methods for accessing the Iceberg REST API. * - * @see PolarisClient#catalogApi(ClientCredentials) + * @see PolarisClient#catalogApi(String) */ -public class CatalogApi extends RestApi { +public class CatalogApi extends PolarisRestApi { public CatalogApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { super(client, endpoints, authToken, uri); } - public String obtainToken(ClientCredentials credentials) { - try (Response response = - request("v1/oauth/tokens") - .post( - Entity.form( - new MultivaluedHashMap<>( - Map.of( - "grant_type", - "client_credentials", - "scope", - "PRINCIPAL_ROLE:ALL", - "client_id", - credentials.clientId(), - "client_secret", - credentials.clientSecret()))))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - return response.readEntity(OAuthTokenResponse.class).token(); - } - } - public void createNamespace(String catalogName, String namespaceName) { try (Response response = request("v1/{cat}/namespaces", Map.of("cat", catalogName)) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java index afa97bd577..58b3d7ac4a 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.it.env; +import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; /** @@ -25,4 +26,13 @@ * representing an admin user is injected into test parameters by {@link * PolarisIntegrationTestExtension}. */ -public record ClientCredentials(String clientId, String clientSecret) {} +public record ClientCredentials(String clientId, String clientSecret) { + + /** + * Creates a {@link ClientCredentials} from an instance of the Admin API model {@link + * PrincipalWithCredentialsCredentials}. + */ + public ClientCredentials(PrincipalWithCredentialsCredentials credentials) { + this(credentials.getClientId(), credentials.getClientSecret()); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java index 7090b41475..bafaeca5f6 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.it.env; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; /** @@ -27,4 +28,13 @@ * * @see Server#adminCredentials() */ -public record ClientPrincipal(String principalName, ClientCredentials credentials) {} +public record ClientPrincipal(String principalName, ClientCredentials credentials) { + + /** + * Creates a {@link ClientPrincipal} from an instance of the Admin API model {@link + * PrincipalWithCredentials}. + */ + public ClientPrincipal(PrincipalWithCredentials principal) { + this(principal.getPrincipal().getName(), new ClientCredentials(principal.getCredentials())); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java index a31fd0cd2b..5a993b2497 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java @@ -38,9 +38,9 @@ /** * A simple, non-exhaustive set of helper methods for accessing the generic tables REST API * - * @see PolarisClient#genericTableApi(ClientCredentials) + * @see PolarisClient#genericTableApi(String) */ -public class GenericTableApi extends RestApi { +public class GenericTableApi extends PolarisRestApi { GenericTableApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { super(client, endpoints, authToken, uri); } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java index a119bf615d..740307c14b 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -22,18 +22,15 @@ import java.util.Map; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.auth.OAuth2Properties; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; public final class IcebergHelper { private IcebergHelper() {} public static RESTCatalog restCatalog( - PolarisClient client, PolarisApiEndpoints endpoints, - PrincipalWithCredentials credentials, String catalog, - Map extraProperties) { - String authToken = client.obtainToken(credentials); + Map extraProperties, + String authToken) { RESTCatalog restCatalog = new RESTCatalog(); ImmutableMap.Builder propertiesBuilder = diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergTokenAccessManager.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergTokenAccessManager.java deleted file mode 100644 index 86054160a1..0000000000 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergTokenAccessManager.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.it.env; - -import jakarta.ws.rs.client.Client; -import org.apache.polaris.service.it.ext.PolarisAccessManager; - -/** - * This class obtains access tokens from the {@code v1/oauth/tokens} endpoint defined by the Iceberg - * REST Catalog spec. - * - *

Note: even though this endpoint is still part of the Iceberg REST Catalog spec it has been - * deprecated per Iceberg PR#10603. - */ -public class IcebergTokenAccessManager implements PolarisAccessManager { - private final Client client; - - public IcebergTokenAccessManager(Client client) { - this.client = client; - } - - @Override - public String obtainAccessToken(PolarisApiEndpoints endpoints, ClientCredentials credentials) { - CatalogApi anon = new CatalogApi(client, endpoints, null, endpoints.catalogApiEndpoint()); - return anon.obtainToken(credentials); - } -} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java index 72c66cd128..9bd1ccded1 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java @@ -51,9 +51,9 @@ /** * A simple, non-exhaustive set of helper methods for accessing the Polaris Management API. * - * @see PolarisClient#managementApi(ClientCredentials) + * @see PolarisClient#managementApi(String) */ -public class ManagementApi extends RestApi { +public class ManagementApi extends PolarisRestApi { public ManagementApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { super(client, endpoints, authToken, uri); } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/OAuth2Api.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/OAuth2Api.java new file mode 100644 index 0000000000..f7174b4e37 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/OAuth2Api.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.Map; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; + +/** + * A simple facade to an OAuth2 token endpoint. It works with both Polaris internal token endpoint + * and with external identity providers. + */ +public class OAuth2Api extends RestApi { + + private final String endpointPath; + + public OAuth2Api(Client client, URI issuerUrl, String endpointPath) { + super(client, issuerUrl); + this.endpointPath = endpointPath; + } + + public String obtainAccessToken(ClientCredentials credentials, String scope) { + return obtainAccessToken( + Map.of( + "grant_type", + "client_credentials", + "client_id", + credentials.clientId(), + "client_secret", + credentials.clientSecret(), + "scope", + scope)); + } + + public String obtainAccessToken(Map requestBody) { + try (Response response = + request(endpointPath).post(Entity.form(new MultivaluedHashMap<>(requestBody)))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + String token = response.readEntity(OAuthTokenResponse.class).token(); + return token; + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java index baec590f91..6537b234f2 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java @@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import jakarta.ws.rs.client.Client; +import java.net.URI; +import java.util.Map; import java.util.Random; import org.apache.iceberg.rest.RESTSerializers; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; @@ -73,7 +75,7 @@ public static ObjectMapper buildObjectMapper() { /** * This method should be used by test code to make top-level entity names. The purpose of this * method is two-fold: - *

  • Identify top-level entities for latger clean-up by {@link #cleanUp(ClientCredentials)}. + *
  • Identify top-level entities for later clean-up by {@link #cleanUp(String)}. *
  • Allow {@link PolarisServerManager}s to customize top-level entities per environment. */ public String newEntityName(String hint) { @@ -84,70 +86,53 @@ public ManagementApi managementApi(String authToken) { return new ManagementApi(client, endpoints, authToken, endpoints.managementApiEndpoint()); } - public ManagementApi managementApi(ClientCredentials credentials) { - return managementApi(obtainToken(credentials)); - } - - public ManagementApi managementApi(PrincipalWithCredentials principal) { - return managementApi(obtainToken(principal)); - } - - public CatalogApi catalogApi(PrincipalWithCredentials principal) { - return new CatalogApi( - client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint()); - } - - public CatalogApi catalogApi(ClientCredentials credentials) { - return new CatalogApi( - client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint()); + public CatalogApi catalogApi(String authToken) { + return new CatalogApi(client, endpoints, authToken, endpoints.catalogApiEndpoint()); } public CatalogApi catalogApiPlain() { return new CatalogApi(client, endpoints, null, endpoints.catalogApiEndpoint()); } - public GenericTableApi genericTableApi(PrincipalWithCredentials principal) { - return new GenericTableApi( - client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint()); + public GenericTableApi genericTableApi(String authToken) { + return new GenericTableApi(client, endpoints, authToken, endpoints.catalogApiEndpoint()); } - public GenericTableApi genericTableApi(ClientCredentials credentials) { - return new GenericTableApi( - client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint()); + public PolicyApi policyApi(String authToken) { + return new PolicyApi(client, endpoints, authToken, endpoints.catalogApiEndpoint()); } - public PolicyApi policyApi(PrincipalWithCredentials principal) { - return new PolicyApi(client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint()); + /** Requests an access token from the Polaris server for the given principal. */ + public String obtainToken(PrincipalWithCredentials credentials) { + return obtainToken(new ClientPrincipal(credentials)); } - public PolicyApi policyApi(ClientCredentials credentials) { - return new PolicyApi( - client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint()); + /** Requests an access token from the Polaris server for the given principal. */ + public String obtainToken(ClientPrincipal principal) { + return obtainToken(principal.credentials()); } - /** - * Requests an access token from the Polaris server for the client ID/secret pair that is part of - * the given principal data object. - */ - public String obtainToken(PrincipalWithCredentials principal) { - return obtainToken( - new ClientCredentials( - principal.getCredentials().getClientId(), - principal.getCredentials().getClientSecret())); + /** Requests an access token from the Polaris server for the given credentials. */ + public String obtainToken(ClientCredentials credentials) { + OAuth2Api api = new OAuth2Api(client, endpoints.catalogApiEndpoint(), "v1/oauth/tokens"); + return api.obtainAccessToken(credentials, "PRINCIPAL_ROLE:ALL"); } - /** Requests an access token from the Polaris server for the given {@link ClientCredentials}. */ - public String obtainToken(ClientCredentials credentials) { - return polarisServerManager().accessManager(client).obtainAccessToken(endpoints, credentials); + /** + * Requests an access token from the authorization server denoted by the issuer URL and token + * endpoint path. + */ + public String obtainToken(URI issuerUrl, String endpointPath, Map requestBody) { + return new OAuth2Api(client, issuerUrl, endpointPath).obtainAccessToken(requestBody); } - private boolean ownedName(String name) { + public boolean ownedName(String name) { return name != null && name.contains(clientId); } - public void cleanUp(ClientCredentials credentials) { - ManagementApi managementApi = managementApi(credentials); - CatalogApi catalogApi = catalogApi(credentials); + public void cleanUp(String authToken) { + ManagementApi managementApi = managementApi(authToken); + CatalogApi catalogApi = catalogApi(authToken); managementApi.listCatalogs().stream() .filter(c -> ownedName(c.getName())) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisRestApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisRestApi.java new file mode 100644 index 0000000000..ddf9e0c50b --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisRestApi.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Invocation; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +/** + * A base class for all Polaris APIs. This class assumes that the client is already authenticated, + * and therefore, possesses a valid token. + */ +public class PolarisRestApi extends RestApi { + private final PolarisApiEndpoints endpoints; + private final String authToken; + + PolarisRestApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + super(client, uri); + this.endpoints = endpoints; + this.authToken = authToken; + } + + protected Map defaultHeaders() { + Map headers = new HashMap<>(); + headers.put(endpoints.realmHeaderName(), endpoints.realmId()); + if (authToken != null) { + headers.put("Authorization", "Bearer " + authToken); + } + return headers; + } + + @Override + public Invocation.Builder request( + String path, Map templateValues, Map queryParams) { + return request(path, templateValues, queryParams, defaultHeaders()); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolicyApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolicyApi.java index 6061ef8c94..54d0204e15 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolicyApi.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolicyApi.java @@ -42,7 +42,7 @@ import org.apache.polaris.service.types.UpdatePolicyRequest; import org.assertj.core.api.Assertions; -public class PolicyApi extends RestApi { +public class PolicyApi extends PolarisRestApi { PolicyApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { super(client, endpoints, authToken, uri); } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java index 419458fe1a..13964af0ac 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java @@ -22,20 +22,15 @@ import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.client.WebTarget; import java.net.URI; -import java.util.HashMap; import java.util.Map; /** Base class for API helper classes. */ public class RestApi { private final Client client; - private final PolarisApiEndpoints endpoints; - private final String authToken; private final URI uri; - RestApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + RestApi(Client client, URI uri) { this.client = client; - this.endpoints = endpoints; - this.authToken = authToken; this.uri = uri; } @@ -47,18 +42,9 @@ public Invocation.Builder request(String path, Map templateValue return request(path, templateValues, Map.of()); } - protected Map defaultHeaders() { - Map headers = new HashMap<>(); - headers.put(endpoints.realmHeaderName(), endpoints.realmId()); - if (authToken != null) { - headers.put("Authorization", "Bearer " + authToken); - } - return headers; - } - public Invocation.Builder request( String path, Map templateValues, Map queryParams) { - return request(path, templateValues, queryParams, defaultHeaders()); + return request(path, templateValues, queryParams, Map.of()); } public Invocation.Builder request( diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java index 85e99e0c11..c23c1f6c9c 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java @@ -25,7 +25,6 @@ import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.ClientBuilder; import java.util.ServiceLoader; -import org.apache.polaris.service.it.env.IcebergTokenAccessManager; import org.apache.polaris.service.it.env.Server; import org.junit.jupiter.api.extension.ExtensionContext; @@ -51,10 +50,6 @@ public interface PolarisServerManager { */ Server serverForContext(ExtensionContext context); - default PolarisAccessManager accessManager(Client client) { - return new IcebergTokenAccessManager(client); - } - /** Create a new HTTP client for accessing the server targeted by tests. */ default Client createClient() { return ClientBuilder.newBuilder() diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java index 466af53476..922278f3ce 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java @@ -81,8 +81,8 @@ public void before( endpoints = apiEndpoints; client = polarisClient(endpoints); sparkToken = client.obtainToken(credentials); - managementApi = client.managementApi(credentials); - catalogApi = client.catalogApi(credentials); + managementApi = client.managementApi(sparkToken); + catalogApi = client.catalogApi(sparkToken); warehouseDir = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve("spark-warehouse"); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index d647893255..2e0acc5a32 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -69,7 +69,6 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.ClientPrincipal; import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.PolarisApiEndpoints; @@ -111,7 +110,6 @@ public class PolarisApplicationIntegrationTest { private static RestApi managementApi; private static PolarisApiEndpoints endpoints; private static PolarisClient client; - private static ClientCredentials clientCredentials; private static ClientPrincipal admin; private static String authToken; private static URI baseLocation; @@ -126,9 +124,8 @@ public static void setup( client = polarisClient(endpoints); realm = endpoints.realmId(); admin = adminCredentials; - clientCredentials = adminCredentials.credentials(); - authToken = client.obtainToken(clientCredentials); - managementApi = client.managementApi(clientCredentials); + authToken = client.obtainToken(adminCredentials.credentials()); + managementApi = client.managementApi(authToken); baseLocation = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve(realm + "/"); } @@ -165,7 +162,7 @@ public void before(TestInfo testInfo) { @AfterEach public void cleanUp() { - client.cleanUp(clientCredentials); + client.cleanUp(authToken); } private static void createCatalog( @@ -592,7 +589,7 @@ public void testRequestHeaderTooLarge() throws Exception { .untilAsserted( () -> { Invocation.Builder request = - localClient.managementApi(clientCredentials).request("v1/principal-roles"); + localClient.managementApi(authToken).request("v1/principal-roles"); // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 // definitely exceeds the limit for (int i = 0; i < 1500; i++) { @@ -628,7 +625,7 @@ public void testRequestBodyTooLarge() throws Exception { Entity.json(new PrincipalRole("r".repeat(1000001))); try (Response response = localClient - .managementApi(clientCredentials) + .managementApi(authToken) .request("v1/principal-roles") .post(largeRequest)) { // Note we only validate the status code here because per RFC 9110, the server MAY diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java index 1d969c6bc2..e79fe78887 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java @@ -113,12 +113,14 @@ public class PolarisManagementServiceIntegrationTest { private static ManagementApi managementApi; private static CatalogApi catalogApi; private static ClientCredentials rootCredentials; + private static String authToken; @BeforeAll public static void setup(PolarisApiEndpoints endpoints, ClientCredentials credentials) { client = polarisClient(endpoints); - managementApi = client.managementApi(credentials); - catalogApi = client.catalogApi(credentials); + authToken = client.obtainToken(credentials); + managementApi = client.managementApi(authToken); + catalogApi = client.catalogApi(authToken); rootCredentials = credentials; } @@ -129,7 +131,7 @@ public static void close() throws Exception { @AfterEach public void tearDown() { - client.cleanUp(rootCredentials); + client.cleanUp(authToken); } @Test @@ -177,7 +179,8 @@ public void testListCatalogs() { public void testListCatalogsUnauthorized() { PrincipalWithCredentials principal = managementApi.createPrincipal(client.newEntityName("a_new_user")); - try (Response response = client.managementApi(principal).request("v1/catalogs").get()) { + String authToken = client.obtainToken(principal); + try (Response response = client.managementApi(authToken).request("v1/catalogs").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -807,7 +810,8 @@ public void testCatalogRoleInvalidName() { public void testListPrincipalsUnauthorized() { PrincipalWithCredentials principal = managementApi.createPrincipal(client.newEntityName("new_admin")); - try (Response response = client.managementApi(principal).request("v1/principals").get()) { + String authToken = client.obtainToken(principal); + try (Response response = client.managementApi(authToken).request("v1/principals").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java index 415f41806c..6f987e2b68 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java @@ -119,8 +119,7 @@ public class PolarisPolicyServiceIntegrationTest { private static URI s3BucketBase; private static String principalRoleName; - private static ClientCredentials adminCredentials; - private static PrincipalWithCredentials principalCredentials; + private static String adminToken; private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; @@ -155,17 +154,19 @@ String[] properties() default { @BeforeAll public static void setup( PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) { - adminCredentials = credentials; endpoints = apiEndpoints; client = polarisClient(endpoints); - managementApi = client.managementApi(credentials); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); String principalName = client.newEntityName("snowman-rest"); principalRoleName = client.newEntityName("rest-admin"); - principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + PrincipalWithCredentials principalCredentials = + managementApi.createPrincipalWithRole(principalName, principalRoleName); URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir); s3BucketBase = testRootUri.resolve("my-bucket"); - policyApi = client.policyApi(principalCredentials); + String principalToken = client.obtainToken(principalCredentials); + policyApi = client.policyApi(principalToken); } @AfterAll @@ -236,13 +237,10 @@ public void before(TestInfo testInfo) { } }); + String principalToken = client.obtainToken(principalCredentials); restCatalog = IcebergHelper.restCatalog( - client, - endpoints, - principalCredentials, - currentCatalogName, - extraPropertiesBuilder.build()); + endpoints, currentCatalogName, extraPropertiesBuilder.build(), principalToken); CatalogGrant catalogGrant = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); managementApi.createCatalogRole(currentCatalogName, CATALOG_ROLE_1); @@ -251,12 +249,12 @@ public void before(TestInfo testInfo) { managementApi.grantCatalogRoleToPrincipalRole( principalRoleName, currentCatalogName, catalogRole); - policyApi = client.policyApi(principalCredentials); + policyApi = client.policyApi(principalToken); } @AfterEach public void cleanUp() { - client.cleanUp(adminCredentials); + client.cleanUp(adminToken); } @Test diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java index 78817fe926..469ca37ce4 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java @@ -94,6 +94,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.service.it.env.CatalogApi; import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ClientPrincipal; import org.apache.polaris.service.it.env.GenericTableApi; import org.apache.polaris.service.it.env.IcebergHelper; import org.apache.polaris.service.it.env.ManagementApi; @@ -135,12 +136,14 @@ public abstract class PolarisRestCatalogIntegrationBase extends CatalogTestsBy default, this method uses the {@link PolarisClient} to obtain the token from the Polaris + * internal OAuth2 token endpoint. + * + *

    Subclasses can override this method to customize the token acquisition process, e.g. when + * using external identity providers. + */ + protected String obtainToken(PolarisClient client, ClientPrincipal principal) { + return client.obtainToken(principal); + } + @AfterEach public void cleanUp() { - client.cleanUp(adminCredentials); + cleanUp(client, adminToken); + } + + /** + * Cleans up the Polaris environment after each test. + * + *

    Subclasses can override this method to perform additional cleanup actions. + */ + protected void cleanUp(PolarisClient client, String adminToken) { + client.cleanUp(adminToken); } @Override @@ -309,11 +352,7 @@ protected RESTCatalog initCatalog(String catalogName, Map additi extraPropertiesBuilder.putAll(restCatalogConfig); extraPropertiesBuilder.putAll(additionalProperties); return IcebergHelper.restCatalog( - client, - endpoints, - principalCredentials, - currentCatalogName, - extraPropertiesBuilder.buildKeepingLast()); + endpoints, currentCatalogName, extraPropertiesBuilder.buildKeepingLast(), principalToken); } @Override diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java index 25fac69ac1..8cf61a5c44 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -67,6 +67,7 @@ */ @ExtendWith(PolarisIntegrationTestExtension.class) public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogTests { + static { Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5); } @@ -81,7 +82,7 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT org.apache.iceberg.CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4"); - private static ClientCredentials adminCredentials; + private static String adminToken; private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; @@ -90,10 +91,10 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT @BeforeAll static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { - adminCredentials = credentials; endpoints = apiEndpoints; client = polarisClient(endpoints); - managementApi = client.managementApi(credentials); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); } @AfterAll @@ -140,12 +141,15 @@ public void before(TestInfo testInfo) { restCatalog = IcebergHelper.restCatalog( - client, endpoints, principalCredentials, catalogName, DEFAULT_REST_CATALOG_CONFIG); + endpoints, + catalogName, + DEFAULT_REST_CATALOG_CONFIG, + client.obtainToken(principalCredentials)); } @AfterEach public void cleanUp() { - client.cleanUp(adminCredentials); + client.cleanUp(adminToken); } /** diff --git a/runtime/defaults/src/main/resources/application-test.properties b/runtime/defaults/src/main/resources/application-test.properties index 3ba01dfb03..997b12d51e 100644 --- a/runtime/defaults/src/main/resources/application-test.properties +++ b/runtime/defaults/src/main/resources/application-test.properties @@ -21,16 +21,19 @@ # Per-test specific configuration should use QuarkusTestProfile quarkus.datasource.devservices.enabled=false +quarkus.keycloak.devservices.enabled=false quarkus.log.level=ERROR quarkus.log.file.enable=false quarkus.console.color=true # Useful loggers for debugging purposes. -# quarkus.log.category."org.apache.polaris".level=INFO +quarkus.log.category."org.apache.polaris".level=DEBUG # quarkus.log.category."org.apache.iceberg".level=INFO # quarkus.log.category."io.quarkus.http.access-log".level=INFO # quarkus.log.category."org.apache.hc.client5.http".level=INFO +# quarkus.log.category."org.apache.polaris.test.commons.keycloak".level=DEBUG +quarkus.log.category."io.quarkus.oidc".level=DEBUG # Silence a few verbose loggers that are not useful for unit tests. quarkus.log.category."org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl".level=ERROR diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index ad650f5862..699b0a8c88 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -184,6 +184,10 @@ dependencies { testFixturesImplementation("com.azure:azure-core") testFixturesImplementation("com.azure:azure-storage-blob") testFixturesImplementation("com.azure:azure-storage-file-datalake") + + // This dependency brings in RESTEasy Classic, which conflicts with Quarkus RESTEasy Reactive; + // it must not be present during Quarkus augmentation otherwise Quarkus tests won't start. + intTestRuntimeOnly(libs.keycloak.admin.client) } tasks.named("javadoc") { dependsOn("jandex") } diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java new file mode 100644 index 0000000000..9f7bbd7626 --- /dev/null +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import java.util.Map; +import org.apache.polaris.service.it.env.ClientPrincipal; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.test.PolarisRestCatalogFileIntegrationTest; +import org.apache.polaris.test.commons.keycloak.KeycloakAccess; +import org.apache.polaris.test.commons.keycloak.KeycloakProfile; + +@QuarkusIntegrationTest +@TestProfile(KeycloakProfile.class) +public class RestCatalogKeycloakFileIT extends PolarisRestCatalogFileIntegrationTest { + + KeycloakAccess keycloak; + + @Override + protected ClientPrincipal createTestPrincipal( + PolarisClient client, String principalName, String principalRole) { + ClientPrincipal principal = super.createTestPrincipal(client, principalName, principalRole); + keycloak.createRole(principalRole); + keycloak.createUser( + principalName, + // Use the same password as the client secret + principal.credentials().clientSecret()); + keycloak.assignRoleToUser(principalRole, principalName); + keycloak.createServiceAccount( + principal.credentials().clientId(), principal.credentials().clientSecret()); + return principal; + } + + @Override + protected String obtainToken(PolarisClient client, ClientPrincipal principal) { + // Use password grant type to obtain a token that is tied to the principal user, + // not just to the service account. + Map request = + Map.of( + "grant_type", "password", + "client_id", principal.credentials().clientId(), + "client_secret", principal.credentials().clientSecret(), + "username", principal.principalName(), + "password", principal.credentials().clientSecret()); + return client.obtainToken(keycloak.getIssuerUrl(), keycloak.getTokenPath(), request); + } + + @Override + protected void cleanUp(PolarisClient client, String adminToken) { + ManagementApi managementApi = client.managementApi(adminToken); + managementApi.listPrincipals().stream() + .filter(p -> client.ownedName(p.getName())) + .forEach( + p -> { + keycloak.deleteUser(p.getName()); + keycloak.deleteServiceAccount(p.getClientId()); + }); + managementApi.listPrincipalRoles().stream() + .filter(r -> client.ownedName(r.getName())) + .forEach(role -> keycloak.deleteRole(role.getName())); + super.cleanUp(client, adminToken); + } +} diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java index 726faa1639..6a93da886e 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java @@ -92,6 +92,7 @@ public class RestCatalogMinIOSpecialIT { private static final String BUCKET_URI_PREFIX = "/minio-test"; private static final String MINIO_ACCESS_KEY = "test-ak-123"; private static final String MINIO_SECRET_KEY = "test-sk-123"; + private static String adminToken; public static class Profile implements QuarkusTestProfile { @@ -132,7 +133,8 @@ static void setup( adminCredentials = credentials; endpoints = apiEndpoints; client = polarisClient(endpoints); - managementApi = client.managementApi(credentials); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); endpoint = minioAccess.s3endpoint(); } @@ -148,7 +150,8 @@ public void before(TestInfo testInfo) { principalRoleName = client.newEntityName("test-admin"); principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); - catalogApi = client.catalogApi(principalCredentials); + String principalToken = client.obtainToken(principalCredentials); + catalogApi = client.catalogApi(principalToken); catalogName = client.newEntityName(testInfo.getTestMethod().orElseThrow().getName()); } @@ -203,7 +206,7 @@ private RESTCatalog createCatalog( @AfterEach public void cleanUp() { - client.cleanUp(adminCredentials); + client.cleanUp(adminToken); } @ParameterizedTest diff --git a/runtime/test-common/build.gradle.kts b/runtime/test-common/build.gradle.kts index 2772ae6262..51433665aa 100644 --- a/runtime/test-common/build.gradle.kts +++ b/runtime/test-common/build.gradle.kts @@ -32,10 +32,21 @@ configurations.all { } dependencies { + implementation(project(":polaris-core")) + implementation(libs.jakarta.ws.rs.api) implementation(enforcedPlatform(libs.quarkus.bom)) implementation("io.quarkus:quarkus-junit5") + implementation(platform(libs.testcontainers.bom)) implementation("org.testcontainers:testcontainers") implementation("org.testcontainers:postgresql") + + implementation(libs.testcontainers.keycloak) { + exclude(group = "org.keycloak", module = "keycloak-admin-client") + } + // Keycloak Admin Client brings RESTEasy Classic, which conflicts with Quarkus RESTEasy Reactive; + // it must not be present during Quarkus augmentation otherwise Quarkus tests won't start. + compileOnly(libs.keycloak.admin.client) + implementation(project(":polaris-container-spec-helper")) } diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java new file mode 100644 index 0000000000..d3a1d3539c --- /dev/null +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.test.commons.keycloak; + +import java.net.URI; + +/** A facade interface for accessing Keycloak server functionalities. */ +public interface KeycloakAccess { + + String PRINCIPAL_NAME_CLAIM = "preferred_username"; + + /** + * Returns the URL of the Keycloak issuer. This is typically {@code + * https:///realms/}. + */ + URI getIssuerUrl(); + + /** + * Returns the URL of the Keycloak token endpoint. This is typically {@code + * https:///realms//protocol/openid-connect/token}. + */ + URI getTokenEndpoint(); + + /** + * Returns the path of the Keycloak token endpoint. This is typically {@code + * /realms//protocol/openid-connect/token}. + */ + default String getTokenPath() { + return getIssuerUrl().relativize(getTokenEndpoint()).getPath(); + } + + /** + * Creates a new role in Keycloak with the specified name. The role should not have the {@code + * PRINCIPAL_ROLE:} prefix. + */ + void createRole(String name); + + /** + * Creates a new user in Keycloak with the specified name, password, and assigns the given role to + * them. + */ + void createUser(String name, String password); + + /** + * Assigns a role to a user in Keycloak. The role should not have the {@code PRINCIPAL_ROLE:} + * prefix. Both the role and the user must exist. + */ + void assignRoleToUser(String role, String user); + + /** Creates a new service account in Keycloak with the specified client ID and client secret. */ + void createServiceAccount(String clientId, String clientSecret); + + /** + * Deletes a role in Keycloak with the specified name. The role should not have the {@code + * PRINCIPAL_ROLE:} prefix. + */ + void deleteRole(String name); + + /** Deletes a user in Keycloak with the specified name. */ + void deleteUser(String name); + + /** Deletes a service account in Keycloak with the specified client ID. */ + void deleteServiceAccount(String clientId); +} diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java new file mode 100644 index 0000000000..195dd56e81 --- /dev/null +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.test.commons.keycloak; + +import com.google.common.base.Preconditions; +import dasniko.testcontainers.keycloak.ExtendableKeycloakContainer; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import org.apache.polaris.containerspec.ContainerSpecHelper; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.keycloak.admin.client.Keycloak; +import org.keycloak.admin.client.resource.RealmResource; +import org.keycloak.admin.client.resource.UserResource; +import org.keycloak.representations.idm.ClientRepresentation; +import org.keycloak.representations.idm.CredentialRepresentation; +import org.keycloak.representations.idm.RoleRepresentation; +import org.keycloak.representations.idm.UserRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.output.Slf4jLogConsumer; + +public class KeycloakContainer extends ExtendableKeycloakContainer + implements KeycloakAccess { + + private static final Logger LOGGER = LoggerFactory.getLogger(KeycloakContainer.class); + + private static final String CONTEXT_PATH = "/realms/master/"; + + private URI issuerUrl; + private URI tokenEndpoint; + private Keycloak keycloakAdminClient; + + @SuppressWarnings("resource") + public KeycloakContainer() { + super( + ContainerSpecHelper.containerSpecHelper("keycloak", KeycloakContainer.class) + .dockerImageName(null) + .asCanonicalNameString()); + withLogConsumer(new Slf4jLogConsumer(LOGGER)); + withEnv("KC_LOG_LEVEL", getRootLoggerLevel() + ",org.keycloak:" + getKeycloakLoggerLevel()); + } + + @Override + public void start() { + super.start(); + keycloakAdminClient = getKeycloakAdminClient(); + URI rootUrl = URI.create(getAuthServerUrl()); + issuerUrl = rootUrl.resolve(CONTEXT_PATH); + tokenEndpoint = issuerUrl.resolve("protocol/openid-connect/token"); + createRole("ALL"); + RootCredentialsSet.fromEnvironment().credentials().values().stream() + .findFirst() + .ifPresent( + creds -> { + createUser("root", creds.clientSecret()); + createServiceAccount(creds.clientId(), creds.clientSecret()); + }); + } + + @Override + public void stop() { + super.stop(); + if (keycloakAdminClient != null) { + keycloakAdminClient.close(); + keycloakAdminClient = null; + } + } + + @Override + public URI getIssuerUrl() { + return issuerUrl; + } + + @Override + public URI getTokenEndpoint() { + return tokenEndpoint; + } + + @Override + public void createRole(String roleName) { + RealmResource master = keycloakAdminClient.realms().realm("master"); + RoleRepresentation role = new RoleRepresentation(); + role.setName("PRINCIPAL_ROLE:" + roleName); + master.roles().create(role); + } + + @Override + public void createUser(String name, String password) { + RealmResource master = keycloakAdminClient.realms().realm("master"); + UserRepresentation user = new UserRepresentation(); + user.setEnabled(true); + user.setUsername(name); + user.setFirstName(name); + user.setLastName(name); + user.setEmail(name + "@polaris.local"); + user.setEmailVerified(true); + user.setRequiredActions(List.of()); + CredentialRepresentation credential = new CredentialRepresentation(); + credential.setType(CredentialRepresentation.PASSWORD); + credential.setValue(password); + credential.setTemporary(false); + user.setCredentials(List.of(credential)); + try (Response response = master.users().create(user)) { + Preconditions.checkState( + response.getStatus() == 201, "Failed to create user, status: " + response.getStatus()); + } + } + + @Override + public void assignRoleToUser(String role, String user) { + RealmResource master = keycloakAdminClient.realms().realm("master"); + List users = master.users().search(user); + UserResource userResource = master.users().get(users.getFirst().getId()); + RoleRepresentation roleRepresentation = + master.roles().get("PRINCIPAL_ROLE:" + role).toRepresentation(); + userResource.roles().realmLevel().add(List.of(roleRepresentation)); + } + + @Override + public void createServiceAccount(String clientId, String clientSecret) { + RealmResource master = keycloakAdminClient.realms().realm("master"); + ClientRepresentation client = new ClientRepresentation(); + client.setClientId(clientId); + client.setSecret(clientSecret); + client.setPublicClient(false); + client.setServiceAccountsEnabled(true); + client.setDirectAccessGrantsEnabled(true); // required for password grant + try (Response response = master.clients().create(client)) { + Preconditions.checkState( + response.getStatus() == 201, "Failed to create client, status: " + response.getStatus()); + } + } + + @Override + public void deleteRole(String name) { + RealmResource master = keycloakAdminClient.realms().realm("master"); + master.roles().deleteRole("PRINCIPAL_ROLE:" + name); + } + + @Override + public void deleteUser(String name) { + RealmResource master = keycloakAdminClient.realms().realm("master"); + List users = master.users().search(name); + for (UserRepresentation user : users) { + master.users().get(user.getId()).remove(); + } + } + + @Override + public void deleteServiceAccount(String clientId) { + RealmResource master = keycloakAdminClient.realms().realm("master"); + List clients = master.clients().findByClientId(clientId); + for (ClientRepresentation client : clients) { + master.clients().get(client.getId()).remove(); + } + } + + private static String getRootLoggerLevel() { + return LOGGER.isInfoEnabled() ? "INFO" : LOGGER.isWarnEnabled() ? "WARN" : "ERROR"; + } + + private static String getKeycloakLoggerLevel() { + return LOGGER.isDebugEnabled() ? "DEBUG" : getRootLoggerLevel(); + } +} diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakLifecycleManager.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakLifecycleManager.java new file mode 100644 index 0000000000..e29642fdae --- /dev/null +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakLifecycleManager.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.test.commons.keycloak; + +import io.quarkus.test.common.DevServicesContext; +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; + +public class KeycloakLifecycleManager + implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { + + private KeycloakContainer keycloak; + private DevServicesContext context; + + @Override + public void setIntegrationTestContext(DevServicesContext context) { + this.context = context; + } + + @Override + public void inject(TestInjector testInjector) { + testInjector.injectIntoFields(keycloak, new TestInjector.MatchesType(KeycloakAccess.class)); + } + + @Override + public Map start() { + keycloak = new KeycloakContainer(); + context.containerNetworkId().ifPresent(keycloak::withNetworkMode); + keycloak.start(); + return Map.of("quarkus.oidc.auth-server-url", keycloak.getIssuerUrl().toString()); + } + + @Override + public void stop() { + KeycloakContainer k = keycloak; + try (k) { + } finally { + keycloak = null; + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisAccessManager.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakProfile.java similarity index 50% rename from integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisAccessManager.java rename to runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakProfile.java index 2104665bda..ac31d086ce 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisAccessManager.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakProfile.java @@ -16,12 +16,31 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.it.ext; +package org.apache.polaris.test.commons.keycloak; -import org.apache.polaris.service.it.env.ClientCredentials; -import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.List; +import java.util.Map; -public interface PolarisAccessManager { +public class KeycloakProfile implements QuarkusTestProfile { - String obtainAccessToken(PolarisApiEndpoints endpoints, ClientCredentials credentials); + @Override + public Map getConfigOverrides() { + return Map.of( + "quarkus.oidc.tenant-enabled", + "true", + "quarkus.oidc.client-id", + "polaris", + "polaris.authentication.type", + "external", + "polaris.oidc.principal-mapper.name-claim-path", + KeycloakAccess.PRINCIPAL_NAME_CLAIM, + "polaris.oidc.principal-roles-mapper.filter", + "PRINCIPAL_ROLE:.*"); + } + + @Override + public List testResources() { + return List.of(new TestResourceEntry(KeycloakLifecycleManager.class, Map.of())); + } } diff --git a/runtime/test-common/src/main/resources/org/apache/polaris/test/commons/keycloak/Dockerfile-keycloak-version b/runtime/test-common/src/main/resources/org/apache/polaris/test/commons/keycloak/Dockerfile-keycloak-version new file mode 100644 index 0000000000..d6af8d4f64 --- /dev/null +++ b/runtime/test-common/src/main/resources/org/apache/polaris/test/commons/keycloak/Dockerfile-keycloak-version @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM keycloak/keycloak:26.2.5 From 845537bec77ed9c38937f640713c3e3ac9d055fa Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Thu, 14 Aug 2025 12:04:01 +0200 Subject: [PATCH 2/3] review + nits --- .../main/resources/application-test.properties | 4 ++-- .../service/it/RestCatalogKeycloakFileIT.java | 7 ++----- .../test/commons/keycloak/KeycloakAccess.java | 11 ++++++----- .../test/commons/keycloak/KeycloakContainer.java | 16 +++++++--------- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/runtime/defaults/src/main/resources/application-test.properties b/runtime/defaults/src/main/resources/application-test.properties index 997b12d51e..0e6aecedfd 100644 --- a/runtime/defaults/src/main/resources/application-test.properties +++ b/runtime/defaults/src/main/resources/application-test.properties @@ -28,12 +28,12 @@ quarkus.log.file.enable=false quarkus.console.color=true # Useful loggers for debugging purposes. -quarkus.log.category."org.apache.polaris".level=DEBUG +# quarkus.log.category."org.apache.polaris".level=INFO # quarkus.log.category."org.apache.iceberg".level=INFO # quarkus.log.category."io.quarkus.http.access-log".level=INFO # quarkus.log.category."org.apache.hc.client5.http".level=INFO # quarkus.log.category."org.apache.polaris.test.commons.keycloak".level=DEBUG -quarkus.log.category."io.quarkus.oidc".level=DEBUG +# quarkus.log.category."io.quarkus.oidc".level=DEBUG # Silence a few verbose loggers that are not useful for unit tests. quarkus.log.category."org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl".level=ERROR diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java index 9f7bbd7626..842ed4ffbe 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogKeycloakFileIT.java @@ -39,10 +39,7 @@ protected ClientPrincipal createTestPrincipal( PolarisClient client, String principalName, String principalRole) { ClientPrincipal principal = super.createTestPrincipal(client, principalName, principalRole); keycloak.createRole(principalRole); - keycloak.createUser( - principalName, - // Use the same password as the client secret - principal.credentials().clientSecret()); + keycloak.createUser(principalName); keycloak.assignRoleToUser(principalRole, principalName); keycloak.createServiceAccount( principal.credentials().clientId(), principal.credentials().clientSecret()); @@ -59,7 +56,7 @@ protected String obtainToken(PolarisClient client, ClientPrincipal principal) { "client_id", principal.credentials().clientId(), "client_secret", principal.credentials().clientSecret(), "username", principal.principalName(), - "password", principal.credentials().clientSecret()); + "password", KeycloakAccess.USER_PASSWORD); return client.obtainToken(keycloak.getIssuerUrl(), keycloak.getTokenPath(), request); } diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java index d3a1d3539c..c1c67b4e30 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakAccess.java @@ -24,8 +24,12 @@ /** A facade interface for accessing Keycloak server functionalities. */ public interface KeycloakAccess { + /** The claim name used to identify the principal in Keycloak tokens. */ String PRINCIPAL_NAME_CLAIM = "preferred_username"; + /** The password used for all users in Keycloak. */ + String USER_PASSWORD = "s3cr3t"; + /** * Returns the URL of the Keycloak issuer. This is typically {@code * https:///realms/}. @@ -52,11 +56,8 @@ default String getTokenPath() { */ void createRole(String name); - /** - * Creates a new user in Keycloak with the specified name, password, and assigns the given role to - * them. - */ - void createUser(String name, String password); + /** Creates a new user in Keycloak. The password is always {@value #USER_PASSWORD} */ + void createUser(String name); /** * Assigns a role to a user in Keycloak. The role should not have the {@code PRINCIPAL_ROLE:} diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java index 195dd56e81..672e11cf81 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java @@ -65,13 +65,11 @@ public void start() { issuerUrl = rootUrl.resolve(CONTEXT_PATH); tokenEndpoint = issuerUrl.resolve("protocol/openid-connect/token"); createRole("ALL"); - RootCredentialsSet.fromEnvironment().credentials().values().stream() - .findFirst() - .ifPresent( - creds -> { - createUser("root", creds.clientSecret()); - createServiceAccount(creds.clientId(), creds.clientSecret()); - }); + createUser("root"); + RootCredentialsSet.fromEnvironment() + .credentials() + .values() + .forEach(creds -> createServiceAccount(creds.clientId(), creds.clientSecret())); } @Override @@ -102,7 +100,7 @@ public void createRole(String roleName) { } @Override - public void createUser(String name, String password) { + public void createUser(String name) { RealmResource master = keycloakAdminClient.realms().realm("master"); UserRepresentation user = new UserRepresentation(); user.setEnabled(true); @@ -114,7 +112,7 @@ public void createUser(String name, String password) { user.setRequiredActions(List.of()); CredentialRepresentation credential = new CredentialRepresentation(); credential.setType(CredentialRepresentation.PASSWORD); - credential.setValue(password); + credential.setValue(KeycloakAccess.USER_PASSWORD); credential.setTemporary(false); user.setCredentials(List.of(credential)); try (Response response = master.users().create(user)) { From 22241becdc5886d0d88d617883485e61f8d43e54 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Thu, 14 Aug 2025 12:11:56 +0200 Subject: [PATCH 3/3] nit --- .../apache/polaris/test/commons/keycloak/KeycloakContainer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java index 672e11cf81..9e75a04afd 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/keycloak/KeycloakContainer.java @@ -66,6 +66,7 @@ public void start() { tokenEndpoint = issuerUrl.resolve("protocol/openid-connect/token"); createRole("ALL"); createUser("root"); + assignRoleToUser("ALL", "root"); RootCredentialsSet.fromEnvironment() .credentials() .values()