Add integration tests with Keycloak

gradle/libs.versions.toml

@@ -72,6 +72,7 @@ jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api",
 jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" }
 javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" }
 junit-bom = { module = "org.junit:junit-bom", version = "5.13.4" }
+keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version = "26.0.6" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" }
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.15.3" }
 microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" }
@@ -94,6 +95,7 @@ spark35-sql-scala212 = { module = "org.apache.spark:spark-sql_2.12", version.ref
 swagger-annotations = { module = "io.swagger:swagger-annotations", version.ref = "swagger" }
 swagger-jaxrs = { module = "io.swagger:swagger-jaxrs", version.ref = "swagger" }
 testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version = "1.21.3" }
+testcontainers-keycloak = { module = "com.github.dasniko:testcontainers-keycloak", version = "3.8.0" }
 threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" }
 
 [plugins]

integration-tests/build.gradle.kts

@@ -24,6 +24,7 @@ dependencies {
   implementation(project(":polaris-api-management-model"))
   implementation(project(":polaris-api-catalog-service"))
 
+  implementation(libs.jakarta.annotation.api)
   implementation(libs.jakarta.ws.rs.api)
   implementation(libs.guava)
 

integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java

@@ -25,7 +25,6 @@
 import com.google.common.base.Joiner;
 import jakarta.ws.rs.client.Client;
 import jakarta.ws.rs.client.Entity;
-import jakarta.ws.rs.core.MultivaluedHashMap;
 import jakarta.ws.rs.core.Response;
 import java.net.URI;
 import java.util.ArrayList;
@@ -42,38 +41,17 @@
 import org.apache.iceberg.rest.responses.ListNamespacesResponse;
 import org.apache.iceberg.rest.responses.ListTablesResponse;
 import org.apache.iceberg.rest.responses.LoadTableResponse;
-import org.apache.iceberg.rest.responses.OAuthTokenResponse;
 
 /**
  * A simple, non-exhaustive set of helper methods for accessing the Iceberg REST API.
  *
- * @see PolarisClient#catalogApi(ClientCredentials)
+ * @see PolarisClient#catalogApi(String)
  */
-public class CatalogApi extends RestApi {
+public class CatalogApi extends PolarisRestApi {
   public CatalogApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) {
     super(client, endpoints, authToken, uri);
   }
 
-  public String obtainToken(ClientCredentials credentials) {
-    try (Response response =
-        request("v1/oauth/tokens")
-            .post(
-                Entity.form(
-                    new MultivaluedHashMap<>(
-                        Map.of(
-                            "grant_type",
-                            "client_credentials",
-                            "scope",
-                            "PRINCIPAL_ROLE:ALL",
-                            "client_id",
-                            credentials.clientId(),
-                            "client_secret",
-                            credentials.clientSecret()))))) {
-      assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
-      return response.readEntity(OAuthTokenResponse.class).token();
-    }
-  }
-
   public void createNamespace(String catalogName, String namespaceName) {
     try (Response response =
         request("v1/{cat}/namespaces", Map.of("cat", catalogName))

integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java

@@ -18,11 +18,21 @@
  */
 package org.apache.polaris.service.it.env;
 
+import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials;
 import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
 
 /**
  * This class holds credentials for accessing the test Polaris Server. An instance of this class
  * representing an admin user is injected into test parameters by {@link
  * PolarisIntegrationTestExtension}.
  */
-public record ClientCredentials(String clientId, String clientSecret) {}
+public record ClientCredentials(String clientId, String clientSecret) {
+
+  /**
+   * Creates a {@link ClientCredentials} from an instance of the Admin API model {@link
+   * PrincipalWithCredentialsCredentials}.
+   */
+  public ClientCredentials(PrincipalWithCredentialsCredentials credentials) {
+    this(credentials.getClientId(), credentials.getClientSecret());
+  }
+}

integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java

@@ -18,6 +18,7 @@
  */
 package org.apache.polaris.service.it.env;
 
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
 import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
 
 /**
@@ -27,4 +28,13 @@
  *
  * @see Server#adminCredentials()
  */
-public record ClientPrincipal(String principalName, ClientCredentials credentials) {}
+public record ClientPrincipal(String principalName, ClientCredentials credentials) {
+
+  /**
+   * Creates a {@link ClientPrincipal} from an instance of the Admin API model {@link
+   * PrincipalWithCredentials}.
+   */
+  public ClientPrincipal(PrincipalWithCredentials principal) {
+    this(principal.getPrincipal().getName(), new ClientCredentials(principal.getCredentials()));
+  }
+}

integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java

@@ -38,9 +38,9 @@
 /**
  * A simple, non-exhaustive set of helper methods for accessing the generic tables REST API
  *
- * @see PolarisClient#genericTableApi(ClientCredentials)
+ * @see PolarisClient#genericTableApi(String)
  */
-public class GenericTableApi extends RestApi {
+public class GenericTableApi extends PolarisRestApi {
   GenericTableApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) {
     super(client, endpoints, authToken, uri);
   }

integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java

@@ -22,18 +22,15 @@
 import java.util.Map;
 import org.apache.iceberg.rest.RESTCatalog;
 import org.apache.iceberg.rest.auth.OAuth2Properties;
-import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
 
 public final class IcebergHelper {
   private IcebergHelper() {}
 
   public static RESTCatalog restCatalog(
-      PolarisClient client,
       PolarisApiEndpoints endpoints,
-      PrincipalWithCredentials credentials,
       String catalog,
-      Map<String, String> extraProperties) {
-    String authToken = client.obtainToken(credentials);
+      Map<String, String> extraProperties,
+      String authToken) {
     RESTCatalog restCatalog = new RESTCatalog();
 
     ImmutableMap.Builder<String, String> propertiesBuilder =

integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java

@@ -51,9 +51,9 @@
 /**
  * A simple, non-exhaustive set of helper methods for accessing the Polaris Management API.
  *
- * @see PolarisClient#managementApi(ClientCredentials)
+ * @see PolarisClient#managementApi(String)
  */
-public class ManagementApi extends RestApi {
+public class ManagementApi extends PolarisRestApi {
   public ManagementApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) {
     super(client, endpoints, authToken, uri);
   }

integration-tests/src/main/java/org/apache/polaris/service/it/env/OAuth2Api.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.env;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.core.MultivaluedHashMap;
+import jakarta.ws.rs.core.Response;
+import java.net.URI;
+import java.util.Map;
+import org.apache.iceberg.rest.responses.OAuthTokenResponse;
+
+/**
+ * A simple facade to an OAuth2 token endpoint. It works with both Polaris internal token endpoint
+ * and with external identity providers.
+ */
+public class OAuth2Api extends RestApi {
+
+  private final String endpointPath;
+
+  public OAuth2Api(Client client, URI issuerUrl, String endpointPath) {
+    super(client, issuerUrl);
+    this.endpointPath = endpointPath;
+  }
+
+  public String obtainAccessToken(ClientCredentials credentials, String scope) {
+    return obtainAccessToken(
+        Map.of(
+            "grant_type",
+            "client_credentials",
+            "client_id",
+            credentials.clientId(),
+            "client_secret",
+            credentials.clientSecret(),
+            "scope",
+            scope));
+  }
+
+  public String obtainAccessToken(Map<String, String> requestBody) {
+    try (Response response =
+        request(endpointPath).post(Entity.form(new MultivaluedHashMap<>(requestBody)))) {
+      assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+      String token = response.readEntity(OAuthTokenResponse.class).token();
+      return token;
+    }
+  }
+}

integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java

@@ -26,6 +26,8 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 import jakarta.ws.rs.client.Client;
+import java.net.URI;
+import java.util.Map;
 import java.util.Random;
 import org.apache.iceberg.rest.RESTSerializers;
 import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
@@ -73,7 +75,7 @@ public static ObjectMapper buildObjectMapper() {
   /**
    * This method should be used by test code to make top-level entity names. The purpose of this
    * method is two-fold:
-   * <li>Identify top-level entities for latger clean-up by {@link #cleanUp(ClientCredentials)}.
+   * <li>Identify top-level entities for later clean-up by {@link #cleanUp(String)}.
    * <li>Allow {@link PolarisServerManager}s to customize top-level entities per environment.
    */
   public String newEntityName(String hint) {
@@ -84,70 +86,53 @@ public ManagementApi managementApi(String authToken) {
     return new ManagementApi(client, endpoints, authToken, endpoints.managementApiEndpoint());
   }
 
-  public ManagementApi managementApi(ClientCredentials credentials) {
-    return managementApi(obtainToken(credentials));
-  }
-
-  public ManagementApi managementApi(PrincipalWithCredentials principal) {
-    return managementApi(obtainToken(principal));
-  }
-
-  public CatalogApi catalogApi(PrincipalWithCredentials principal) {
-    return new CatalogApi(
-        client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint());
-  }
-
-  public CatalogApi catalogApi(ClientCredentials credentials) {
-    return new CatalogApi(
-        client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint());
+  public CatalogApi catalogApi(String authToken) {
+    return new CatalogApi(client, endpoints, authToken, endpoints.catalogApiEndpoint());
   }
 
   public CatalogApi catalogApiPlain() {
     return new CatalogApi(client, endpoints, null, endpoints.catalogApiEndpoint());
   }
 
-  public GenericTableApi genericTableApi(PrincipalWithCredentials principal) {
-    return new GenericTableApi(
-        client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint());
+  public GenericTableApi genericTableApi(String authToken) {
+    return new GenericTableApi(client, endpoints, authToken, endpoints.catalogApiEndpoint());
   }
 
-  public GenericTableApi genericTableApi(ClientCredentials credentials) {
-    return new GenericTableApi(
-        client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint());
+  public PolicyApi policyApi(String authToken) {
+    return new PolicyApi(client, endpoints, authToken, endpoints.catalogApiEndpoint());
   }
 
-  public PolicyApi policyApi(PrincipalWithCredentials principal) {
-    return new PolicyApi(client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint());
+  /** Requests an access token from the Polaris server for the given principal. */
+  public String obtainToken(PrincipalWithCredentials credentials) {
+    return obtainToken(new ClientPrincipal(credentials));
   }
 
-  public PolicyApi policyApi(ClientCredentials credentials) {
-    return new PolicyApi(
-        client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint());
+  /** Requests an access token from the Polaris server for the given principal. */
+  public String obtainToken(ClientPrincipal principal) {
+    return obtainToken(principal.credentials());
   }
 
-  /**
-   * Requests an access token from the Polaris server for the client ID/secret pair that is part of
-   * the given principal data object.
-   */
-  public String obtainToken(PrincipalWithCredentials principal) {
-    return obtainToken(
-        new ClientCredentials(
-            principal.getCredentials().getClientId(),
-            principal.getCredentials().getClientSecret()));
+  /** Requests an access token from the Polaris server for the given credentials. */
+  public String obtainToken(ClientCredentials credentials) {
+    OAuth2Api api = new OAuth2Api(client, endpoints.catalogApiEndpoint(), "v1/oauth/tokens");
+    return api.obtainAccessToken(credentials, "PRINCIPAL_ROLE:ALL");
   }
 
-  /** Requests an access token from the Polaris server for the given {@link ClientCredentials}. */
-  public String obtainToken(ClientCredentials credentials) {
-    return polarisServerManager().accessManager(client).obtainAccessToken(endpoints, credentials);
+  /**
+   * Requests an access token from the authorization server denoted by the issuer URL and token
+   * endpoint path.
+   */
+  public String obtainToken(URI issuerUrl, String endpointPath, Map<String, String> requestBody) {
+    return new OAuth2Api(client, issuerUrl, endpointPath).obtainAccessToken(requestBody);
   }
 
-  private boolean ownedName(String name) {
+  public boolean ownedName(String name) {
     return name != null && name.contains(clientId);
   }
 
-  public void cleanUp(ClientCredentials credentials) {
-    ManagementApi managementApi = managementApi(credentials);
-    CatalogApi catalogApi = catalogApi(credentials);
+  public void cleanUp(String authToken) {
+    ManagementApi managementApi = managementApi(authToken);
+    CatalogApi catalogApi = catalogApi(authToken);
 
     managementApi.listCatalogs().stream()
         .filter(c -> ownedName(c.getName()))