From d7d47570f31b4b04f0b7a1b0957061c911a50c0e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 18 Aug 2025 13:21:19 -0700 Subject: [PATCH 1/4] refactor complete --- .../hadoop/HadoopFederatedCatalogFactory.java | 7 +++ .../hive/HiveFederatedCatalogFactory.java | 7 +++ .../core/catalog/ExternalCatalogFactory.java | 11 ++++ .../core/catalog}/GenericTableCatalog.java | 2 +- .../catalog/common/CatalogHandler.java | 15 ++++- .../generic/GenericTableCatalogHandler.java | 59 +++++++++++++++++-- .../generic/PolarisGenericTableCatalog.java | 1 + .../iceberg/IcebergCatalogHandler.java | 11 +--- .../IcebergRESTExternalCatalogFactory.java | 7 +++ 9 files changed, 103 insertions(+), 17 deletions(-) rename {runtime/service/src/main/java/org/apache/polaris/service/catalog/generic => polaris-core/src/main/java/org/apache/polaris/core/catalog}/GenericTableCatalog.java (97%) diff --git a/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java b/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java index 50294da99c..34eeddb2b6 100644 --- a/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java +++ b/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java @@ -24,6 +24,7 @@ import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.polaris.core.catalog.ExternalCatalogFactory; +import org.apache.polaris.core.catalog.GenericTableCatalog; import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.connection.AuthenticationType; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; @@ -58,4 +59,10 @@ public Catalog createCatalog( warehouse, connectionConfigInfoDpo.asIcebergCatalogProperties(userSecretsManager)); return hadoopCatalog; } + + @Override + public GenericTableCatalog createGenericCatalog(ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { + // TODO implement + throw new UnsupportedOperationException("Generic table federation to this catalog is not supported."); + } } diff --git a/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java b/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java index 3e607e12ec..ea4a7a0011 100644 --- a/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java +++ b/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java @@ -23,6 +23,7 @@ import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.hive.HiveCatalog; import org.apache.polaris.core.catalog.ExternalCatalogFactory; +import org.apache.polaris.core.catalog.GenericTableCatalog; import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.connection.AuthenticationType; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; @@ -71,4 +72,10 @@ public Catalog createCatalog( warehouse, connectionConfigInfoDpo.asIcebergCatalogProperties(userSecretsManager)); return hiveCatalog; } + + @Override + public GenericTableCatalog createGenericCatalog(ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { + // TODO implement + throw new UnsupportedOperationException("Generic table federation to this catalog is not supported."); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/ExternalCatalogFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/ExternalCatalogFactory.java index 59c8903753..039a64ccd3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/ExternalCatalogFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/ExternalCatalogFactory.java @@ -40,4 +40,15 @@ public interface ExternalCatalogFactory { */ Catalog createCatalog( ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager); + + /** + * Creates a generic table catalog for the given connection configuration. + * + * @param connectionConfig the connection configuration + * @param userSecretsManager the user secrets manager for handling credentials + * @return the initialized catalog + * @throws IllegalStateException if the connection configuration is invalid + */ + GenericTableCatalog createGenericCatalog( + ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/GenericTableCatalog.java similarity index 97% rename from runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java rename to polaris-core/src/main/java/org/apache/polaris/core/catalog/GenericTableCatalog.java index ff16612907..7b418ac617 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/GenericTableCatalog.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.catalog.generic; +package org.apache.polaris.core.catalog; import java.util.List; import java.util.Map; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java index 9cc6c93e83..5f3a0425fb 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java @@ -20,6 +20,7 @@ import static org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_TABLE; +import jakarta.enterprise.inject.Instance; import jakarta.ws.rs.core.SecurityContext; import java.util.Arrays; import java.util.List; @@ -34,6 +35,7 @@ import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.catalog.ExternalCatalogFactory; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -43,6 +45,7 @@ import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.service.types.PolicyIdentifier; /** @@ -58,6 +61,8 @@ public abstract class CatalogHandler { protected final ResolutionManifestFactory resolutionManifestFactory; protected final String catalogName; protected final PolarisAuthorizer authorizer; + protected final UserSecretsManager userSecretsManager; + protected final Instance externalCatalogFactories; protected final CallContext callContext; protected final PolarisPrincipal polarisPrincipal; @@ -68,7 +73,9 @@ public CatalogHandler( ResolutionManifestFactory resolutionManifestFactory, SecurityContext securityContext, String catalogName, - PolarisAuthorizer authorizer) { + PolarisAuthorizer authorizer, + UserSecretsManager userSecretsManager, + Instance externalCatalogFactories) { this.callContext = callContext; this.resolutionManifestFactory = resolutionManifestFactory; this.catalogName = catalogName; @@ -82,8 +89,14 @@ public CatalogHandler( this.securityContext = securityContext; this.polarisPrincipal = (PolarisPrincipal) securityContext.getUserPrincipal(); this.authorizer = authorizer; + this.userSecretsManager = userSecretsManager; + this.externalCatalogFactories = externalCatalogFactories; } + protected UserSecretsManager getUserSecretsManager() { + return userSecretsManager; +} + /** Initialize the catalog once authorized. Called after all `authorize...` methods. */ protected abstract void initializeCatalog(); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java index cf5879fe6f..b8b8db1c65 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java @@ -18,24 +18,38 @@ */ package org.apache.polaris.service.catalog.generic; +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.inject.Instance; import jakarta.ws.rs.core.SecurityContext; import java.util.LinkedHashSet; import java.util.Map; + +import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.ExternalCatalogFactory; +import org.apache.polaris.core.catalog.GenericTableCatalog; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.connection.ConnectionType; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.table.GenericTableEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.service.catalog.common.CatalogHandler; import org.apache.polaris.service.types.GenericTable; import org.apache.polaris.service.types.ListGenericTablesResponse; import org.apache.polaris.service.types.LoadGenericTableResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GenericTableCatalogHandler extends CatalogHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(GenericTableCatalogHandler.class); private PolarisMetaStoreManager metaStoreManager; @@ -47,16 +61,51 @@ public GenericTableCatalogHandler( PolarisMetaStoreManager metaStoreManager, SecurityContext securityContext, String catalogName, - PolarisAuthorizer authorizer) { - super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer); + PolarisAuthorizer authorizer, + UserSecretsManager userSecretsManager, + Instance externalCatalogFactories) { + super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer, userSecretsManager, externalCatalogFactories); this.metaStoreManager = metaStoreManager; } @Override protected void initializeCatalog() { - this.genericTableCatalog = - new PolarisGenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest); - this.genericTableCatalog.initialize(catalogName, Map.of()); + CatalogEntity resolvedCatalogEntity = + CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); + ConnectionConfigInfoDpo connectionConfigInfoDpo = + resolvedCatalogEntity.getConnectionConfigInfoDpo(); + if (connectionConfigInfoDpo != null) { + LOGGER + .atInfo() + .addKeyValue("remoteUrl", connectionConfigInfoDpo.getUri()) + .log("Initializing federated catalog"); + FeatureConfiguration.enforceFeatureEnabledOrThrow( + callContext.getRealmConfig(), FeatureConfiguration.ENABLE_CATALOG_FEDERATION); + + GenericTableCatalog federatedCatalog; + ConnectionType connectionType = + ConnectionType.fromCode(connectionConfigInfoDpo.getConnectionTypeCode()); + + // Use the unified factory pattern for all external catalog types + Instance externalCatalogFactory = + externalCatalogFactories.select( + Identifier.Literal.of(connectionType.getFactoryIdentifier())); + if (externalCatalogFactory.isResolvable()) { + federatedCatalog = + externalCatalogFactory + .get() + .createGenericCatalog(connectionConfigInfoDpo, getUserSecretsManager()); + } else { + throw new UnsupportedOperationException( + "External catalog factory for type '" + connectionType + "' is unavailable."); + } + this.genericTableCatalog = federatedCatalog; + } else { + LOGGER.atInfo().log("Initializing non-federated catalog"); + this.genericTableCatalog = + new PolarisGenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest); + this.genericTableCatalog.initialize(catalogName, Map.of()); + } } public ListGenericTablesResponse listGenericTables(Namespace parent) { diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java index a198c6cb34..293b73c1d5 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java @@ -25,6 +25,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.polaris.core.catalog.GenericTableCatalog; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index c7705934c2..92b442fa9a 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -121,13 +121,10 @@ public class IcebergCatalogHandler extends CatalogHandler implements AutoCloseab private static final Logger LOGGER = LoggerFactory.getLogger(IcebergCatalogHandler.class); private final PolarisMetaStoreManager metaStoreManager; - private final UserSecretsManager userSecretsManager; private final CallContextCatalogFactory catalogFactory; private final ReservedProperties reservedProperties; private final CatalogHandlerUtils catalogHandlerUtils; - private final Instance externalCatalogFactories; - // Catalog instance will be initialized after authorizing resolver successfully resolves // the catalog entity. protected Catalog baseCatalog = null; @@ -149,13 +146,11 @@ public IcebergCatalogHandler( ReservedProperties reservedProperties, CatalogHandlerUtils catalogHandlerUtils, Instance externalCatalogFactories) { - super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer); + super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer, userSecretsManager, externalCatalogFactories); this.metaStoreManager = metaStoreManager; - this.userSecretsManager = userSecretsManager; this.catalogFactory = catalogFactory; this.reservedProperties = reservedProperties; this.catalogHandlerUtils = catalogHandlerUtils; - this.externalCatalogFactories = externalCatalogFactories; } /** @@ -196,10 +191,6 @@ public ListNamespacesResponse listNamespaces( } } - private UserSecretsManager getUserSecretsManager() { - return userSecretsManager; - } - @Override protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java index 05de201c37..e65fc428eb 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java @@ -25,6 +25,7 @@ import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTCatalog; import org.apache.polaris.core.catalog.ExternalCatalogFactory; +import org.apache.polaris.core.catalog.GenericTableCatalog; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; import org.apache.polaris.core.connection.ConnectionType; import org.apache.polaris.core.connection.iceberg.IcebergRestConnectionConfigInfoDpo; @@ -62,4 +63,10 @@ public Catalog createCatalog( return federatedCatalog; } + + @Override + public GenericTableCatalog createGenericCatalog(ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { + // TODO implement + throw new UnsupportedOperationException("Generic table federation to this catalog is not supported."); + } } From d711067ca66b0155eb67e10a435a633e18f5aaed Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 18 Aug 2025 13:21:57 -0700 Subject: [PATCH 2/4] autolint --- .../hadoop/HadoopFederatedCatalogFactory.java | 6 ++++-- .../federation/hive/HiveFederatedCatalogFactory.java | 6 ++++-- .../service/catalog/common/CatalogHandler.java | 2 +- .../catalog/generic/GenericTableCatalogHandler.java | 11 ++++++++--- .../catalog/iceberg/IcebergCatalogHandler.java | 9 ++++++++- .../iceberg/IcebergRESTExternalCatalogFactory.java | 6 ++++-- 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java b/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java index 34eeddb2b6..8da714072d 100644 --- a/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java +++ b/extensions/federation/hadoop/src/main/java/org/apache/polaris/extensions/federation/hadoop/HadoopFederatedCatalogFactory.java @@ -61,8 +61,10 @@ public Catalog createCatalog( } @Override - public GenericTableCatalog createGenericCatalog(ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { + public GenericTableCatalog createGenericCatalog( + ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { // TODO implement - throw new UnsupportedOperationException("Generic table federation to this catalog is not supported."); + throw new UnsupportedOperationException( + "Generic table federation to this catalog is not supported."); } } diff --git a/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java b/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java index ea4a7a0011..12c8d80f63 100644 --- a/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java +++ b/extensions/federation/hive/src/main/java/org/apache/polaris/extensions/federation/hive/HiveFederatedCatalogFactory.java @@ -74,8 +74,10 @@ public Catalog createCatalog( } @Override - public GenericTableCatalog createGenericCatalog(ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { + public GenericTableCatalog createGenericCatalog( + ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { // TODO implement - throw new UnsupportedOperationException("Generic table federation to this catalog is not supported."); + throw new UnsupportedOperationException( + "Generic table federation to this catalog is not supported."); } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java index 5f3a0425fb..75a0130ad5 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java @@ -95,7 +95,7 @@ public CatalogHandler( protected UserSecretsManager getUserSecretsManager() { return userSecretsManager; -} + } /** Initialize the catalog once authorized. Called after all `authorize...` methods. */ protected abstract void initializeCatalog(); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java index b8b8db1c65..8881fb19dd 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java @@ -23,8 +23,6 @@ import jakarta.ws.rs.core.SecurityContext; import java.util.LinkedHashSet; import java.util.Map; - -import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; @@ -64,7 +62,14 @@ public GenericTableCatalogHandler( PolarisAuthorizer authorizer, UserSecretsManager userSecretsManager, Instance externalCatalogFactories) { - super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer, userSecretsManager, externalCatalogFactories); + super( + callContext, + resolutionManifestFactory, + securityContext, + catalogName, + authorizer, + userSecretsManager, + externalCatalogFactories); this.metaStoreManager = metaStoreManager; } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 92b442fa9a..79967fa1dc 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -146,7 +146,14 @@ public IcebergCatalogHandler( ReservedProperties reservedProperties, CatalogHandlerUtils catalogHandlerUtils, Instance externalCatalogFactories) { - super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer, userSecretsManager, externalCatalogFactories); + super( + callContext, + resolutionManifestFactory, + securityContext, + catalogName, + authorizer, + userSecretsManager, + externalCatalogFactories); this.metaStoreManager = metaStoreManager; this.catalogFactory = catalogFactory; this.reservedProperties = reservedProperties; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java index e65fc428eb..8584da753a 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRESTExternalCatalogFactory.java @@ -65,8 +65,10 @@ public Catalog createCatalog( } @Override - public GenericTableCatalog createGenericCatalog(ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { + public GenericTableCatalog createGenericCatalog( + ConnectionConfigInfoDpo connectionConfig, UserSecretsManager userSecretsManager) { // TODO implement - throw new UnsupportedOperationException("Generic table federation to this catalog is not supported."); + throw new UnsupportedOperationException( + "Generic table federation to this catalog is not supported."); } } From 8613122dfbe253871c90a66b19fbcaf50923087b Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 18 Aug 2025 13:52:21 -0700 Subject: [PATCH 3/4] various build fixes --- .../core/connection/ConnectionConfigInfoDpo.java | 9 ++++++--- .../hive/HiveConnectionConfigInfoDpo.java | 14 ++++++++++++-- .../generic/GenericTableCatalogAdapter.java | 16 ++++++++++++++-- .../catalog/policy/PolicyCatalogAdapter.java | 16 ++++++++++++++-- .../catalog/policy/PolicyCatalogHandler.java | 9 +++++++-- ...larisGenericTableCatalogHandlerAuthzTest.java | 4 +++- .../catalog/PolicyCatalogHandlerAuthzTest.java | 4 +++- 7 files changed, 59 insertions(+), 13 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java index 33a263635c..cd3d0ab335 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java @@ -76,7 +76,7 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie public ConnectionConfigInfoDpo( @JsonProperty(value = "connectionTypeCode", required = true) int connectionTypeCode, @JsonProperty(value = "uri", required = true) @Nonnull String uri, - @JsonProperty(value = "authenticationParameters", required = true) @Nonnull + @JsonProperty(value = "authenticationParameters", required = true) @Nullable AuthenticationParametersDpo authenticationParameters, @JsonProperty(value = "serviceIdentity", required = false) @Nullable ServiceIdentityInfoDpo serviceIdentity) { @@ -86,7 +86,7 @@ public ConnectionConfigInfoDpo( protected ConnectionConfigInfoDpo( int connectionTypeCode, @Nonnull String uri, - @Nonnull AuthenticationParametersDpo authenticationParameters, + @Nullable AuthenticationParametersDpo authenticationParameters, @Nullable ServiceIdentityInfoDpo serviceIdentity, boolean validateUri) { this.connectionTypeCode = connectionTypeCode; @@ -203,7 +203,10 @@ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( hiveConfigModel.getAuthenticationParameters(), secretReferences); config = new HiveConnectionConfigInfoDpo( - hiveConfigModel.getUri(), authenticationParameters, hiveConfigModel.getWarehouse()); + hiveConfigModel.getUri(), + authenticationParameters, + hiveConfigModel.getWarehouse(), + null /*Service Identity Info*/); break; default: throw new IllegalStateException( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java index aaf5753776..1f9027f2be 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java @@ -30,6 +30,7 @@ import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; import org.apache.polaris.core.connection.ConnectionType; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.secrets.UserSecretsManager; /** @@ -44,8 +45,10 @@ public HiveConnectionConfigInfoDpo( @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "authenticationParameters", required = false) @Nullable AuthenticationParametersDpo authenticationParameters, - @JsonProperty(value = "warehouse", required = false) @Nullable String warehouse) { - super(ConnectionType.HIVE.getCode(), uri, authenticationParameters); + @JsonProperty(value = "warehouse", required = false) @Nullable String warehouse, + @JsonProperty(value = "serviceIdentity", required = false) @Nullable + ServiceIdentityInfoDpo serviceIdentity) { + super(ConnectionType.HIVE.getCode(), uri, authenticationParameters, serviceIdentity); this.warehouse = warehouse; } @@ -77,6 +80,13 @@ public String toString() { return properties; } + @Override + public ConnectionConfigInfoDpo withServiceIdentity( + @Nonnull ServiceIdentityInfoDpo serviceIdentityInfo) { + return new HiveConnectionConfigInfoDpo( + getUri(), getAuthenticationParameters(), warehouse, serviceIdentityInfo); + } + @Override public ConnectionConfigInfo asConnectionConfigInfoModel() { return HiveConnectionConfigInfo.builder() diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java index ddf6f7697e..2a4a903b96 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java @@ -19,16 +19,20 @@ package org.apache.polaris.service.catalog.generic; import jakarta.enterprise.context.RequestScoped; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.ExternalCatalogFactory; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.api.PolarisCatalogGenericTableApiService; import org.apache.polaris.service.catalog.common.CatalogAdapter; @@ -52,6 +56,8 @@ public class GenericTableCatalogAdapter private final PolarisAuthorizer polarisAuthorizer; private final ReservedProperties reservedProperties; private final CatalogPrefixParser prefixParser; + private final UserSecretsManager userSecretsManager; + private final Instance externalCatalogFactories; @Inject public GenericTableCatalogAdapter( @@ -61,7 +67,9 @@ public GenericTableCatalogAdapter( PolarisMetaStoreManager metaStoreManager, PolarisAuthorizer polarisAuthorizer, CatalogPrefixParser prefixParser, - ReservedProperties reservedProperties) { + ReservedProperties reservedProperties, + UserSecretsManager userSecretsManager, + @Any Instance externalCatalogFactories) { this.realmContext = realmContext; this.callContext = callContext; this.resolutionManifestFactory = resolutionManifestFactory; @@ -69,6 +77,8 @@ public GenericTableCatalogAdapter( this.polarisAuthorizer = polarisAuthorizer; this.prefixParser = prefixParser; this.reservedProperties = reservedProperties; + this.userSecretsManager = userSecretsManager; + this.externalCatalogFactories = externalCatalogFactories; } private GenericTableCatalogHandler newHandlerWrapper( @@ -83,7 +93,9 @@ private GenericTableCatalogHandler newHandlerWrapper( metaStoreManager, securityContext, prefixParser.prefixToCatalogName(realmContext, prefix), - polarisAuthorizer); + polarisAuthorizer, + userSecretsManager, + externalCatalogFactories); } @Override diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java index 9ac52cb1b6..6168e9a523 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java @@ -19,18 +19,22 @@ package org.apache.polaris.service.catalog.policy; import jakarta.enterprise.context.RequestScoped; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.rest.RESTUtil; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.ExternalCatalogFactory; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.policy.PolicyType; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.api.PolarisCatalogPolicyApiService; import org.apache.polaris.service.catalog.common.CatalogAdapter; @@ -55,6 +59,8 @@ public class PolicyCatalogAdapter implements PolarisCatalogPolicyApiService, Cat private final PolarisMetaStoreManager metaStoreManager; private final PolarisAuthorizer polarisAuthorizer; private final CatalogPrefixParser prefixParser; + private final UserSecretsManager userSecretsManager; + private final Instance externalCatalogFactories; @Inject public PolicyCatalogAdapter( @@ -63,13 +69,17 @@ public PolicyCatalogAdapter( ResolutionManifestFactory resolutionManifestFactory, PolarisMetaStoreManager metaStoreManager, PolarisAuthorizer polarisAuthorizer, - CatalogPrefixParser prefixParser) { + CatalogPrefixParser prefixParser, + UserSecretsManager userSecretsManager, + @Any Instance externalCatalogFactories) { this.realmContext = realmContext; this.callContext = callContext; this.resolutionManifestFactory = resolutionManifestFactory; this.metaStoreManager = metaStoreManager; this.polarisAuthorizer = polarisAuthorizer; this.prefixParser = prefixParser; + this.userSecretsManager = userSecretsManager; + this.externalCatalogFactories = externalCatalogFactories; } private PolicyCatalogHandler newHandlerWrapper(SecurityContext securityContext, String prefix) { @@ -83,7 +93,9 @@ private PolicyCatalogHandler newHandlerWrapper(SecurityContext securityContext, metaStoreManager, securityContext, prefixParser.prefixToCatalogName(realmContext, prefix), - polarisAuthorizer); + polarisAuthorizer, + userSecretsManager, + externalCatalogFactories); } @Override diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java index 54b72589b9..7d70c6302b 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java @@ -20,6 +20,7 @@ import com.google.common.base.Strings; import jakarta.annotation.Nullable; +import jakarta.enterprise.inject.Instance; import jakarta.ws.rs.core.SecurityContext; import java.util.Arrays; import java.util.HashSet; @@ -31,6 +32,7 @@ import org.apache.iceberg.exceptions.NotFoundException; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.ExternalCatalogFactory; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -42,6 +44,7 @@ import org.apache.polaris.core.persistence.resolver.ResolverStatus; import org.apache.polaris.core.policy.PolicyType; import org.apache.polaris.core.policy.exceptions.NoSuchPolicyException; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.service.catalog.common.CatalogHandler; import org.apache.polaris.service.types.AttachPolicyRequest; import org.apache.polaris.service.types.CreatePolicyRequest; @@ -65,8 +68,10 @@ public PolicyCatalogHandler( PolarisMetaStoreManager metaStoreManager, SecurityContext securityContext, String catalogName, - PolarisAuthorizer authorizer) { - super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer); + PolarisAuthorizer authorizer, + UserSecretsManager userSecretsManager, + Instance externalCatalogFactories) { + super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer, userSecretsManager, externalCatalogFactories); this.metaStoreManager = metaStoreManager; } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java index f09bd1a783..ab8c429a90 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java @@ -52,7 +52,9 @@ private GenericTableCatalogHandler newWrapper( metaStoreManager, securityContext(authenticatedPrincipal), catalogName, - polarisAuthorizer); + polarisAuthorizer, + null, + null); } /** diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolicyCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolicyCatalogHandlerAuthzTest.java index 066bd54a36..5d4fe957d4 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolicyCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/PolicyCatalogHandlerAuthzTest.java @@ -57,7 +57,9 @@ private PolicyCatalogHandler newWrapper(Set activatedPrincipalRoles, Str metaStoreManager, securityContext(authenticatedPrincipal), catalogName, - polarisAuthorizer); + polarisAuthorizer, + null, + null); } /** From bf29d588ed7e36c6cc2f922809813bcdd9c3cd45 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 18 Aug 2025 13:52:26 -0700 Subject: [PATCH 4/4] autolint --- .../service/catalog/policy/PolicyCatalogHandler.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java index 7d70c6302b..6549295221 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogHandler.java @@ -71,7 +71,14 @@ public PolicyCatalogHandler( PolarisAuthorizer authorizer, UserSecretsManager userSecretsManager, Instance externalCatalogFactories) { - super(callContext, resolutionManifestFactory, securityContext, catalogName, authorizer, userSecretsManager, externalCatalogFactories); + super( + callContext, + resolutionManifestFactory, + securityContext, + catalogName, + authorizer, + userSecretsManager, + externalCatalogFactories); this.metaStoreManager = metaStoreManager; }