diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogConfig.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogConfig.java new file mode 100644 index 0000000000..df5c6150c9 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogConfig.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.it.env; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.apache.polaris.core.admin.model.Catalog; + +/** + * Annotation to configure the catalog type and properties for integration tests. + * + *

This is a server-side setting; it is used to specify the Polaris Catalog type (e.g., INTERNAL, + * EXTERNAL) and any additional properties required for the catalog configuration. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +@Inherited +public @interface CatalogConfig { + + /** The type of the catalog. Defaults to INTERNAL. */ + Catalog.TypeEnum value() default Catalog.TypeEnum.INTERNAL; + + /** Additional properties for the catalog configuration. */ + String[] properties() default {}; +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java index 740307c14b..027c68905a 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.auth.OAuth2Properties; @@ -35,12 +36,8 @@ public static RESTCatalog restCatalog( ImmutableMap.Builder propertiesBuilder = ImmutableMap.builder() - .put( - org.apache.iceberg.CatalogProperties.URI, endpoints.catalogApiEndpoint().toString()) + .put(CatalogProperties.URI, endpoints.catalogApiEndpoint().toString()) .put(OAuth2Properties.TOKEN, authToken) - .put( - org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, - "org.apache.iceberg.inmemory.InMemoryFileIO") .put("warehouse", catalog) .put("header." + endpoints.realmHeaderName(), endpoints.realmId()) .putAll(extraProperties); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java index 7e2a8e41aa..7d3e657c03 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java @@ -19,9 +19,16 @@ package org.apache.polaris.service.it.env; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import java.lang.annotation.Annotation; +import java.lang.reflect.AnnotatedElement; import java.net.URI; import java.nio.file.Path; +import java.util.Arrays; +import java.util.Map; import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.jupiter.api.TestInfo; public final class IntegrationTestsHelper { @@ -54,4 +61,66 @@ static URI getTemporaryDirectory(Function getenv, Path defaultLo envVar = envVar.startsWith("/") ? "file://" + envVar : envVar; return URI.create(envVar + "/").normalize(); } + + /** + * Extract a value from the annotated elements of the test method and class. + * + *

This method looks for annotations of the specified type on both the test method and the test + * class, extracts the value using the provided extractor function, and returns it. + * + *

If the value is present in both the method and class annotations, the value from the method + * annotation will be used. If the value is not present in either the method or class annotations, + * this method returns the default value. + */ + public static T extractFromAnnotatedElements( + TestInfo testInfo, Class annotationClass, Function extractor, T defaultValue) { + return testInfo + .getTestMethod() + .map(AnnotatedElement.class::cast) + .or(testInfo::getTestClass) + .map(clz -> clz.getAnnotation(annotationClass)) + .map(extractor) + .orElse(defaultValue); + } + + /** + * Collect properties from annotated elements in the test method and class. + * + *

This method looks for annotations of the specified type on both the test method and the test + * class, extracts properties using the provided extractor function, and combines them into a map. + * If a property appears in both the method and class annotations, the value from the method + * annotation will be used. + */ + public static Map mergeFromAnnotatedElements( + TestInfo testInfo, + Class annotationClass, + Function propertiesExtractor, + Map defaults) { + String[] methodProperties = + testInfo + .getTestMethod() + .map(m -> m.getAnnotation(annotationClass)) + .map(propertiesExtractor) + .orElse(new String[0]); + String[] classProperties = + testInfo + .getTestClass() + .map(clz -> clz.getAnnotation(annotationClass)) + .map(propertiesExtractor) + .orElse(new String[0]); + String[] properties = + Stream.concat(Arrays.stream(methodProperties), Arrays.stream(classProperties)) + .toArray(String[]::new); + if (properties.length % 2 != 0) { + throw new IllegalArgumentException( + "Properties must be in key-value pairs, but found an odd number of elements: " + + Arrays.toString(properties)); + } + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.putAll(defaults); + for (int i = 0; i < properties.length; i += 2) { + builder.put(properties[i], properties[i + 1]); + } + return builder.buildKeepingLast(); + } } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestCatalogConfig.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestCatalogConfig.java new file mode 100644 index 0000000000..1260f110b4 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestCatalogConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.it.env; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to configure the REST catalog for integration tests. + * + *

This is a client-side setting; it is used to configure the client-side REST catalog that is + * used in test code to connect to the Polaris REST API and to the storage layer. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +@Inherited +public @interface RestCatalogConfig { + String[] value() default {}; +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java index 6f987e2b68..2557466de2 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java @@ -22,11 +22,9 @@ import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; -import com.google.common.collect.ImmutableMap; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; +import java.io.IOException; import java.lang.reflect.Method; import java.net.URI; import java.nio.file.Path; @@ -64,6 +62,7 @@ import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.policy.PredefinedPolicyTypes; import org.apache.polaris.core.policy.exceptions.PolicyInUseException; +import org.apache.polaris.service.it.env.CatalogConfig; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.IcebergHelper; import org.apache.polaris.service.it.env.IntegrationTestsHelper; @@ -71,6 +70,7 @@ import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.env.PolarisClient; import org.apache.polaris.service.it.env.PolicyApi; +import org.apache.polaris.service.it.env.RestCatalogConfig; import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.apache.polaris.service.types.ApplicablePolicy; import org.apache.polaris.service.types.AttachPolicyRequest; @@ -131,25 +131,10 @@ public class PolarisPolicyServiceIntegrationTest { private final String catalogBaseLocation = s3BucketBase + "/" + System.getenv("USER") + "/path/to/data"; - private static final String[] DEFAULT_CATALOG_PROPERTIES = { - "polaris.config.allow.unstructured.table.location", "true", - "polaris.config.allow.external.table.location", "true" - }; - - @Retention(RetentionPolicy.RUNTIME) - private @interface CatalogConfig { - Catalog.TypeEnum value() default Catalog.TypeEnum.INTERNAL; - - String[] properties() default { - "polaris.config.allow.unstructured.table.location", "true", - "polaris.config.allow.external.table.location", "true" - }; - } - - @Retention(RetentionPolicy.RUNTIME) - private @interface RestCatalogConfig { - String[] value() default {}; - } + private static final Map DEFAULT_CATALOG_PROPERTIES = + Map.of( + "polaris.config.allow.unstructured.table.location", "true", + "polaris.config.allow.external.table.location", "true"); @BeforeAll public static void setup( @@ -188,28 +173,26 @@ public void before(TestInfo testInfo) { .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) .build(); - Optional catalogConfig = - Optional.ofNullable( - method.getAnnotation(PolarisPolicyServiceIntegrationTest.CatalogConfig.class)); CatalogProperties.Builder catalogPropsBuilder = CatalogProperties.builder(catalogBaseLocation); - String[] properties = - catalogConfig - .map(PolarisPolicyServiceIntegrationTest.CatalogConfig::properties) - .orElse(DEFAULT_CATALOG_PROPERTIES); - for (int i = 0; i < properties.length; i += 2) { - catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); - } + + Map catalogProperties = + IntegrationTestsHelper.mergeFromAnnotatedElements( + testInfo, CatalogConfig.class, CatalogConfig::properties, DEFAULT_CATALOG_PROPERTIES); + catalogPropsBuilder.putAll(catalogProperties); + if (!s3BucketBase.getScheme().equals("file")) { catalogPropsBuilder.addProperty( CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); } + + Catalog.TypeEnum catalogType = + IntegrationTestsHelper.extractFromAnnotatedElements( + testInfo, CatalogConfig.class, CatalogConfig::value, Catalog.TypeEnum.INTERNAL); + Catalog catalog = PolarisCatalog.builder() - .setType( - catalogConfig - .map(PolarisPolicyServiceIntegrationTest.CatalogConfig::value) - .orElse(Catalog.TypeEnum.INTERNAL)) + .setType(catalogType) .setName(currentCatalogName) .setProperties(catalogPropsBuilder.build()) .setStorageConfigInfo( @@ -221,26 +204,14 @@ public void before(TestInfo testInfo) { managementApi.createCatalog(principalRoleName, catalog); - Optional restCatalogConfig = - testInfo - .getTestMethod() - .flatMap( - m -> - Optional.ofNullable( - m.getAnnotation( - PolarisPolicyServiceIntegrationTest.RestCatalogConfig.class))); - ImmutableMap.Builder extraPropertiesBuilder = ImmutableMap.builder(); - restCatalogConfig.ifPresent( - config -> { - for (int i = 0; i < config.value().length; i += 2) { - extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); - } - }); + Map restCatalogProperties = + IntegrationTestsHelper.mergeFromAnnotatedElements( + testInfo, RestCatalogConfig.class, RestCatalogConfig::value, Map.of()); String principalToken = client.obtainToken(principalCredentials); restCatalog = IcebergHelper.restCatalog( - endpoints, currentCatalogName, extraPropertiesBuilder.build(), principalToken); + endpoints, currentCatalogName, restCatalogProperties, principalToken); CatalogGrant catalogGrant = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); managementApi.createCatalogRole(currentCatalogName, CATALOG_ROLE_1); @@ -253,8 +224,14 @@ public void before(TestInfo testInfo) { } @AfterEach - public void cleanUp() { - client.cleanUp(adminToken); + public void cleanUp() throws IOException { + try { + if (restCatalog != null) { + restCatalog.close(); + } + } finally { + client.cleanUp(adminToken); + } } @Test diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java index 469ca37ce4..ff0d9b7f11 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java @@ -31,8 +31,7 @@ import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; +import java.io.IOException; import java.lang.reflect.Method; import java.net.URI; import java.util.Arrays; @@ -93,13 +92,16 @@ import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.CatalogConfig; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.ClientPrincipal; import org.apache.polaris.service.it.env.GenericTableApi; import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.ManagementApi; import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.env.RestCatalogConfig; import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.apache.polaris.service.types.CreateGenericTableRequest; import org.apache.polaris.service.types.GenericTable; @@ -149,11 +151,13 @@ public abstract class PolarisRestCatalogIntegrationBase extends CatalogTests restCatalogConfig; - private URI externalCatalogBase; + private URI externalCatalogBaseLocation; private String catalogBaseLocation; private static final Map DEFAULT_REST_CATALOG_CONFIG = Map.of( + org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO", org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1", "catalog-default-key1", org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2", @@ -165,26 +169,11 @@ public abstract class PolarisRestCatalogIntegrationBase extends CatalogTests DEFAULT_CATALOG_PROPERTIES = + Map.of( + "polaris.config.allow.unstructured.table.location", "true", + "polaris.config.allow.external.table.location", "true", + "polaris.config.list-pagination-enabled", "true"); /** * Get the storage configuration information for the catalog. @@ -194,11 +183,29 @@ String[] properties() default { protected abstract StorageConfigInfo getStorageConfigInfo(); /** - * A hook for test sub-classes to initialize {@link FileIO} objects used by test cases thenselves + * A hook for test subclasses to initialize {@link FileIO} objects used by test cases themselves * for accessing test storage. + * + *

Important: this FileIO instance is not tied to a REST Catalog; it is created by test code + * performing adhoc, direct access to the storage; thus, it may not be configured with the same + * properties as the ones used by {@link #catalog()}! */ - protected void initializeClientFileIO(FileIO fileIO) { - fileIO.initialize(Map.of()); + protected T initializeClientFileIO(T fileIO) { + fileIO.initialize(clientFileIOProperties().build()); + return fileIO; + } + + /** + * Get the properties to be used for {@linkplain #initializeClientFileIO(FileIO) initializing} the + * {@link FileIO} instance used by test code for accessing test storage. + */ + protected ImmutableMap.Builder clientFileIOProperties() { + return ImmutableMap.builder(); + } + + /** Get the base URI for the external catalog. */ + protected URI externalCatalogBaseLocation() { + return externalCatalogBaseLocation; } @BeforeAll @@ -232,31 +239,36 @@ public void before(TestInfo testInfo) { Method method = testInfo.getTestMethod().orElseThrow(); currentCatalogName = client.newEntityName(method.getName()); + StorageConfigInfo storageConfig = getStorageConfigInfo(); URI testRuntimeURI = URI.create(storageConfig.getAllowedLocations().getFirst()); catalogBaseLocation = testRuntimeURI + "/" + CATALOG_LOCATION_SUBPATH; - externalCatalogBase = + externalCatalogBaseLocation = URI.create( testRuntimeURI + "/" + EXTERNAL_CATALOG_LOCATION_SUBPATH + "/" + method.getName()); - Optional catalogConfig = - Optional.ofNullable(method.getAnnotation(CatalogConfig.class)); - CatalogProperties.Builder catalogPropsBuilder = CatalogProperties.builder(catalogBaseLocation); - String[] properties = - catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); - for (int i = 0; i < properties.length; i += 2) { - catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); - } + + Map catalogProperties = + IntegrationTestsHelper.mergeFromAnnotatedElements( + testInfo, CatalogConfig.class, CatalogConfig::properties, DEFAULT_CATALOG_PROPERTIES); + catalogPropsBuilder.putAll(catalogProperties); + catalogPropsBuilder.addProperty( FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true"); + if (!testRuntimeURI.getScheme().equals("file")) { catalogPropsBuilder.addProperty( CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); } + + Catalog.TypeEnum catalogType = + IntegrationTestsHelper.extractFromAnnotatedElements( + testInfo, CatalogConfig.class, CatalogConfig::value, Catalog.TypeEnum.INTERNAL); + Catalog catalog = PolarisCatalog.builder() - .setType(catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) + .setType(catalogType) .setName(currentCatalogName) .setProperties(catalogPropsBuilder.build()) .setStorageConfigInfo(storageConfig) @@ -264,30 +276,12 @@ public void before(TestInfo testInfo) { managementApi.createCatalog(principalRoleName, catalog); - Map dynamicConfig = - testInfo - .getTestMethod() - .map(m -> m.getAnnotation(RestCatalogConfig.class)) - .map(RestCatalogConfig::value) - .map( - values -> { - if (values.length % 2 != 0) { - throw new IllegalArgumentException( - String.format("Missing value for config '%s'", values[values.length - 1])); - } - Map config = new HashMap<>(); - for (int i = 0; i < values.length; i += 2) { - config.put(values[i], values[i + 1]); - } - return config; - }) - .orElse(ImmutableMap.of()); - restCatalogConfig = - ImmutableMap.builder() - .putAll(DEFAULT_REST_CATALOG_CONFIG) - .putAll(dynamicConfig) - .build(); + IntegrationTestsHelper.mergeFromAnnotatedElements( + testInfo, + RestCatalogConfig.class, + RestCatalogConfig::value, + DEFAULT_REST_CATALOG_CONFIG); restCatalog = initCatalog(currentCatalogName, ImmutableMap.of()); } @@ -321,8 +315,14 @@ protected String obtainToken(PolarisClient client, ClientPrincipal principal) { } @AfterEach - public void cleanUp() { - cleanUp(client, adminToken); + public void cleanUp() throws IOException { + try { + if (restCatalog != null) { + restCatalog.close(); + } + } finally { + cleanUp(client, adminToken); + } } /** @@ -651,12 +651,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - externalCatalogBase + "/ns1/my_table", + externalCatalogBaseLocation + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { initializeClientFileIO(resolvingFileIO); resolvingFileIO.setConf(new Configuration()); - String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBaseLocation + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -686,12 +686,12 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - externalCatalogBase + "/ns1/my_table", + externalCatalogBaseLocation + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { initializeClientFileIO(resolvingFileIO); resolvingFileIO.setConf(new Configuration()); - String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBaseLocation + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -720,12 +720,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - externalCatalogBase + "/ns1/my_table", + externalCatalogBaseLocation + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { initializeClientFileIO(resolvingFileIO); resolvingFileIO.setConf(new Configuration()); - String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBaseLocation + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -748,12 +748,12 @@ public void testLoadTableTwiceWithETag() { TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - externalCatalogBase + "/ns1/my_table", + externalCatalogBaseLocation + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { initializeClientFileIO(resolvingFileIO); resolvingFileIO.setConf(new Configuration()); - String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBaseLocation + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table_etagged"), fileLocation); Invocation invocation = @@ -792,12 +792,12 @@ public void testRegisterAndLoadTableWithReturnedETag() { TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - externalCatalogBase + "/ns1/my_table", + externalCatalogBaseLocation + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { initializeClientFileIO(resolvingFileIO); resolvingFileIO.setConf(new Configuration()); - String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBaseLocation + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); Invocation registerInvocation = @@ -835,12 +835,12 @@ public void testCreateAndLoadTableWithReturnedEtag() { TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - externalCatalogBase + "/ns1/my_table", + externalCatalogBaseLocation + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { initializeClientFileIO(resolvingFileIO); resolvingFileIO.setConf(new Configuration()); - String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBaseLocation + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); Invocation createInvocation = diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java index bbd66512d8..7b1e38b83e 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogMinIOIT.java @@ -25,7 +25,6 @@ import java.net.URI; import java.util.List; import java.util.Map; -import org.apache.iceberg.io.FileIO; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.storage.StorageAccessProperty; @@ -70,14 +69,12 @@ static void setup( } @Override - protected void initializeClientFileIO(FileIO fileIO) { - fileIO.initialize( - ImmutableMap.builder() - .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), endpoint) - .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName(), "true") - .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY) - .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY) - .build()); + protected ImmutableMap.Builder clientFileIOProperties() { + return super.clientFileIOProperties() + .put(StorageAccessProperty.AWS_ENDPOINT.getPropertyName(), endpoint) + .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS.getPropertyName(), "true") + .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY) + .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); } @Override