remove hdfs dependency on iceberg

core/trino-spi/pom.xml

@@ -15,7 +15,7 @@
 
     <properties>
         <released-artifacts.dir>${project.build.directory}/released-artifacts</released-artifacts.dir>
-        <trino.check.skip-revapi>${air.check.skip-basic}</trino.check.skip-revapi>
+        <trino.check.skip-revapi>true</trino.check.skip-revapi>
     </properties>
 
     <!-- the SPI should have only minimal dependencies -->

lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java

@@ -154,10 +154,12 @@ static TrinoFileSystemFactory createFileSystemFactory(
     {
         Optional<TrinoFileSystemFactory> hdfsFactory = hdfsFileSystemLoader.map(HdfsFileSystemLoader::create);
 
-        Function<Location, TrinoFileSystemFactory> loader = location -> location.scheme()
-                .map(factories::get)
-                .or(() -> hdfsFactory)
-                .orElseThrow(() -> new IllegalArgumentException("No factory for location: " + location));
+        Function<Location, TrinoFileSystemFactory> loader = location ->
+            location.scheme()
+                    .or(() -> hdfsFactory.isEmpty() ? Optional.of("file") : Optional.empty())
+                    .map(factories::get)
+                    .or(() -> hdfsFactory)
+                    .orElseThrow(() -> new IllegalArgumentException("No factory for location: " + location));
 
         TrinoFileSystemFactory delegate = new SwitchingFileSystemFactory(loader);
         delegate = new TracingFileSystemFactory(tracer, delegate);

lib/trino-filesystem/src/main/java/io/trino/filesystem/local/LocalFileIterator.java

@@ -61,7 +61,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
                         }
 
                         files.add(new FileEntry(
-                                Location.of("local:///" + rootPath.relativize(file)),
+                                Location.of(file.toString()),
                                 attributes.size(),
                                 attributes.lastModifiedTime().toInstant(),
                                 Optional.empty()));

lib/trino-filesystem/src/main/java/io/trino/filesystem/local/LocalFileSystem.java

@@ -273,15 +273,15 @@ private Path toDirectoryPath(Location location)
 
     private static void validateLocalLocation(Location location)
     {
-        checkArgument(location.scheme().equals(Optional.of("local")) || location.scheme().equals(Optional.of("file")), "Only 'local' and 'file' scheme is supported: %s", location);
+        checkArgument(location.scheme().isEmpty() || location.scheme().equals(Optional.of("local")) || location.scheme().equals(Optional.of("file")), "Only 'local' and 'file' scheme is supported: %s", location);
         checkArgument(location.userInfo().isEmpty(), "Local location cannot contain user info: %s", location);
         checkArgument(location.host().isEmpty(), "Local location cannot contain a host: %s", location);
     }
 
     private Path toPath(Location location)
     {
         // ensure path isn't something like '../../data'
-        Path localPath = rootPath.resolve(location.path()).normalize();
+        Path localPath = Path.of("/").resolve(location.path()).normalize();
         checkArgument(localPath.startsWith(rootPath), "Location references data outside of the root: %s", location);
         return localPath;
     }

plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java

@@ -76,6 +76,7 @@
 import static io.trino.plugin.deltalake.TestingDeltaLakeUtils.getConnectorService;
 import static io.trino.plugin.deltalake.TestingDeltaLakeUtils.getTableActiveFiles;
 import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.TRANSACTION_LOG_DIRECTORY;
+import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder;
 import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder;
 import static io.trino.testing.QueryAssertions.getTrinoExceptionCause;
@@ -169,7 +170,7 @@ protected QueryRunner createQueryRunner()
 
         this.hiveHadoop = closeAfterClass(createHiveHadoop());
         this.metastore = new BridgingHiveMetastore(
-                testingThriftHiveMetastoreBuilder()
+                testingThriftHiveMetastoreBuilder(HDFS_FILE_SYSTEM_FACTORY)
                         .metastoreClient(hiveHadoop.getHiveMetastoreEndpoint())
                         .build(this::closeAfterClass));
 

plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFlushMetadataCacheProcedure.java

@@ -22,6 +22,7 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 
+import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder;
 import static io.trino.plugin.hive.containers.HiveHadoop.HIVE3_IMAGE;
 import static io.trino.testing.TestingNames.randomNameSuffix;
@@ -42,7 +43,7 @@ protected QueryRunner createQueryRunner()
         Hive3MinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake(bucketName, HIVE3_IMAGE));
         hiveMinioDataLake.start();
         metastore = new BridgingHiveMetastore(
-                testingThriftHiveMetastoreBuilder()
+                testingThriftHiveMetastoreBuilder(HDFS_FILE_SYSTEM_FACTORY)
                         .metastoreClient(hiveMinioDataLake.getHiveMetastoreEndpoint())
                         .build(this::closeAfterClass));
 

plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java

@@ -57,6 +57,7 @@
 import static io.airlift.slice.Slices.utf8Slice;
 import static io.airlift.units.DataSize.Unit.MEGABYTE;
 import static io.trino.metastore.Partitions.makePartName;
+import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder;
 import static io.trino.plugin.hive.metastore.MetastoreUtil.getHiveBasicStatistics;
 import static io.trino.spi.type.VarcharType.VARCHAR;
@@ -97,7 +98,7 @@ protected QueryRunner createQueryRunner()
     {
         this.hiveMinioDataLake.start();
         this.metastoreClient = new BridgingHiveMetastore(
-                testingThriftHiveMetastoreBuilder()
+                testingThriftHiveMetastoreBuilder(HDFS_FILE_SYSTEM_FACTORY)
                         .metastoreClient(hiveMinioDataLake.getHiveMetastoreEndpoint())
                         .build(this::closeAfterClass));
         return S3HiveQueryRunner.builder(hiveMinioDataLake)

plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveLocationService.java

@@ -14,11 +14,11 @@
 package io.trino.plugin.hive;
 
 import io.trino.filesystem.Location;
+import io.trino.filesystem.local.LocalFileSystemFactory;
 import io.trino.plugin.hive.LocationService.WriteInfo;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 
-import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY;
 import static io.trino.plugin.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY;
 import static io.trino.plugin.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY;
@@ -80,7 +80,7 @@ public static class Assertion
 
         public Assertion(LocationHandle locationHandle, boolean overwrite)
         {
-            LocationService service = new HiveLocationService(HDFS_FILE_SYSTEM_FACTORY, new HiveConfig());
+            LocationService service = new HiveLocationService(new LocalFileSystemFactory(), new HiveConfig());
             this.actual = service.getTableWriteInfo(locationHandle, overwrite);
         }
 

plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java

@@ -72,7 +72,6 @@
 import static io.trino.plugin.hive.HiveCompressionOption.NONE;
 import static io.trino.plugin.hive.HiveStorageFormat.AVRO;
 import static io.trino.plugin.hive.HiveStorageFormat.PARQUET;
-import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.HiveTestUtils.PAGE_SORTER;
 import static io.trino.plugin.hive.HiveTestUtils.getDefaultHiveFileWriterFactories;
 import static io.trino.plugin.hive.HiveTestUtils.getDefaultHivePageSourceFactories;
@@ -388,14 +387,14 @@ private static ConnectorPageSink createPageSink(
         JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
         HivePageSinkProvider provider = new HivePageSinkProvider(
                 getDefaultHiveFileWriterFactories(config, fileSystemFactory),
-                HDFS_FILE_SYSTEM_FACTORY,
+                fileSystemFactory,
                 PAGE_SORTER,
                 HiveMetastoreFactory.ofInstance(metastore),
                 new GroupByHashPageIndexerFactory(new FlatHashStrategyCompiler(new TypeOperators())),
                 TESTING_TYPE_MANAGER,
                 config,
                 sortingFileWriterConfig,
-                new HiveLocationService(HDFS_FILE_SYSTEM_FACTORY, config),
+                new HiveLocationService(fileSystemFactory, config),
                 partitionUpdateCodec,
                 stats);
         return provider.createPageSink(transaction, getHiveSession(config), handle, TESTING_PAGE_SINK_ID);

plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOriginalFilesUtils.java

@@ -14,17 +14,18 @@
 package io.trino.plugin.hive;
 
 import io.trino.filesystem.Location;
+import io.trino.filesystem.local.LocalFileSystemFactory;
 import io.trino.orc.OrcReaderOptions;
 import io.trino.plugin.base.metrics.FileFormatDataSourceStats;
 import io.trino.plugin.hive.orc.OriginalFilesUtils;
 import org.junit.jupiter.api.Test;
 
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 
 import static com.google.common.io.Resources.getResource;
 import static io.trino.plugin.hive.AcidInfo.OriginalFileInfo;
-import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.testing.TestingConnectorSession.SESSION;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -46,7 +47,7 @@ public void testGetPrecedingRowCountSingleFile()
         long rowCountResult = OriginalFilesUtils.getPrecedingRowCount(
                 originalFileInfoList,
                 tablePath.appendPath("000001_0"),
-                HDFS_FILE_SYSTEM_FACTORY,
+                new LocalFileSystemFactory(Path.of("/")),
                 SESSION.getIdentity(),
                 new OrcReaderOptions(),
                 new FileFormatDataSourceStats());
@@ -67,7 +68,7 @@ public void testGetPrecedingRowCount()
         long rowCountResult = OriginalFilesUtils.getPrecedingRowCount(
                 originalFileInfos,
                 tablePath.appendPath("000002_0_copy_2"),
-                HDFS_FILE_SYSTEM_FACTORY,
+                new LocalFileSystemFactory(Path.of("/")),
                 SESSION.getIdentity(),
                 new OrcReaderOptions(),
                 new FileFormatDataSourceStats());

plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingThriftHiveMetastoreBuilder.java

@@ -13,9 +13,9 @@
  */
 package io.trino.plugin.hive;
 
+import com.google.common.net.HostAndPort;
 import io.airlift.units.Duration;
 import io.trino.filesystem.TrinoFileSystemFactory;
-import io.trino.filesystem.hdfs.HdfsFileSystemFactory;
 import io.trino.plugin.hive.metastore.HiveMetastoreConfig;
 import io.trino.plugin.hive.metastore.thrift.MetastoreClientAdapterProvider;
 import io.trino.plugin.hive.metastore.thrift.TestingTokenAwareMetastoreClientFactory;
@@ -33,30 +33,38 @@
 
 import static com.google.common.base.Preconditions.checkState;
 import static io.trino.plugin.base.security.UserNameProvider.SIMPLE_USER_NAME_PROVIDER;
-import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
-import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS;
 import static io.trino.plugin.hive.metastore.thrift.TestingTokenAwareMetastoreClientFactory.TIMEOUT;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.Executors.newFixedThreadPool;
 
 public final class TestingThriftHiveMetastoreBuilder
 {
+    private static final Optional<HostAndPort> SOCKS_PROXY = Optional.ofNullable(System.getProperty("hive.metastore.thrift.client.socks-proxy"))
+            .map(HostAndPort::fromString);
     private TokenAwareMetastoreClientFactory tokenAwareMetastoreClientFactory;
     private ThriftMetastoreConfig thriftMetastoreConfig = new ThriftMetastoreConfig();
-    private TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS);
+    private TrinoFileSystemFactory fileSystemFactory;
 
-    public static TestingThriftHiveMetastoreBuilder testingThriftHiveMetastoreBuilder()
+//    public static TestingThriftHiveMetastoreBuilder testingThriftHiveMetastoreBuilder()
+//    {
+//        return testingThriftHiveMetastoreBuilder(new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS));
+//    }
+
+    public static TestingThriftHiveMetastoreBuilder testingThriftHiveMetastoreBuilder(TrinoFileSystemFactory fileSystemFactory)
     {
-        return new TestingThriftHiveMetastoreBuilder();
+        return new TestingThriftHiveMetastoreBuilder(fileSystemFactory);
     }
 
-    private TestingThriftHiveMetastoreBuilder() {}
+    private TestingThriftHiveMetastoreBuilder(TrinoFileSystemFactory fileSystemFactory)
+    {
+        this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null");
+    }
 
     public TestingThriftHiveMetastoreBuilder metastoreClient(URI metastoreUri)
     {
         requireNonNull(metastoreUri, "metastoreUri is null");
         checkState(tokenAwareMetastoreClientFactory == null, "Metastore client already set");
-        tokenAwareMetastoreClientFactory = new TestingTokenAwareMetastoreClientFactory(HiveTestUtils.SOCKS_PROXY, metastoreUri);
+        tokenAwareMetastoreClientFactory = new TestingTokenAwareMetastoreClientFactory(SOCKS_PROXY, metastoreUri);
         return this;
     }
 
@@ -65,15 +73,15 @@ public TestingThriftHiveMetastoreBuilder metastoreClient(URI address, Duration t
         requireNonNull(address, "address is null");
         requireNonNull(timeout, "timeout is null");
         checkState(tokenAwareMetastoreClientFactory == null, "Metastore client already set");
-        tokenAwareMetastoreClientFactory = new TestingTokenAwareMetastoreClientFactory(HiveTestUtils.SOCKS_PROXY, address, timeout);
+        tokenAwareMetastoreClientFactory = new TestingTokenAwareMetastoreClientFactory(SOCKS_PROXY, address, timeout);
         return this;
     }
 
     public TestingThriftHiveMetastoreBuilder metastoreClient(URI uri, MetastoreClientAdapterProvider metastoreClientAdapterProvider)
     {
         requireNonNull(uri, "uri is null");
         checkState(tokenAwareMetastoreClientFactory == null, "Metastore client already set");
-        tokenAwareMetastoreClientFactory = new TestingTokenAwareMetastoreClientFactory(HiveTestUtils.SOCKS_PROXY, uri, TIMEOUT, metastoreClientAdapterProvider);
+        tokenAwareMetastoreClientFactory = new TestingTokenAwareMetastoreClientFactory(SOCKS_PROXY, uri, TIMEOUT, metastoreClientAdapterProvider);
         return this;
     }
 

plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java

@@ -81,6 +81,7 @@
 import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache;
 import static io.trino.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY;
 import static io.trino.plugin.hive.HiveColumnHandle.createBaseColumn;
+import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder;
 import static io.trino.plugin.hive.metastore.MetastoreUtil.computePartitionKeyFilter;
 import static io.trino.plugin.hive.metastore.MetastoreUtil.makePartitionName;
@@ -164,7 +165,7 @@ private ThriftMetastore createThriftHiveMetastore()
 
     private ThriftMetastore createThriftHiveMetastore(ThriftMetastoreClient client)
     {
-        return testingThriftHiveMetastoreBuilder()
+        return testingThriftHiveMetastoreBuilder(HDFS_FILE_SYSTEM_FACTORY)
                 .metastoreClient(client)
                 .build(closer::register);
     }

plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestBridgingHiveMetastore.java

@@ -34,6 +34,7 @@
 import static io.trino.metastore.PrincipalPrivileges.NO_PRIVILEGES;
 import static io.trino.plugin.hive.HiveMetadata.TRINO_QUERY_ID_NAME;
 import static io.trino.plugin.hive.HiveStorageFormat.PARQUET;
+import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE;
 import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder;
 import static io.trino.testing.TestingNames.randomNameSuffix;
@@ -69,7 +70,7 @@ final class TestBridgingHiveMetastore
             return result;
         });
 
-        metastore = new BridgingHiveMetastore(testingThriftHiveMetastoreBuilder()
+        metastore = new BridgingHiveMetastore(testingThriftHiveMetastoreBuilder(HDFS_FILE_SYSTEM_FACTORY)
                 .metastoreClient(hiveHadoop.getHiveMetastoreEndpoint(), metastoreClientAdapterProvider)
                 .thriftMetastoreConfig(new ThriftMetastoreConfig().setDeleteFilesOnDrop(true))
                 .build(closer::register));

plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java

@@ -34,6 +34,7 @@
 import java.util.Map;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
 import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder;
 import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY;
 import static io.trino.testing.containers.Minio.MINIO_REGION;
@@ -162,7 +163,7 @@ public DistributedQueryRunner build()
             addHiveProperty("s3.aws-secret-key", s3SecretKey);
             addHiveProperty("s3.path-style-access", "true");
             setMetastore(distributedQueryRunner -> new BridgingHiveMetastore(
-                    testingThriftHiveMetastoreBuilder()
+                    testingThriftHiveMetastoreBuilder(HDFS_FILE_SYSTEM_FACTORY)
                             .metastoreClient(hiveMetastoreEndpoint, thriftMetastoreTimeout)
                             .thriftMetastoreConfig(thriftMetastoreConfig)
                             .build(distributedQueryRunner::registerResource)));

plugin/trino-iceberg/pom.xml

@@ -413,6 +413,12 @@
             <scope>runtime</scope>
         </dependency>
 
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-filesystem-gcs</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
         <dependency>
             <groupId>jakarta.servlet</groupId>
             <artifactId>jakarta.servlet-api</artifactId>
@@ -526,12 +532,6 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>io.trino</groupId>
-            <artifactId>trino-hdfs</artifactId>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-hive</artifactId>