Trino: Add Trino Docker Compose for integration testing

Makefile

@@ -103,6 +103,10 @@ test-integration-rebuild: ## Rebuild integration Docker services from scratch
 	docker compose -f dev/docker-compose-integration.yml rm -f
 	docker compose -f dev/docker-compose-integration.yml build --no-cache
 
+test-integration-trino: ## Run tests marked with @pytest.mark.integration_trino
+	sh ./dev/run-trino.sh
+	$(TEST_RUNNER) pytest tests/ -m integration_trino $(PYTEST_ARGS)
+
 test-s3: ## Run tests marked with @pytest.mark.s3
 	sh ./dev/run-minio.sh
 	$(TEST_RUNNER) pytest tests/ -m s3 $(PYTEST_ARGS)
@@ -116,7 +120,7 @@ test-gcs: ## Run tests marked with @pytest.mark.gcs
 	$(TEST_RUNNER) pytest tests/ -m gcs $(PYTEST_ARGS)
 
 test-coverage: COVERAGE=1
-test-coverage: test test-integration test-s3 test-adls test-gcs coverage-report ## Run all tests with coverage and report
+test-coverage: test test-integration test-integration-trino test-s3 test-adls test-gcs coverage-report ## Run all tests with coverage and report
 
 coverage-report: ## Combine and report coverage
 	poetry run coverage combine

dev/docker-compose-integration.yml

@@ -54,6 +54,21 @@ services:
       - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
       - CATALOG_S3_ENDPOINT=http://minio:9000
       - CATALOG_JDBC_STRICT__MODE=true
+
+  trino:
+    image: trinodb/trino:476
+    container_name: pyiceberg-trino
+    networks:
+      iceberg_net:
+    ports:
+      - 8082:8080
+    environment:
+      - CATALOG_MANAGEMENT=dynamic
+    depends_on:
+      - rest
+      - hive
+    volumes:
+      - ./trino/catalog:/etc/trino/catalog
   minio:
     image: minio/minio
     container_name: pyiceberg-minio

dev/docker-compose-trino.yml

@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+services:
+  rest:
+    image: apache/iceberg-rest-fixture
+    container_name: pyiceberg-rest
+    networks:
+      iceberg_net:
+    ports:
+      - 8181:8181
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+      - CATALOG_WAREHOUSE=s3://warehouse/
+      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+      - CATALOG_S3_ENDPOINT=http://minio:9000
+
+  trino:
+    image: trinodb/trino:476
+    container_name: pyiceberg-trino
+    networks:
+      iceberg_net:
+    ports:
+      - 8082:8080
+    environment:
+      - CATALOG_MANAGEMENT=dynamic
+    depends_on:
+      - rest
+      - hive
+    volumes:
+      - ./trino/catalog:/etc/trino/catalog
+      - ./trino/config.properties:/etc/trino/config.properties
+
+  minio:
+    image: minio/minio
+    container_name: pyiceberg-minio
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DOMAIN=minio
+    networks:
+      iceberg_net:
+        aliases:
+          - warehouse.minio
+    ports:
+      - 9001:9001
+      - 9000:9000
+    command: ["server", "/data", "--console-address", ":9001"]
+  mc:
+    depends_on:
+      - minio
+    image: minio/mc
+    container_name: pyiceberg-mc
+    networks:
+      iceberg_net:
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+    entrypoint: >
+      /bin/sh -c "
+      until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc mb minio/warehouse;
+      /usr/bin/mc policy set public minio/warehouse;
+      tail -f /dev/null
+      "
+
+  hive:
+    build: hive/
+    container_name: hive
+    hostname: hive
+    networks:
+      iceberg_net:
+    ports:
+      - 9083:9083
+    environment:
+      SERVICE_NAME: "metastore"
+      SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/"
+
+networks:
+  iceberg_net:

dev/run-trino.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -ex
+
+if [ $(docker ps -q --filter "name=pyiceberg-trino" --filter "status=running" ) ]; then
+    echo "Trino service running"
+else
+    docker compose -f dev/docker-compose-trino.yml kill
+    docker compose -f dev/docker-compose-trino.yml up -d
+    while [ -z $(docker ps -q --filter "name=pyiceberg-trino" --filter "status=running" ) ]
+    do
+      echo "Waiting for Trino"
+      sleep 1
+    done
+fi

dev/trino/catalog/warehouse_hive.properties

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+connector.name=iceberg
+iceberg.catalog.type=hive_metastore
+iceberg.expire-snapshots.min-retention=0d
+iceberg.remove-orphan-files.min-retention=0d
+iceberg.register-table-procedure.enabled=true
+hive.metastore.uri=thrift://hive:9083
+iceberg.hive-catalog-name=hive
+fs.native-s3.enabled=true
+s3.region=us-east-1
+s3.aws-access-key=admin
+s3.aws-secret-key=password
+s3.endpoint=http://minio:9000
+s3.path-style-access=false

dev/trino/catalog/warehouse_rest.properties

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+connector.name=iceberg
+iceberg.catalog.type=rest
+iceberg.rest-catalog.uri=http://rest:8181
+iceberg.rest-catalog.warehouse=s3://warehouse/default
+iceberg.rest-catalog.nested-namespace-enabled=true
+iceberg.rest-catalog.case-insensitive-name-matching=true
+iceberg.expire-snapshots.min-retention=0d
+iceberg.remove-orphan-files.min-retention=0d
+iceberg.register-table-procedure.enabled=true
+fs.native-s3.enabled=true
+s3.region=us-east-1
+s3.aws-access-key=admin
+s3.aws-secret-key=password
+s3.endpoint=http://minio:9000
+s3.path-style-access=false

dev/trino/config.properties

@@ -0,0 +1,8 @@
+#single node install config
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+discovery.uri=http://localhost:8080
+http-server.process-forwarded=true
+http-server.https.enabled=false
+catalog.management=${ENV:CATALOG_MANAGEMENT}