Skip to content

Commit 2377440

Browse files
electrumelectrum
committed
Use AWS v2 SDK for S3 in product tests
1 parent af38a3c commit 2377440

12 files changed

+124
-124
lines changed

‎testing/trino-product-tests/pom.xml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,6 @@
1919
</properties>
2020

2121
<dependencies>
22-
<dependency>
23-
<groupId>com.amazonaws</groupId>
24-
<artifactId>aws-java-sdk-core</artifactId>
25-
</dependency>
26-
27-
<dependency>
28-
<groupId>com.amazonaws</groupId>
29-
<artifactId>aws-java-sdk-s3</artifactId>
30-
</dependency>
31-
3222
<dependency>
3323
<groupId>com.google.guava</groupId>
3424
<artifactId>guava</artifactId>
@@ -229,6 +219,11 @@
229219
<artifactId>glue</artifactId>
230220
</dependency>
231221

222+
<dependency>
223+
<groupId>software.amazon.awssdk</groupId>
224+
<artifactId>s3</artifactId>
225+
</dependency>
226+
232227
<dependency>
233228
<groupId>com.clickhouse</groupId>
234229
<artifactId>clickhouse-jdbc</artifactId>

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/S3ClientFactory.java‎

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,15 @@
1313
*/
1414
package io.trino.tests.product.deltalake;
1515

16-
import com.amazonaws.ClientConfiguration;
17-
import com.amazonaws.Protocol;
18-
import com.amazonaws.auth.AWSCredentials;
19-
import com.amazonaws.auth.AWSCredentialsProvider;
20-
import com.amazonaws.auth.AWSStaticCredentialsProvider;
21-
import com.amazonaws.auth.BasicAWSCredentials;
22-
import com.amazonaws.client.builder.AwsClientBuilder;
23-
import com.amazonaws.services.s3.AmazonS3;
24-
import com.amazonaws.services.s3.AmazonS3Client;
25-
import com.amazonaws.services.s3.model.Region;
2616
import io.trino.testing.minio.MinioClient;
17+
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
18+
import software.amazon.awssdk.auth.credentials.AwsCredentials;
19+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
20+
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
21+
import software.amazon.awssdk.regions.Region;
22+
import software.amazon.awssdk.services.s3.S3Client;
23+
24+
import java.net.URI;
2725

2826
import static io.trino.testing.SystemEnvironmentUtils.requireEnv;
2927
import static io.trino.testing.minio.MinioClient.DEFAULT_MINIO_ACCESS_KEY;
@@ -34,7 +32,9 @@ final class S3ClientFactory
3432
public static final String AWS_S3_SERVER_TYPE = "aws";
3533
public static final String MINIO_S3_SERVER_TYPE = "minio";
3634

37-
public AmazonS3 createS3Client(String serverType)
35+
private S3ClientFactory() {}
36+
37+
public static S3Client createS3Client(String serverType)
3838
{
3939
return switch (serverType) {
4040
case AWS_S3_SERVER_TYPE -> createAwsS3Client();
@@ -43,25 +43,22 @@ public AmazonS3 createS3Client(String serverType)
4343
};
4444
}
4545

46-
private AmazonS3 createAwsS3Client()
46+
private static S3Client createAwsS3Client()
4747
{
4848
String region = requireEnv("AWS_REGION");
49-
return AmazonS3Client.builder().withRegion(region).build();
49+
return S3Client.builder().region(Region.of(region)).build();
5050
}
5151

52-
private AmazonS3 createMinioS3Client()
52+
private static S3Client createMinioS3Client()
5353
{
54-
AWSCredentials credentials = new BasicAWSCredentials(DEFAULT_MINIO_ACCESS_KEY, DEFAULT_MINIO_SECRET_KEY);
55-
AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(credentials);
56-
ClientConfiguration clientConfiguration = new ClientConfiguration()
57-
.withProtocol(Protocol.HTTP)
58-
.withSignerOverride("AWSS3V4SignerType");
54+
AwsCredentials credentials = AwsBasicCredentials.create(DEFAULT_MINIO_ACCESS_KEY, DEFAULT_MINIO_SECRET_KEY);
55+
AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials);
5956

60-
return AmazonS3Client.builder()
61-
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(MinioClient.DEFAULT_MINIO_ENDPOINT, Region.US_East_2.name()))
62-
.withPathStyleAccessEnabled(true)
63-
.withClientConfiguration(clientConfiguration)
64-
.withCredentials(credentialsProvider)
57+
return S3Client.builder()
58+
.endpointOverride(URI.create(MinioClient.DEFAULT_MINIO_ENDPOINT))
59+
.region(Region.US_EAST_2)
60+
.forcePathStyle(true)
61+
.credentialsProvider(credentialsProvider)
6562
.build();
6663
}
6764
}

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeActiveFilesCache.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,18 @@
1313
*/
1414
package io.trino.tests.product.deltalake;
1515

16-
import com.amazonaws.services.s3.AmazonS3;
1716
import com.google.inject.Inject;
1817
import com.google.inject.name.Named;
1918
import io.trino.tempto.BeforeMethodWithContext;
2019
import org.testng.annotations.Test;
20+
import software.amazon.awssdk.services.s3.S3Client;
2121

2222
import static io.trino.tempto.assertions.QueryAssert.Row.row;
2323
import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure;
2424
import static io.trino.testing.TestingNames.randomNameSuffix;
2525
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
2626
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
27+
import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client;
2728
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry;
2829
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.removeS3Directory;
2930
import static io.trino.tests.product.utils.QueryExecutors.onDelta;
@@ -38,12 +39,12 @@ public class TestDeltaLakeActiveFilesCache
3839
@Named("s3.server_type")
3940
private String s3ServerType;
4041

41-
private AmazonS3 s3;
42+
private S3Client s3;
4243

4344
@BeforeMethodWithContext
4445
public void setup()
4546
{
46-
s3 = new S3ClientFactory().createS3Client(s3ServerType);
47+
s3 = createS3Client(s3ServerType);
4748
}
4849

4950
@Test(groups = {DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
*/
1414
package io.trino.tests.product.deltalake;
1515

16-
import com.amazonaws.services.s3.AmazonS3;
17-
import com.amazonaws.services.s3.model.ListObjectsV2Result;
1816
import com.google.common.collect.ImmutableList;
1917
import com.google.inject.Inject;
2018
import com.google.inject.name.Named;
@@ -23,6 +21,8 @@
2321
import io.trino.testng.services.Flaky;
2422
import org.testng.annotations.DataProvider;
2523
import org.testng.annotations.Test;
24+
import software.amazon.awssdk.services.s3.S3Client;
25+
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
2626

2727
import java.util.List;
2828
import java.util.Map;
@@ -38,6 +38,7 @@
3838
import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113;
3939
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
4040
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
41+
import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client;
4142
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE;
4243
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH;
4344
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry;
@@ -55,12 +56,12 @@ public class TestDeltaLakeChangeDataFeedCompatibility
5556
@Named("s3.server_type")
5657
private String s3ServerType;
5758

58-
private AmazonS3 s3Client;
59+
private S3Client s3Client;
5960

6061
@BeforeMethodWithContext
6162
public void setup()
6263
{
63-
s3Client = new S3ClientFactory().createS3Client(s3ServerType);
64+
s3Client = createS3Client(s3ServerType);
6465
}
6566

6667
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider")
@@ -886,7 +887,7 @@ private void assertThereIsNoCdfFileGenerated(String tableName, String tablePrope
886887
private void assertThatThereIsNoChangeDataFiles(String tableName)
887888
{
888889
String prefix = "databricks-compatibility-test-" + tableName + "/_change_data/";
889-
ListObjectsV2Result listResult = s3Client.listObjectsV2(bucketName, prefix);
890-
assertThat(listResult.getObjectSummaries()).isEmpty();
890+
ListObjectsV2Response response = s3Client.listObjectsV2(request -> request.bucket(bucketName).prefix(prefix));
891+
assertThat(response.contents()).isEmpty();
891892
}
892893
}

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
*/
1414
package io.trino.tests.product.deltalake;
1515

16-
import com.amazonaws.services.s3.AmazonS3;
17-
import com.amazonaws.services.s3.model.ObjectListing;
18-
import com.amazonaws.services.s3.model.S3ObjectSummary;
1916
import com.google.common.collect.ImmutableList;
2017
import com.google.inject.Inject;
2118
import com.google.inject.name.Named;
@@ -26,6 +23,8 @@
2623
import io.trino.tests.product.deltalake.util.DatabricksVersion;
2724
import org.testng.annotations.DataProvider;
2825
import org.testng.annotations.Test;
26+
import software.amazon.awssdk.services.s3.S3Client;
27+
import software.amazon.awssdk.services.s3.model.S3Object;
2928

3029
import java.math.BigDecimal;
3130
import java.util.List;
@@ -43,6 +42,7 @@
4342
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154;
4443
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
4544
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
45+
import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client;
4646
import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertLastEntryIsCheckpointed;
4747
import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertTransactionLogVersion;
4848
import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION;
@@ -62,13 +62,13 @@ public class TestDeltaLakeCheckpointsCompatibility
6262
@Named("s3.server_type")
6363
private String s3ServerType;
6464

65-
private AmazonS3 s3;
65+
private S3Client s3;
6666
private Optional<DatabricksVersion> databricksRuntimeVersion;
6767

6868
@BeforeMethodWithContext
6969
public void setup()
7070
{
71-
s3 = new S3ClientFactory().createS3Client(s3ServerType);
71+
s3 = createS3Client(s3ServerType);
7272
databricksRuntimeVersion = getDatabricksRuntimeVersion();
7373
}
7474

@@ -731,13 +731,9 @@ private List<String> listSidecarFiles(String bucketName, String tableDirectory)
731731

732732
private List<String> listS3Directory(String bucketName, String directory)
733733
{
734-
ImmutableList.Builder<String> result = ImmutableList.builder();
735-
ObjectListing listing = s3.listObjects(bucketName, directory);
736-
do {
737-
listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).forEach(result::add);
738-
listing = s3.listNextBatchOfObjects(listing);
739-
}
740-
while (listing.isTruncated());
741-
return result.build();
734+
return s3.listObjectsV2Paginator(request -> request.bucket(bucketName).prefix(directory))
735+
.contents().stream()
736+
.map(S3Object::key)
737+
.toList();
742738
}
743739
}

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@
1313
*/
1414
package io.trino.tests.product.deltalake;
1515

16-
import com.amazonaws.services.s3.AmazonS3;
1716
import com.google.common.collect.ImmutableList;
1817
import com.google.inject.Inject;
1918
import com.google.inject.name.Named;
2019
import io.trino.tempto.BeforeMethodWithContext;
2120
import io.trino.tempto.assertions.QueryAssert.Row;
2221
import io.trino.testng.services.Flaky;
2322
import org.testng.annotations.Test;
23+
import software.amazon.awssdk.services.s3.S3Client;
24+
import software.amazon.awssdk.services.s3.model.S3Object;
2425

2526
import java.sql.Date;
2627
import java.util.List;
@@ -31,6 +32,7 @@
3132
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS;
3233
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
3334
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
35+
import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client;
3436
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE;
3537
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH;
3638
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry;
@@ -46,13 +48,13 @@ public class TestDeltaLakeCloneTableCompatibility
4648
@Named("s3.server_type")
4749
private String s3ServerType;
4850

49-
private AmazonS3 s3;
51+
private S3Client s3;
5052

5153
@BeforeMethodWithContext
5254
public void setup()
5355
{
5456
super.setUp();
55-
s3 = new S3ClientFactory().createS3Client(s3ServerType);
57+
s3 = createS3Client(s3ServerType);
5658
}
5759

5860
@Test(groups = {DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
@@ -546,9 +548,11 @@ private List<String> getToBeVacuumedDataFilesFromDryRun(String tableName)
546548

547549
private List<String> getFilesFromTableDirectory(String directory)
548550
{
549-
return s3.listObjectsV2(bucketName, directory).getObjectSummaries().stream()
550-
.filter(s3ObjectSummary -> !s3ObjectSummary.getKey().contains("/_delta_log"))
551-
.map(s3ObjectSummary -> format("s3://%s/%s", bucketName, s3ObjectSummary.getKey()))
551+
return s3.listObjectsV2Paginator(request -> request.bucket(bucketName).prefix(directory))
552+
.contents().stream()
553+
.map(S3Object::key)
554+
.filter(key -> !key.contains("/_delta_log"))
555+
.map(key -> format("s3://%s/%s", bucketName, key))
552556
.collect(toImmutableList());
553557
}
554558

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateOrReplaceTableAsSelectCompatibility.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,22 @@
1313
*/
1414
package io.trino.tests.product.deltalake;
1515

16-
import com.amazonaws.services.s3.AmazonS3;
1716
import com.google.common.collect.ImmutableList;
1817
import com.google.inject.Inject;
1918
import com.google.inject.name.Named;
2019
import io.trino.tempto.BeforeMethodWithContext;
2120
import io.trino.tempto.assertions.QueryAssert.Row;
2221
import io.trino.tempto.query.QueryExecutor;
2322
import org.testng.annotations.Test;
23+
import software.amazon.awssdk.services.s3.S3Client;
2424

2525
import java.util.List;
2626

2727
import static io.trino.tempto.assertions.QueryAssert.Row.row;
2828
import static io.trino.testing.TestingNames.randomNameSuffix;
2929
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
3030
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
31+
import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client;
3132
import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertLastEntryIsCheckpointed;
3233
import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertTransactionLogVersion;
3334
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry;
@@ -42,13 +43,13 @@ public class TestDeltaLakeCreateOrReplaceTableAsSelectCompatibility
4243
@Named("s3.server_type")
4344
private String s3ServerType;
4445

45-
private AmazonS3 s3;
46+
private S3Client s3;
4647

4748
@BeforeMethodWithContext
4849
public void setup()
4950
{
5051
super.setUp();
51-
s3 = new S3ClientFactory().createS3Client(s3ServerType);
52+
s3 = createS3Client(s3ServerType);
5253
}
5354

5455
@Test(groups = {DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})

0 commit comments

Comments
 (0)