HIVE-29305: Migrate from the AWS SDK for Java v1 to v2

pom.xml

@@ -233,8 +233,7 @@
     <datasketches.version>2.0.0</datasketches.version>
     <spotbugs.version>4.8.6</spotbugs.version>
     <validation-api.version>1.1.0.Final</validation-api.version>
-    <aws-secretsmanager-caching.version>1.0.1</aws-secretsmanager-caching.version>
-    <aws-java-sdk.version>1.12.720</aws-java-sdk.version>
+    <aws-java-sdk.version>2.35.11</aws-java-sdk.version>
     <jansi.version>2.4.0</jansi.version>
     <!-- If upgrading, upgrade atlas as well in ql/pom.xml, which brings in some springframework dependencies transitively -->
     <spring.version>5.3.39</spring.version>
@@ -273,26 +272,9 @@
     <dependencies>
       <!-- dependencies are always listed in sorted order by groupId, artifactId -->
       <dependency>
-        <groupId>com.amazonaws</groupId>
-        <artifactId>aws-java-sdk-bundle</artifactId>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>secretsmanager</artifactId>
         <version>${aws-java-sdk.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>io.netty</groupId>
-            <artifactId>*</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <dependency>
-        <groupId>com.amazonaws.secretsmanager</groupId>
-        <artifactId>aws-secretsmanager-caching-java</artifactId>
-        <version>${aws-secretsmanager-caching.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-secretsmanager</artifactId>
-          </exclusion>
-        </exclusions>
       </dependency>
       <dependency>
         <groupId>com.esotericsoftware.kryo</groupId>

ql/pom.xml

@@ -168,13 +168,8 @@
     </dependency>
     <!-- inter-project -->
     <dependency>
-      <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk-bundle</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.amazonaws.secretsmanager</groupId>
-      <artifactId>aws-secretsmanager-caching-java</artifactId>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>secretsmanager</artifactId>
     </dependency>
     <dependency>
       <groupId>com.esotericsoftware.kryo</groupId>

ql/src/java/org/apache/hadoop/hive/ql/secrets/AWSSecretsManagerSecretSource.java

@@ -17,27 +17,35 @@
  */
 package org.apache.hadoop.hive.ql.secrets;
 
-import com.amazonaws.secretsmanager.caching.SecretCache;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
+import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest;
+import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Implementation of SecretSource which loads secrets from AWS Secrets Manager.
  * The format of the uri is "aws-sm:///{key-name-or-arn}"
- * It uses aws secrets cache sdk to fetch and refresh the secret, the environment must be setup so that the default
+ * It uses AWS SDK v2 with Guava cache to fetch and refresh the secret, the environment must be setup so that the default
  * client can load the secret else it will fail.
  * It expects the secret fetched to be a json string with "password" as the key for password, this is default for
  * redshift, rds or external database configs. It does not make use of any other fields.
  */
 public class AWSSecretsManagerSecretSource implements SecretSource {
-  // Do not create SecretCache here, it fails to initialize in non-aws aware environments.
-  private volatile SecretCache cache = null;
+  // Do not create cache here, it fails to initialize in non-aws aware environments.
+  private volatile LoadingCache<String, String> cache = null;
+  private volatile SecretsManagerClient client = null;
   private final ObjectMapper mapper = new ObjectMapper();
 
   /**
@@ -63,10 +71,10 @@ public String getSecret(URI uri) throws IOException {
 
     String secretsString;
     try {
-      secretsString = cache.getSecretString(key);
-    } catch (Exception e) {
+      secretsString = cache.get(key);
+    } catch (ExecutionException e) {
       // Wrap any exception from the above service call to IOException.
-      throw new IOException("Error trying to get secret", e);
+      throw new IOException("Error trying to get secret", e.getCause());
     }
     if (secretsString == null) {
       throw new IOException("secret was not found");
@@ -84,26 +92,50 @@ public String getSecret(URI uri) throws IOException {
   }
 
   private void initCache() {
-    // DCL based SecretCache setup, to ensure thread safety.
+    // DCL based cache setup, to ensure thread safety.
     if (cache == null) {
       synchronized (this) {
         if (cache == null) {
-          cache = new SecretCache();
+          client = SecretsManagerClient.builder().build();
+          cache = CacheBuilder.newBuilder()
+              .expireAfterWrite(1, TimeUnit.HOURS)
+              .maximumSize(100)
+              .build(new CacheLoader<String, String>() {
+                @Override
+                public String load(String secretId) throws Exception {
+                  GetSecretValueRequest request = GetSecretValueRequest.builder()
+                      .secretId(secretId)
+                      .build();
+                  GetSecretValueResponse response = client.getSecretValue(request);
+                  return response.secretString();
+                }
+              });
         }
       }
     }
   }
 
   @VisibleForTesting
-  void setCache(SecretCache cache) {
+  void setCache(LoadingCache<String, String> cache) {
     this.cache = cache;
   }
 
+  @VisibleForTesting
+  void setClient(SecretsManagerClient client) {
+    this.client = client;
+  }
+
   /**
    * Clean the resources in the class.
    */
   @Override
   public void close() {
-    cache.close();
+    if (cache != null) {
+      cache.invalidateAll();
+      cache.cleanUp();
+    }
+    if (client != null) {
+      client.close();
+    }
   }
 }

ql/src/test/org/apache/hadoop/hive/ql/secrets/AWSSecretsManagerSecretSourceTest.java

@@ -19,21 +19,22 @@
 
 import static org.junit.Assert.assertEquals;
 
-import com.amazonaws.secretsmanager.caching.SecretCache;
+import com.google.common.cache.LoadingCache;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.concurrent.ExecutionException;
 
 public class AWSSecretsManagerSecretSourceTest {
   private AWSSecretsManagerSecretSource source = new AWSSecretsManagerSecretSource();
-  private SecretCache cache;
+  private LoadingCache<String, String> cache;
 
   @Before
   public void setupTest() {
-    cache = Mockito.mock(SecretCache.class);
+    cache = Mockito.mock(LoadingCache.class);
     source.setCache(cache);
   }
 
@@ -50,25 +51,25 @@ public void testAWSSecretsManagerInvalidScheme() throws Exception {
   @Test
   public void testAWSSecretsManagerSuccess() throws Exception {
     // Test good case.
-    Mockito.when(cache.getSecretString("test")).thenReturn("{\"password\":\"super-secret\"}");
+    Mockito.when(cache.get("test")).thenReturn("{\"password\":\"super-secret\"}");
     assertEquals("super-secret", source.getSecret(new URI("aws-sm:///test")));
   }
 
   @Test(expected = IOException.class)
   public void testAWSSecretsManagerServiceException() throws Exception {
-    Mockito.when(cache.getSecretString("bad")).thenThrow(RuntimeException.class);
+    Mockito.when(cache.get("bad")).thenThrow(new ExecutionException(new RuntimeException()));
     source.getSecret(new URI("aws-sm:///bad"));
   }
 
   @Test(expected = IOException.class)
   public void testAWSSecretsManagerInvalidJson() throws Exception {
-    Mockito.when(cache.getSecretString("test")).thenReturn("{\"password\":\"super-secret\"");
+    Mockito.when(cache.get("test")).thenReturn("{\"password\":\"super-secret\"");
     source.getSecret(new URI("aws-sm:///test"));
   }
 
   @Test(expected = IOException.class)
   public void testAWSSecretsManagerArrayJson() throws Exception {
-    Mockito.when(cache.getSecretString("test")).thenReturn("[]");
+    Mockito.when(cache.get("test")).thenReturn("[]");
     source.getSecret(new URI("aws-sm:///test"));
   }
-}
+}

standalone-metastore/pom.xml

@@ -100,7 +100,7 @@
     <protobuf.version>3.25.5</protobuf.version>
     <protoc-jar-maven-plugin.version>3.11.4</protoc-jar-maven-plugin.version>
     <protoc.path>${env.PROTOC_PATH}</protoc.path>
-    <io.grpc.version>1.72.0</io.grpc.version>
+    <io.grpc.version>1.76.0</io.grpc.version>
     <sqlline.version>1.9.0</sqlline.version>
     <netty.version>4.1.127.Final</netty.version>
     <!-- HIVE-28992: only upgrade to newer than 3.25.0 if you tested the prompt -->