Implement exponential backoff when creating a database connection

core/src/main/java/de/bluecolored/bluemap/core/storage/sql/Database.java

@@ -122,11 +122,16 @@ public void close() throws IOException {
     }
 
     private DataSource createDataSource(ConnectionFactory connectionFactory, int maxPoolSize) {
+        ConnectionFactory retryingConnectionFactory = new RetryingConnectionFactory(
+                connectionFactory,
+                5,
+                1000,
+                16000,
+                2
+        );
+
         PoolableConnectionFactory poolableConnectionFactory =
-                new PoolableConnectionFactory(() -> {
-                    Logger.global.logDebug("Creating new SQL-Connection...");
-                    return connectionFactory.createConnection();
-                }, null);
+                new PoolableConnectionFactory(retryingConnectionFactory, null);
         poolableConnectionFactory.setPoolStatements(true);
         poolableConnectionFactory.setMaxOpenPreparedStatements(20);
         poolableConnectionFactory.setDefaultAutoCommit(false);

core/src/main/java/de/bluecolored/bluemap/core/storage/sql/RetryingConnectionFactory.java

@@ -0,0 +1,58 @@
+package de.bluecolored.bluemap.core.storage.sql;
+
+import de.bluecolored.bluemap.core.logger.Logger;
+import org.apache.commons.dbcp2.ConnectionFactory;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+
+public class RetryingConnectionFactory implements ConnectionFactory {
+
+    private final ConnectionFactory delegate;
+    private final int maxAttempts;
+    private final long initialDelayMs;
+    private final long maxDelayMs;
+    private final double backoffMultiplier;
+
+    public RetryingConnectionFactory(ConnectionFactory delegate, int maxAttempts, long initialDelayMs, long maxDelayMs, double backoffMultiplier) {
+        this.delegate = delegate;
+        this.maxAttempts = maxAttempts;
+        this.initialDelayMs = initialDelayMs;
+        this.maxDelayMs = maxDelayMs;
+        this.backoffMultiplier = backoffMultiplier;
+    }
+
+    @Override
+    public Connection createConnection() throws SQLException {
+        long delay = initialDelayMs;
+
+        for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+            try {
+                Logger.global.logDebug("Creating new SQL-Connection...");
+                return delegate.createConnection();
+            } catch (SQLException ex) {
+                if (attempt == maxAttempts) {
+                    throw ex;
+                }
+
+                Logger.global.logWarning(String.format(
+                        "Exception caught while attempting to create an SQL connection. Waiting for %.1f second(s) before retrying. " + ex,
+                        delay / 1000.0
+                ));
+
+                safeSleep(delay);
+                delay = Math.min((long) (delay * backoffMultiplier), maxDelayMs);
+            }
+        }
+
+        throw new IllegalStateException();
+    }
+
+    private void safeSleep(long delay) throws SQLException {
+        try {
+            Thread.sleep(delay);
+        } catch (InterruptedException e) {
+            throw new SQLException("Interrupted while waiting to retry connection.", e);
+        }
+    }
+}