From d2efc61d341351a3d8a395e92f5a0ca110804aa5 Mon Sep 17 00:00:00 2001 From: Aleksei Balan <28422872+abalanonline@users.noreply.github.com> Date: Mon, 7 Jul 2025 15:39:26 -0400 Subject: [PATCH 1/5] catching errors in DefaultServerMonitor --- .../com/mongodb/internal/connection/DefaultServerMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java b/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java index 4842a6c8a2..acceb92fa6 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java +++ b/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java @@ -261,7 +261,7 @@ private ServerDescription lookupServerDescription(final ServerDescription curren // Get existing connection return doHeartbeat(currentServerDescription, shouldStreamResponses); - } catch (Throwable t) { + } catch (Exception t) { roundTripTimeSampler.reset(); InternalConnection localConnection = withLock(lock, () -> { InternalConnection result = connection; From d5beaf660fac51b0c48ef3332b558a900b52bc02 Mon Sep 17 00:00:00 2001 From: Valentin Kovalenko Date: Wed, 23 Jul 2025 21:59:13 -0600 Subject: [PATCH 2/5] Properly handle exceptions in threads created by MongoClient JAVA-5913 --- .../AsynchronousClusterEventListener.java | 25 +++-- .../internal/connection/BaseCluster.java | 61 ++++++------ .../connection/DefaultConnectionPool.java | 2 +- .../DefaultDnsSrvRecordMonitor.java | 65 +++++++------ .../connection/DefaultServerMonitor.java | 13 ++- .../connection/LoadBalancedCluster.java | 95 ++++++++++--------- .../connection/PowerOfTwoBufferPool.java | 10 +- .../TlsChannelStreamFactoryFactory.java | 3 + .../src/main/com/mongodb/MongoClient.java | 37 ++++---- 9 files changed, 178 insertions(+), 133 deletions(-) diff --git a/driver-core/src/main/com/mongodb/internal/connection/AsynchronousClusterEventListener.java b/driver-core/src/main/com/mongodb/internal/connection/AsynchronousClusterEventListener.java index 22d577d9b7..77f827f01f 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/AsynchronousClusterEventListener.java +++ b/driver-core/src/main/com/mongodb/internal/connection/AsynchronousClusterEventListener.java @@ -31,6 +31,8 @@ import com.mongodb.event.ServerMonitorListener; import com.mongodb.event.ServerOpeningEvent; import com.mongodb.internal.VisibleForTesting; +import com.mongodb.internal.diagnostics.logging.Logger; +import com.mongodb.internal.diagnostics.logging.Loggers; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -53,6 +55,8 @@ */ @ThreadSafe final class AsynchronousClusterEventListener implements ClusterListener, ServerListener, ServerMonitorListener { + private static final Logger LOGGER = Loggers.getLogger("cluster"); + private final BlockingQueue> eventPublishers = new LinkedBlockingQueue<>(); private final ClusterListener clusterListener; private final ServerListener serverListener; @@ -162,16 +166,21 @@ private void addEvent(final Supplier supplier) { } private void publishEvents() { - while (true) { - try { - Supplier eventPublisher = eventPublishers.take(); - boolean isLastEvent = eventPublisher.get(); - if (isLastEvent) { - break; + try { + while (true) { + try { + Supplier eventPublisher = eventPublishers.take(); + boolean isLastEvent = eventPublisher.get(); + if (isLastEvent) { + break; + } + } catch (Exception e) { + // ignore exceptions thrown from listeners, also ignore interrupts that user code may cause } - } catch (Exception e) { - // ignore exceptions thrown from listeners, also ignore interrupts that user code may cause } + } catch (Throwable t) { + LOGGER.error(this + " stopped working. You may want to recreate the MongoClient", t); + throw t; } } } diff --git a/driver-core/src/main/com/mongodb/internal/connection/BaseCluster.java b/driver-core/src/main/com/mongodb/internal/connection/BaseCluster.java index fb840d9ad0..b7b5660ae6 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/BaseCluster.java +++ b/driver-core/src/main/com/mongodb/internal/connection/BaseCluster.java @@ -519,39 +519,44 @@ private final class WaitQueueHandler implements Runnable { } public void run() { - while (!isClosed) { - CountDownLatch currentPhase = phase.get(); - ClusterDescription curDescription = description; + try { + while (!isClosed) { + CountDownLatch currentPhase = phase.get(); + ClusterDescription curDescription = description; + + Timeout timeout = Timeout.infinite(); + boolean someWaitersNotSatisfied = false; + for (Iterator iter = waitQueue.iterator(); iter.hasNext();) { + ServerSelectionRequest currentRequest = iter.next(); + if (handleServerSelectionRequest(currentRequest, currentPhase, curDescription)) { + iter.remove(); + } else { + someWaitersNotSatisfied = true; + timeout = Timeout.earliest( + timeout, + currentRequest.getTimeout(), + startMinWaitHeartbeatTimeout()); + } + } - Timeout timeout = Timeout.infinite(); - boolean someWaitersNotSatisfied = false; - for (Iterator iter = waitQueue.iterator(); iter.hasNext();) { - ServerSelectionRequest currentRequest = iter.next(); - if (handleServerSelectionRequest(currentRequest, currentPhase, curDescription)) { - iter.remove(); - } else { - someWaitersNotSatisfied = true; - timeout = Timeout.earliest( - timeout, - currentRequest.getTimeout(), - startMinWaitHeartbeatTimeout()); + if (someWaitersNotSatisfied) { + connect(); } - } - if (someWaitersNotSatisfied) { - connect(); + try { + timeout.awaitOn(currentPhase, () -> "ignored"); + } catch (MongoInterruptedException closed) { + // The cluster has been closed and the while loop will exit. + } } - - try { - timeout.awaitOn(currentPhase, () -> "ignored"); - } catch (MongoInterruptedException closed) { - // The cluster has been closed and the while loop will exit. + // Notify all remaining waiters that a shutdown is in progress + for (Iterator iter = waitQueue.iterator(); iter.hasNext();) { + iter.next().onResult(null, new MongoClientException("Shutdown in progress")); + iter.remove(); } - } - // Notify all remaining waiters that a shutdown is in progress - for (Iterator iter = waitQueue.iterator(); iter.hasNext();) { - iter.next().onResult(null, new MongoClientException("Shutdown in progress")); - iter.remove(); + } catch (Throwable t) { + LOGGER.error(this + " stopped working. You may want to recreate the MongoClient", t); + throw t; } } } diff --git a/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java b/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java index 0ef94d559c..fc4d2eb21d 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java +++ b/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java @@ -1369,7 +1369,7 @@ private void runAndLogUncaught(final Runnable runnable) { try { runnable.run(); } catch (Throwable t) { - LOGGER.error("The pool is not going to work correctly from now on. You may want to recreate the MongoClient", t); + LOGGER.error(this + "stopped working. You may want to recreate the MongoClient", t); throw t; } } diff --git a/driver-core/src/main/com/mongodb/internal/connection/DefaultDnsSrvRecordMonitor.java b/driver-core/src/main/com/mongodb/internal/connection/DefaultDnsSrvRecordMonitor.java index d535cb0aec..9a17f2eb4f 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/DefaultDnsSrvRecordMonitor.java +++ b/driver-core/src/main/com/mongodb/internal/connection/DefaultDnsSrvRecordMonitor.java @@ -75,41 +75,46 @@ private class DnsSrvRecordMonitorRunnable implements Runnable { @Override public void run() { - while (!isClosed && shouldContinueMonitoring()) { - try { - List resolvedHostNames = dnsResolver.resolveHostFromSrvRecords(hostName, srvServiceName); - Set hosts = createServerAddressSet(resolvedHostNames); - - if (isClosed) { - return; - } + try { + while (!isClosed && shouldContinueMonitoring()) { + try { + List resolvedHostNames = dnsResolver.resolveHostFromSrvRecords(hostName, srvServiceName); + Set hosts = createServerAddressSet(resolvedHostNames); + + if (isClosed) { + return; + } - if (!hosts.equals(currentHosts)) { - try { - dnsSrvRecordInitializer.initialize(unmodifiableSet(hosts)); - currentHosts = hosts; - } catch (Exception e) { - LOGGER.warn("Exception in monitor thread during notification of DNS resolution state change", e); + if (!hosts.equals(currentHosts)) { + try { + dnsSrvRecordInitializer.initialize(unmodifiableSet(hosts)); + currentHosts = hosts; + } catch (Exception e) { + LOGGER.warn("Exception in monitor thread during notification of DNS resolution state change", e); + } } + } catch (MongoException e) { + if (currentHosts.isEmpty()) { + dnsSrvRecordInitializer.initialize(e); + } + LOGGER.info("Exception while resolving SRV records", e); + } catch (Exception e) { + if (currentHosts.isEmpty()) { + dnsSrvRecordInitializer.initialize(new MongoInternalException("Unexpected runtime exception", e)); + } + LOGGER.info("Unexpected runtime exception while resolving SRV record", e); } - } catch (MongoException e) { - if (currentHosts.isEmpty()) { - dnsSrvRecordInitializer.initialize(e); - } - LOGGER.info("Exception while resolving SRV records", e); - } catch (Exception e) { - if (currentHosts.isEmpty()) { - dnsSrvRecordInitializer.initialize(new MongoInternalException("Unexpected runtime exception", e)); - } - LOGGER.info("Unexpected runtime exception while resolving SRV record", e); - } - try { - Thread.sleep(getRescanFrequencyMillis()); - } catch (InterruptedException closed) { - // fall through + try { + Thread.sleep(getRescanFrequencyMillis()); + } catch (InterruptedException closed) { + // fall through + } + clusterType = dnsSrvRecordInitializer.getClusterType(); } - clusterType = dnsSrvRecordInitializer.getClusterType(); + } catch (Throwable t) { + LOGGER.error(this + " stopped working. You may want to recreate the MongoClient", t); + throw t; } } diff --git a/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java b/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java index acceb92fa6..8a72e47428 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java +++ b/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java @@ -231,8 +231,9 @@ public void run() { } } catch (InterruptedException | MongoInterruptedException closed) { // stop the monitor - } catch (RuntimeException e) { - LOGGER.error(format("Server monitor for %s exiting with exception", serverId), e); + } catch (Throwable t) { + LOGGER.error(format("%s for %s stopped working. You may want to recreate the MongoClient", this, serverId), t); + throw t; } finally { if (connection != null) { connection.close(); @@ -532,7 +533,7 @@ public void run() { } else { pingServer(connection); } - } catch (Throwable t) { + } catch (Exception t) { if (connection != null) { connection.close(); connection = null; @@ -542,7 +543,11 @@ public void run() { } } catch (InterruptedException closed) { // stop the monitor - } finally { + } catch (Throwable t) { + LOGGER.error(format("%s for %s stopped working. You may want to recreate the MongoClient", this, serverId), t); + throw t; + } + finally { if (connection != null) { connection.close(); } diff --git a/driver-core/src/main/com/mongodb/internal/connection/LoadBalancedCluster.java b/driver-core/src/main/com/mongodb/internal/connection/LoadBalancedCluster.java index 588bd9f609..c6211e8ff1 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/LoadBalancedCluster.java +++ b/driver-core/src/main/com/mongodb/internal/connection/LoadBalancedCluster.java @@ -363,55 +363,60 @@ private void notifyWaitQueueHandler(final ServerSelectionRequest request) { private final class WaitQueueHandler implements Runnable { public void run() { - List timeoutList = new ArrayList<>(); - while (!(isClosed() || initializationCompleted)) { - lock.lock(); - try { - if (isClosed() || initializationCompleted) { - break; - } - Timeout waitTimeNanos = Timeout.infinite(); - - for (Iterator iterator = waitQueue.iterator(); iterator.hasNext();) { - ServerSelectionRequest next = iterator.next(); - - Timeout nextTimeout = next.getTimeout(); - Timeout waitTimeNanosFinal = waitTimeNanos; - waitTimeNanos = nextTimeout.call(NANOSECONDS, - () -> Timeout.earliest(waitTimeNanosFinal, nextTimeout), - (ns) -> Timeout.earliest(waitTimeNanosFinal, nextTimeout), - () -> { - timeoutList.add(next); - iterator.remove(); - return waitTimeNanosFinal; - }); - } - if (timeoutList.isEmpty()) { - try { - waitTimeNanos.awaitOn(condition, () -> "ignored"); - } catch (MongoInterruptedException unexpected) { - fail(); + try { + List timeoutList = new ArrayList<>(); + while (!(isClosed() || initializationCompleted)) { + lock.lock(); + try { + if (isClosed() || initializationCompleted) { + break; + } + Timeout waitTimeNanos = Timeout.infinite(); + + for (Iterator iterator = waitQueue.iterator(); iterator.hasNext();) { + ServerSelectionRequest next = iterator.next(); + + Timeout nextTimeout = next.getTimeout(); + Timeout waitTimeNanosFinal = waitTimeNanos; + waitTimeNanos = nextTimeout.call(NANOSECONDS, + () -> Timeout.earliest(waitTimeNanosFinal, nextTimeout), + (ns) -> Timeout.earliest(waitTimeNanosFinal, nextTimeout), + () -> { + timeoutList.add(next); + iterator.remove(); + return waitTimeNanosFinal; + }); + } + if (timeoutList.isEmpty()) { + try { + waitTimeNanos.awaitOn(condition, () -> "ignored"); + } catch (MongoInterruptedException unexpected) { + fail(); + } } + } finally { + lock.unlock(); } - } finally { - lock.unlock(); + timeoutList.forEach(request -> request.onError(createTimeoutException(request + .getOperationContext() + .getTimeoutContext()))); + timeoutList.clear(); } - timeoutList.forEach(request -> request.onError(createTimeoutException(request - .getOperationContext() - .getTimeoutContext()))); - timeoutList.clear(); - } - // This code is executed either after closing the LoadBalancedCluster or after initializing it. In the latter case, - // waitQueue is guaranteed to be empty (as DnsSrvRecordInitializer.initialize clears it and no thread adds new elements to - // it after that). So shutdownList is not empty iff LoadBalancedCluster is closed, in which case we need to complete the - // requests in it. - List shutdownList = Locks.withLock(lock, () -> { - ArrayList result = new ArrayList<>(waitQueue); - waitQueue.clear(); - return result; - }); - shutdownList.forEach(request -> request.onError(createShutdownException())); + // This code is executed either after closing the LoadBalancedCluster or after initializing it. In the latter case, + // waitQueue is guaranteed to be empty (as DnsSrvRecordInitializer.initialize clears it and no thread adds new elements to + // it after that). So shutdownList is not empty iff LoadBalancedCluster is closed, in which case we need to complete the + // requests in it. + List shutdownList = Locks.withLock(lock, () -> { + ArrayList result = new ArrayList<>(waitQueue); + waitQueue.clear(); + return result; + }); + shutdownList.forEach(request -> request.onError(createShutdownException())); + } catch (Throwable t) { + LOGGER.error(this + " stopped working. You may want to recreate the MongoClient", t); + throw t; + } } } diff --git a/driver-core/src/main/com/mongodb/internal/connection/PowerOfTwoBufferPool.java b/driver-core/src/main/com/mongodb/internal/connection/PowerOfTwoBufferPool.java index 15a319157d..a8c7f87a24 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/PowerOfTwoBufferPool.java +++ b/driver-core/src/main/com/mongodb/internal/connection/PowerOfTwoBufferPool.java @@ -16,6 +16,8 @@ package com.mongodb.internal.connection; +import com.mongodb.internal.diagnostics.logging.Logger; +import com.mongodb.internal.diagnostics.logging.Loggers; import com.mongodb.internal.thread.DaemonThreadFactory; import org.bson.ByteBuf; import org.bson.ByteBufNIO; @@ -34,6 +36,7 @@ *

This class is not part of the public API and may be removed or changed at any time

*/ public class PowerOfTwoBufferPool implements BufferProvider { + private static final Logger LOGGER = Loggers.getLogger("connection"); /** * The global default pool. Pruning is enabled on this pool. Idle buffers are pruned after one minute. @@ -137,7 +140,12 @@ public void release(final ByteBuffer buffer) { } private void prune() { - powerOfTwoToPoolMap.values().forEach(BufferPool::prune); + try { + powerOfTwoToPoolMap.values().forEach(BufferPool::prune); + } catch (Throwable t) { + LOGGER.error(this + " stopped pruning idle buffer pools. You may want to recreate the MongoClient", t); + throw t; + } } static int log2(final int powerOfTwo) { diff --git a/driver-core/src/main/com/mongodb/internal/connection/TlsChannelStreamFactoryFactory.java b/driver-core/src/main/com/mongodb/internal/connection/TlsChannelStreamFactoryFactory.java index df8b3c2fe4..b0fae1d044 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/TlsChannelStreamFactoryFactory.java +++ b/driver-core/src/main/com/mongodb/internal/connection/TlsChannelStreamFactoryFactory.java @@ -162,6 +162,9 @@ void start() { LOGGER.warn("Exception in selector loop", e); } } + } catch (Throwable t) { + LOGGER.error(this + " stopped working. You may want to recreate the MongoClient", t); + throw t; } finally { try { selector.close(); diff --git a/driver-legacy/src/main/com/mongodb/MongoClient.java b/driver-legacy/src/main/com/mongodb/MongoClient.java index 31da4c1b9e..951622cf20 100644 --- a/driver-legacy/src/main/com/mongodb/MongoClient.java +++ b/driver-legacy/src/main/com/mongodb/MongoClient.java @@ -856,29 +856,34 @@ private ExecutorService createCursorCleaningService() { } private void cleanCursors() { - ServerCursorAndNamespace cur; - while ((cur = orphanedCursors.poll()) != null) { - ReadWriteBinding binding = new SingleServerBinding(delegate.getCluster(), cur.serverCursor.getAddress(), - new OperationContext(IgnorableRequestContext.INSTANCE, NoOpSessionContext.INSTANCE, - new TimeoutContext(getTimeoutSettings()), options.getServerApi())); - try { - ConnectionSource source = binding.getReadConnectionSource(); + try { + ServerCursorAndNamespace cur; + while ((cur = orphanedCursors.poll()) != null) { + ReadWriteBinding binding = new SingleServerBinding(delegate.getCluster(), cur.serverCursor.getAddress(), + new OperationContext(IgnorableRequestContext.INSTANCE, NoOpSessionContext.INSTANCE, + new TimeoutContext(getTimeoutSettings()), options.getServerApi())); try { - Connection connection = source.getConnection(); + ConnectionSource source = binding.getReadConnectionSource(); try { - BsonDocument killCursorsCommand = new BsonDocument("killCursors", new BsonString(cur.namespace.getCollectionName())) - .append("cursors", new BsonArray(singletonList(new BsonInt64(cur.serverCursor.getId())))); - connection.command(cur.namespace.getDatabaseName(), killCursorsCommand, NoOpFieldNameValidator.INSTANCE, - ReadPreference.primary(), new BsonDocumentCodec(), source.getOperationContext()); + Connection connection = source.getConnection(); + try { + BsonDocument killCursorsCommand = new BsonDocument("killCursors", new BsonString(cur.namespace.getCollectionName())) + .append("cursors", new BsonArray(singletonList(new BsonInt64(cur.serverCursor.getId())))); + connection.command(cur.namespace.getDatabaseName(), killCursorsCommand, NoOpFieldNameValidator.INSTANCE, + ReadPreference.primary(), new BsonDocumentCodec(), source.getOperationContext()); + } finally { + connection.release(); + } } finally { - connection.release(); + source.release(); } } finally { - source.release(); + binding.release(); } - } finally { - binding.release(); } + } catch (Throwable t) { + LOGGER.error(this + " stopped cleaning cursors. You may want to recreate the MongoClient", t); + throw t; } } From 3f27bf9916f74aadc0fc03c03b7e7ad97fa1f223 Mon Sep 17 00:00:00 2001 From: Valentin Kovalenko Date: Wed, 23 Jul 2025 22:53:18 -0600 Subject: [PATCH 3/5] Fix a typo --- .../com/mongodb/internal/connection/DefaultConnectionPool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java b/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java index fc4d2eb21d..3c829e9d9b 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java +++ b/driver-core/src/main/com/mongodb/internal/connection/DefaultConnectionPool.java @@ -1369,7 +1369,7 @@ private void runAndLogUncaught(final Runnable runnable) { try { runnable.run(); } catch (Throwable t) { - LOGGER.error(this + "stopped working. You may want to recreate the MongoClient", t); + LOGGER.error(this + " stopped working. You may want to recreate the MongoClient", t); throw t; } } From 4f739ef5699ba5ee97601cd0a200f9f7964ac997 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jul 2025 16:23:04 +0100 Subject: [PATCH 4/5] Update driver-legacy/src/main/com/mongodb/MongoClient.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- driver-legacy/src/main/com/mongodb/MongoClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-legacy/src/main/com/mongodb/MongoClient.java b/driver-legacy/src/main/com/mongodb/MongoClient.java index 951622cf20..09d58e1b49 100644 --- a/driver-legacy/src/main/com/mongodb/MongoClient.java +++ b/driver-legacy/src/main/com/mongodb/MongoClient.java @@ -881,7 +881,7 @@ private void cleanCursors() { binding.release(); } } - } catch (Throwable t) { + } catch (Throwable t) { LOGGER.error(this + " stopped cleaning cursors. You may want to recreate the MongoClient", t); throw t; } From 3e7bea7d2c4b314e7bfa004cc970bfac21f957af Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jul 2025 16:23:20 +0100 Subject: [PATCH 5/5] Update driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../com/mongodb/internal/connection/DefaultServerMonitor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java b/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java index 8a72e47428..fe61183d90 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java +++ b/driver-core/src/main/com/mongodb/internal/connection/DefaultServerMonitor.java @@ -546,8 +546,7 @@ public void run() { } catch (Throwable t) { LOGGER.error(format("%s for %s stopped working. You may want to recreate the MongoClient", this, serverId), t); throw t; - } - finally { + } finally { if (connection != null) { connection.close(); }