Merge pull request #1727 from Caltech-IPAC/FIREFLY-1683-thread-leak

FIREFLY-1683: prevent excessive thread use

Author: loitly

src/firefly/java/edu/caltech/ipac/firefly/server/ServerContext.java

@@ -855,11 +855,12 @@ public void contextInitialized(ServletContextEvent servletContextEvent) {
                 ServletContext cntx = servletContextEvent.getServletContext();
                 ServerContext.init(cntx.getContextPath(), cntx.getServletContextName(), cntx.getRealPath(WEBAPP_CONFIG_LOC));
                 VersionUtil.initVersion(cntx);  // can be called multiple times, only inits on the first call
-                SCHEDULE_TASK_EXEC.scheduleAtFixedRate(
-                        () -> DbMonitor.cleanup(false),
-                        DbMonitor.CLEANUP_INTVL,
-                        DbMonitor.CLEANUP_INTVL,
-                        TimeUnit.MILLISECONDS);
+                // we're using DuckDB exclusively for embedded database.  No need for cleanup.
+//                SCHEDULE_TASK_EXEC.scheduleAtFixedRate(
+//                        () -> DbMonitor.cleanup(false),
+//                        DbMonitor.CLEANUP_INTVL,
+//                        DbMonitor.CLEANUP_INTVL,
+//                        TimeUnit.SECONDS);
             } catch (Throwable e) {
                 e.printStackTrace();
             }
@@ -868,7 +869,7 @@ public void contextInitialized(ServletContextEvent servletContextEvent) {
         public void contextDestroyed(ServletContextEvent servletContextEvent) {
             try {
                 System.out.println("contextDestroyed...");
-                DbMonitor.cleanup(true, false);
+//                DbMonitor.cleanup(true, false);
                 ((EhcacheProvider)CacheManager.getCacheProvider()).shutdown();
                 try {
                     SHORT_TASK_EXEC.shutdownNow();

src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java

@@ -274,12 +274,14 @@ class DbStats {
         int rowCnt = -1;
         int totalRows = -1;
         long memory = -1;
+        final long created = System.currentTimeMillis();
 
         public int tblCnt() { return tblCnt; }
         public int colCnt() { return colCnt;}
         public int rowCnt() { return rowCnt; }
         public int totalRows() { return totalRows;}
         public long memory() { return memory;}
+        public long created() { return created;}
     }
 
     /**
@@ -348,8 +350,13 @@ public ReentrantLock getLock() {
         public void setCompact(boolean compact) { isCompact = compact;}
         public boolean isCompact() { return isCompact; }
 
-        public void updateStats() { this.dbStats = dbAdapter.getDbStats(); }
         public DbStats getDbStats() { return dbStats == null ? new DbStats() : dbStats; }
+
+        public void updateStats() {
+            if (dbStats == null || dbStats.created < lastAccessed) {
+                this.dbStats = dbAdapter.getDbStats();
+            }
+        }
     }
 
     /**

src/firefly/java/edu/caltech/ipac/firefly/server/db/DbMonitor.java

@@ -12,11 +12,12 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import static edu.caltech.ipac.firefly.core.Util.Try;
-import static edu.caltech.ipac.firefly.server.ServerContext.SHORT_TASK_EXEC;
 
 /**
  * Date: 5/3/24
@@ -71,6 +72,7 @@ private static long maxMemory() {
     private static final ConcurrentHashMap<String, DbAdapter.EmbeddedDbInstance> dbInstances = new ConcurrentHashMap<>();
     private static final DbAdapter.EmbeddedDbStats dbStats = new DbAdapter.EmbeddedDbStats();
     private static final Logger.LoggerImpl LOGGER = Logger.getLogger();
+    private static final ExecutorService DB_STATS_THREADS = Executors.newFixedThreadPool(5);    // up to 5 threads for gathering DB stats
 
     public static ConcurrentHashMap<String, DbAdapter.EmbeddedDbInstance> getDbInstances() {
         return dbInstances;
@@ -99,9 +101,9 @@ public static void updateDbStats() {
         LOGGER.trace("DbAdapter -> updateDbStats");
         Ref<Future<?>> t = new Ref<>();
         for (DbAdapter.EmbeddedDbInstance db : dbInstances.values()) {
-            t.set(SHORT_TASK_EXEC.submit(db::updateStats));
+            t.set(DB_STATS_THREADS.submit(db::updateStats));
         }
-        Try.it(() -> t.get().get(5, TimeUnit.SECONDS));      // run all in parallel, but wait for up to 5 seconds
+        Try.it(() -> t.get().get(10, TimeUnit.SECONDS));      // run all in parallel, but wait for up to 5 seconds
     }
 
 //====================================================================