GH Issue 1574: Report connection counts in thread dumps and over JMX (#8062)

## Rationale
A connector that hits maxConnections stops accepting while its threads
sit idle, meaning there's no clue of the problem in thread dumps for
asynchronous requests or websockets.

## Related Pull Requests
- LabKey/premiumModules#752

## Changes
- Capture connection counts in thread dumps
- Track websockets

Author: labkey-jeckels

api/src/org/labkey/api/util/DebugInfoDumper.java

@@ -18,6 +18,7 @@
 import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
 import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.action.NullSafeBindException;
 import org.labkey.api.data.ConnectionWrapper;
 import org.labkey.api.data.ContainerManager;
@@ -27,6 +28,7 @@
 import org.labkey.api.data.dialect.BasePostgreSqlDialect;
 import org.labkey.api.files.FileSystemDirectoryListener;
 import org.labkey.api.files.FileSystemWatchers;
+import org.labkey.api.mbean.LabKeyManagement;
 import org.labkey.api.miniprofiler.MiniProfiler;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.query.QueryForm;
@@ -38,9 +40,13 @@
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.HttpView;
 import org.labkey.api.view.ViewContext;
+import org.labkey.api.websocket.WebSocketTracker;
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.vfs.FileLike;
 
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -356,6 +362,8 @@ public static synchronized void dumpThreads(LoggerWriter logWriter)
                 logWriter.debug("CPU count: " + osBean.getAvailableProcessors());
             }
 
+            dumpConnectionCounts(logWriter);
+
             logWriter.debug("*********************************************");
 
             Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
@@ -448,6 +456,82 @@ public static synchronized void dumpThreads(LoggerWriter logWriter)
         }
     }
 
+    /**
+     * Connection counts per connector, plus the long-lived populations that fill them. A connector that reaches
+     * maxConnections stops accepting while its threads sit idle, so the stacks below show an untroubled server.
+     * See GH Issue 1574.
+     */
+    private static void dumpConnectionCounts(LoggerWriter logWriter)
+    {
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+
+        List<ObjectName> connectors;
+
+        try
+        {
+            // Wildcard domain: embedded Tomcat registers these under "Tomcat" (the engine name Tomcat.getEngine()
+            // assigns), a standalone install under "Catalina"
+            connectors = new ArrayList<>(mbs.queryNames(new ObjectName("*:type=ThreadPool,name=*"), null));
+            connectors.sort(Comparator.comparing(ObjectName::getCanonicalName));
+        }
+        catch (Exception e)
+        {
+            logWriter.debug("Failed to find connector ThreadPool MBeans: " + e);
+            connectors = List.of();
+        }
+
+        if (connectors.isEmpty())
+            logWriter.debug("No connector ThreadPool MBeans found; connection counts are unavailable");
+
+        // The saturated connector is the one worth reading, so don't let an earlier one's failure hide it
+        for (ObjectName connector : connectors)
+        {
+            try
+            {
+                logWriter.debug("Connector " + unquote(connector.getKeyProperty("name")) +
+                    ": connections " + getMBeanAttribute(mbs, connector, "connectionCount") +
+                    "/" + getMBeanLimit(mbs, connector, "maxConnections") +
+                    ", threads busy " + getMBeanAttribute(mbs, connector, "currentThreadsBusy") +
+                    "/" + getMBeanLimit(mbs, connector, "maxThreads"));
+            }
+            catch (Exception e)
+            {
+                logWriter.debug("Failed to read connection counts for " + connector.getCanonicalName() + ": " + e);
+            }
+        }
+
+        logWriter.debug("Open WebSocket connections: " + WebSocketTracker.getOpenCount());
+
+        // MCP ships in a premium module, so read its session count over JMX rather than depending on it
+        Object mcpSessions = getMBeanAttribute(mbs, LabKeyManagement.createName("MCP", null), "SessionCount");
+        if (null != mcpSessions)
+            logWriter.debug("Open MCP sessions: " + mcpSessions);
+    }
+
+    private static String unquote(@Nullable String name)
+    {
+        return null != name && name.startsWith("\"") ? ObjectName.unquote(name) : String.valueOf(name);
+    }
+
+    private static @Nullable Object getMBeanAttribute(MBeanServer mbs, ObjectName name, String attribute)
+    {
+        try
+        {
+            return mbs.getAttribute(name, attribute);
+        }
+        catch (Exception e)
+        {
+            return null;
+        }
+    }
+
+    /** Tomcat reports an unbounded maximum as -1 */
+    private static String getMBeanLimit(MBeanServer mbs, ObjectName name, String attribute)
+    {
+        Object value = getMBeanAttribute(mbs, name, attribute);
+        return value instanceof Number number && number.longValue() < 0 ? "unlimited" : String.valueOf(value);
+    }
+
     private static void writeTable(LoggerWriter logWriter, UserSchema schema, String tableName, String header)
     {
         QueryForm form = new QueryForm();

api/src/org/labkey/api/websocket/BrowserEndpoint.java

@@ -35,6 +35,7 @@
 import org.labkey.api.security.AuthenticationManager;
 import org.labkey.api.security.SecurityManager;
 import org.labkey.api.security.User;
+import org.labkey.api.util.QuietCloser;
 import org.labkey.api.util.UnexpectedException;
 import org.labkey.api.util.logging.LogHelper;
 
@@ -54,6 +55,7 @@ public abstract class BrowserEndpoint extends Endpoint
 
     protected Session browserSession;
     ServerEndpoint serverEndpoint = null;
+    private QuietCloser tracked;
 
     private void close()
     {
@@ -90,6 +92,9 @@ public void modifyHandshake(ServerEndpointConfig config, HandshakeRequest reques
     @Override
     public void onOpen(Session session, EndpointConfig endpointConfig)
     {
+        // Only the inbound browser socket counts against the connector; serverEndpoint's socket is outbound
+        tracked = WebSocketTracker.opened();
+
         String uri = null;
         try
         {
@@ -113,6 +118,8 @@ public void onOpen(Session session, EndpointConfig endpointConfig)
     public void onClose(Session session, CloseReason closeReason)
     {
         LOG.debug("BrowserEndpoint.onClose()");
+        if (null != tracked)
+            tracked.close();
         if (null != serverEndpoint)
             serverEndpoint.close();
     }

api/src/org/labkey/api/websocket/WebSocketTracker.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.websocket;
+
+import org.labkey.api.util.QuietCloser;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Live count of inbound WebSocket connections, guests included. Each holds a Tomcat connector connection slot for as
+ * long as the client keeps the socket open, so it competes with every other request against maxConnections.
+ * See GH Issue 1574.
+ */
+public class WebSocketTracker
+{
+    private static final AtomicInteger OPEN_COUNT = new AtomicInteger();
+
+    private WebSocketTracker()
+    {
+    }
+
+    /** Call from onOpen(); close the result from onClose(). Closing more than once is a no-op. */
+    public static QuietCloser opened()
+    {
+        OPEN_COUNT.incrementAndGet();
+        AtomicBoolean closed = new AtomicBoolean();
+
+        return () -> {
+            if (closed.compareAndSet(false, true))
+                OPEN_COUNT.decrementAndGet();
+        };
+    }
+
+    public static int getOpenCount()
+    {
+        return OPEN_COUNT.get();
+    }
+}

core/src/org/labkey/core/CoreModule.java

@@ -90,6 +90,7 @@
 import org.labkey.api.files.FileBrowserConfigWriter;
 import org.labkey.api.files.FileContentService;
 import org.labkey.api.markdown.MarkdownService;
+import org.labkey.api.mbean.LabKeyManagement;
 import org.labkey.api.mcp.McpService;
 import org.labkey.api.message.settings.MessageConfigService;
 import org.labkey.api.migration.DatabaseMigrationService;
@@ -261,6 +262,7 @@
 import org.labkey.core.login.LoginController;
 import org.labkey.core.metrics.SimpleMetricsServiceImpl;
 import org.labkey.core.metrics.WebSocketConnectionManager;
+import org.labkey.core.metrics.WebSocketMXBean;
 import org.labkey.core.notification.EmailPreferenceConfigServiceImpl;
 import org.labkey.core.notification.EmailPreferenceContainerListener;
 import org.labkey.core.notification.EmailPreferenceUserListener;
@@ -346,6 +348,8 @@
 import org.radeox.test.macro.list.ExampleListFormatterTest;
 import org.radeox.test.macro.list.SimpleListTest;
 
+import javax.management.StandardMBean;
+
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -1292,6 +1296,7 @@ public void moduleStartupComplete(ServletContext servletContext)
         });
 
         UsageMetricsService.get().registerUsageMetrics(getName(), WebSocketConnectionManager.getInstance());
+        LabKeyManagement.register(new StandardMBean(WebSocketConnectionManager.getInstance(), WebSocketMXBean.class, true), "WebSockets");
         UsageMetricsService.get().registerUsageMetrics(getName(), DbLoginManager.getMetricsProvider());
         UsageMetricsService.get().registerUsageMetrics(getName(), SecurityManager.getMetricsProvider());
         UsageMetricsService.get().registerUsageMetrics(getName(), DisplayFormatAnalyzer.getMetricsProvider());

core/src/org/labkey/core/metrics/WebSocketConnectionManager.java

@@ -16,11 +16,12 @@
 package org.labkey.core.metrics;
 
 import org.labkey.api.usageMetrics.UsageMetricsProvider;
+import org.labkey.api.websocket.WebSocketTracker;
 
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
-public class WebSocketConnectionManager implements UsageMetricsProvider
+public class WebSocketConnectionManager implements UsageMetricsProvider, WebSocketMXBean
 {
     private static final WebSocketConnectionManager _instance = new WebSocketConnectionManager();
 
@@ -44,6 +45,24 @@ public void incrementCounter(boolean success)
             failureCounter.incrementAndGet();
     }
 
+    @Override
+    public int getOpenConnectionCount()
+    {
+        return WebSocketTracker.getOpenCount();
+    }
+
+    @Override
+    public int getSuccessCount()
+    {
+        return successCounter.get();
+    }
+
+    @Override
+    public int getFailureCount()
+    {
+        return failureCounter.get();
+    }
+
     public boolean showWarning()
     {
         return successCounter.get() == 0 && failureCounter.get() > 0;
@@ -55,7 +74,8 @@ public Map<String, Object> getUsageMetrics()
         return Map.of(
             "webSocketConnections", Map.of(
                     "success", successCounter.get(),
-                    "failure", failureCounter.get()
+                    "failure", failureCounter.get(),
+                    "open", WebSocketTracker.getOpenCount()
                 )
         );
     }

core/src/org/labkey/core/metrics/WebSocketMXBean.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.core.metrics;
+
+/** WebSocket connection counts, exported over JMX as LabKey:name=WebSockets. See GH Issue 1574. */
+public interface WebSocketMXBean
+{
+    /** @return inbound WebSocket connections the server is currently holding, guests included */
+    int getOpenConnectionCount();
+
+    /** @return connections opened since startup by signed-in users */
+    int getSuccessCount();
+
+    /** @return connection attempts that browsers have reported back as failed */
+    int getFailureCount();
+}

core/src/org/labkey/core/notification/NotificationEndpoint.java

@@ -21,7 +21,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.labkey.api.security.User;
 import org.labkey.api.util.MemTracker;
+import org.labkey.api.util.QuietCloser;
 import org.labkey.api.security.SecurityManager;
+import org.labkey.api.websocket.WebSocketTracker;
 import org.labkey.core.metrics.WebSocketConnectionManager;
 
 import jakarta.servlet.http.HttpSession;
@@ -57,6 +59,7 @@ public class NotificationEndpoint extends Endpoint
     private Session session;
     private int userId;
     private boolean errored;
+    private QuietCloser tracked;
 
     public NotificationEndpoint()
     {
@@ -75,6 +78,10 @@ public void onOpen(Session session, EndpointConfig endpointConfig)
         this.userId = null==id ? 0 : id;
 
         LOG.debug("{} onOpen", this);
+        // endpointsMap holds only authenticated endpoints, but a guest's socket occupies a connection slot just the
+        // same - on a public site they're nearly all guests. GH Issue 1574
+        tracked = WebSocketTracker.opened();
+
         synchronized (endpointsMap)
         {
             if (this.userId > 0)
@@ -101,6 +108,8 @@ public void onClose(Session session, CloseReason closeReason)
         {
             endpointsMap.removeMapping(this.userId, this);
         }
+        if (null != tracked)
+            tracked.close();
         super.onClose(session, closeReason);
     }