Check external data sources in a separate health check category (#8198)

## Rationale
The default health check should fail only when the LabKey database is
unreachable, while external data sources stay visible through a separate
category. Today an unreachable external data source makes the default
check return 503, so load balancers and probes pull or restart a server
that is otherwise working, and neither action can fix an external
database.

## Changes
- The default health check covers only the LabKey database.
- External data sources are checked under a new `externalDataSources`
category, for monitoring that wants an explicit signal; it also reports
data sources that failed to connect. Their connection failure/reconnect
logging now happens only when that category (or `all`) is polled.
- Callers requesting `categories=all` see external data source failures
as before.

## Tasks
- [x] Claude Code Review
- [x] Manual Testing - not needed
- [x] Test Automation

Author: labkey-jeckels

api/src/org/labkey/api/data/DbScope.java

@@ -25,6 +25,7 @@
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.TestOnly;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Test;
@@ -1856,6 +1857,17 @@ public static void addScope(String dsName, LabKeyDataSource dataSource)
         }
     }
 
+    @TestOnly
+    public static void removeScope(String dsName)
+    {
+        synchronized (_scopeLoaders)
+        {
+            _scopeLoaders.remove(dsName);
+        }
+
+        _dataSourceFailures.remove(dsName);
+    }
+
     public boolean isRds()
     {
         return _rds;
@@ -2214,6 +2226,17 @@ public static DbScope getDbScope(String dsName)
                 .toList();
     }
 
+    /**
+     * Returns the names of data sources whose connection attempt failed, which {@link #getDbScopes()} omits
+     */
+    public static @NotNull Set<String> getFailedDataSourceNames()
+    {
+        return getLoaders().stream()
+            .filter(DbScopeLoader::isFailed)
+            .map(DbScopeLoader::getDsName)
+            .collect(Collectors.toCollection(LinkedHashSet::new));
+    }
+
     /**
      * Clear out the DbScopeLoaders that previously failed to connect to their data source. Next call to getDbScopes()
      * will attempt to retry those failed connections.

core/api-src/org/labkey/api/admin/HealthCheckRegistry.java

@@ -37,6 +37,7 @@ public class HealthCheckRegistry
 
     public static final String DEFAULT_CATEGORY = "default";
     public static final String TRIAL_INSTANCES_CATEGORY = "trial";
+    public static final String EXTERNAL_DATA_SOURCES_CATEGORY = "externalDataSources";
 
     private final Map<String, Map<String, HealthCheck>> _healthCheckCategories = new HashMap<>();
     private static final HealthCheckRegistry _instance = new HealthCheckRegistry();

core/src/org/labkey/core/CoreModule.java

@@ -591,41 +591,65 @@ private void deleteTempFiles()
         }
     }
 
+    private static boolean isConnected(DbScope dbScope, Set<String> failedDataSources)
+    {
+        boolean dbConnected;
+        try (Connection conn = dbScope.getConnection())
+        {
+            dbConnected = conn != null;
+        }
+        // Some failures come as ConfigurationException, not SQLException. Cast a wide net to ensure
+        // we return a 200 saying we're not healthy instead of a 500
+        catch (Exception e)
+        {
+            if (failedDataSources.add(dbScope.getDataSourceName()))
+                LOG.warn("Failed to get connection for data source {}", dbScope.getDataSourceName(), e);
+            dbConnected = false;
+        }
+
+        if (dbConnected && failedDataSources.remove(dbScope.getDataSourceName()))
+            LOG.info("Reconnected to data source {}", dbScope.getDataSourceName());
+
+        return dbConnected;
+    }
+
     private void registerHealthChecks()
     {
         // Data sources that were unreachable on the previous check, so we can log transitions instead of every poll
         Set<String> failedDataSources = ConcurrentHashMap.newKeySet();
 
-        HealthCheckRegistry.get().registerHealthCheck("database",  HealthCheckRegistry.DEFAULT_CATEGORY, () ->
-            {
-                Map<String, Object> healthValues = new HashMap<>();
-                boolean allConnected = true;
-                for (DbScope dbScope : DbScope.getDbScopes())
-                {
-                    boolean dbConnected;
-                    try (Connection conn = dbScope.getConnection())
-                    {
-                        dbConnected = conn != null;
-                    }
-                    // Some failures come as ConfigurationException, not SQLException. Cast a wide net to ensure
-                    // we return a 200 saying we're not healthy instead of a 500
-                    catch (Exception e)
-                    {
-                        if (failedDataSources.add(dbScope.getDataSourceName()))
-                            LOG.warn("Failed to get connection for data source {}", dbScope.getDataSourceName(), e);
-                        dbConnected = false;
-                    }
+        // Default-category failures take the server out of rotation, so check only the LabKey scope; getDbScopes() would
+        // also connect to external data sources, blocking until an unreachable one times out
+        HealthCheckRegistry.get().registerHealthCheck("database",  HealthCheckRegistry.DEFAULT_CATEGORY, () -> {
+            DbScope labkeyScope = DbScope.getLabKeyScope();
+            boolean dbConnected = isConnected(labkeyScope, failedDataSources);
+            Map<String, Object> healthValues = new HashMap<>();
+            healthValues.put(labkeyScope.getDatabaseName(), dbConnected);
+            return new HealthCheck.Result(dbConnected, healthValues);
+        });
 
-                    if (dbConnected && failedDataSources.remove(dbScope.getDataSourceName()))
-                        LOG.info("Reconnected to data source {}", dbScope.getDataSourceName());
+        HealthCheckRegistry.get().registerHealthCheck("database",  HealthCheckRegistry.EXTERNAL_DATA_SOURCES_CATEGORY, () -> {
+            Map<String, Object> healthValues = new HashMap<>();
+            boolean allConnected = true;
+            for (DbScope dbScope : DbScope.getDbScopes())
+            {
+                if (dbScope.isLabKeyScope())
+                    continue;
 
-                    healthValues.put(dbScope.getDatabaseName(), dbConnected);
-                    allConnected &= dbConnected;
-                }
+                boolean dbConnected = isConnected(dbScope, failedDataSources);
+                healthValues.put(dbScope.getDatabaseName(), dbConnected);
+                allConnected &= dbConnected;
+            }
 
-                return new HealthCheck.Result(allConnected, healthValues);
+            // getDbScopes() omits data sources that failed to connect, and those have no database name
+            for (String dsName : DbScope.getFailedDataSourceNames())
+            {
+                healthValues.put(dsName, false);
+                allConnected = false;
             }
-        );
+
+            return new HealthCheck.Result(allConnected, healthValues);
+        });
 
         HealthCheckRegistry.get().registerHealthCheck("modules", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> {
             Map<String, Throwable> failures =  ModuleLoader.getInstance().getModuleFailures();
@@ -1484,6 +1508,7 @@ public TabDisplayMode getTabDisplayMode()
             DavController.MoveActionContainerScopingTestCase.class,
             EmailServiceImpl.TestCase.class,
             FilesSiteSettingsAction.TestCase.class,
+            HealthCheckTestCase.class,
             LoggerController.TestCase.class,
             LoggingTestCase.class,
             LoginController.TestCase.class,

core/src/org/labkey/core/HealthCheckTestCase.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.core;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.junit.Assert;
+import org.junit.Test;
+import org.labkey.api.admin.HealthCheck;
+import org.labkey.api.admin.HealthCheckRegistry;
+import org.labkey.api.data.DbScope;
+
+import javax.sql.DataSource;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class HealthCheckTestCase extends Assert
+{
+    private static final String UNREACHABLE_DS = "healthCheckTestDataSource";
+    private static final String SCOPE_LOADER_LOGGER = "org.labkey.api.data.DbScopeLoader";
+
+    private static Map<String, Object> databaseDetails(String category)
+    {
+        HealthCheck.Result result = (HealthCheck.Result) HealthCheckRegistry.get().checkHealth(category).getDetails().get("database");
+        return result.getDetails();
+    }
+
+    @Test
+    public void testDefaultCoversOnlyLabKeyScope()
+    {
+        assertEquals(Set.of(DbScope.getLabKeyScope().getDatabaseName()), databaseDetails(HealthCheckRegistry.DEFAULT_CATEGORY).keySet());
+    }
+
+    @Test
+    public void testExternalCoversOtherScopes()
+    {
+        Set<String> expected = new HashSet<>();
+        DbScope.getDbScopes().stream()
+            .filter(scope -> !scope.isLabKeyScope())
+            .forEach(scope -> expected.add(scope.getDatabaseName()));
+        expected.addAll(DbScope.getFailedDataSourceNames());
+        assertEquals(expected, databaseDetails(HealthCheckRegistry.EXTERNAL_DATA_SOURCES_CATEGORY).keySet());
+    }
+
+    @Test
+    public void testUnreachableExternalDataSource() throws Exception
+    {
+        // Core doesn't compile against tomcat-dbcp, but it's the only pool LabKeyDataSource accepts
+        Class<?> poolClass = Class.forName("org.apache.tomcat.dbcp.dbcp2.BasicDataSource");
+        DataSource ds = (DataSource) poolClass.getConstructor().newInstance();
+        poolClass.getMethod("setDriverClassName", String.class).invoke(ds, "org.postgresql.Driver");
+        // Port 1 refuses immediately, so the connection attempt fails fast instead of waiting out a timeout
+        poolClass.getMethod("setUrl", String.class).invoke(ds, "jdbc:postgresql://127.0.0.1:1/healthchecktest");
+        DbScope.addScope(UNREACHABLE_DS, new DbScope.LabKeyDataSource(ds, UNREACHABLE_DS));
+
+        // The loader logs the expected connection failure as an error, which the test harness would report as a server error
+        Level previousLevel = LogManager.getLogger(SCOPE_LOADER_LOGGER).getLevel();
+        Configurator.setLevel(SCOPE_LOADER_LOGGER, Level.OFF);
+        try
+        {
+            assertTrue(HealthCheckRegistry.get().checkHealth(HealthCheckRegistry.DEFAULT_CATEGORY).isHealthy());
+            assertFalse("Default check connected to an external data source", DbScope.getFailedDataSourceNames().contains(UNREACHABLE_DS));
+
+            assertFalse(HealthCheckRegistry.get().checkHealth(HealthCheckRegistry.EXTERNAL_DATA_SOURCES_CATEGORY).isHealthy());
+            assertEquals(false, databaseDetails(HealthCheckRegistry.EXTERNAL_DATA_SOURCES_CATEGORY).get(UNREACHABLE_DS));
+
+            assertFalse(HealthCheckRegistry.get().checkHealth(List.of("all")).isHealthy());
+        }
+        finally
+        {
+            Configurator.setLevel(SCOPE_LOADER_LOGGER, previousLevel);
+            DbScope.removeScope(UNREACHABLE_DS);
+        }
+    }
+}