Retry cat-log on unreachable platform hosts

CONTRIBUTING.md

@@ -115,6 +115,7 @@ requests_).
  - Samuel Denton
  - Scott Owen James
  - James Frost
+ - ahmad
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version

changes.d/4263.fix.md

@@ -0,0 +1 @@
+`cylc cat-log` now retries another configured platform host when the selected host is unreachable.

cylc/flow/scripts/cat_log.py

@@ -96,7 +96,10 @@
     get_workflow_run_job_dir,
     get_workflow_run_pub_db_path,
 )
-from cylc.flow.platforms import get_platform
+from cylc.flow.platforms import (
+    get_host_from_platform,
+    get_platform,
+)
 from cylc.flow.remote import (
     remote_cylc_cmd,
     watch_and_kill,
@@ -453,20 +456,34 @@ async def _get_remote_log(
     if prepend_path:
         cmd.append('--prepend-path')
     cmd.append(workflow_id)
-    # TODO: Add Intelligent Host selection to this
-    # https://github.com/cylc/cylc-flow/issues/4263
+    bad_hosts = set()
     with suppress(KeyboardInterrupt):
-        # (Ctrl-C while tailing)
-        # NOTE: This will raise NoHostsError if the platform is not
-        # contactable
-        # For testing purposes
-        return await remote_cylc_cmd(
-            cmd,
-            platform,
-            capture_process=(mode == LISTDIR),
-            manage=(mode == TAIL),
-            text=(mode == LISTDIR),
-        )
+        while True:
+            # (Ctrl-C while tailing)
+            # NOTE: This will raise NoHostsError if the platform is not
+            # contactable.
+            host = get_host_from_platform(platform, bad_hosts=bad_hosts)
+            result = await remote_cylc_cmd(
+                cmd,
+                platform,
+                host=host,
+                capture_process=(mode == LISTDIR),
+                manage=(mode == TAIL),
+                text=(mode == LISTDIR),
+            )
+
+            # LISTDIR returns a process so the caller can consume its output.
+            # Wait for this short-lived command here so an SSH failure can be
+            # retried before returning the process to the caller.
+            if isinstance(result, Popen):
+                result.wait()
+                return_code = result.returncode
+            else:
+                return_code = result
+
+            if return_code != 255:
+                return result
+            bad_hosts.add(host)
     return 1
 
 

tests/unit/scripts/test_cat_log.py

@@ -23,6 +23,8 @@
 
 from cylc.flow.loggingutil import CylcLogFormatter
 from cylc.flow.scripts.cat_log import (
+    CAT,
+    _get_remote_log,
     colorise_cat_log,
 )
 
@@ -88,3 +90,46 @@ def test_colorise_cat_log_colour(log_file):
             ]
         ])
     )
+
+
+@pytest.mark.asyncio
+async def test_get_remote_log_retries_unreachable_host(monkeypatch):
+    """It should try another platform host after an SSH failure."""
+    from cylc.flow.scripts import cat_log
+
+    hosts = []
+
+    def select_host(platform, bad_hosts=None):
+        host = next(
+            item for item in platform['hosts']
+            if item not in (bad_hosts or set())
+        )
+        hosts.append(host)
+        return host
+
+    async def remote_cmd(*args, **kwargs):
+        return 255 if kwargs['host'] == 'foo' else 0
+
+    monkeypatch.setattr(cat_log, 'get_host_from_platform', select_host)
+    monkeypatch.setattr(cat_log, 'remote_cylc_cmd', remote_cmd)
+    monkeypatch.setattr(
+        cat_log,
+        'get_remote_workflow_run_job_dir',
+        lambda *args: '/remote/log/job.out',
+    )
+
+    result = await _get_remote_log(
+        'workflow',
+        {
+            'hosts': ['foo', 'bar'],
+            'tail command template': 'tail -f %(filename)s',
+        },
+        '1',
+        'task',
+        '01',
+        'job.out',
+        CAT,
+    )
+
+    assert result == 0
+    assert hosts == ['foo', 'bar']