Allow configurable Cylc run directory

CONTRIBUTING.md

@@ -115,6 +115,7 @@ requests_).
  - Samuel Denton
  - Scott Owen James
  - James Frost
+ - Patrick Mulrooney
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version

changes.d/7411.feat.md

@@ -0,0 +1 @@
+Allow CYLC_RUN_DIR to override the default $HOME/cylc-run directory.

cylc/flow/cfgspec/globalcfg.py

@@ -810,6 +810,20 @@ def default_for(
           `-- flow/
               `-- global.cylc
 
+    The workflow run directory can also be configured by environment:
+
+    .. envvar:: CYLC_RUN_DIR
+
+       Override the root directory in which workflows are installed and run.
+       The default is ``$HOME/cylc-run``.
+
+       All Cylc commands targeting workflows in this directory must be run
+       with the same value of ``CYLC_RUN_DIR``.
+
+       This variable is forwarded to scheduler run hosts and job platforms,
+       and written into generated job scripts. The configured path must be
+       accessible at the same location on these hosts.
+
     .. note::
 
        The ``global.cylc`` file can be templated using Jinja2 variables.
@@ -1358,8 +1372,9 @@ def default_for(
                   desc="""
             Configure alternate workflow run directory locations.
 
-            Symlinks from the the standard ``$HOME/cylc-run`` locations will be
-            created.
+            Symlinks from the standard run directory locations will be
+            created. The run directory defaults to ``$HOME/cylc-run`` and can
+            be overridden by :envvar:`CYLC_RUN_DIR`.
 
             .. note::
 
@@ -1380,12 +1395,15 @@ def default_for(
                     If specified, the workflow run directory will
                     be created in ``<this-path>/cylc-run/<workflow-id>``
                     and a symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>``.
+                    ``<run-dir>/<workflow-id>``.
                     If not specified, the workflow run directory will be
-                    created in ``$HOME/cylc-run/<workflow-id>``.
+                    created in ``<run-dir>/<workflow-id>``.
                     All the workflow files and the ``.service`` directory get
                     installed into this directory.
 
+                    Here, ``<run-dir>`` is ``$HOME/cylc-run`` by default or
+                    the value of :envvar:`CYLC_RUN_DIR` if set.
+
                     .. versionadded:: 8.0.0
                 """)
                 for folder, versionadded in SYMLINKABLE_LOCATIONS.items():
@@ -1396,9 +1414,12 @@ def default_for(
                         be created in
                         ``<this-path>/cylc-run/<workflow-id>/{folder}``
                         and a symbolic link will be created from
-                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+                        ``<run-dir>/<workflow-id>/{folder}``.
                         If not specified, the directory will be created in
-                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+                        ``<run-dir>/<workflow-id>/{folder}``.
+
+                        Here, ``<run-dir>`` is ``$HOME/cylc-run`` by default
+                        or the value of :envvar:`CYLC_RUN_DIR` if set.
 
                         .. versionadded:: {versionadded}
                     """)

cylc/flow/job_file.py

@@ -190,8 +190,8 @@ def _write_prelude(self, handle, job_conf):
         )
         env_vars = (
             (job_conf['platform']['copyable environment variables'] or [])
-            # pass CYLC_COVERAGE into the job execution environment
-            + ['CYLC_COVERAGE']
+            # pass core Cylc variables into the job execution environment
+            + ['CYLC_COVERAGE', 'CYLC_RUN_DIR']
         )
         for key in env_vars:
             if key in os.environ:

cylc/flow/pathutil.py

@@ -65,6 +65,15 @@
 }
 
 
+def _get_cylc_run_dir() -> str:
+    """Return the unexpanded Cylc run directory.
+
+    ``CYLC_RUN_DIR`` can be used to override the standard
+    ``$HOME/cylc-run`` location.
+    """
+    return os.getenv('CYLC_RUN_DIR') or _CYLC_RUN_DIR
+
+
 def expand_path(*args: Union[Path, str]) -> str:
     """Expand both vars and user in path and normalise it, joining any
     extra args."""
@@ -78,7 +87,7 @@ def get_remote_workflow_run_dir(
 ) -> str:
     """Return remote workflow run directory, joining any extra args,
     NOT expanding vars or user."""
-    return os.path.join(_CYLC_RUN_DIR, workflow_id, *args)
+    return os.path.join(_get_cylc_run_dir(), workflow_id, *args)
 
 
 def get_remote_workflow_run_job_dir(
@@ -91,7 +100,7 @@ def get_remote_workflow_run_job_dir(
 
 def get_cylc_run_dir(alt_run_dir: Optional[str] = None) -> str:
     """Return the cylc-run dir, or alt path, with vars/user expanded."""
-    return expand_path(alt_run_dir or _CYLC_RUN_DIR)
+    return expand_path(alt_run_dir or _get_cylc_run_dir())
 
 
 def get_alt_workflow_run_dir(
@@ -115,7 +124,7 @@ def get_workflow_run_dir(
     Join any extra args, and expand vars and user.
     Does not check that the directory exists.
     """
-    return expand_path(_CYLC_RUN_DIR, workflow_id, *args)
+    return expand_path(_get_cylc_run_dir(), workflow_id, *args)
 
 
 def get_workflow_run_job_dir(workflow, *args):

cylc/flow/remote.py

@@ -347,6 +347,7 @@ def construct_ssh_cmd(
         'CYLC_COVERAGE',
         'CLIENT_COMMS_METH',
         'CYLC_ENV_NAME',
+        'CYLC_RUN_DIR',
         *platform['ssh forward environment variables'],
     ]:
         if envvar in os.environ:

tests/unit/test_job_file.py

@@ -301,10 +301,13 @@ def test_write_prelude(
         "CYLC_WORKFLOW_INITIAL_CYCLE_POINT": "20200101T0000Z",
         "CYLC_WORKFLOW_NAME": "test_write_prelude",
         "CYLC_WORKFLOW_NAME_BASE": "test_write_prelude",
-        'CYLC_COVERAGE': '1'
+        'CYLC_COVERAGE': '1',
+        'CYLC_RUN_DIR': '/data/cylc-run',
     }.items():
         monkeypatch.setenv(k, v)
 
+    expected.append("export CYLC_RUN_DIR='/data/cylc-run'")
+
     with io.StringIO() as fake_file:
         # copyable environment variables
         JobFileWriter()._write_prelude(fake_file, job_conf)

tests/unit/test_pathutil.py

@@ -28,6 +28,7 @@
 from cylc.flow.pathutil import (
     EXPLICIT_RELATIVE_PATH_REGEX,
     expand_path,
+    get_cylc_run_dir,
     get_dirs_to_symlink,
     get_next_rundir_number,
     get_remote_workflow_run_dir,
@@ -122,6 +123,30 @@ def test_get_remote_workflow_run_dirs(
     assert result == expected
 
 
+def test_cylc_run_dir_env(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """CYLC_RUN_DIR overrides the standard $HOME/cylc-run location."""
+    cylc_run_dir = tmp_path / 'custom-cylc-run'
+    monkeypatch.setenv('CYLC_RUN_DIR', str(cylc_run_dir))
+
+    assert get_cylc_run_dir() == str(cylc_run_dir)
+    assert get_workflow_run_dir('foo') == str(cylc_run_dir / 'foo')
+    assert get_remote_workflow_run_dir('foo') == str(
+        cylc_run_dir / 'foo'
+    )
+
+
+def test_alt_run_dir_overrides_env(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """An explicit alternate run dir takes priority over CYLC_RUN_DIR."""
+    monkeypatch.setenv('CYLC_RUN_DIR', str(tmp_path / 'from-env'))
+    alt_run_dir = tmp_path / 'explicit'
+
+    assert get_cylc_run_dir(str(alt_run_dir)) == str(alt_run_dir)
+
+
 @pytest.mark.parametrize(
     'func, tail1',
     [(get_workflow_run_dir, ''),

tests/unit/test_remote.py

@@ -141,6 +141,22 @@ def test_construct_ssh_cmd_forward_env(monkeypatch: pytest.MonkeyPatch):
     cmd = construct_ssh_cmd(['play'], config, host)
     assert cmd == expect
 
+    # CYLC_RUN_DIR is a core variable and does not need to be configured for
+    # forwarding.
+    monkeypatch.setenv('CYLC_RUN_DIR', '/data/cylc-run')
+    expect = [
+        'ssh',
+        host,
+        'env',
+        f'CYLC_VERSION={cylc.flow.__version__}',
+        'CYLC_RUN_DIR=/data/cylc-run',
+        'FOO=BAR',
+        'cylc',
+        'play',
+    ]
+    cmd = construct_ssh_cmd(['play'], config, host)
+    assert cmd == expect
+
 
 def test_get_proc_ancestors__basic():
     """It should return a list of ancestor PIDs, starting with the parent."""