Cylc Config json output mode

cylc/flow/parsec/OrderedDict.py

@@ -100,6 +100,19 @@ def prepend(self, key, value):
         self[key] = value
         self.move_to_end(key, last=False)
 
+    @staticmethod
+    def repl_val(target, replace, replacement):
+        """Replace dictionary values with a string.
+
+        Designed to be used recursively.
+        """
+        for key, val in target.items():
+            if isinstance(val, dict):
+                OrderedDictWithDefaults.repl_val(
+                    val, replace, replacement)
+            elif val == replace:
+                target[key] = replacement
+
 
 class DictTree:
     """An object providing a single point of access to a tree of dicts.

cylc/flow/parsec/config.py

@@ -15,9 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from copy import deepcopy
+import json
 import re
+import sys
 from textwrap import dedent
-from typing import TYPE_CHECKING, Callable, Iterable, List, Optional
+from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, TextIO
 
 from cylc.flow.context_node import ContextNode
 from cylc.flow.parsec.exceptions import (
@@ -33,6 +35,7 @@
 
 if TYPE_CHECKING:
     from optparse import Values
+    from typing_extensions import Literal
 
 
 class DefaultList(list):
@@ -41,6 +44,7 @@ class DefaultList(list):
 
 class ParsecConfig:
     """Object wrapper for parsec functions."""
+    META: "Literal['meta']" = 'meta'
 
     def __init__(
         self,
@@ -166,7 +170,7 @@ def get(self, keys: Optional[Iterable[str]] = None, sparse: bool = False):
         return cfg
 
     def idump(self, items=None, sparse=False, prefix='',
-              oneline=False, none_str='', handle=None):
+              oneline=False, none_str='', handle=None, json=False):
         """
         items is a list of --item style inputs:
            '[runtime][foo]script'.
@@ -182,7 +186,40 @@ def idump(self, items=None, sparse=False, prefix='',
                 mkeys.append(j)
         if null:
             mkeys = [[]]
-        self.mdump(mkeys, sparse, prefix, oneline, none_str, handle=handle)
+        if json:
+            self.jdump(mkeys, sparse, oneline, none_str, handle=handle)
+        else:
+            self.mdump(mkeys, sparse, prefix, oneline, none_str, handle=handle)
+
+    def jdump(
+        self,
+        mkeys: Optional[Iterable] = None,
+        sparse: bool = False,
+        oneline: bool = False,
+        none_str: Optional[str] = None,
+        handle: Optional[TextIO] = None
+    ) -> None:
+        """Dump a config to JSON format.
+
+        Args:
+            mkeys: Items to display.
+            sparse: Only display user set items, not defaults.
+            oneline: Output on a single line.
+            none_str: Value to give instead of null.
+            handle: Where to write the output.
+        """
+        # Use json indent to control online output:
+        indent = None if oneline else 4
+
+        for keys in mkeys or []:
+            if not keys:
+                keys = []
+            cfg = self.get(keys, sparse)
+            if none_str:
+                OrderedDictWithDefaults.repl_val(cfg, None, none_str)
+            data = json.dumps(cfg, indent=indent)
+
+        print(data, file=handle or sys.stdout)
 
     def mdump(self, mkeys=None, sparse=False, prefix='',
               oneline=False, none_str='', handle=None):

cylc/flow/scripts/config.py

@@ -111,6 +111,15 @@ def get_option_parser() -> COP:
             "overrides any settings it shares with those higher up."),
         action="store_true", default=False, dest="print_hierarchy")
 
+    parser.add_option(
+        '--json',
+        help=(
+            'Returns config as JSON rather than Cylc Config format.'),
+        default=False,
+        action='store_true',
+        dest='json'
+    )
+
     parser.add_option(icp_option)
 
     platform_listing_options_group = parser.add_option_group(
@@ -140,6 +149,28 @@ def get_option_parser() -> COP:
     return parser
 
 
+def json_opt_check(parser, options):
+    """Return an error if --json and incompatible options used.
+    """
+    not_with_json = {
+        '--print-hierarchy': 'print_hierarchy',
+        '--platform-names': 'print_platform_names',
+        '--platforms': 'print_platforms'
+    }
+
+    if not options.json:
+        return
+
+    not_with_json = [
+        name for name, dest
+        in not_with_json.items()
+        if options.__dict__[dest]]
+
+    if not_with_json:
+        parser.error(
+            f'--json incompatible with {" or ".join(not_with_json)}')
+
+
 def get_config_file_hierarchy(workflow_id: Optional[str] = None) -> List[str]:
     filepaths = [os.path.join(path, glbl_cfg().CONF_BASENAME)
                  for _, path in glbl_cfg().conf_dir_hierarchy]
@@ -164,6 +195,7 @@ async def _main(
     options: 'Values',
     *ids,
 ) -> None:
+    json_opt_check(parser, options)
 
     if options.print_platform_names and options.print_platforms:
         options.print_platform_names = False
@@ -189,7 +221,8 @@ async def _main(
             options.item,
             not options.defaults,
             oneline=options.oneline,
-            none_str=options.none_str
+            none_str=options.none_str,
+            json=options.json,
         )
         return
 
@@ -219,5 +252,6 @@ async def _main(
         options.item,
         not options.defaults,
         oneline=options.oneline,
-        none_str=options.none_str
+        none_str=options.none_str,
+        json=options.json
     )

tests/integration/scripts/test_config.py

@@ -0,0 +1,145 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import pytest
+
+from cylc.flow.option_parsers import Options
+from cylc.flow.scripts.config import _main, get_option_parser
+
+
+@pytest.fixture(scope='module')
+def setup(mod_one_conf, mod_flow):
+    parser = get_option_parser()
+    ConfigOptions = Options(parser)
+    opts = ConfigOptions()
+    opts.json = True
+    wid = mod_flow(mod_one_conf)
+    yield parser, opts, wid
+
+
+async def test_json_basic(setup, capsys):
+    """Test that the output is in JSON format."""
+    await _main(*setup)
+
+    result = capsys.readouterr()
+    assert result.err == ''
+    assert json.loads(result.out)['scheduling']['graph'] == {
+        'R1': 'one'
+    }
+
+
+async def test_json_workflow_cfg(flow, capsys):
+    """It fills config in values from CLI."""
+    wid = flow(
+        {
+            'scheduling': {'graph': {'P1D': 'foo'}},
+            'runtime': {'foo': {}},
+        }
+    )
+    parser = get_option_parser()
+    ConfigOptions = Options(parser)
+    opts = ConfigOptions()
+    opts.json = True
+    opts.icp = '1000'
+
+    await _main(parser, opts, wid)
+
+    returned_config = json.loads(capsys.readouterr().out)
+    assert returned_config['scheduling']['initial cycle point'] == '1000'
+    assert returned_config['runtime']['foo'] == {
+        'completion': 'succeeded',
+        'simulation': {'default run length': 0.0}
+    }
+
+
+@pytest.mark.parametrize(
+    'not_with',
+    [
+        (['print_platforms']),
+        (['print_platforms', 'print_platform_names']),
+        (['print_platforms', 'print_platform_names', 'print_hierarchy']),
+    ],
+)
+async def test_json_and_not_other_option(
+    setup, capsys, not_with
+):
+    """It fails if incompatible options provided."""
+    parser, opts, wid = setup
+    for key in not_with:
+        setattr(opts, key, True)
+
+    with pytest.raises(SystemExit):
+        await _main(parser, opts, wid)
+
+    result = capsys.readouterr()
+    assert result.out == ''
+    assert '--json incompatible with' in result.err
+    for key in not_with:
+        if 'platform' in key:
+            key = key.strip('print_')
+        assert key.replace('_', '-') in result.err
+
+    # Clean up, since setup object is shared:
+    for key in not_with:
+        setattr(opts, key, False)
+
+
+async def test_json_global_cfg(setup, mock_glbl_cfg, capsys):
+    """It returns the global configuration in JSON format."""
+    mock_glbl_cfg(
+        'cylc.flow.scripts.config.glbl_cfg',
+        '''
+            [platforms]
+                [[golders_green]]
+                    [[[meta]]]
+                        can = "Test lots of things"
+                        because = metadata, is, not, fussy
+                        number = 99
+        ''',
+    )
+    parser, opts, _ = setup
+
+    await _main(parser, opts)
+
+    returned_config = json.loads(capsys.readouterr().out)
+    assert returned_config == {
+        'platforms': {
+            'golders_green': {
+                'meta': {
+                    'can': 'Test lots of things',
+                    'because': 'metadata, is, not, fussy',
+                    'number': '99',
+                }
+            }
+        }
+    }
+
+
+async def test_json_global_cfg_empty(setup, mock_glbl_cfg, capsys):
+    """It returns an empty global configuration in JSON format."""
+    parser, opts, _ = setup
+    mock_glbl_cfg('cylc.flow.scripts.config.glbl_cfg', '')
+    opts.item = ['scheduler][mail]']
+    opts.json = True
+    opts.defaults = True
+    opts.none_str = 'zilch'
+
+    await _main(parser, opts)
+
+    returned_config = json.loads(capsys.readouterr().out)
+    for key in ['footer', 'from', 'smtp', 'to']:
+        assert returned_config[key] == 'zilch'

tests/unit/parsec/test_ordered_dict.py

@@ -146,3 +146,16 @@ def test_prepend(self):
 
         self.assertEqual('year', next(iterator))
         self.assertEqual('key', next(iterator))
+
+    def test_repl_val(self):
+        """it recursively replaces dict values with str."""
+        level1, level2, level3 = (OrderedDictWithDefaults() for i in range(3))
+        level3['none'] = None
+        level2['level3'] = level3
+        level2['general'] = None
+        level1['level2'] = level2
+
+        level1.repl_val(level1, None, 'Monk')
+
+        assert level1['level2']['level3']['none'] == 'Monk'
+        assert level1['level2']['general'] == 'Monk'