Jinja2: catch and log warnings

changes.d/7365.feat.md

@@ -0,0 +1 @@
+Python warnings raised in users' custom jinja2 filters, globals and tests are now logged.

cylc/flow/parsec/fileparse.py

@@ -35,24 +35,34 @@
 from pathlib import Path
 import re
 import sys
-import typing as t
+from typing import (
+    TYPE_CHECKING,
+    Any,
+)
+import warnings
 
-from cylc.flow import __version__
-from cylc.flow import LOG
+from cylc.flow import (
+    LOG,
+    __version__,
+)
+from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
 from cylc.flow.parsec.exceptions import (
-    FileParseError, ParsecError, TemplateVarLanguageClash
+    FileParseError,
+    ParsecError,
+    TemplateVarLanguageClash,
 )
 from cylc.flow.parsec.include import inline
-from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
-from cylc.flow.plugins import run_plugins
 from cylc.flow.parsec.util import itemstr
+from cylc.flow.plugins import run_plugins
 from cylc.flow.templatevars import get_template_vars_from_db
 from cylc.flow.workflow_files import (
-    get_workflow_source_dir, check_flow_file)
+    check_flow_file,
+    get_workflow_source_dir,
+)
 
-if t.TYPE_CHECKING:
+
+if TYPE_CHECKING:
     from optparse import Values
-    from typing import Union
 
 
 # heading/sections can contain commas (namespace name lists) and any
@@ -109,14 +119,14 @@
 )
 TEMPLATING_DETECTED = 'templating_detected'
 TEMPLATE_VARIABLES = 'template_variables'
-EXTRA_VARS_TEMPLATE: t.Dict[str, t.Any] = {
+EXTRA_VARS_TEMPLATE: dict[str, Any] = {
     'env': {},
     TEMPLATE_VARIABLES: {},
     TEMPLATING_DETECTED: None
 }
 
 
-def get_cylc_env_vars() -> t.Dict[str, str]:
+def get_cylc_env_vars() -> dict[str, str]:
     """Return a restricted dict of CYLC_ environment variables for templating.
 
     The following variables are ignored because the do not necessarily reflect
@@ -251,7 +261,7 @@ def multiline(flines, value, index, maxline):
     return quot + newvalue + line, index
 
 
-def process_plugins(fpath: 'Union[str, Path]', opts: 'Values'):
+def process_plugins(fpath: 'str | Path', opts: 'Values'):
     """Run a Cylc pre-configuration plugin.
 
     Plugins should return a dictionary containing:
@@ -320,9 +330,9 @@ def process_plugins(fpath: 'Union[str, Path]', opts: 'Values'):
 
 
 def merge_template_vars(
-    native_tvars: t.Dict[str, t.Any],
-    plugin_result: t.Dict[str, t.Any]
-) -> t.Dict[str, t.Any]:
+    native_tvars: dict[str, Any],
+    plugin_result: dict[str, Any]
+) -> dict[str, Any]:
     """Manage the merger of Cylc Native and Plugin template variables.
 
     Args:
@@ -363,7 +373,9 @@ def merge_template_vars(
         return native_tvars
 
 
-def _prepend_old_templatevars(fpath: str, template_vars: t.Dict) -> t.Dict:
+def _prepend_old_templatevars(
+    fpath: str, template_vars: dict[str, Any]
+) -> dict[str, Any]:
     """If the fpath is in a rundir, extract template variables from database.
 
     Args:
@@ -402,10 +414,10 @@ def _get_fpath_for_source(fpath: str, opts: "Values") -> str:
 
 def read_and_proc(
     fpath: str,
-    template_vars: t.Optional[t.Dict[str, t.Any]] = None,
-    viewcfg: t.Any = None,
-    opts: t.Any = None,
-) -> t.List[str]:
+    template_vars: dict[str, Any] | None = None,
+    viewcfg: Any = None,
+    opts: Any = None,
+) -> list[str]:
     """
     Read a cylc parsec config file (at fpath), inline any include files,
     process with Jinja2, and concatenate continuation lines.
@@ -495,9 +507,24 @@ def read_and_proc(
                     'Jinja2 Python package must be installed '
                     'to process file: ' + fpath
                 ) from None
-            flines = jinja2process(
-                fpath, flines, fdir, template_vars
-            )
+            with warnings.catch_warnings(
+                record=True, action='default'
+            ) as warns:
+                flines = jinja2process(
+                    fpath, flines, fdir, template_vars
+                )
+            if warns:
+                LOG.warning(
+                    "The following warnings were raised during Jinja2 "
+                    "preprocessing (note: any Jinja 3.1 deprecations will "
+                    "break at Cylc 8.7):\n"
+                    + "\n".join(
+                        warnings.formatwarning(
+                            w.message, w.category, w.filename, w.lineno
+                        )
+                        for w in warns
+                    )
+                )
 
     # concatenate continuation lines
     if do_contin:
@@ -512,8 +539,8 @@ def read_and_proc(
 
 
 def hashbang_and_plugin_templating_clash(
-    templating: str, flines: t.List[str]
-) -> t.Optional[str]:
+    templating: str, flines: list[str]
+) -> str | None:
     """Return file's hashbang/shebang, but raise TemplateVarLanguageClash
     if plugin-set template engine and hashbang do not match.
 
@@ -539,7 +566,7 @@ def hashbang_and_plugin_templating_clash(
                 ...
             cylc.flow.parsec.exceptions.TemplateVarLanguageClash: ...
     """
-    hashbang: t.Optional[str] = None
+    hashbang: str | None = None
     # Get hashbang if possible:
     if flines:
         match = re.match(r'^#!(\S+)', flines[0])
@@ -559,9 +586,9 @@ def hashbang_and_plugin_templating_clash(
 
 def parse(
     fpath: str,
-    output_fname: t.Optional[str] = None,
-    template_vars: t.Optional[t.Dict[str, t.Any]] = None,
-    opts: t.Any = None,
+    output_fname: str | None = None,
+    template_vars: dict[str, Any] | None = None,
+    opts: Any = None,
 ) -> OrderedDictWithDefaults:
     """Parse file items line-by-line into a corresponding nested dict."""
 
@@ -574,7 +601,7 @@ def parse(
 
     nesting_level = 0
     config = OrderedDictWithDefaults()
-    parents: t.List[str] = []
+    parents: list[str] = []
 
     maxline = len(flines) - 1
     index = -1

cylc/flow/parsec/jinja2support.py

@@ -26,7 +26,7 @@
 import re
 import sys
 import traceback
-import typing as t
+from typing import Any
 
 from jinja2 import (
     BaseLoader,
@@ -35,14 +35,16 @@
     FileSystemLoader,
     StrictUndefined,
     TemplateNotFound,
-    TemplateSyntaxError)
+    TemplateSyntaxError,
+)
 
 from cylc.flow import LOG
 from cylc.flow.exceptions import InputError
 import cylc.flow.flags
 from cylc.flow.parsec.exceptions import Jinja2Error
 from cylc.flow.parsec.fileparse import get_cylc_env_vars
 
+
 TRACEBACK_LINENO = re.compile(
     r'\s+?File "(?P<file>.*)", line (?P<line>\d+), in .*template'
 )
@@ -206,8 +208,8 @@ def jinja2environment(dir_=None):
 
 def get_error_lines(
     base_template_file: str,
-    template_lines: t.List[str],
-) -> t.Dict[str, t.List[str]]:
+    template_lines: list[str],
+) -> dict[str, list[str]]:
     """Extract exception lines from Jinja2 tracebacks.
 
     Returns:
@@ -219,7 +221,7 @@ def get_error_lines(
     ret = {}
     for line in reversed(traceback.format_exc().splitlines()):
         match = TRACEBACK_LINENO.match(line)
-        lines: t.List[str] = []
+        lines: list[str] = []
         if match:
             filename = match.groupdict()['file']
             lineno = int(match.groupdict()['line'])
@@ -246,10 +248,10 @@ def get_error_lines(
 
 def jinja2process(
     fpath: str,
-    flines: t.List[str],
+    flines: list[str],
     dir_: str,
-    template_vars: t.Optional[t.Dict[str, t.Any]] = None,
-) -> t.List[str]:
+    template_vars: dict[str, Any] | None = None,
+) -> list[str]:
     """Pass configure file through Jinja2 processor.
 
     Args:

tests/unit/parsec/test_fileparse.py

@@ -14,35 +14,38 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from tempfile import NamedTemporaryFile
 from contextlib import suppress
-
 import os
-import pytest
-from pytest import param
+from pathlib import Path
+import re
 import sqlite3
+from tempfile import NamedTemporaryFile
 from types import SimpleNamespace
+import warnings
+
+import pytest
+from pytest import param
 
 from cylc.flow import __version__ as cylc_version
+from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
 from cylc.flow.parsec.exceptions import (
     FileParseError,
     IncludeFileNotFoundError,
     Jinja2Error,
     ParsecError,
 )
-from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
 from cylc.flow.parsec.fileparse import (
     EXTRA_VARS_TEMPLATE,
-    _prepend_old_templatevars,
     _get_fpath_for_source,
-    get_cylc_env_vars,
+    _prepend_old_templatevars,
     addict,
     addsect,
+    get_cylc_env_vars,
+    merge_template_vars,
     multiline,
     parse,
     process_plugins,
     read_and_proc,
-    merge_template_vars
 )
 
 
@@ -447,6 +450,39 @@ def test_read_and_proc_jinja2_error_missing_shebang():
         assert r == ['a={{ name }}']
 
 
+def test_read_and_proc_jinja2_warnings(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+):
+    """Warnings from users' custom jinja2 filters etc should be logged."""
+    (fpath := tmp_path / 'flow.cylc').write_text(r"#!jinja2\n")
+    msg = "Mock deprecation warning"
+
+    def mock_jinja2process(fpath, *a, **k):
+        if fpath.endswith('flow.cylc'):
+            warnings.warn(msg, category=DeprecationWarning)
+        return []
+
+    monkeypatch.setattr(
+        'cylc.flow.parsec.jinja2support.jinja2process', mock_jinja2process
+    )
+    read_and_proc(
+        fpath=str(fpath),
+        viewcfg={'jinja2': True, 'contin': False, 'inline': False},
+    )
+    assert len(caplog.records) == 1
+    rec = caplog.records[0]
+    assert rec.levelname == 'WARNING'
+    assert re.match(
+        (
+            r"The following warnings .* during Jinja2 preprocessing.*\s+"
+            rf"{__file__}:\d+: DeprecationWarning: {msg}"
+        ),
+        rec.message,
+    )
+
+
 def test_parse_keys_only_singleline():
     with NamedTemporaryFile() as of, NamedTemporaryFile() as tf:
         fpath = tf.name