start cleaning up the ast/interpreter.py type mess

mesonbuild/ast/interpreter.py

@@ -30,8 +30,6 @@
     InterpreterObject,
 )
 
-from ..interpreterbase.helpers import flatten
-
 from ..interpreter import (
     StringHolder,
     BooleanHolder,
@@ -41,7 +39,6 @@
 )
 
 from ..mparser import (
-    ArgumentNode,
     ArithmeticNode,
     ArrayNode,
     AssignmentNode,
@@ -60,7 +57,7 @@
 if T.TYPE_CHECKING:
     from .visitor import AstVisitor
     from ..interpreter import Interpreter
-    from ..interpreterbase import TYPE_var, TYPE_nvar
+    from ..interpreterbase import TYPE_var, TYPE_kwargs
     from ..mparser import (
         AndNode,
         ComparisonNode,
@@ -71,7 +68,13 @@
         TestCaseClauseNode,
         UMinusNode,
     )
-    from ..mesonlib import SubProject
+    from ..mesonlib import HoldableObject, SubProject
+
+    TYPE_ivar = T.Union[str, int, bool, 'HoldableObject', 'MesonInterpreterObject',
+                        'UnknownValue', 'IntrospectionBuildTarget', 'IntrospectionFile',
+                        'IntrospectionDependency', list['TYPE_ivar'], dict[str | UnknownValue, 'TYPE_ivar']]
+    TYPE_nvar = T.Union[TYPE_ivar, mparser.BaseNode]
+    TYPE_nkwargs = T.Dict[str, TYPE_nvar]
 
 _T = T.TypeVar('_T')
 _V = T.TypeVar('_V')
@@ -271,7 +274,7 @@ def load_root_meson_file(self) -> None:
             self.ast.accept(i)
 
     def func_subdir(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> None:
-        args = self.flatten_args(args)
+        args = self.flatten_args_hack(args)
         if len(args) != 1 or not isinstance(args[0], str):
             sys.stderr.write(f'Unable to evaluate subdir({args}) in AstInterpreter --> Skipping\n')
             return
@@ -285,24 +288,28 @@ def func_subdir(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str
             buildfilename = os.path.join(subdir, environment.build_filename)
             sys.stderr.write(f'Unable to find build file {buildfilename} --> Skipping\n')
 
-    def inner_method_call(self, obj: BaseNode, method_name: str, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> T.Any:
-        for arg in itertools.chain(args, kwargs.values()):
+    def inner_method_call(self, iobj: TYPE_ivar, method_name: str, iargs: T.List[TYPE_ivar], nkwargs: TYPE_nkwargs) -> TYPE_ivar:
+        for arg in itertools.chain(iargs, nkwargs.values()):
             if isinstance(arg, UnknownValue):
                 return UnknownValue()
 
-        if isinstance(obj, str):
-            result = StringHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
-        elif isinstance(obj, bool):
-            result = BooleanHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
-        elif isinstance(obj, int):
-            result = IntegerHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
-        elif isinstance(obj, list):
+        args = T.cast('list[TYPE_var]', iargs)
+        kwargs = T.cast('TYPE_kwargs', nkwargs)
+        if isinstance(iobj, str):
+            result = StringHolder(iobj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
+        elif isinstance(iobj, bool):
+            result = BooleanHolder(iobj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
+        elif isinstance(iobj, int):
+            result = IntegerHolder(iobj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
+        elif isinstance(iobj, list):
+            obj = T.cast('list[TYPE_var]', iobj) # assuming ArrayHolder is fine with receiving TYPE_ivars...
             result = ArrayHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
-        elif isinstance(obj, dict):
+        elif isinstance(iobj, dict):
+            obj = T.cast('dict[str, TYPE_var]', iobj) # assuming DictHolder is fine with receiving TYPE_ivars...
             result = DictHolder(obj, T.cast('Interpreter', self)).method_call(method_name, args, kwargs)
         else:
             return UnknownValue()
-        return result
+        return T.cast('TYPE_ivar', result)
 
     def method_call(self, node: mparser.MethodNode) -> None:
         invocable = node.source_object
@@ -345,25 +352,24 @@ def evaluate_indexing(self, node: IndexNode) -> None:
         self.evaluate_statement(node.iobject)
         self.evaluate_statement(node.index)
 
-    def reduce_arguments(
+    # this returns a different type than the supertype; TYPE_nvar and TYPE_nkwargs
+    # are limited to the AstInterpreter
+    def reduce_arguments( # type: ignore[override]
                 self,
                 args: mparser.ArgumentNode,
                 key_resolver: T.Callable[[mparser.BaseNode], str] = default_resolve_key,
                 duplicate_key_error: T.Optional[str] = None,
-            ) -> T.Tuple[T.List[T.Any], T.Any]:
+            ) -> T.Tuple[T.List[TYPE_ivar], TYPE_nkwargs]:
         for arg in args.arguments:
             self.evaluate_statement(arg)
         for value in args.kwargs.values():
             self.evaluate_statement(value)
-        if isinstance(args, ArgumentNode):
-            kwargs = {}
-            for key, val in args.kwargs.items():
-                kwargs[key_resolver(key)] = val
-            if args.incorrect_order():
-                raise InvalidArguments('All keyword arguments must be after positional arguments.')
-            return self.flatten_args(args.arguments), kwargs
-        else:
-            return self.flatten_args(args), {}
+        kwargs: TYPE_nkwargs = {}
+        for key, val in args.kwargs.items():
+            kwargs[key_resolver(key)] = val
+        if args.incorrect_order():
+            raise InvalidArguments('All keyword arguments must be after positional arguments.')
+        return self.flatten_args(args.arguments), kwargs
 
     def evaluate_comparison(self, node: ComparisonNode) -> None:
         self.evaluate_statement(node.left)
@@ -543,7 +549,7 @@ def get_cur_value(self, var_name: str) -> T.Union[BaseNode, UnknownValue]:
     # node = [123, somedep.found()]
     # ```
     # `node_to_runtime_value` will return `[123, UnknownValue()]`.
-    def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var]) -> T.Any:
+    def node_to_runtime_value(self, node: T.Union[UnknownValue, TYPE_ivar, mparser.BaseNode]) -> TYPE_ivar:
         if isinstance(node, (mparser.StringNode, mparser.BooleanNode, mparser.NumberNode)):
             return node.value
         elif isinstance(node, mparser.StringNode):
@@ -556,7 +562,17 @@ def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var])
         elif isinstance(node, ArrayNode):
             return [self.node_to_runtime_value(x) for x in node.args.arguments]
         elif isinstance(node, mparser.DictNode):
-            return {self.node_to_runtime_value(k): self.node_to_runtime_value(v) for k, v in node.args.kwargs.items()}
+            result: T.Dict[str | UnknownValue, TYPE_ivar] = {}
+            for raw_k, raw_v in node.args.kwargs.items():
+                k = self.node_to_runtime_value(raw_k)
+                if not isinstance(k, (str, UnknownValue)):
+                    k = UnknownValue()
+                if isinstance(k, UnknownValue):
+                    # there could be more than one unknown key, so add an unknown mapping
+                    result[k] = UnknownValue()
+                else:
+                    result[k] = self.node_to_runtime_value(raw_v)
+            return result
         elif isinstance(node, IdNode):
             assert len(self.dataflow_dag.tgt_to_srcs[node]) == 1
             val = next(iter(self.dataflow_dag.tgt_to_srcs[node]))
@@ -568,8 +584,8 @@ def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var])
             else:
                 return self.node_to_runtime_value(funcval)
         elif isinstance(node, ArithmeticNode):
-            left = self.node_to_runtime_value(node.left)
-            right = self.node_to_runtime_value(node.right)
+            left: TYPE_ivar = self.node_to_runtime_value(node.left)
+            right: TYPE_ivar = self.node_to_runtime_value(node.right)
             if isinstance(left, list) and isinstance(right, UnknownValue):
                 return left + [right]
             if isinstance(right, list) and isinstance(left, UnknownValue):
@@ -586,11 +602,11 @@ def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var])
                     if not isinstance(right, list):
                         right = [right]
                     return left + right
-                return left + right
+                return left + right # type: ignore[operator]
             elif node.operation == '-':
-                return left - right
+                return left - right # type: ignore[operator]
             elif node.operation == '*':
-                return left * right
+                return left * right # type: ignore[operator]
             elif node.operation == '/':
                 if isinstance(left, int) and isinstance(right, int):
                     return left // right
@@ -606,7 +622,7 @@ def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var])
             index = self.node_to_runtime_value(node.index)
             if isinstance(iobject, UnknownValue) or isinstance(index, UnknownValue):
                 return UnknownValue()
-            return iobject[index]
+            return iobject[index] # type: ignore[index]
         elif isinstance(node, mparser.ComparisonNode):
             left = self.node_to_runtime_value(node.left)
             right = self.node_to_runtime_value(node.right)
@@ -617,9 +633,9 @@ def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var])
             elif node.ctype == '!=':
                 return left != right
             elif node.ctype == 'in':
-                return left in right
+                return left in right # type: ignore[operator]
             elif node.ctype == 'not in':
-                return left not in right
+                return left not in right # type: ignore[operator]
         elif isinstance(node, mparser.TernaryNode):
             cond = self.node_to_runtime_value(node.condition)
             if isinstance(cond, UnknownValue):
@@ -631,15 +647,19 @@ def node_to_runtime_value(self, node: T.Union[UnknownValue, BaseNode, TYPE_var])
         elif isinstance(node, mparser.OrNode):
             left = self.node_to_runtime_value(node.left)
             right = self.node_to_runtime_value(node.right)
+            if left is True or right is True:
+                return True
             if isinstance(left, UnknownValue) or isinstance(right, UnknownValue):
                 return UnknownValue()
-            return left or right
+            return False
         elif isinstance(node, mparser.AndNode):
             left = self.node_to_runtime_value(node.left)
             right = self.node_to_runtime_value(node.right)
+            if left is False or right is False:
+                return False
             if isinstance(left, UnknownValue) or isinstance(right, UnknownValue):
                 return UnknownValue()
-            return left and right
+            return True
         elif isinstance(node, mparser.UMinusNode):
             val = self.node_to_runtime_value(node.value)
             if isinstance(val, UnknownValue):
@@ -716,7 +736,7 @@ def func_unset_variable(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.
         self.cur_assignments[var_name].append((self.nesting.copy(), node))
 
     def nodes_to_pretty_filelist(self, root_path: Path, subdir: str, nodes: T.List[BaseNode]) -> T.List[T.Union[str, UnknownValue]]:
-        def src_to_abs(src: T.Union[str, IntrospectionFile, UnknownValue]) -> T.Union[str, UnknownValue]:
+        def src_to_abs(src: TYPE_ivar) -> T.Union[str, UnknownValue]:
             if isinstance(src, str):
                 return os.path.normpath(os.path.join(root_path, subdir, src))
             elif isinstance(src, IntrospectionFile):
@@ -726,18 +746,11 @@ def src_to_abs(src: T.Union[str, IntrospectionFile, UnknownValue]) -> T.Union[st
             else:
                 raise TypeError
 
-        rtvals: T.List[T.Any] = flatten([self.node_to_runtime_value(sn) for sn in nodes])
+        rtvals: T.List[T.Any] = self.flatten_args(nodes)
         return [src_to_abs(x) for x in rtvals]
 
-    def flatten_args(self, args_raw: T.Union[TYPE_nvar, T.Sequence[TYPE_nvar]], include_unknown_args: bool = False) -> T.List[TYPE_var]:
-        # Make sure we are always dealing with lists
-        if isinstance(args_raw, list):
-            args = args_raw
-        else:
-            args = [args_raw]
-
-        # BaseNode resolves to Any. :/
-        flattened_args: T.List[T.Union[TYPE_var, T.Any]] = []
+    def flatten_args(self, args: T.Sequence[TYPE_nvar]) -> T.List[TYPE_ivar]:
+        flattened_args: T.List[TYPE_ivar] = []
 
         # Resolve the contents of args
         for i in args:
@@ -746,13 +759,26 @@ def flatten_args(self, args_raw: T.Union[TYPE_nvar, T.Sequence[TYPE_nvar]], incl
                 if resolved is not None:
                     if not isinstance(resolved, list):
                         resolved = [resolved]
-                    flattened_args += resolved
-            elif isinstance(i, (str, bool, int, float, UnknownValue, IntrospectionFile)) or include_unknown_args:
+                    flattened_args += self.flatten_args(resolved)
+            elif isinstance(i, (str, bool, int, float, UnknownValue, IntrospectionFile,
+                                IntrospectionBuildTarget, IntrospectionDependency)):
                 flattened_args += [i]
+            elif isinstance(i, list):
+                flattened_args += self.flatten_args(i)
             else:
-                raise NotImplementedError
+                raise NotImplementedError(f'flatten_args is missing a case for {type(i)}')
         return flattened_args
 
+    def flatten_args_hack(self, args: T.List[TYPE_var]) -> T.List[TYPE_var]:
+        # Unlike method calls, functions are invoked even if one or more values
+        # are unknown.  The return type is actually T.List[TYPE_ivar], and
+        # while callers declare args as T.List[TYPE_var], they receive a
+        # list of TYPE_ivar too.
+        # The right solution would involve making TYPE_var an argument to
+        # InterpreterBase, so that FunctionType is also changed to not
+        # use TYPE_var.
+        return T.cast('T.List[TYPE_var]', self.flatten_args(T.cast('T.List[TYPE_ivar]', args)))
+
     def evaluate_testcase(self, node: TestCaseClauseNode) -> Disabler | None:
         return Disabler(subproject=self.subproject)
 

mesonbuild/ast/introspection.py

@@ -121,7 +121,7 @@ def _str_list(node: T.Any) -> T.Optional[T.List[str]]:
             proj_vers = proj_vers.value
         if not isinstance(proj_vers, str):
             proj_vers = 'undefined'
-        proj_langs = self.flatten_args(args[1:])
+        proj_langs = self.flatten_args_hack(args[1:])
         # Match the value returned by ``meson.project_license()`` when
         # no ``license`` argument is specified in the ``project()`` call.
         proj_license = _str_list(kwargs.get('license', None)) or ['unknown']
@@ -174,7 +174,7 @@ def func_add_languages(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.D
 
     def _add_languages(self, raw_langs: T.List[TYPE_var], required: T.Union[bool, UnknownValue], for_machine: MachineChoice) -> None:
         langs: T.List[Language] = []
-        for l in self.flatten_args(raw_langs):
+        for l in self.flatten_args_hack(raw_langs):
             # we need to call .lower() here because `project('foo', 'CpP')` is valid.
             if isinstance(l, str):
                 langs.append(T.cast('Language', l.lower()))
@@ -194,7 +194,7 @@ def _add_languages(self, raw_langs: T.List[TYPE_var], required: T.Union[bool, Un
 
     def func_dependency(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str, TYPE_var]) -> T.Optional[IntrospectionDependency]:
         assert isinstance(node, FunctionNode)
-        args = self.flatten_args(args)
+        args = self.flatten_args_hack(args)
         kwargs = self.flatten_kwargs(kwargs)
         if not args:
             return None
@@ -223,7 +223,7 @@ def func_dependency(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict
 
     def build_target(self, node: BaseNode, args: T.List[TYPE_var], kwargs_raw: T.Dict[str, TYPE_var], targetclass: T.Type[BuildTarget]) -> T.Union[IntrospectionBuildTarget, UnknownValue]:
         assert isinstance(node, FunctionNode)
-        args = self.flatten_args(args)
+        args = self.flatten_args_hack(args)
         if not args or not isinstance(args[0], str):
             return UnknownValue()
         name = args[0]

mesonbuild/interpreter/interpreterobjects.py

@@ -35,7 +35,7 @@
     from ..cmake.interpreter import CMakeInterpreter
     from ..dependencies.base import IncludeType
     from ..envconfig import MachineInfo
-    from ..interpreterbase import FeatureCheckBase, TYPE_var, TYPE_kwargs, TYPE_nvar, TYPE_nkwargs
+    from ..interpreterbase import FeatureCheckBase, TYPE_var, TYPE_kwargs
     from ..mesonlib import SubProject
     from .interpreter import Interpreter
 
@@ -563,7 +563,7 @@ def configtool_method(self, args: T.Tuple[str], kwargs: TYPE_kwargs) -> str:
     @noPosargs
     @typed_kwargs('dependency.partial_dependency', *_PARTIAL_DEP_KWARGS)
     @InterpreterObject.method('partial_dependency')
-    def partial_dependency_method(self, args: T.List[TYPE_nvar], kwargs: 'kwargs.DependencyMethodPartialDependency') -> Dependency:
+    def partial_dependency_method(self, args: T.List[TYPE_var], kwargs: 'kwargs.DependencyMethodPartialDependency') -> Dependency:
         pdep = self.held_object.get_partial_dependency(**kwargs)
         return pdep
 
@@ -740,7 +740,7 @@ def found_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> bool:
     @noPosargs
     @typed_kwargs('dependency.partial_dependency', *_PARTIAL_DEP_KWARGS)
     @InterpreterObject.method('partial_dependency')
-    def partial_dependency_method(self, args: T.List[TYPE_nvar], kwargs: 'kwargs.DependencyMethodPartialDependency') -> Dependency:
+    def partial_dependency_method(self, args: T.List[TYPE_var], kwargs: 'kwargs.DependencyMethodPartialDependency') -> Dependency:
         pdep = self.held_object.get_partial_dependency(**kwargs)
         return pdep
 
@@ -1034,7 +1034,7 @@ def outdir_method(self, args: T.List[TYPE_var], kwargs: TYPE_kwargs) -> str:
     @noKwargs
     @typed_pos_args('extract_objects', varargs=(mesonlib.File, str, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList))
     @InterpreterObject.method('extract_objects')
-    def extract_objects_method(self, args: T.Tuple[T.List[str | build.TargetSources]], kwargs: TYPE_nkwargs) -> build.ExtractedObjects:
+    def extract_objects_method(self, args: T.Tuple[T.List[str | build.TargetSources]], kwargs: TYPE_kwargs) -> build.ExtractedObjects:
         if self.subproject != self.held_object.subproject:
             raise InterpreterException('Tried to extract objects from a different subproject.')
         tobj = self._target_object
@@ -1067,7 +1067,7 @@ def extract_objects_method(self, args: T.Tuple[T.List[str | build.TargetSources]
         )
     )
     @InterpreterObject.method('extract_all_objects')
-    def extract_all_objects_method(self, args: T.List[TYPE_nvar], kwargs: 'kwargs.BuildTargeMethodExtractAllObjects') -> build.ExtractedObjects:
+    def extract_all_objects_method(self, args: T.List[TYPE_var], kwargs: 'kwargs.BuildTargeMethodExtractAllObjects') -> build.ExtractedObjects:
         return self._target_object.extract_all_objects(kwargs['recursive'])
 
     @noPosargs

mesonbuild/interpreterbase/__init__.py

@@ -51,9 +51,7 @@
     'TV_func',
     'TYPE_elementary',
     'TYPE_var',
-    'TYPE_nvar',
     'TYPE_kwargs',
-    'TYPE_nkwargs',
     'TYPE_key_resolver',
     'TYPE_HoldableTypes',
 
@@ -74,9 +72,7 @@
     TV_func,
     TYPE_elementary,
     TYPE_var,
-    TYPE_nvar,
     TYPE_kwargs,
-    TYPE_nkwargs,
     TYPE_key_resolver,
     TYPE_HoldableTypes,