diff --git a/pyinfra-metadata.toml b/pyinfra-metadata.toml index 96e5e3ac6..1fb90d298 100644 --- a/pyinfra-metadata.toml +++ b/pyinfra-metadata.toml @@ -81,6 +81,24 @@ path = "src/pyinfra/facts/freebsd.py" type = "fact" tags = ["package-manager"] +[pyinfra.plugins."freebsd-ops"] +name = "freebsd" +path = "src/pyinfra/operations/freebsd" +type = "operation" +tags = ["system"] + +[pyinfra.plugins."openwrt-ops"] +name = "openwrt" +path = "src/pyinfra/operations/openwrt" +type = "operation" +tags = ["system"] + +[pyinfra.plugins."openwrt-facts"] +name = "openwrt" +path = "src/pyinfra/facts/openwrt" +type = "fact" +tags = ["system"] + [pyinfra.plugins."opkg-ops"] name = "opkg" path = "src/pyinfra/operations/opkg.py" diff --git a/repros/brew-fixes-2026-02-repro.sh b/repros/brew-fixes-2026-02-repro.sh new file mode 100755 index 000000000..159497779 --- /dev/null +++ b/repros/brew-fixes-2026-02-repro.sh @@ -0,0 +1,5 @@ +#! /bin/bash +pyinfra --version +brew list --versions | grep openssl +pyinfra @local fact brew.BrewPackages |& grep openssl -A 3 +pyinfra --dry @local brew.packages openssl@3.6.1 diff --git a/scripts/docs_utils.py b/scripts/docs_utils.py index 01cf71b9c..29dc9e190 100644 --- a/scripts/docs_utils.py +++ b/scripts/docs_utils.py @@ -1,9 +1,10 @@ import re -from inspect import cleandoc, getmembers, ismodule +from collections.abc import Callable +from inspect import cleandoc, getmembers, isfunction, ismodule from pathlib import Path from types import ModuleType -from typing import Any -from collections.abc import Generator + +from pyinfra.api import FactBase, ShortFactBase def format_doc_line(line: str) -> str: @@ -114,19 +115,6 @@ def prepare_docstring(doc: str | None) -> str: return rst_to_md_docstring(cleandoc(doc)) -def including_sub_modules(module: ModuleType) -> Generator[ModuleType, None, None]: - """Yield all modules to be examined, including the base modules.""" - yield module - module_name = module.__name__ - for key, value in getmembers(module): - if ( - ismodule(value) - and value.__name__.startswith(module_name) - and (not key.startswith("__")) - ): - yield from including_sub_modules(value) - - def get_module_names( src_dir: Path, *, @@ -148,11 +136,63 @@ def get_module_names( return module_names -def remove_dups(all: list[tuple[str, Any]]) -> list[tuple[str, Any]]: - """Remove items with duplicate values, i.e. the same function or module found again.""" +def is_fact_class(m: ModuleType, _key: str, value: object) -> bool: + return ( + isinstance(value, type) + and (issubclass(value, FactBase) or issubclass(value, ShortFactBase)) + and value.__module__.startswith(m.__name__) + and value is not FactBase + and not value.__name__.endswith("Base") # hacky! + ) + + +def is_fact_class_in_op(_m: ModuleType, _key: str, value: object) -> bool: + return isinstance(value, type) and ( + issubclass(value, FactBase) or issubclass(value, ShortFactBase) + ) + + +def function_of_interest(module: ModuleType, key: str, value: object) -> bool: + return ( + isfunction(value) + and value.__module__.startswith(module.__name__) + and getattr(value, "_inner", False) + and not value.__name__.startswith("_") + and not key.startswith("_") + ) + + +def get_objects_from_module( + module: ModuleType, + predicate: Callable[[object], bool], + filt: Callable[[ModuleType, str, object], bool], +) -> list[tuple[str, type]]: + if not hasattr(module, "__path__"): # not a package thus a single file + found = [ + (key, value) for key, value in getmembers(module, predicate) if filt(module, key, value) + ] + else: + # for packages, deferred import mechanism gives zero members so need to use __all__ + found = [ + (name, item) + for sym in module.__all__ + for name, item in ( + ( + (f"{sym}.{key}", value) + for key, value in getmembers(getattr(module, sym), predicate) + if filt(module, key, value) + ) + if ismodule(getattr(module, sym)) + else ([(sym, getattr(module, sym))] if predicate(getattr(module, sym)) else []) + ) + if filt(module, name, item) + ] + + # Remove items with duplicate values, i.e. the same function or module found again unique, seen = [], set() - for key, value in all: + for key, value in found: if value not in seen: seen.add(value) unique.append((key, value)) + return unique diff --git a/scripts/generate_facts_docs.py b/scripts/generate_facts_docs.py index 1439d380d..36d3b4cd6 100755 --- a/scripts/generate_facts_docs.py +++ b/scripts/generate_facts_docs.py @@ -2,21 +2,20 @@ import sys from importlib import import_module -from inspect import getfullargspec, getmembers, isclass +from inspect import getfullargspec, isclass from os import makedirs, path from pathlib import Path from types import FunctionType, MethodType -from pyinfra.api.facts import FactBase, ShortFactBase from pyinfra.api.metadata import ALLOWED_TAGS, parse_plugins sys.path.append(path.dirname(path.realpath(__file__))) from docs_utils import ( format_doc_line, get_module_names, - including_sub_modules, + get_objects_from_module, + is_fact_class, prepare_docstring, - remove_dups, ) # noqa: E402 CARD_SCRIPT = """\ @@ -119,20 +118,8 @@ def build_facts_docs(): lines.append(f"See also: [operations/{module_name}](../operations/{module_name}.md).") lines.append("") - all_fact_classes = [ - (key, value) - for m in including_sub_modules(module) - for key, value in getmembers(m) - if ( - isclass(value) - and (issubclass(value, FactBase) or issubclass(value, ShortFactBase)) - and value.__module__.startswith(m.__name__) - and value is not FactBase - and not value.__name__.endswith("Base") # hacky! - ) - ] + fact_classes = get_objects_from_module(module, isclass, is_fact_class) - fact_classes = remove_dups(all_fact_classes) for fact, cls in fact_classes: name = fact args_string_and_brackets = "" diff --git a/scripts/generate_llms_txt.py b/scripts/generate_llms_txt.py old mode 100644 new mode 100755 index a64fb8e97..1c5a19f37 --- a/scripts/generate_llms_txt.py +++ b/scripts/generate_llms_txt.py @@ -11,19 +11,22 @@ from __future__ import annotations import sys -from inspect import getdoc, getfullargspec, getmembers, isclass, signature +from importlib import import_module +from inspect import getdoc, getfullargspec, isclass, isfunction, signature from os import environ, makedirs, path from pathlib import Path from types import FunctionType, MethodType, ModuleType -from importlib import import_module from pyinfra.api import metadata from pyinfra.api.connectors import get_all_connectors -from pyinfra.api.facts import FactBase, ShortFactBase sys.path.append(path.dirname(path.realpath(__file__))) -from docs_utils import including_sub_modules, prepare_docstring, remove_dups # noqa: E402 - +from docs_utils import ( + function_of_interest, + get_objects_from_module, + is_fact_class, + prepare_docstring, +) # noqa: E402 BASE_URL = "https://docs.pyinfra.com/en" VERSION = environ.get("DOCS_VERSION", "latest") @@ -199,39 +202,11 @@ def section(title: str, pages: list[tuple[str, str, str]]) -> None: def _extract_operation_funcs(module: ModuleType) -> list[tuple[str, FunctionType]]: - funcs = [ - ( - f"{m.__name__.split('.')[-1]}.{key}" if m != module else key, - getattr(value, "_inner"), - ) - for m in including_sub_modules(module) - for key, value in getmembers(m) - if ( - isinstance(value, FunctionType) - and value.__module__.startswith(m.__name__) - and getattr(value, "_inner", False) - and not value.__name__.startswith("_") - and not key.startswith("_") - ) - ] - return remove_dups(funcs) + return get_objects_from_module(module, isfunction, function_of_interest) def _extract_fact_classes(module: ModuleType) -> list[tuple[str, type]]: - classes = [ - (key, value) - for m in including_sub_modules(module) - for key, value in getmembers(m) - if ( - isclass(value) - and (issubclass(value, FactBase) or issubclass(value, ShortFactBase)) - and value.__module__.startswith(m.__name__) - and value is not FactBase - and value is not ShortFactBase - and not value.__name__.endswith("Base") - ) - ] - return remove_dups(classes) + return get_objects_from_module(module, isclass, is_fact_class) def render_operation_section(module_name: str) -> list[str]: diff --git a/scripts/generate_operations_docs.py b/scripts/generate_operations_docs.py index 853932a54..16742f0e7 100755 --- a/scripts/generate_operations_docs.py +++ b/scripts/generate_operations_docs.py @@ -2,21 +2,20 @@ import sys from importlib import import_module -from inspect import getmembers, isclass, signature +from inspect import isclass, isfunction, signature from os import makedirs, path from pathlib import Path -from types import FunctionType -from pyinfra.api.facts import FactBase from pyinfra.api.metadata import ALLOWED_TAGS, parse_plugins sys.path.append(path.dirname(path.realpath(__file__))) from docs_utils import ( format_doc_line, + function_of_interest, get_module_names, - including_sub_modules, + get_objects_from_module, + is_fact_class_in_op, prepare_docstring, - remove_dups, ) # noqa: E402 MODULE_DEF_LINE_MAX = 90 @@ -118,39 +117,21 @@ def build_operations_docs(): lines.append(module_doc) lines.append("") - operation_facts = [ - (key, value) - for m in including_sub_modules(module) - for key, value in getmembers(m) - if (isclass(value) and issubclass(value, FactBase)) - ] + unique_facts = get_objects_from_module(module, isclass, is_fact_class_in_op) - unique_facts = remove_dups(operation_facts) if unique_facts: items = [] for key, value in unique_facts: - fact_module = value.__module__.replace("pyinfra.facts.", "") + fact_module = value.__module__.replace("pyinfra.facts.", "").split(".")[0] items.append( f"[`{fact_module}.{key}`](../facts/{fact_module}.md#{fact_module}-{key})" ) lines.append("Facts used in these operations: {}.".format(", ".join(items))) lines.append("") - all_operation_functions = [ - (f"{m.__name__.split('.')[-1]}.{key}" if m != module else key, value._inner) - for m in including_sub_modules(module) - for key, value in getmembers(m) - if ( - isinstance(value, FunctionType) - and value.__module__.startswith(m.__name__) - and getattr(value, "_inner", False) - and not value.__name__.startswith("_") - and not key.startswith("_") - ) - ] - operation_functions = remove_dups(all_operation_functions) + operation_functions = get_objects_from_module(module, isfunction, function_of_interest) - for name, func in operation_functions: + for name, func in sorted(operation_functions, key=lambda x: (x[0].count("."), x[0])): decorated_func = getattr(func, "_inner", None) while decorated_func: func = decorated_func diff --git a/src/pyinfra/api/facts.py b/src/pyinfra/api/facts.py index 9d7dc62ba..45f4a92f0 100644 --- a/src/pyinfra/api/facts.py +++ b/src/pyinfra/api/facts.py @@ -53,6 +53,8 @@ T = TypeVar("T") +already_logged_as_deprecated = set() # used to ensure only one warning per deprecated fact + class FactBase(Generic[T]): name: str @@ -63,6 +65,10 @@ class FactBase(Generic[T]): command: Callable[..., str | StringCommand] + is_deprecated = False + + deprecated_for: str | None = None + def requires_command(self, *args, **kwargs) -> str | None: """Return the binary name that must exist on the remote host for this fact to run. If the binary is absent the fact returns its ``default()`` value silently. @@ -118,6 +124,8 @@ def process_pipeline(self, args, output): class ShortFactBase(Generic[T]): name: str fact: type[FactBase] + is_deprecated = False + deprecated_for: str | None = None @override def __init_subclass__(cls) -> None: @@ -195,6 +203,15 @@ def get_fact( ensure_hosts: Any | None = None, apply_failed_hosts: bool = True, ) -> Any: + global already_logged_as_deprecated + + if cls.is_deprecated and (cls not in already_logged_as_deprecated): + already_logged_as_deprecated.add(cls) + msg = f"The {cls.__name__} fact is deprecated" + if cls.deprecated_for is not None: + msg = f"{msg}, please use: {cls.deprecated_for}" + logger.warning(msg) + if issubclass(cls, ShortFactBase): return get_short_facts( state, diff --git a/src/pyinfra/facts/openwrt/__init__.py b/src/pyinfra/facts/openwrt/__init__.py new file mode 100644 index 000000000..0a389c460 --- /dev/null +++ b/src/pyinfra/facts/openwrt/__init__.py @@ -0,0 +1,31 @@ +""" +Facts specific to the [OpenWrt](https://www.openwrt.org) distribution. +""" + +import importlib +import sys +from typing import Any + +__ALL__ = { + "OpenWrtFeature": "features.OpenWrtFeature", + "OpenWrtHasFeature": "features.OpenWrtHasFeature", + "opkg": "opkg", +} + +__all__ = list(__ALL__.keys()) + + +def __getattr__(name: str) -> Any: + # On-demand import of OpenWrt facts, so we don't have to import them all at once + # this forces py3.7>=, but that's fine as py2 is EOL and py3.6 is also EOL + # Also, pyinfra is py3.11>=, so this is not a breaking change. + if name in __all__: + pieces = __ALL__[name].split(".") + module = importlib.import_module(f".{pieces[0]}", package=__name__) + if len(pieces) < 2: + return module + del sys.modules[module.__name__] + del sys.modules[__name__].__dict__[pieces[0]] + return getattr(module, pieces[1]) + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/pyinfra/facts/openwrt/features.py b/src/pyinfra/facts/openwrt/features.py new file mode 100644 index 000000000..e22a5472d --- /dev/null +++ b/src/pyinfra/facts/openwrt/features.py @@ -0,0 +1,135 @@ +""" +Provides a fact that tells whether a host supports an OpenWrt feature or not. + + + whether the host uses ``apk`` (vs. ``opkg``) + + whether the host has ``DSA`` (vs. ``swconfig``) + + whether the host has ``FW4`` (and ``nftables`` vs. ``FW3/iptables``) + +note: this does _not_ show up in the online documentation; the file header in __init__.py does +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum, unique + +from typing_extensions import override + +from pyinfra import logger +from pyinfra.api import FactBase + + +@unique +class OpenWrtFeature(Enum): + USES_APK = "uses_apk" + HAS_DSA = "has_dsa" + HAS_FW4 = "has_fw4" + + +@dataclass(frozen=True) +class Release: + """ + A release with major and minor components + """ + + major: int + minor: int + + +@dataclass(frozen=True) +class ReleaseRange: + """ + A range of releases, usually used for validity. + Any release is newer than a start of None and older than an end of None + """ + + start: Release | None + end: Release | None + + def contains(self, release: Release) -> bool: + return ( + (self.start is None) + or (release.major > self.start.major) + or ((release.major == self.start.major) and (release.minor >= self.start.minor)) + ) and ( + (self.end is None) + or (release.major < self.end.major) + or ((release.major == self.end.major) and (release.minor <= self.end.minor)) + ) + + +# +# References for release ranges in FEATURES table +# +# Feature.USES_APK - https://openwrt.org/releases/25.12/notes-25.12.0#switch_package_manager_from_opkg_to_apk +# Feature.HAS_DSA - https://openwrt.org/releases/21.02/notes-21.02.0#initial_dsa_support +# Feature.HAS_FW4 - https://openwrt.org/releases/22.03/notes-22.03.0#firewall4_based_on_nftables + +FEATURES = { + OpenWrtFeature.USES_APK: ReleaseRange(Release(25, 12), None), + # the following is a bit optimistic as it is really target-specific + OpenWrtFeature.HAS_DSA: ReleaseRange(Release(21, 2), None), + OpenWrtFeature.HAS_FW4: ReleaseRange(Release(22, 3), None), +} + +DOES_NOT_EXIST = ReleaseRange(Release(9999, 0), Release(9999, 0)) + +# line in /etc/openwrt-release with release number looks like: DISTRIB_RELEASE='19.07.2' +THE_FILE = "/etc/openwrt_release" +PREFIX = "DISTRIB_RELEASE='" +SUFFIX = "'" + + +class OpenWrtHasFeature(FactBase[bool]): + """ + Returns `true` if the running version of OpenWrt supports a specific feature and `false` + otherwise. + + .. code:: python + from pyinfra.facts.openwrt import OpenWrtFeature + + if host.get_fact(OpenWrtHasFeature, OpenWrtFeature.HAS_DSA): + # setup configuration using the Distributed Switching Architecture + else: + # setup configuration using swconfig + """ + + # this isn't a ShortFact using LinuxDistribution because short facts can't have parameters + + @override + def command(self, feature: OpenWrtFeature) -> str: + # TODO: remove this once CLI and fact tests from fixtures support enums + if isinstance(feature, str): + feature = OpenWrtFeature(feature) + + return f"echo {feature.value} && cat {THE_FILE}" + + @override + @staticmethod + def default() -> bool: + return False + + @override + def process(self, output: list[str]) -> bool: + if len(output) < 2: # usually will be 8 but conceptually we just need 2 + logger.error(f"not enough lines of output from {THE_FILE}") + return False + + feature = OpenWrtFeature(output[0]) + for line in output[1:]: + if not line.startswith(PREFIX): + continue + if len(pieces := line.removeprefix(PREFIX).strip("'").split(".")) >= 2: + try: + major, minor = int(pieces[0]), int(pieces[1]) + except (ValueError, TypeError): + logger.error(f"could not decode OpenWrt release from '{'.'.join(pieces)}'") + else: + return FEATURES.get(feature, DOES_NOT_EXIST).contains(Release(major, minor)) + else: + logger.error(f"unrecognized format for OpenWrt release: '{'.'.join(pieces)}'") + break + else: + logger.error(f"'{PREFIX}' not found in {THE_FILE}") + + return False diff --git a/src/pyinfra/facts/openwrt/opkg.py b/src/pyinfra/facts/openwrt/opkg.py new file mode 100644 index 000000000..bfea62222 --- /dev/null +++ b/src/pyinfra/facts/openwrt/opkg.py @@ -0,0 +1,303 @@ +""" +Gather the information provided by ``opkg`` on OpenWrt systems: + + ``opkg`` configuration + + feeds configuration + + list of installed packages + + list of packages with available upgrades + +See https://openwrt.org/docs/guide-user/additional-software/opkg + +.. note:: + as of OpenWrt [Release 25.12](https://openwrt.org/releases/25.12/notes-25.12.0#switch_package_manager_from_opkg_to_apk) + OpenWrt uses [apk](../facts/apk.md) + +note: this does _not_ show up in the online documentation; the file header in __init__.py does +and thus the note above is repeated in each fact. +""" + +from __future__ import annotations + +import re +from typing import NamedTuple + +from typing_extensions import override + +from pyinfra import logger +from pyinfra.api import FactBase +from pyinfra.facts.util.packaging import PackageVersionDict, parse_packages + +OpkgArchInstallInfo = dict[str, int] + + +class OpkgPkgUpgradeInfo(NamedTuple): + installed: str + available: str + + +OpkgPkgUpgradeMap = dict[str, OpkgPkgUpgradeInfo] + + +class OpkgConfInfo(NamedTuple): + paths: dict[str, str] # list of paths, e.g. {'root':'/', 'ram':'/tmp} + list_dir: str # where package lists are stored, e.g. /var/opkg-lists + options: dict[str, str | bool] # mapping from option to value, e.g. {'check_signature': True} + arch_cfg: dict[str, int] # priorities for architectures + + +class OpkgFeedInfo(NamedTuple): + url: str # url for the feed + fmt: str # format of the feed, e.g. "src/gz" + kind: str # whether it comes from the 'distribution' or is 'custom' + + +OpkgFeedMap = dict[str, OpkgFeedInfo] + + +class OpkgConf(FactBase[OpkgConfInfo]): + """ + Returns a ``NamedTuple`` with the current ``opkg`` configuration: + + .. code:: python + + OpkgConfInfo( + paths = { + "root": "/", + "ram": "/tmp", + }, + list_dir = "/opt/opkg-lists", + options = { + "overlay_root": "/overlay" + }, + arch_cfg = { + "all": 1, + "noarch": 1, + "i386_pentium": 10 + } + ) + + .. note:: + as of OpenWrt [Release 25.12](https://openwrt.org/releases/25.12/notes-25.12.0#switch_package_manager_from_opkg_to_apk) + OpenWrt uses [apk](../facts/apk.md) + """ + + @override + def requires_command(self) -> str: + return "opkg" + + regex = re.compile( + r""" + ^(?:\s*) + (?: + (?:arch\s+(?P\w+)\s+(?P\d+))| + (?:dest\s+(?P\w+)\s+(?P[\w/\-]+))| + (?:lists_dir\s+(?Pext)\s+(?P[\w/\-]+))| + (?:option\s+(?P