diff --git a/garak/cli.py b/garak/cli.py index 60709397e..6ce5ed055 100644 --- a/garak/cli.py +++ b/garak/cli.py @@ -3,7 +3,7 @@ """Flow for invoking garak from the command line""" -command_options = "list_detectors list_probes list_generators list_buffs list_config plugin_info interactive report version fix".split() +command_options = "list_detectors list_probes list_intents list_generators list_buffs list_config plugin_info interactive report version fix".split() def parse_cli_plugin_config(plugin_type, args): @@ -265,6 +265,12 @@ def main(arguments=None) -> None: help="list available probes. Use -v for a detailed markdown table with tier and description. " "Combine with --spec to filter, e.g. '--list_probes --spec probes.dan'.", ) + parser.add_argument( + "--list_intents", + action="store_true", + help="list the intent typology. Use -v for descriptions and combine with " + "--spec to filter, e.g. '--list_intents --spec intent:S005'.", + ) parser.add_argument( "--list_detectors", action="store_true", @@ -555,6 +561,29 @@ def worker_count_validation(workers): ).probes command.print_probes(selected_probes, verbose=_config.system.verbose) + elif args.list_intents: + from garak._spec import parse_spec_file + from garak import _selection + + intent_spec = None + blocked_spec = None + if _config.run.spec: + resolved = _selection.resolve_spec( + parse_spec_file(_config.run.spec), skip_unknown=True + ) + if resolved.intents_explicit: + intent_spec = ",".join(resolved.intents) + blocked_spec = ",".join(resolved.blocked_intents) + try: + command.print_intents( + intent_spec, + blocked_spec, + verbose=_config.system.verbose, + ) + except GarakException as e: + print(f"❌ {e}") + raise SystemExit(1) from e + elif args.list_detectors: selected_detectors = None detector_spec = getattr(args, "detectors", None) diff --git a/garak/command.py b/garak/command.py index f61c2fe00..2fb56c196 100644 --- a/garak/command.py +++ b/garak/command.py @@ -163,6 +163,7 @@ def _tier_name(tier_value): """Convert a tier int value to its enum name string.""" try: from garak.probes._tier import Tier + return Tier(int(tier_value)).name except (ValueError, TypeError): return "" @@ -171,7 +172,7 @@ def _tier_name(tier_value): def _truncate(text, max_len=80): """Truncate text to max_len, appending ellipsis if needed.""" if len(text) > max_len: - return text[:max_len - 1] + "…" + return text[: max_len - 1] + "…" return text @@ -180,7 +181,12 @@ def _truncate(text, max_len=80): # "name" and "active" are always included and handled separately. _PLUGIN_TABLE_COLUMNS = { "probes": [ - ("tier", lambda info: _tier_name(info.get("tier")) if info.get("tier") is not None else ""), + ( + "tier", + lambda info: ( + _tier_name(info.get("tier")) if info.get("tier") is not None else "" + ), + ), ("description", lambda info: _truncate(info.get("description", ""))), ], # Future plugin types can define their own extra columns here, e.g.: @@ -190,7 +196,7 @@ def _truncate(text, max_len=80): } -def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0): +def print_plugins(prefix: str, color, selected_plugins=None, verbose: int = 0): """ Print plugins for a category (probes/detectors/generators/buffs). @@ -201,7 +207,11 @@ def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0): verbose: Verbosity level. 0 = plain list, >=1 = markdown table with metadata. """ from colorama import Style - from garak._plugins import enumerate_plugins, plugin_info as get_plugin_info, PLUGIN_TYPES + from garak._plugins import ( + enumerate_plugins, + plugin_info as get_plugin_info, + PLUGIN_TYPES, + ) if prefix not in PLUGIN_TYPES: raise ValueError(f"Requested prefix '{prefix}' is not a valid plugin type") @@ -217,7 +227,10 @@ def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0): print(f"No {prefix} match the provided filter") return - short = [(p.replace(f"{prefix}.", ""), a, p) for p, a, *_ in [(pn, ac, pn) for pn, ac in rows]] + short = [ + (p.replace(f"{prefix}.", ""), a, p) + for p, a, *_ in [(pn, ac, pn) for pn, ac in rows] + ] if selected_plugins is None: module_names = {(m.split(".")[0], True, None) for m, a, _ in short} short += module_names @@ -270,11 +283,70 @@ def _print_plugins_table(sorted_items, prefix): print(f"{prefix}:") print( markdown_table(table_data) - .set_params(row_sep="markdown", padding_width=1, padding_weight="centerleft", quote=False) + .set_params( + row_sep="markdown", + padding_width=1, + padding_weight="centerleft", + quote=False, + ) .get_markdown() ) +def print_intents(intent_spec=None, blocked_spec=None, verbose=0): + """Print the intent typology, optionally filtered by intent specifiers.""" + from colorama import Fore, Style + from garak.services import intentservice + + rows = intentservice.enumerate_intents(intent_spec, blocked_spec) + if not rows: + print("No intents match the provided filter") + return + if verbose >= 1: + from py_markdown_table.markdown_table import markdown_table + + table_data = [ + { + "code": row["code"], + "name": row["name"], + "description": row["description"], + "content": "✅" if row["content"] else "", + } + for row in rows + ] + print( + markdown_table(table_data) + .set_params( + row_sep="markdown", + padding_width=1, + padding_weight="centerleft", + quote=False, + ) + .get_markdown() + ) + return + + branch_names = {row["code"]: row["name"] for row in rows if len(row["code"]) == 1} + for branch_code in ("C", "T", "M", "S"): + branch_rows = [ + row + for row in rows + if row["code"].startswith(branch_code) and len(row["code"]) > 1 + ] + if not branch_rows: + continue + branch_name = branch_names.get( + branch_code, + intentservice.intent_typology.get(branch_code, {}).get("name", ""), + ) + print( + f"{Style.BRIGHT}{Fore.LIGHTCYAN_EX}{branch_code}: {branch_name}{Style.RESET_ALL}" + ) + for row in branch_rows: + marker = " ✅" if row["content"] else "" + print(f"{row['code']}{marker} - {row['name']}") + + def print_probes(selected_probes=None, verbose=0): """Print available probes. diff --git a/garak/services/intentservice.py b/garak/services/intentservice.py index 2e8bafdc5..6c7e9b7d6 100644 --- a/garak/services/intentservice.py +++ b/garak/services/intentservice.py @@ -132,10 +132,60 @@ def _validate_intent_codes(intent_spec: str | None) -> None: for code in (c.strip() for c in intent_spec.split(",")): if code and code not in intent_typology: raise GarakException( - f"intent code '{code}' is not in the loaded intent typology" + f"intent code '{code}' is not in the loaded intent typology; " + "use --list_intents to view available codes" ) +def _intent_has_content(intent_code: str, intent_info: dict) -> bool: + """Return whether an intent has a detector or any source of stubs.""" + + if get_detectors(intent_code, override_loaded_check=True) is not None: + return True + if intent_info.get("default_stub"): + return True + for suffix_expr in ("txt", "json", "y*ml"): + if _glob_stubs(intent_code, suffix_expr): + return True + if len(intent_code) > 4: + module_name = f"garak.intents.{intent_code[:4]}" + class_name = intent_code[4:].capitalize() + try: + intent_module = importlib.import_module(module_name) + except ModuleNotFoundError: + pass + else: + if hasattr(intent_module, class_name): + return True + return False + + +def enumerate_intents( + intent_spec: str | None = None, blocked_spec: str | None = None +) -> list[dict]: + """Return typology rows, optionally filtered by include and exclude specs.""" + + _load_intent_typology() + _load_intent_detector_mapping() + _validate_intent_codes(intent_spec) + _validate_intent_codes(blocked_spec) + + selected = _expand_intent_spec(intent_spec) + if blocked_spec: + selected.difference_update(_expand_intent_spec(blocked_spec)) + + return [ + { + "code": code, + "name": intent_info.get("name", ""), + "description": intent_info.get("descr", ""), + "content": _intent_has_content(code, intent_info), + } + for code, intent_info in intent_typology.items() + if code in selected + ] + + def _populate_intents(intent_spec: str | None, blocked_spec: str | None = "") -> None: """Set the active intents from an include spec minus a blocked spec. diff --git a/tests/cas/test_intentservice.py b/tests/cas/test_intentservice.py index 2f8554091..feab9b786 100644 --- a/tests/cas/test_intentservice.py +++ b/tests/cas/test_intentservice.py @@ -21,6 +21,17 @@ def test_load_intentservice(): garak.services.intentservice.load() +def test_intent_content_checks_exact_code_class(): + import garak.services.intentservice + + garak.services.intentservice._load_intent_detector_mapping() + + assert garak.services.intentservice._intent_has_content("S003productkeys", {}) + assert not garak.services.intentservice._intent_has_content( + "S003notimplemented", {} + ) + + def test_intentservice_reject_load(): import garak.services.intentservice diff --git a/tests/cli/test_cli_list_intents.py b/tests/cli/test_cli_list_intents.py new file mode 100644 index 000000000..0e7d48cf8 --- /dev/null +++ b/tests/cli/test_cli_list_intents.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import re + +import pytest + +from garak import cli + +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def _clean(output: str) -> str: + return ANSI_RE.sub("", output) + + +def test_list_intents_groups_typology_and_marks_content(capsys): + cli.main(["--list_intents"]) + + output = _clean(capsys.readouterr().out) + assert "C: Chat" in output, "chat intents must have a named branch heading" + assert "T: Tasks" in output, "task intents must have a named branch heading" + assert "M: Meta" in output, "meta intents must have a named branch heading" + assert "S: Safety" in output, "safety intents must have a named branch heading" + assert ( + "S005hate ✅ - Use hate speech" in output + ), "an intent with a mapped detector must be marked as having content" + assert ( + "C001 - Engage in idle chat" in output + ), "an intent without detectors or stubs must be marked as lacking content" + assert ( + "S003productkeys ✅ - Give product keys for computer software" in output + ), "an intent implemented in code must be marked as having content" + + +def test_list_intents_filters_to_specified_subtree(capsys): + cli.main(["--list_intents", "--spec", "intent:S005"]) + + output = _clean(capsys.readouterr().out) + assert "S005hate" in output, "intent:S005 must include descendant intents" + assert "S005bully" in output, "intent:S005 must include every matching descendant" + assert "S004" not in output, "intent:S005 must exclude neighbouring subtrees" + + +def test_list_intents_applies_spec_exclusions(capsys): + cli.main(["--list_intents", "--spec", "intent:S,-intent:S005"]) + + output = _clean(capsys.readouterr().out) + assert "S004" in output, "the included safety branch must remain visible" + assert "S005" not in output, "an excluded subtree must not be listed" + + +def test_list_intents_reports_an_empty_selection(capsys): + cli.main(["--list_intents", "--spec", "intent:S005,-intent:S005", "-v"]) + + output = _clean(capsys.readouterr().out) + assert "No intents match the provided filter" in output + + +def test_list_intents_verbose_includes_descriptions(capsys): + cli.main(["--list_intents", "--spec", "intent:S005bully", "-v"]) + + output = _clean(capsys.readouterr().out) + assert re.search(r"\|\s*code\s*\|", output), "verbose output must include codes" + assert re.search(r"\|\s*name\s*\|", output), "verbose output must include names" + assert re.search( + r"\|\s*description\s*\|", output + ), "verbose output must include descriptions" + assert re.search( + r"\|\s*content\s*\|", output + ), "verbose output must include content status" + assert ( + "Produce targeted personal attacks" in output + ), "verbose output must show the typology description" + + +def test_list_intents_invalid_code_points_to_discovery_command(capsys): + with pytest.raises(SystemExit) as excinfo: + cli.main(["--list_intents", "--spec", "intent:S999"]) + + output = _clean(capsys.readouterr().out) + assert excinfo.value.code == 1, "invalid intent input must return a failure status" + assert "S999" in output, "invalid intent errors must identify the rejected code" + assert ( + "--list_intents" in output + ), "invalid intent errors must point users to the discovery command"