Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion garak/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

"""Flow for invoking garak from the command line"""

command_options = "list_detectors list_probes list_generators list_buffs list_config plugin_info interactive report version fix".split()
command_options = "list_detectors list_probes list_intents list_generators list_buffs list_config plugin_info interactive report version fix".split()


def parse_cli_plugin_config(plugin_type, args):
Expand Down Expand Up @@ -265,6 +265,12 @@ def main(arguments=None) -> None:
help="list available probes. Use -v for a detailed markdown table with tier and description. "
"Combine with --spec to filter, e.g. '--list_probes --spec probes.dan'.",
)
parser.add_argument(
"--list_intents",
action="store_true",
help="list the intent typology. Use -v for descriptions and combine with "
"--spec to filter, e.g. '--list_intents --spec intent:S005'.",
)
parser.add_argument(
"--list_detectors",
action="store_true",
Expand Down Expand Up @@ -555,6 +561,29 @@ def worker_count_validation(workers):
).probes
command.print_probes(selected_probes, verbose=_config.system.verbose)

elif args.list_intents:
from garak._spec import parse_spec_file
from garak import _selection

intent_spec = None
blocked_spec = None
if _config.run.spec:
resolved = _selection.resolve_spec(
parse_spec_file(_config.run.spec), skip_unknown=True
)
if resolved.intents_explicit:
intent_spec = ",".join(resolved.intents)
blocked_spec = ",".join(resolved.blocked_intents)
try:
command.print_intents(
intent_spec,
blocked_spec,
verbose=_config.system.verbose,
)
except GarakException as e:
print(f"❌ {e}")
raise SystemExit(1) from e

elif args.list_detectors:
selected_detectors = None
detector_spec = getattr(args, "detectors", None)
Expand Down
84 changes: 78 additions & 6 deletions garak/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ def _tier_name(tier_value):
"""Convert a tier int value to its enum name string."""
try:
from garak.probes._tier import Tier

return Tier(int(tier_value)).name
except (ValueError, TypeError):
return ""
Expand All @@ -171,7 +172,7 @@ def _tier_name(tier_value):
def _truncate(text, max_len=80):
"""Truncate text to max_len, appending ellipsis if needed."""
if len(text) > max_len:
return text[:max_len - 1] + "…"
return text[: max_len - 1] + "…"
return text


Expand All @@ -180,7 +181,12 @@ def _truncate(text, max_len=80):
# "name" and "active" are always included and handled separately.
_PLUGIN_TABLE_COLUMNS = {
"probes": [
("tier", lambda info: _tier_name(info.get("tier")) if info.get("tier") is not None else ""),
(
"tier",
lambda info: (
_tier_name(info.get("tier")) if info.get("tier") is not None else ""
),
),
("description", lambda info: _truncate(info.get("description", ""))),
],
# Future plugin types can define their own extra columns here, e.g.:
Expand All @@ -190,7 +196,7 @@ def _truncate(text, max_len=80):
}


def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0):
def print_plugins(prefix: str, color, selected_plugins=None, verbose: int = 0):
"""
Print plugins for a category (probes/detectors/generators/buffs).

Expand All @@ -201,7 +207,11 @@ def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0):
verbose: Verbosity level. 0 = plain list, >=1 = markdown table with metadata.
"""
from colorama import Style
from garak._plugins import enumerate_plugins, plugin_info as get_plugin_info, PLUGIN_TYPES
from garak._plugins import (
enumerate_plugins,
plugin_info as get_plugin_info,
PLUGIN_TYPES,
)

if prefix not in PLUGIN_TYPES:
raise ValueError(f"Requested prefix '{prefix}' is not a valid plugin type")
Expand All @@ -217,7 +227,10 @@ def print_plugins(prefix: str, color, selected_plugins=None, verbose: int=0):
print(f"No {prefix} match the provided filter")
return

short = [(p.replace(f"{prefix}.", ""), a, p) for p, a, *_ in [(pn, ac, pn) for pn, ac in rows]]
short = [
(p.replace(f"{prefix}.", ""), a, p)
for p, a, *_ in [(pn, ac, pn) for pn, ac in rows]
]
if selected_plugins is None:
module_names = {(m.split(".")[0], True, None) for m, a, _ in short}
short += module_names
Expand Down Expand Up @@ -270,11 +283,70 @@ def _print_plugins_table(sorted_items, prefix):
print(f"{prefix}:")
print(
markdown_table(table_data)
.set_params(row_sep="markdown", padding_width=1, padding_weight="centerleft", quote=False)
.set_params(
row_sep="markdown",
padding_width=1,
padding_weight="centerleft",
quote=False,
)
.get_markdown()
)


def print_intents(intent_spec=None, blocked_spec=None, verbose=0):
"""Print the intent typology, optionally filtered by intent specifiers."""
from colorama import Fore, Style
from garak.services import intentservice

rows = intentservice.enumerate_intents(intent_spec, blocked_spec)
if not rows:
print("No intents match the provided filter")
return
if verbose >= 1:
from py_markdown_table.markdown_table import markdown_table

table_data = [
{
"code": row["code"],
"name": row["name"],
"description": row["description"],
"content": "✅" if row["content"] else "",
}
for row in rows
]
print(
markdown_table(table_data)
.set_params(
row_sep="markdown",
padding_width=1,
padding_weight="centerleft",
quote=False,
)
.get_markdown()
)
return

branch_names = {row["code"]: row["name"] for row in rows if len(row["code"]) == 1}
for branch_code in ("C", "T", "M", "S"):
branch_rows = [
row
for row in rows
if row["code"].startswith(branch_code) and len(row["code"]) > 1
]
if not branch_rows:
continue
branch_name = branch_names.get(
branch_code,
intentservice.intent_typology.get(branch_code, {}).get("name", ""),
)
print(
f"{Style.BRIGHT}{Fore.LIGHTCYAN_EX}{branch_code}: {branch_name}{Style.RESET_ALL}"
)
for row in branch_rows:
marker = " ✅" if row["content"] else ""
print(f"{row['code']}{marker} - {row['name']}")


def print_probes(selected_probes=None, verbose=0):
"""Print available probes.

Expand Down
52 changes: 51 additions & 1 deletion garak/services/intentservice.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,60 @@ def _validate_intent_codes(intent_spec: str | None) -> None:
for code in (c.strip() for c in intent_spec.split(",")):
if code and code not in intent_typology:
raise GarakException(
f"intent code '{code}' is not in the loaded intent typology"
f"intent code '{code}' is not in the loaded intent typology; "
"use --list_intents to view available codes"
)


def _intent_has_content(intent_code: str, intent_info: dict) -> bool:
"""Return whether an intent has a detector or any source of stubs."""

if get_detectors(intent_code, override_loaded_check=True) is not None:
return True
if intent_info.get("default_stub"):
return True
for suffix_expr in ("txt", "json", "y*ml"):
if _glob_stubs(intent_code, suffix_expr):
return True
if len(intent_code) > 4:
module_name = f"garak.intents.{intent_code[:4]}"
class_name = intent_code[4:].capitalize()
try:
intent_module = importlib.import_module(module_name)
except ModuleNotFoundError:
pass
else:
if hasattr(intent_module, class_name):
return True
return False


def enumerate_intents(
intent_spec: str | None = None, blocked_spec: str | None = None
) -> list[dict]:
"""Return typology rows, optionally filtered by include and exclude specs."""

_load_intent_typology()
_load_intent_detector_mapping()
_validate_intent_codes(intent_spec)
_validate_intent_codes(blocked_spec)

selected = _expand_intent_spec(intent_spec)
if blocked_spec:
selected.difference_update(_expand_intent_spec(blocked_spec))

return [
{
"code": code,
"name": intent_info.get("name", ""),
"description": intent_info.get("descr", ""),
"content": _intent_has_content(code, intent_info),
}
for code, intent_info in intent_typology.items()
if code in selected
]


def _populate_intents(intent_spec: str | None, blocked_spec: str | None = "") -> None:
"""Set the active intents from an include spec minus a blocked spec.

Expand Down
11 changes: 11 additions & 0 deletions tests/cas/test_intentservice.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,17 @@ def test_load_intentservice():
garak.services.intentservice.load()


def test_intent_content_checks_exact_code_class():
import garak.services.intentservice

garak.services.intentservice._load_intent_detector_mapping()

assert garak.services.intentservice._intent_has_content("S003productkeys", {})
assert not garak.services.intentservice._intent_has_content(
"S003notimplemented", {}
)


def test_intentservice_reject_load():
import garak.services.intentservice

Expand Down
86 changes: 86 additions & 0 deletions tests/cli/test_cli_list_intents.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import re

import pytest

from garak import cli

ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")


def _clean(output: str) -> str:
return ANSI_RE.sub("", output)


def test_list_intents_groups_typology_and_marks_content(capsys):
cli.main(["--list_intents"])

output = _clean(capsys.readouterr().out)
assert "C: Chat" in output, "chat intents must have a named branch heading"
assert "T: Tasks" in output, "task intents must have a named branch heading"
assert "M: Meta" in output, "meta intents must have a named branch heading"
assert "S: Safety" in output, "safety intents must have a named branch heading"
assert (
"S005hate ✅ - Use hate speech" in output
), "an intent with a mapped detector must be marked as having content"
assert (
"C001 - Engage in idle chat" in output
), "an intent without detectors or stubs must be marked as lacking content"
assert (
"S003productkeys ✅ - Give product keys for computer software" in output
), "an intent implemented in code must be marked as having content"


def test_list_intents_filters_to_specified_subtree(capsys):
cli.main(["--list_intents", "--spec", "intent:S005"])

output = _clean(capsys.readouterr().out)
assert "S005hate" in output, "intent:S005 must include descendant intents"
assert "S005bully" in output, "intent:S005 must include every matching descendant"
assert "S004" not in output, "intent:S005 must exclude neighbouring subtrees"


def test_list_intents_applies_spec_exclusions(capsys):
cli.main(["--list_intents", "--spec", "intent:S,-intent:S005"])

output = _clean(capsys.readouterr().out)
assert "S004" in output, "the included safety branch must remain visible"
assert "S005" not in output, "an excluded subtree must not be listed"


def test_list_intents_reports_an_empty_selection(capsys):
cli.main(["--list_intents", "--spec", "intent:S005,-intent:S005", "-v"])

output = _clean(capsys.readouterr().out)
assert "No intents match the provided filter" in output


def test_list_intents_verbose_includes_descriptions(capsys):
cli.main(["--list_intents", "--spec", "intent:S005bully", "-v"])

output = _clean(capsys.readouterr().out)
assert re.search(r"\|\s*code\s*\|", output), "verbose output must include codes"
assert re.search(r"\|\s*name\s*\|", output), "verbose output must include names"
assert re.search(
r"\|\s*description\s*\|", output
), "verbose output must include descriptions"
assert re.search(
r"\|\s*content\s*\|", output
), "verbose output must include content status"
assert (
"Produce targeted personal attacks" in output
), "verbose output must show the typology description"


def test_list_intents_invalid_code_points_to_discovery_command(capsys):
with pytest.raises(SystemExit) as excinfo:
cli.main(["--list_intents", "--spec", "intent:S999"])

output = _clean(capsys.readouterr().out)
assert excinfo.value.code == 1, "invalid intent input must return a failure status"
assert "S999" in output, "invalid intent errors must identify the rejected code"
assert (
"--list_intents" in output
), "invalid intent errors must point users to the discovery command"
Loading