diff --git a/garak/cli.py b/garak/cli.py index 60709397e..ba1bcd3b4 100644 --- a/garak/cli.py +++ b/garak/cli.py @@ -740,10 +740,18 @@ def _check_selection(rejected, namespace, inactive=()): evaluator = garak.evaluators.ThresholdEvaluator(_config.run.eval_threshold) from garak import _plugins + from garak.exception import GarakException, GeneratorError - generator = _plugins.load_plugin( - f"generators.{_config.plugins.target_type}", config_root=_config - ) + try: + generator = _plugins.load_plugin( + f"generators.{_config.plugins.target_type}", config_root=_config + ) + except GarakException as e: + if isinstance(e, GeneratorError): + raise + raise GeneratorError(str(e)) from e + except Exception as e: + raise GeneratorError(str(e)) from e if ( not _cli_config_supplied @@ -786,8 +794,17 @@ def _check_selection(rejected, namespace, inactive=()): logging.exception(e) logging.info(msg) print(msg) + from garak.exit_codes import ExitCode + + sys.exit(ExitCode.INTERRUPTED) except (ValueError, GarakException) as e: logging.exception(e) print(e) - - _config.set_http_lib_agents(prior_user_agents) + from garak.exit_codes import ExitCode + + exit_code = ExitCode.UNSPECIFIED + if isinstance(e, GarakException) and e.exit_code is not None: + exit_code = e.exit_code + sys.exit(exit_code) + finally: + _config.set_http_lib_agents(prior_user_agents) diff --git a/garak/command.py b/garak/command.py index f61c2fe00..17aacd843 100644 --- a/garak/command.py +++ b/garak/command.py @@ -153,6 +153,9 @@ def end_run(): logging.exception(e) logging.info(msg) print(msg) + from garak.exception import ReportingError + + raise ReportingError(msg) from e msg = f"garak run complete in {timetaken:.2f}s" print(f"✔️ {msg}") diff --git a/garak/exception.py b/garak/exception.py index d333d5793..806f0b177 100644 --- a/garak/exception.py +++ b/garak/exception.py @@ -1,10 +1,14 @@ # SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from garak.exit_codes import ExitCode + class GarakException(Exception): """Base class for all garak exceptions""" + exit_code: int | None = None + class APIKeyMissingError(GarakException): """Exception to be raised if a required API key is not found""" @@ -40,3 +44,35 @@ class PayloadFailure(GarakException): class ReportIncompatibleError(GarakException): """Report references plugins unknown to the current garak install; the report is not compatible with this version""" + + +class ProbeError(GarakException): + exit_code = ExitCode.PROBE + + +class GeneratorError(GarakException): + exit_code = ExitCode.GENERATOR + + +class DetectorError(GarakException): + exit_code = ExitCode.DETECTOR + + +class BuffError(GarakException): + exit_code = ExitCode.BUFF + + +class EvaluatorError(GarakException): + exit_code = ExitCode.EVALUATOR + + +class HarnessError(GarakException): + exit_code = ExitCode.HARNESS + + +class LangProviderError(GarakException): + exit_code = ExitCode.LANGPROVIDER + + +class ReportingError(GarakException): + exit_code = ExitCode.REPORTING diff --git a/garak/exit_codes.py b/garak/exit_codes.py new file mode 100644 index 000000000..72576e723 --- /dev/null +++ b/garak/exit_codes.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Structured exit codes for garak CLI runs. + +Positive codes are used because many shells truncate negative exit statuses. +See https://github.com/NVIDIA/garak/issues/1221 +""" + +from enum import IntEnum + + +class ExitCode(IntEnum): + OK = 0 + INTERRUPTED = 1 + PROBE = 2 + GENERATOR = 3 + DETECTOR = 4 + BUFF = 5 + EVALUATOR = 6 + HARNESS = 7 + LANGPROVIDER = 8 + REPORTING = 9 + OUT_OF_RESOURCES = 10 + UNSPECIFIED = 127 diff --git a/garak/harnesses/base.py b/garak/harnesses/base.py index df6b2d91f..7e7a3f015 100644 --- a/garak/harnesses/base.py +++ b/garak/harnesses/base.py @@ -23,13 +23,12 @@ from garak.configurable import Configurable import garak.attempt import garak.probes.base +from garak.exception import GarakException def _initialize_runtime_services(): """Initialize and validate runtime services required for a successful test""" - from garak.exception import GarakException - # TODO: this block may be gated in the future to ensure it is only run once. At this time # only one harness will execute per run so the output here is reasonable. service_names = ["langservice", "intentservice"] @@ -45,6 +44,10 @@ def _initialize_runtime_services(): service.load() except GarakException as e: logging.critical("❌ %s setup failed!" % service_name, exc_info=e) + from garak.exception import LangProviderError + + if service_name == "langservice" and not isinstance(e, LangProviderError): + raise LangProviderError(str(e)) from e raise e @@ -124,7 +127,9 @@ def _load_buffs(self, buff_names: List) -> None: if err_msg is not None: print(err_msg) logging.warning(err_msg) - continue + from garak.exception import BuffError + + raise BuffError(err_msg) from None def _start_run_hook(self): self._http_lib_user_agents = _config.get_http_lib_agents() @@ -152,9 +157,20 @@ def _run_detector(self, probe_result_attempts, detector_instance) -> None: for attempt in attempt_iterator: if detector_instance.skip: continue - attempt.detector_results[detector_probe_name] = list( - detector_instance.detect(attempt) - ) + try: + attempt.detector_results[detector_probe_name] = list( + detector_instance.detect(attempt) + ) + except GarakException as e: + from garak.exception import DetectorError + + if isinstance(e, DetectorError): + raise + raise DetectorError(str(e)) from e + except Exception as e: + from garak.exception import DetectorError + + raise DetectorError(str(e)) from e def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None: """Core harness method @@ -210,7 +226,18 @@ def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None: ) continue - attempt_results = probe.probe(model) + try: + attempt_results = probe.probe(model) + except GarakException as e: + from garak.exception import ProbeError + + if isinstance(e, ProbeError): + raise + raise ProbeError(str(e)) from e + except Exception as e: + from garak.exception import ProbeError + + raise ProbeError(str(e)) from e assert isinstance( attempt_results, (list, types.GeneratorType) ), "probing should always return an ordered iterable" @@ -283,7 +310,18 @@ def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None: if len(attempt_results) == 0: logging.warning("zero attempt results: probe %s" % probe.probename) - evaluator.evaluate(attempt_results) + try: + evaluator.evaluate(attempt_results) + except GarakException as e: + from garak.exception import EvaluatorError + + if isinstance(e, EvaluatorError): + raise + raise EvaluatorError(str(e)) from e + except Exception as e: + from garak.exception import EvaluatorError + + raise EvaluatorError(str(e)) from e self._end_run_hook() diff --git a/garak/services/langservice.py b/garak/services/langservice.py index beba94e1e..47c3fc650 100644 --- a/garak/services/langservice.py +++ b/garak/services/langservice.py @@ -8,7 +8,7 @@ from typing import List from garak import _config, _plugins -from garak.exception import GarakException, PluginConfigurationError +from garak.exception import GarakException, LangProviderError, PluginConfigurationError from garak.langproviders.base import LangProvider from garak.langproviders.local import Passthru @@ -99,7 +99,7 @@ def load(): msg = f"The language provision configuration provided is missing language: {target_lang},{source_lang}. Configuration must specify language providers for each required direction." logging.error(msg) - raise GarakException(msg) + raise LangProviderError(msg) def get_langprovider(source: str, *, reverse: bool = False): diff --git a/tests/test_exit_codes.py b/tests/test_exit_codes.py new file mode 100644 index 000000000..fba229fbc --- /dev/null +++ b/tests/test_exit_codes.py @@ -0,0 +1,22 @@ +from garak.exception import ( + DetectorError, + EvaluatorError, + GeneratorError, + ProbeError, + ReportingError, +) +from garak.exit_codes import ExitCode + + +def test_exit_code_values_are_positive(): + assert ExitCode.OK == 0 + assert ExitCode.INTERRUPTED == 1 + assert ExitCode.UNSPECIFIED == 127 + + +def test_component_errors_carry_exit_codes(): + assert ProbeError("probe failed").exit_code == ExitCode.PROBE + assert GeneratorError("generator failed").exit_code == ExitCode.GENERATOR + assert DetectorError("detector failed").exit_code == ExitCode.DETECTOR + assert EvaluatorError("evaluator failed").exit_code == ExitCode.EVALUATOR + assert ReportingError("report failed").exit_code == ExitCode.REPORTING