feat(cli): add structured exit codes for improper termination

garak/cli.py

@@ -740,10 +740,18 @@ def _check_selection(rejected, namespace, inactive=()):
             evaluator = garak.evaluators.ThresholdEvaluator(_config.run.eval_threshold)
 
             from garak import _plugins
+            from garak.exception import GarakException, GeneratorError
 
-            generator = _plugins.load_plugin(
-                f"generators.{_config.plugins.target_type}", config_root=_config
-            )
+            try:
+                generator = _plugins.load_plugin(
+                    f"generators.{_config.plugins.target_type}", config_root=_config
+                )
+            except GarakException as e:
+                if isinstance(e, GeneratorError):
+                    raise
+                raise GeneratorError(str(e)) from e
+            except Exception as e:
+                raise GeneratorError(str(e)) from e
 
             if (
                 not _cli_config_supplied
@@ -786,8 +794,17 @@ def _check_selection(rejected, namespace, inactive=()):
         logging.exception(e)
         logging.info(msg)
         print(msg)
+        from garak.exit_codes import ExitCode
+
+        sys.exit(ExitCode.INTERRUPTED)
     except (ValueError, GarakException) as e:
         logging.exception(e)
         print(e)
-
-    _config.set_http_lib_agents(prior_user_agents)
+        from garak.exit_codes import ExitCode
+
+        exit_code = ExitCode.UNSPECIFIED
+        if isinstance(e, GarakException) and e.exit_code is not None:
+            exit_code = e.exit_code
+        sys.exit(exit_code)
+    finally:
+        _config.set_http_lib_agents(prior_user_agents)

garak/command.py

@@ -153,6 +153,9 @@ def end_run():
         logging.exception(e)
         logging.info(msg)
         print(msg)
+        from garak.exception import ReportingError
+
+        raise ReportingError(msg) from e
 
     msg = f"garak run complete in {timetaken:.2f}s"
     print(f"✔️  {msg}")

garak/exception.py

@@ -1,10 +1,14 @@
 # SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+from garak.exit_codes import ExitCode
+
 
 class GarakException(Exception):
     """Base class for all  garak exceptions"""
 
+    exit_code: int | None = None
+
 
 class APIKeyMissingError(GarakException):
     """Exception to be raised if a required API key is not found"""
@@ -40,3 +44,35 @@ class PayloadFailure(GarakException):
 
 class ReportIncompatibleError(GarakException):
     """Report references plugins unknown to the current garak install; the report is not compatible with this version"""
+
+
+class ProbeError(GarakException):
+    exit_code = ExitCode.PROBE
+
+
+class GeneratorError(GarakException):
+    exit_code = ExitCode.GENERATOR
+
+
+class DetectorError(GarakException):
+    exit_code = ExitCode.DETECTOR
+
+
+class BuffError(GarakException):
+    exit_code = ExitCode.BUFF
+
+
+class EvaluatorError(GarakException):
+    exit_code = ExitCode.EVALUATOR
+
+
+class HarnessError(GarakException):
+    exit_code = ExitCode.HARNESS
+
+
+class LangProviderError(GarakException):
+    exit_code = ExitCode.LANGPROVIDER
+
+
+class ReportingError(GarakException):
+    exit_code = ExitCode.REPORTING

garak/exit_codes.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Structured exit codes for garak CLI runs.
+
+Positive codes are used because many shells truncate negative exit statuses.
+See https://github.com/NVIDIA/garak/issues/1221
+"""
+
+from enum import IntEnum
+
+
+class ExitCode(IntEnum):
+    OK = 0
+    INTERRUPTED = 1
+    PROBE = 2
+    GENERATOR = 3
+    DETECTOR = 4
+    BUFF = 5
+    EVALUATOR = 6
+    HARNESS = 7
+    LANGPROVIDER = 8
+    REPORTING = 9
+    OUT_OF_RESOURCES = 10
+    UNSPECIFIED = 127

garak/harnesses/base.py

@@ -23,13 +23,12 @@
 from garak.configurable import Configurable
 import garak.attempt
 import garak.probes.base
+from garak.exception import GarakException
 
 
 def _initialize_runtime_services():
     """Initialize and validate runtime services required for a successful test"""
 
-    from garak.exception import GarakException
-
     # TODO: this block may be gated in the future to ensure it is only run once. At this time
     # only one harness will execute per run so the output here is reasonable.
     service_names = ["langservice", "intentservice"]
@@ -45,6 +44,10 @@ def _initialize_runtime_services():
                 service.load()
         except GarakException as e:
             logging.critical("❌ %s setup failed!" % service_name, exc_info=e)
+            from garak.exception import LangProviderError
+
+            if service_name == "langservice" and not isinstance(e, LangProviderError):
+                raise LangProviderError(str(e)) from e
             raise e
 
 
@@ -124,7 +127,9 @@ def _load_buffs(self, buff_names: List) -> None:
                 if err_msg is not None:
                     print(err_msg)
                     logging.warning(err_msg)
-                    continue
+                    from garak.exception import BuffError
+
+                    raise BuffError(err_msg) from None
 
     def _start_run_hook(self):
         self._http_lib_user_agents = _config.get_http_lib_agents()
@@ -152,9 +157,20 @@ def _run_detector(self, probe_result_attempts, detector_instance) -> None:
         for attempt in attempt_iterator:
             if detector_instance.skip:
                 continue
-            attempt.detector_results[detector_probe_name] = list(
-                detector_instance.detect(attempt)
-            )
+            try:
+                attempt.detector_results[detector_probe_name] = list(
+                    detector_instance.detect(attempt)
+                )
+            except GarakException as e:
+                from garak.exception import DetectorError
+
+                if isinstance(e, DetectorError):
+                    raise
+                raise DetectorError(str(e)) from e
+            except Exception as e:
+                from garak.exception import DetectorError
+
+                raise DetectorError(str(e)) from e
 
     def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None:
         """Core harness method
@@ -210,7 +226,18 @@ def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None:
                 )
                 continue
 
-            attempt_results = probe.probe(model)
+            try:
+                attempt_results = probe.probe(model)
+            except GarakException as e:
+                from garak.exception import ProbeError
+
+                if isinstance(e, ProbeError):
+                    raise
+                raise ProbeError(str(e)) from e
+            except Exception as e:
+                from garak.exception import ProbeError
+
+                raise ProbeError(str(e)) from e
             assert isinstance(
                 attempt_results, (list, types.GeneratorType)
             ), "probing should always return an ordered iterable"
@@ -283,7 +310,18 @@ def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None:
             if len(attempt_results) == 0:
                 logging.warning("zero attempt results: probe %s" % probe.probename)
 
-            evaluator.evaluate(attempt_results)
+            try:
+                evaluator.evaluate(attempt_results)
+            except GarakException as e:
+                from garak.exception import EvaluatorError
+
+                if isinstance(e, EvaluatorError):
+                    raise
+                raise EvaluatorError(str(e)) from e
+            except Exception as e:
+                from garak.exception import EvaluatorError
+
+                raise EvaluatorError(str(e)) from e
 
         self._end_run_hook()
 

garak/services/langservice.py

@@ -8,7 +8,7 @@
 from typing import List
 
 from garak import _config, _plugins
-from garak.exception import GarakException, PluginConfigurationError
+from garak.exception import GarakException, LangProviderError, PluginConfigurationError
 from garak.langproviders.base import LangProvider
 from garak.langproviders.local import Passthru
 
@@ -99,7 +99,7 @@ def load():
 
     msg = f"The language provision configuration provided is missing language: {target_lang},{source_lang}. Configuration must specify language providers for each required direction."
     logging.error(msg)
-    raise GarakException(msg)
+    raise LangProviderError(msg)
 
 
 def get_langprovider(source: str, *, reverse: bool = False):

tests/test_exit_codes.py

@@ -0,0 +1,22 @@
+from garak.exception import (
+    DetectorError,
+    EvaluatorError,
+    GeneratorError,
+    ProbeError,
+    ReportingError,
+)
+from garak.exit_codes import ExitCode
+
+
+def test_exit_code_values_are_positive():
+    assert ExitCode.OK == 0
+    assert ExitCode.INTERRUPTED == 1
+    assert ExitCode.UNSPECIFIED == 127
+
+
+def test_component_errors_carry_exit_codes():
+    assert ProbeError("probe failed").exit_code == ExitCode.PROBE
+    assert GeneratorError("generator failed").exit_code == ExitCode.GENERATOR
+    assert DetectorError("detector failed").exit_code == ExitCode.DETECTOR
+    assert EvaluatorError("evaluator failed").exit_code == ExitCode.EVALUATOR
+    assert ReportingError("report failed").exit_code == ExitCode.REPORTING