From 26e2870bfbae67095d8af2d89a172f4790c8ba42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B5=E8=BD=A6=E5=8F=B8=E6=9C=BA=E5=B0=8F=E6=9D=8E?= <39351936+motodriver@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:04:41 +0800 Subject: [PATCH] docs: update deprecated CLI flag references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: TRAE Signed-off-by: 电车司机小李 <39351936+motodriver@users.noreply.github.com> --- FAQ.md | 4 ++-- README.md | 8 +++---- docs/source/generators/llm.rst | 2 +- docs/source/generators/rest.rst | 2 +- docs/source/generators/websocket.rst | 19 ++++++++--------- docs/source/translation.rst | 10 ++++----- tests/test_docs.py | 32 ++++++++++++++++++++++++++++ 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/FAQ.md b/FAQ.md index 80243f72c..254c7c61f 100644 --- a/FAQ.md +++ b/FAQ.md @@ -69,9 +69,9 @@ Some garak probes still check for traditional cybersecurity vulnerabilities with This sounds like hitting a resource limit - something external to garak, e.g. the kernel, has taken action. Does your process have access to the required system RAM and GPU memory -## How can I use garak to scan a NIM of an LLM? What should the "model_type" be? And how do we pass the NIM endpoint url to garak? +## How can I use garak to scan a NIM of an LLM? What should the target type be? And how do we pass the NIM endpoint URL to garak? -`model_type` should be "nim" for chat-type models (which is most of them - this selects the right class automatically. Then, set model_name to [organisation]/[model name] from [build.nvidia.com](https://build.nvidia.com) (the JSON example is authoritative). For example, `--model_type nim --model_name meta/llama-3.1-8b-instruct`. You will need to put the API key in the `NIM_API_KEY` environment variable, or in the config. +`target_type` should be "nim" for chat-type models (which is most of them - this selects the right class automatically). Then, set `target_name` to [organisation]/[model name] from [build.nvidia.com](https://build.nvidia.com) (the JSON example is authoritative). For example, `--target_type nim --target_name meta/llama-3.1-8b-instruct`. You will need to put the API key in the `NIM_API_KEY` environment variable, or in the config. ## If I have already scanned a model on HuggingFace, and I use the same model somewhere else, say in a container, is it necessary for me to scan the container with garak as well? diff --git a/README.md b/README.md index 1efac843f..dd3783f29 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ The general syntax is: To specify a generator, use the `--target_type` and, optionally, the `--target_name` options. Model type specifies a model family/interface; model name specifies the exact model to be used. The "Intro to generators" section below describes some of the generators supported. A straightforward generator family is Hugging Face models; to load one of these, set `--target_type` to `huggingface` and `--target_name` to the model's name on Hub (e.g. `"RWKV/rwkv-4-169m-pile"`). Some generators might need an API key to be set as an environment variable, and they'll let you know if they need that. -`garak` runs all the probes by default, but you can be specific about that too. `--probes promptinject` will use only the [PromptInject](https://github.com/agencyenterprise/promptinject) framework's methods, for example. You can also specify one specific plugin instead of a plugin family by adding the plugin name after a `.`; for example, `--probes lmrc.SlurUsage` will use an implementation of checking for models generating slurs based on the [Language Model Risk Cards](https://arxiv.org/abs/2303.18190) framework. +`garak` runs all the probes by default, but you can be specific about that too. `--spec probes.promptinject` will use only the [PromptInject](https://github.com/agencyenterprise/promptinject) framework's methods, for example. You can also specify one specific plugin instead of a plugin family by adding the plugin name after a `.`; for example, `--spec probes.lmrc.SlurUsage` will use an implementation of checking for models generating slurs based on the [Language Model Risk Cards](https://arxiv.org/abs/2303.18190) framework. For help and inspiration, find us on [Twitter](https://twitter.com/garak_llm) or [discord](https://discord.gg/uVch4puUCs)! @@ -106,13 +106,13 @@ Probe a commercial model for encoding-based prompt injection (OSX/\*nix) (replac ``` export OPENAI_API_KEY="sk-123XXXXXXXXXXXX" -python3 -m garak --target_type openai --target_name gpt-5-nano --probes encoding +python3 -m garak --target_type openai --target_name gpt-5-nano --spec probes.encoding ``` See if the Hugging Face version of GPT2 is vulnerable to DAN 11.0 ``` -python3 -m garak --target_type huggingface --target_name gpt2 --probes dan.Dan_11_0 +python3 -m garak --target_type huggingface --target_name gpt2 --spec probes.dan.Dan_11_0 ``` @@ -219,7 +219,7 @@ Example usage: ``` export BEDROCK_API_KEY="your-api-key" export BEDROCK_REGION="us-east-1" -garak --target_type bedrock --target_name claude-3-sonnet --probes dan +garak --target_type bedrock --target_name claude-3-sonnet --spec probes.dan ``` ### Test diff --git a/docs/source/generators/llm.rst b/docs/source/generators/llm.rst index 11172bfc8..efbbb439e 100644 --- a/docs/source/generators/llm.rst +++ b/docs/source/generators/llm.rst @@ -15,7 +15,7 @@ Then invoke garak with the ``llm`` model id: .. code-block:: bash - garak --model_type llm --model_name gpt-4o-mini + garak --target_type llm --target_name gpt-4o-mini .. automodule:: garak.generators.llm :members: diff --git a/docs/source/generators/rest.rst b/docs/source/generators/rest.rst index 24c5059e3..c6dcca49f 100644 --- a/docs/source/generators/rest.rst +++ b/docs/source/generators/rest.rst @@ -101,7 +101,7 @@ Set the passphrase securely via environment variable: .. code-block:: bash export MTLS_KEY_PASSPHRASE="your_key_passphrase" - garak --target_type rest -G mtls_config.json --probes dan + garak --target_type rest -G mtls_config.json --spec probes.dan If using a combined PEM file containing both certificate and key, omit ``client_key``: diff --git a/docs/source/generators/websocket.rst b/docs/source/generators/websocket.rst index e254054b8..b01a3c12b 100644 --- a/docs/source/generators/websocket.rst +++ b/docs/source/generators/websocket.rst @@ -8,7 +8,7 @@ real-time bidirectional communication, similar to modern chat applications. Uses the following options from ``_config.plugins.generators["websocket"]["WebSocketGenerator"]``: -* ``uri`` - the WebSocket URI (ws:// or wss://); can also be passed in --model_name +* ``uri`` - the WebSocket URI (ws:// or wss://); can also be passed with ``--target_name`` * ``name`` - a short name for this service; defaults to "WebSocket Generator" * ``auth_type`` - authentication method: "none", "basic", "bearer", or "custom" * ``username`` - username for basic authentication @@ -167,9 +167,9 @@ Usage Examples # Set password securely via environment variable export WEBSOCKET_API_KEY="your_secure_password" - garak --model_type websocket.WebSocketGenerator \ + garak --target_type websocket.WebSocketGenerator \ --generator_options '{"websocket": {"WebSocketGenerator": {"uri": "ws://localhost:3000", "auth_type": "basic", "username": "user"}}}' \ - --probes dan + --spec probes.dan **Configuration File:** @@ -177,17 +177,17 @@ Save configuration to ``websocket_config.json`` and use: .. code-block:: bash - garak --model_type websocket.WebSocketGenerator \ + garak --target_type websocket.WebSocketGenerator \ -G websocket_config.json \ - --probes encoding + --spec probes.encoding **Testing with Public Echo Server:** .. code-block:: bash - garak --model_type websocket.WebSocketGenerator \ + garak --target_type websocket.WebSocketGenerator \ --generator_options '{"websocket": {"WebSocketGenerator": {"uri": "wss://echo.websocket.org", "response_after_typing": false}}}' \ - --probes dan --generations 1 + --spec probes.dan --generations 1 SSH Tunnel Support ------------------ @@ -200,9 +200,9 @@ The generator works seamlessly with SSH tunnels for secure remote testing: ssh -L 3000:target-host:3000 jump-host -N -f # Test through tunnel - garak --model_type websocket.WebSocketGenerator \ + garak --target_type websocket.WebSocketGenerator \ --generator_options '{"websocket": {"WebSocketGenerator": {"uri": "ws://localhost:3000"}}}' \ - --probes malwaregen + --spec probes.malwaregen Typing Indicators ----------------- @@ -222,4 +222,3 @@ This enables proper testing of streaming/real-time LLM services. :undoc-members: :show-inheritance: - diff --git a/docs/source/translation.rst b/docs/source/translation.rst index 60634a27e..30fc8300e 100644 --- a/docs/source/translation.rst +++ b/docs/source/translation.rst @@ -130,7 +130,7 @@ You use the following yaml config. .. code-block:: bash export DEEPL_API_KEY=xxxx - python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --probes encoding --config + python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --spec probes.encoding --config Riva @@ -152,7 +152,7 @@ You use the following yaml config. .. code-block:: bash export RIVA_API_KEY=xxxx - python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --probes encoding --config + python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --spec probes.encoding --config Google Cloud Translation @@ -175,7 +175,7 @@ You use the following yaml config. .. code-block:: bash export GOOGLE_APPLICATION_CREDENTIALS= - python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --probes encoding --config + python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --spec probes.encoding --config Local @@ -196,7 +196,7 @@ You use the following yaml config. .. code-block:: bash - python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --probes encoding --config + python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --spec probes.encoding --config The default configuration will load `Helsinki-NLP MarianMT `_ models for local translation. @@ -218,4 +218,4 @@ contain ``m2m100`` to be loaded by garak. .. code-block:: bash - python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --probes encoding --config + python3 -m garak --target_type nim --target_name meta/llama-3.1-8b-instruct --spec probes.encoding --config diff --git a/tests/test_docs.py b/tests/test_docs.py index 17f9ee619..1564b4a63 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -7,6 +7,18 @@ TOP_PATHS = ["probes", "detectors", "harnesses", "generators", "evaluators", "buffs"] DOC_SOURCE = Path("docs/source") +USER_DOC_ROOT_FILES = (Path("README.md"), Path("FAQ.md")) +DEPRECATED_CLI_FLAGS = ( + "--model_type", + "--model_name", + "--probes", + "--probe_tags", + "--buffs", +) +DEPRECATED_CLI_FLAG_DOCS = { + DOC_SOURCE / "cliref.rst", + DOC_SOURCE / "configurable.rst", +} module_names = {} for top_path in TOP_PATHS: @@ -257,3 +269,23 @@ def test_doc_src_no_markdown(rst_file): assert ( canary_match is None ), f"Markdown-like content in rst: {canary_match.group().strip()} use ReStructured Text for garak docs - Markdown won't render" + + +def test_user_docs_avoid_deprecated_cli_flags(): + user_docs = (*USER_DOC_ROOT_FILES, *DOC_SOURCE.rglob("*.rst")) + stale_references = [] + + for doc_path in user_docs: + if doc_path in DEPRECATED_CLI_FLAG_DOCS: + continue + for line_number, line in enumerate( + doc_path.read_text(encoding="utf-8").splitlines(), start=1 + ): + for flag in DEPRECATED_CLI_FLAGS: + if flag in line: + stale_references.append(f"{doc_path}:{line_number}: {flag}") + + assert not stale_references, ( + "User documentation must use current CLI flags; stale references: " + + ", ".join(stale_references) + )