Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
957f3c2
test: add behavioral equivalence harness for LLMGenerationActions
Pouyanpi Jun 25, 2026
4456dd0
refactor(generation): extract shared general-response helper
Pouyanpi Jun 25, 2026
0bb8616
refactor(generation): split intent detection from general bot turn
Pouyanpi Jun 25, 2026
cde60cc
refactor(generation): typed seam for the single-call cache
Pouyanpi Jun 25, 2026
0ab53d2
refactor(generation): encapsulate the streaming handoff
Pouyanpi Jun 25, 2026
214377f
refactor(generation): remove dead branches in intent/next-step genera…
Pouyanpi Jun 25, 2026
b27e44d
fix(generation): normalize multimodal input in single-call generation
Pouyanpi Jun 25, 2026
6986dea
fix(generation): evict streaming handoff handlers when consumed
Pouyanpi Jun 25, 2026
c3eb6a1
fix(generation): converge single-call general turn with the multi-cal…
Pouyanpi Jun 25, 2026
550ea8c
refactor(generation): extract bot-turn output-event packaging
Pouyanpi Jun 25, 2026
5ae9afa
refactor(generation): dedupe SpecOp-to-dict conversion
Pouyanpi Jun 25, 2026
60829dd
refactor(generation): extract single-call cache read in generate_bot_…
Pouyanpi Jun 25, 2026
434f53c
refactor(generation): extract few-shot example builder
Pouyanpi Jun 25, 2026
5f4c248
fix(generation): satisfy ty on llm_params in general-response call sites
Pouyanpi Jul 13, 2026
1d21763
fix(generation): correct single-call cache-miss handling and harden s…
Pouyanpi Jul 14, 2026
a73a242
fix(generation): release single-call streaming handoff on bypass paths
Pouyanpi Jul 14, 2026
bc798ca
test(generation): close audit-found oracle and contract coverage gaps
Pouyanpi Jul 14, 2026
2bc0ffc
test(generation): cover cache type-mismatch and passthrough+dialog br…
Pouyanpi Jul 14, 2026
28f0440
test(recorded): add dialog bot-message generation cassette
Pouyanpi Jul 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,026 changes: 605 additions & 421 deletions nemoguardrails/actions/llm/generation.py

Large diffs are not rendered by default.

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions tests/recorded/rails/public_api/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
NIM_BASELINE_CONFIG = RailsConfigSource.from_path(CONFIGS_DIR, "nim_baseline")
NEMOGUARDS_FULL_CONFIG = RailsConfigSource.from_path(CONFIGS_DIR, "nemoguards_full")
DIALOG_CONFIG = RailsConfigSource.from_path(CONFIGS_DIR, "dialog")
DIALOG_GENERATION_CONFIG = RailsConfigSource.from_path(CONFIGS_DIR, "dialog_generation")
SINGLE_CALL_CONFIG = RailsConfigSource.from_path(CONFIGS_DIR, "single_call")
TASK_MODELS_CONFIG = RailsConfigSource.from_path(CONFIGS_DIR, "task_models")
STREAMING_OUTPUT_RAILS_CONFIG = RailsConfigSource.from_path(CONFIGS_DIR, "streaming_output_rails")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from tests.recorded.rails.public_api.simple_embedding_provider import SimpleEmbeddingSearchProvider


def init(app):
app.register_embedding_search_provider("simple", SimpleEmbeddingSearchProvider)
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
models:
- type: main
engine: openai
model: gpt-5.4-nano
parameters:
max_retries: 0
core:
embedding_search_provider:
name: simple
knowledge_base:
embedding_search_provider:
name: simple
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Product overview

The product is a guardrails toolkit for large language models. It lets
developers add programmable rails that check user input, steer the dialog, and
validate bot output before it reaches the user.
21 changes: 21 additions & 0 deletions tests/recorded/rails/public_api/configs/dialog_generation/rails.co
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
define user express greeting
"hello"
"hi"

define user ask about the product
"tell me about your product"
"what does the product do"

define bot express greeting
"Hey there!"

define bot offer help
"I'm here to help with anything you need."

define flow greeting
user express greeting
bot express greeting

define flow product
user ask about the product
bot describe the product
37 changes: 36 additions & 1 deletion tests/recorded/rails/public_api/test_dialog.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from nemoguardrails.rails.llm.options import RailStatus
from tests.recorded.assertions import assert_generated_message, assert_rails_result
from tests.recorded.normalization import normalize_rails_result
from tests.recorded.rails.public_api.configs import DIALOG_CONFIG, SINGLE_CALL_CONFIG
from tests.recorded.rails.public_api.configs import DIALOG_CONFIG, DIALOG_GENERATION_CONFIG, SINGLE_CALL_CONFIG
from tests.recorded.rails_config import load_config
from tests.recorded.snapshots import snapshot

Expand All @@ -43,6 +43,41 @@ async def test_dialog_generate_async_public_contract(openai_api_key):
)


@pytest.mark.vcr
async def test_dialog_generation_bot_message_public_contract(openai_api_key):
"""Bot-message GENERATION prompt carries few-shot examples + retrieved kb chunks.

"tell me about your product" maps (via a flow) to the bot intent
"describe the product", which has no predefined message, so
``generate_bot_message`` calls the LLM with the bot-message index examples and
the retrieved kb chunks. VCR matches on ``recorded_body``, so this cassette
pins all three dialog prompts (intent, next-step, bot-message) by their exact
request bodies. It was verified to replay byte-identically against both the
pre-refactor ``generation.py`` on ``develop`` and the refactored branch, so
the prompts are unchanged pre/post -- a prompt-fidelity equivalence check the
prompt-independent ``FakeLLMModel`` oracle cannot make.
"""
rails = LLMRails(load_config(DIALOG_GENERATION_CONFIG), verbose=False)

result = await rails.generate_async(messages=[{"role": "user", "content": "tell me about your product"}])

assert_generated_message(result)
assert result == snapshot(
{
"role": "assistant",
"content": """\
Sure! If you mean the **guardrails toolkit for large language models** you mentioned in the context, here’s what it is and what it helps you do:
- **Programmable “rails” for LLMs:** Lets developers define rules and logic that run during the conversation.
- **Check user input:** You can validate or filter what users send (e.g., detect disallowed content, enforce formatting, constrain requests).
- **Steer the dialog:** You can route or modify the assistant’s behavior based on the input or conversation state (e.g., ask clarifying questions, refuse safely, switch modes).
- **Validate bot output:** Before the response is shown to the user, rails can verify it meets requirements (e.g., must include/avoid certain content, follow a policy, adhere to a schema).
- **Reduce unsafe or off-policy responses:** The overall goal is to make LLM behavior more controlled, consistent, and safer.
If you tell me what kind of use case you’re building (customer support, tutoring, legal/health Q&A, internal tools, etc.), I can suggest what kinds of rails you’d typically implement.\
""",
}
)


@pytest.mark.vcr
async def test_single_call_generate_async_public_contract(openai_api_key):
rails = LLMRails(load_config(SINGLE_CALL_CONFIG), verbose=False)
Expand Down
Loading