diff --git a/.claude/skills/update-docs/SKILL.md b/.claude/skills/update-docs/SKILL.md index 8a25caf175b..c9376ce4059 100644 --- a/.claude/skills/update-docs/SKILL.md +++ b/.claude/skills/update-docs/SKILL.md @@ -62,6 +62,7 @@ Filter to files that could affect documentation: - `src/pipecat/turns/**/*.py` (turn management) - `src/pipecat/observers/**/*.py` (observers) - `src/pipecat/pipeline/**/*.py` (pipeline core) +- `src/pipecat/flows/**/*.py` (Pipecat Flows) Ignore `__init__.py`, `__pycache__`, test files, and files that only contain type re-exports. @@ -138,12 +139,12 @@ For each doc page that needs updates, edit **only the sections that need changes ### Step 7: Update guides -Guides at `DOCS_PATH/pipecat/` reference specific class names, parameters, imports, and code patterns. After completing reference doc edits, check if any guides need updates too. +Guides at `DOCS_PATH/pipecat/` and `DOCS_PATH/pipecat-flows/` reference specific class names, parameters, imports, and code patterns. After completing reference doc edits, check if any guides need updates too. For each changed source file, collect the class names, renamed parameters, and changed imports from the diff. Search the guides directory: ```bash -grep -rl "ClassName\|old_param_name" DOCS_PATH/pipecat/ +grep -rl "ClassName\|old_param_name" DOCS_PATH/pipecat/ DOCS_PATH/pipecat-flows/ ``` For each guide that references changed code: @@ -159,6 +160,7 @@ Guide directories: - `pipecat/fundamentals/` — practical how-tos (metrics, recording, transcripts, etc.) - `pipecat/features/` — feature-specific guides (Gemini Live, OpenAI audio, WhatsApp, etc.) - `pipecat/telephony/` — telephony integration guides (Twilio, Plivo, Telnyx, etc.) +- `pipecat-flows/guides/` — Pipecat Flows guides (nodes-and-messages, functions, context-strategies, state-management, actions); check these when `src/pipecat/flows/**` changed ### Step 8: Identify doc gaps @@ -287,9 +289,11 @@ After all edits are complete, print a summary: ### Updated reference pages - `api-reference/server/services/stt/deepgram.mdx` — Updated Configuration (added `new_param`), InputParams (updated `language` default) - `api-reference/server/services/tts/elevenlabs.mdx` — Updated Event Handlers (added `on_connected`) +- `api-reference/pipecat-flows/flow-manager.mdx` — Updated FlowManager constructor (added `new_param`) ### Updated guides - `pipecat/learn/speech-to-text.mdx` — Updated code example (renamed `old_param` → `new_param`) +- `pipecat-flows/guides/state-management.mdx` — Updated FlowManager init example ### New service pages - `api-reference/server/services/tts/newprovider.mdx` — Created page, added to docs.json (Text-to-Speech), added to supported-services.mdx diff --git a/.claude/skills/update-docs/SOURCE_DOC_MAPPING.md b/.claude/skills/update-docs/SOURCE_DOC_MAPPING.md index bba2939d93c..ed22aec0fa8 100644 --- a/.claude/skills/update-docs/SOURCE_DOC_MAPPING.md +++ b/.claude/skills/update-docs/SOURCE_DOC_MAPPING.md @@ -21,6 +21,11 @@ These source paths don't follow the standard `services/{provider}/{type}.py` → | `pipeline/worker.py` | `api-reference/server/pipeline/pipeline-worker.mdx` | | `pipeline/runner.py` | `api-reference/server/utilities/runner/guide.mdx` | | `transports/base_transport.py` | `api-reference/server/services/transport/transport-params.mdx` | +| `flows/types.py` | `api-reference/pipecat-flows/types.mdx` | +| `flows/manager.py` | `api-reference/pipecat-flows/flow-manager.mdx` | +| `flows/actions.py` | `api-reference/pipecat-flows/flow-manager.mdx` and `api-reference/pipecat-flows/types.mdx` | +| `flows/adapters.py` | `api-reference/pipecat-flows/overview.mdx` | +| `flows/exceptions.py` | `api-reference/pipecat-flows/exceptions.mdx` | ## Skip list diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml index ed9f1c4c200..b31f8251e15 100644 --- a/.github/workflows/update-docs.yml +++ b/.github/workflows/update-docs.yml @@ -13,6 +13,7 @@ on: - "src/pipecat/turns/**" - "src/pipecat/observers/**" - "src/pipecat/pipeline/**" + - "src/pipecat/flows/**" workflow_dispatch: inputs: pr_number: diff --git a/README.md b/README.md index 70de62b4c32..3d44731ea1f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ > Want to dive right in? Run `pipecat init quickstart` or follow the [quickstart guide](https://docs.pipecat.ai/getting-started/quickstart). -## 🚀 What You Can Build +## 🚀 What you can build - **Voice Assistants** – natural, streaming conversations with AI - **Multi-Agent Systems** – specialists that hand off, fan out in parallel, or run as sidecars over a shared bus @@ -28,7 +28,7 @@ - **Multi-Agent Ready**: Each pipeline is an agent. Compose them with handoff, parallel fan-out, sidecar workers, or distributed deployments - **Real-Time**: Ultra-low latency interaction with different transports (e.g. WebSockets or WebRTC) -## 🌐 Pipecat Ecosystem +## 🌐 Pipecat ecosystem ### 📱 Client SDKs @@ -39,7 +39,7 @@ Building client applications? You can connect to Pipecat from any platform using ### 🧭 Structured conversations -Looking to build structured conversations? Check out [Pipecat Flows](https://github.com/pipecat-ai/pipecat-flows) for managing complex conversational states and transitions. +Need predefined or dynamic conversation paths with state management? [Pipecat Flows](https://docs.pipecat.ai/guides/features/pipecat-flows) is built into Pipecat. Browse the [examples](https://github.com/pipecat-ai/pipecat/tree/main/examples/flows) to see it in action. ### 🪄 Beautiful UIs @@ -57,7 +57,7 @@ Looking for help debugging your pipeline and processors? Check out [Whisker](htt Love terminal applications? Check out [Tail](https://github.com/pipecat-ai/tail), a terminal dashboard for Pipecat. -### 🤖 Claude Code Skills +### 🤖 Claude Code skills Use [Pipecat Skills](https://github.com/pipecat-ai/skills) with [Claude Code](https://claude.ai/code) to scaffold projects, deploy to Pipecat Cloud, and more. Install the marketplace with: @@ -67,11 +67,11 @@ claude plugin marketplace add pipecat-ai/skills and install any of the available plugins. -### 🧩 Community Integrations +### 🧩 Community integrations Build and share your own Pipecat service integrations! Browse existing [community integrations](https://docs.pipecat.ai/api-reference/server/services/supported-services) or check out our [guide](COMMUNITY_INTEGRATIONS.md) to create your own. -### 📺️ Pipecat TV Channel +### 📺️ Pipecat TV channel Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.youtube.com/playlist?list=PLzU2zoMTQIHjqC3v4q2XVSR3hGSzwKFwH) channel. @@ -87,36 +87,47 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout ## 🧩 Available services -| Category | Services | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Category | Services | +| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/api-reference/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/api-reference/server/services/stt/aws), [Azure](https://docs.pipecat.ai/api-reference/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/api-reference/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/api-reference/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/api-reference/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/api-reference/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/api-reference/server/services/stt/gladia), [Google](https://docs.pipecat.ai/api-reference/server/services/stt/google), [Gradium](https://docs.pipecat.ai/api-reference/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/api-reference/server/services/stt/groq), [Mistral](https://docs.pipecat.ai/api-reference/server/services/stt/mistral), [Moonshine](https://docs.pipecat.ai/api-reference/server/services/stt/moonshine), [NVIDIA](https://docs.pipecat.ai/api-reference/server/services/stt/nvidia), [OpenAI (Whisper)](https://docs.pipecat.ai/api-reference/server/services/stt/openai), [Sarvam](https://docs.pipecat.ai/api-reference/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/api-reference/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/api-reference/server/services/stt/speechmatics), [Together](https://docs.pipecat.ai/api-reference/server/services/stt/together), [Whisper](https://docs.pipecat.ai/api-reference/server/services/stt/whisper), [xAI](https://docs.pipecat.ai/api-reference/server/services/stt/xai) | -| LLMs | [Anthropic](https://docs.pipecat.ai/api-reference/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/api-reference/server/services/llm/aws), [Azure](https://docs.pipecat.ai/api-reference/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/api-reference/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/api-reference/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/api-reference/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/api-reference/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/api-reference/server/services/llm/grok), [Groq](https://docs.pipecat.ai/api-reference/server/services/llm/groq), [Inception](https://docs.pipecat.ai/api-reference/server/services/llm/inception), [Mistral](https://docs.pipecat.ai/api-reference/server/services/llm/mistral), [Nebius](https://docs.pipecat.ai/api-reference/server/services/llm/nebius), [Novita](https://docs.pipecat.ai/api-reference/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/api-reference/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/api-reference/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/api-reference/server/services/llm/openai), [OpenAI Responses](https://docs.pipecat.ai/api-reference/server/services/llm/openai-responses), [OpenRouter](https://docs.pipecat.ai/api-reference/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/api-reference/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/api-reference/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/api-reference/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/api-reference/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/api-reference/server/services/llm/together) | +| LLMs | [Anthropic](https://docs.pipecat.ai/api-reference/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/api-reference/server/services/llm/aws), [Azure](https://docs.pipecat.ai/api-reference/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/api-reference/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/api-reference/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/api-reference/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/api-reference/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/api-reference/server/services/llm/grok), [Groq](https://docs.pipecat.ai/api-reference/server/services/llm/groq), [Inception](https://docs.pipecat.ai/api-reference/server/services/llm/inception), [Mistral](https://docs.pipecat.ai/api-reference/server/services/llm/mistral), [Nebius](https://docs.pipecat.ai/api-reference/server/services/llm/nebius), [Novita](https://docs.pipecat.ai/api-reference/server/services/llm/novita), [NVIDIA NIM](https://docs.pipecat.ai/api-reference/server/services/llm/nvidia), [Ollama](https://docs.pipecat.ai/api-reference/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/api-reference/server/services/llm/openai), [OpenAI Responses](https://docs.pipecat.ai/api-reference/server/services/llm/openai-responses), [OpenRouter](https://docs.pipecat.ai/api-reference/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/api-reference/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/api-reference/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/api-reference/server/services/llm/sambanova), [Sarvam](https://docs.pipecat.ai/api-reference/server/services/llm/sarvam), [Together AI](https://docs.pipecat.ai/api-reference/server/services/llm/together) | | Text-to-Speech | [Async](https://docs.pipecat.ai/api-reference/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/api-reference/server/services/tts/aws), [Azure](https://docs.pipecat.ai/api-reference/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/api-reference/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/api-reference/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/api-reference/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/api-reference/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/api-reference/server/services/tts/fish), [Google](https://docs.pipecat.ai/api-reference/server/services/tts/google), [Gradium](https://docs.pipecat.ai/api-reference/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/api-reference/server/services/tts/groq), [Hume](https://docs.pipecat.ai/api-reference/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/api-reference/server/services/tts/inworld), [Kokoro](https://docs.pipecat.ai/api-reference/server/services/tts/kokoro), [LMNT](https://docs.pipecat.ai/api-reference/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/api-reference/server/services/tts/minimax), [Mistral](https://docs.pipecat.ai/api-reference/server/services/tts/mistral), [Neuphonic](https://docs.pipecat.ai/api-reference/server/services/tts/neuphonic), [NVIDIA](https://docs.pipecat.ai/api-reference/server/services/tts/nvidia), [OpenAI](https://docs.pipecat.ai/api-reference/server/services/tts/openai), [Piper](https://docs.pipecat.ai/api-reference/server/services/tts/piper), [Resemble](https://docs.pipecat.ai/api-reference/server/services/tts/resemble), [Rime](https://docs.pipecat.ai/api-reference/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/api-reference/server/services/tts/sarvam), [Smallest](https://docs.pipecat.ai/api-reference/server/services/tts/smallest), [Soniox](https://docs.pipecat.ai/api-reference/server/services/tts/soniox), [Speechmatics](https://docs.pipecat.ai/api-reference/server/services/tts/speechmatics), [Together](https://docs.pipecat.ai/api-reference/server/services/tts/together), [xAI](https://docs.pipecat.ai/api-reference/server/services/tts/xai), [XTTS](https://docs.pipecat.ai/api-reference/server/services/tts/xtts) | -| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/api-reference/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/api-reference/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/api-reference/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/api-reference/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/api-reference/server/services/s2s/ultravox), | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/api-reference/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/api-reference/server/services/transport/fastapi-websocket), [LiveKit (WebRTC)](https://docs.pipecat.ai/api-reference/server/services/transport/livekit), [SmallWebRTCTransport](https://docs.pipecat.ai/api-reference/server/services/transport/small-webrtc), [Vonage (WebRTC)](https://docs.pipecat.ai/api-reference/server/services/transport/vonage), [WebSocket Server](https://docs.pipecat.ai/api-reference/server/services/transport/websocket-server), [WhatsApp](https://docs.pipecat.ai/api-reference/server/services/transport/whatsapp), Local | -| Serializers | [Exotel](https://docs.pipecat.ai/api-reference/server/services/serializers/exotel), [Genesys](https://docs.pipecat.ai/api-reference/server/services/serializers/genesys), [Plivo](https://docs.pipecat.ai/api-reference/server/services/serializers/plivo), [Twilio](https://docs.pipecat.ai/api-reference/server/services/serializers/twilio), [Telnyx](https://docs.pipecat.ai/api-reference/server/services/serializers/telnyx), [Vonage](https://docs.pipecat.ai/api-reference/server/services/serializers/vonage) | -| Video | [HeyGen](https://docs.pipecat.ai/api-reference/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/api-reference/server/services/transport/lemonslice), [Tavus](https://docs.pipecat.ai/api-reference/server/services/video/tavus), [Simli](https://docs.pipecat.ai/api-reference/server/services/video/simli) | -| Memory | [mem0](https://docs.pipecat.ai/api-reference/server/services/memory/mem0) | -| Vision & Image | [fal](https://docs.pipecat.ai/api-reference/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/api-reference/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/api-reference/server/services/vision/moondream) | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/api-reference/server/utilities/audio/silero-vad-analyzer), [Krisp Viva](https://docs.pipecat.ai/guides/features/krisp-viva), [Koala](https://docs.pipecat.ai/api-reference/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/api-reference/server/utilities/audio/aic-filter), [RNNoise](https://docs.pipecat.ai/api-reference/server/utilities/audio/rnnoise-filter) | -| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/api-reference/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/api-reference/server/services/analytics/sentry) | -| Community | [Browse community integrations →](https://docs.pipecat.ai/api-reference/server/services/supported-services) | +| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/api-reference/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/api-reference/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/api-reference/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/api-reference/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/api-reference/server/services/s2s/ultravox), | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/api-reference/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/api-reference/server/services/transport/fastapi-websocket), [LiveKit (WebRTC)](https://docs.pipecat.ai/api-reference/server/services/transport/livekit), [SmallWebRTCTransport](https://docs.pipecat.ai/api-reference/server/services/transport/small-webrtc), [Vonage (WebRTC)](https://docs.pipecat.ai/api-reference/server/services/transport/vonage), [WebSocket Server](https://docs.pipecat.ai/api-reference/server/services/transport/websocket-server), [WhatsApp](https://docs.pipecat.ai/api-reference/server/services/transport/whatsapp), Local | +| Serializers | [Exotel](https://docs.pipecat.ai/api-reference/server/services/serializers/exotel), [Genesys](https://docs.pipecat.ai/api-reference/server/services/serializers/genesys), [Plivo](https://docs.pipecat.ai/api-reference/server/services/serializers/plivo), [Twilio](https://docs.pipecat.ai/api-reference/server/services/serializers/twilio), [Telnyx](https://docs.pipecat.ai/api-reference/server/services/serializers/telnyx), [Vonage](https://docs.pipecat.ai/api-reference/server/services/serializers/vonage) | +| Video | [HeyGen](https://docs.pipecat.ai/api-reference/server/services/video/heygen), [LemonSlice](https://docs.pipecat.ai/api-reference/server/services/transport/lemonslice), [Tavus](https://docs.pipecat.ai/api-reference/server/services/video/tavus), [Simli](https://docs.pipecat.ai/api-reference/server/services/video/simli) | +| Memory | [mem0](https://docs.pipecat.ai/api-reference/server/services/memory/mem0) | +| Vision & Image | [fal](https://docs.pipecat.ai/api-reference/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/api-reference/server/services/image-generation/google-imagen), [Moondream](https://docs.pipecat.ai/api-reference/server/services/vision/moondream) | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/api-reference/server/utilities/audio/silero-vad-analyzer), [Krisp Viva](https://docs.pipecat.ai/guides/features/krisp-viva), [Koala](https://docs.pipecat.ai/api-reference/server/utilities/audio/koala-filter), [ai-coustics](https://docs.pipecat.ai/api-reference/server/utilities/audio/aic-filter), [RNNoise](https://docs.pipecat.ai/api-reference/server/utilities/audio/rnnoise-filter) | +| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/api-reference/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/api-reference/server/services/analytics/sentry) | +| Community | [Browse community integrations →](https://docs.pipecat.ai/api-reference/server/services/supported-services) | 📚 [View full services documentation →](https://docs.pipecat.ai/api-reference/server/services/supported-services) ## ⚡ Getting started -You can get started with Pipecat running on your local machine, then move your agent processes to the cloud when you're ready. +Run Pipecat on your local machine, then move your agent processes to the cloud when you're ready. Either way, you'll need [uv](https://docs.astral.sh/uv/getting-started/installation/) — install it first: -1. Install uv +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` - ```bash - curl -LsSf https://astral.sh/uv/install.sh | sh - ``` +### Start a new project with the CLI - > **Need help?** Refer to the [uv install documentation](https://docs.astral.sh/uv/getting-started/installation/). +The quickest path: install the Pipecat CLI and scaffold a new phone or web/mobile bot interactively. -2. Install the module +```bash +uv tool install "pipecat-ai[cli]" +pipecat init +``` + +Follow the [quickstart guide](https://docs.pipecat.ai/getting-started/quickstart) to get your very first bot running, or dive a little deeper into [bootstrapping a project](https://docs.pipecat.ai/pipecat/get-started/build-your-next-bot). + +### Manual installation + +Prefer to wire things up yourself? + +1. Install the module ```bash # For new projects @@ -128,13 +139,13 @@ You can get started with Pipecat running on your local machine, then move your a uv add pipecat-ai ``` -3. Set up your environment +2. Set up your environment ```bash cp env.example .env ``` -4. To keep things lightweight, only the core framework is included by default. If you need support for third-party AI services, you can add the necessary dependencies with: +3. To keep things lightweight, only the core framework is included by default. If you need support for third-party AI services, you can add the necessary dependencies with: ```bash uv add "pipecat-ai[option,...]" @@ -142,19 +153,21 @@ You can get started with Pipecat running on your local machine, then move your a > **Using pip?** You can still use `pip install pipecat-ai` and `pip install "pipecat-ai[option,...]"` to get set up. +From here, the code examples below are the best way to learn — agents you can run, read, and adapt. + ## 🧪 Code examples -- [Foundational](https://github.com/pipecat-ai/pipecat/tree/main/examples) — small snippets that build on each other, introducing one or two concepts at a time -- [Example apps](https://github.com/pipecat-ai/pipecat-examples) — complete applications that you can use as starting points for development +- [Focused examples](https://github.com/pipecat-ai/pipecat/tree/main/examples) — small agents that each illustrate one or two specific services or concepts +- [Example apps](https://github.com/pipecat-ai/pipecat-examples) — complete applications you can use as starting points for development -## 🛠️ Contributing to the framework +## 🛠️ Developing Pipecat ### Prerequisites **Minimum Python Version:** 3.11 **Recommended Python Version:** >= 3.12 -### Setup Steps +### Setup steps 1. Clone the repository and navigate to it: @@ -179,7 +192,7 @@ You can get started with Pipecat running on your local machine, then move your a > **Note**: Some extras (local, gstreamer) require system dependencies. See documentation if you encounter build errors. -### Claude Code Skills +### Claude Code skills Install development workflow skills for contributing to Pipecat with [Claude Code](https://claude.ai/code): diff --git a/changelog/4882.added.md b/changelog/4882.added.md new file mode 100644 index 00000000000..4c4d37305ed --- /dev/null +++ b/changelog/4882.added.md @@ -0,0 +1 @@ +- Pipecat Flows is now part of `pipecat-ai`. The conversation-flow framework previously published as the separate `pipecat-ai-flows` package now ships with Pipecat under the `pipecat.flows` namespace — `from pipecat.flows import FlowManager, NodeConfig` — so there is no longer a separate package to install or keep version-matched. Code importing from `pipecat_flows` should switch to `pipecat.flows`. If the deprecated `pipecat-ai-flows` package is still installed alongside this Pipecat, Pipecat logs an error prompting you to remove it. The standalone package's release history remains available in the archived [pipecat-flows repository](https://github.com/pipecat-ai/pipecat-flows/blob/main/CHANGELOG.md). diff --git a/docs/api/conf.py b/docs/api/conf.py index 1350c029f3c..1fb8ee00efb 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -125,6 +125,7 @@ def import_core_modules(): """Import core pipecat modules for autodoc to discover.""" core_modules = [ "pipecat", + "pipecat.flows", "pipecat.frames", "pipecat.pipeline", "pipecat.processors", diff --git a/examples/README.md b/examples/README.md index 3c57a4dd94a..71504b3c55d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,7 +4,7 @@ This directory contains examples showing how to build voice and multimodal agent ## Setup -1. Follow the [README](https://github.com/pipecat-ai/pipecat/blob/main/README.md#%EF%B8%8F-contributing-to-the-framework) steps to get your local environment configured. +1. Follow the [README](https://github.com/pipecat-ai/pipecat/blob/main/README.md#%EF%B8%8F-developing-pipecat) steps to get your local environment configured. > **Run from root directory**: Make sure you are running the steps from the root directory. @@ -61,6 +61,10 @@ uv run getting-started/06-voice-agent.py -t twilio -x NGROK_HOST_NAME Progressive introduction to Pipecat, from minimal TTS to a full voice agent with function calling. +### [`flows/`](./flows/) + +Structured conversations with [Pipecat Flows](../src/pipecat/flows): predefined and dynamic conversation paths with state management, across multiple LLM providers. + ### [`voice/`](./voice/) Full STT + LLM + TTS voice agent pipelines showcasing different speech service providers (Deepgram, ElevenLabs, Cartesia, etc.) diff --git a/examples/flows/README.md b/examples/flows/README.md new file mode 100644 index 00000000000..8e02c1fb0ea --- /dev/null +++ b/examples/flows/README.md @@ -0,0 +1,51 @@ +# Pipecat Flows Examples + +[Pipecat Flows](../../src/pipecat/flows) is the structured-conversation framework built into Pipecat. It lets you build both predefined conversation paths and dynamically generated flows while handling the complexities of state management and LLM interactions. These examples show it in action. + +## Hello, world + +[`hello_world.py`](./hello_world.py) is the smallest possible Flow: a bot that asks for your favorite color and then says goodbye. It's a good first read — it shows the basics of nodes, functions, and transitions. To run it, see Setup below. + +## Setup + +1. Follow the [README](../../README.md#%EF%B8%8F-developing-pipecat) steps to configure your local environment. Run the commands from the repo root. + +2. Copy the [`env.example`](../../env.example) file and add API keys for the services you plan to use: + + ```bash + cp env.example .env + # Edit .env with your API keys + ``` + +3. Run any example: + + ```bash + uv run python examples/flows/food_ordering.py + ``` + +4. Open the web interface at http://localhost:7860/client/ and click "Connect". + +All examples support multiple LLM providers (OpenAI, Anthropic, Google Gemini, AWS Bedrock) to demonstrate cross-provider compatibility. Like the other Pipecat examples, they default to the SmallWebRTC transport and also support Daily (`-t daily`) and telephony providers (`-t twilio -x NGROK_HOST_NAME`) — see the [examples README](../README.md#running-examples-with-other-transports) for transport details. + +## Examples + +### Core flows + +- [`food_ordering.py`](./food_ordering.py) — restaurant order flow demonstrating node and edge functions +- [`restaurant_reservation.py`](./restaurant_reservation.py) — reservation system with availability checking +- [`patient_intake.py`](./patient_intake.py) — medical intake system showing complex state management +- [`insurance_quote.py`](./insurance_quote.py) — insurance quote system with data collection +- [`podcast_interview.py`](./podcast_interview.py) — podcast interview flow + +### Advanced features + +- [`llm_switching.py`](./llm_switching.py) — switching between LLM providers during a conversation +- [`warm_transfer.py`](./warm_transfer.py) — transferring calls between flows (DailyTransport only) +- [`multi_worker_handoff.py`](./multi_worker_handoff.py) — composing Flows with Pipecat's multi-worker framework: a structured Flows reservation worker hands off to and from a free-form `LLMWorker` router over the bus, sharing a single conversation context +- [`food_ordering_advanced_functionschema.py`](./food_ordering_advanced_functionschema.py) — the food-ordering flow defined with `FlowsFunctionSchema`s instead of direct functions, for when you need to specify a function's schema explicitly + +The examples define their functions as "direct functions" — async functions whose schema is derived from the signature and docstring — which is the recommended pattern. `food_ordering_advanced_functionschema.py` shows the alternative `FlowsFunctionSchema` approach. + +## Learn more + +See the [Pipecat Flows guide](https://docs.pipecat.ai/guides/features/pipecat-flows) for a full walkthrough of nodes, functions, context strategies, and actions. diff --git a/examples/flows/assets/hold_music/README.md b/examples/flows/assets/hold_music/README.md new file mode 100644 index 00000000000..47bb2bcf219 --- /dev/null +++ b/examples/flows/assets/hold_music/README.md @@ -0,0 +1,7 @@ +# Hold Music Player + +This project is a hold music player, based on the `wav_audio_send` example from the [daily-python repository](https://github.com/daily-co/daily-python/blob/main/demos/audio/wav_audio_send.py). It is designed to serve as a helper for other examples, providing a reusable component for scenarios that require hold music functionality. + +The hold music WAV file used in this example was sourced from [No Copyright Music](https://www.no-copyright-music.com/). + +To see this hold music player in action, check out the [warm transfer example](../warm_transfer.py). diff --git a/examples/flows/assets/hold_music/hold_music.py b/examples/flows/assets/hold_music/hold_music.py new file mode 100644 index 00000000000..e116a87a9c5 --- /dev/null +++ b/examples/flows/assets/hold_music/hold_music.py @@ -0,0 +1,147 @@ +# +# This demo will join a Daily meeting and send the audio from a WAV file into +# the meeting. It uses the asyncio library. +# +# Usage: python3 hold_music.py -m MEETING_URL -i FILE.wav +# + +import argparse +import asyncio +import signal +import wave + +from daily import * + +SAMPLE_RATE = 16000 +NUM_CHANNELS = 1 + + +class AsyncSendWavApp: + def __init__(self, input_file_name, sample_rate, num_channels): + self.__mic_device = Daily.create_microphone_device( + "my-mic", + sample_rate=sample_rate, + channels=num_channels, + non_blocking=True, + ) + + self.__client = CallClient() + + self.__client.update_subscription_profiles( + {"base": {"camera": "unsubscribed", "microphone": "unsubscribed"}} + ) + + self.__app_error = None + + self.__start_event = asyncio.Event() + self.__task = asyncio.get_running_loop().create_task(self.send_wav_file(input_file_name)) + + async def run(self, meeting_url, meeting_token): + (data, error) = await self.join(meeting_url, meeting_token) + + if error: + print(f"Unable to join meeting: {error}") + self.__app_error = error + + self.__start_event.set() + + await self.__task + + async def join(self, meeting_url, meeting_token): + future = asyncio.get_running_loop().create_future() + + def join_completion(data, error): + future.get_loop().call_soon_threadsafe(future.set_result, (data, error)) + + self.__client.join( + meeting_url, + meeting_token, + client_settings={ + "inputs": { + "camera": False, + "microphone": {"isEnabled": True, "settings": {"deviceId": "my-mic"}}, + } + }, + completion=join_completion, + ) + + return await future + + async def leave(self): + future = asyncio.get_running_loop().create_future() + + def leave_completion(error): + future.get_loop().call_soon_threadsafe(future.set_result, error) + + self.__client.leave(completion=leave_completion) + + await future + + self.__client.release() + + self.__task.cancel() + await self.__task + + async def write_frames(self, frames): + future = asyncio.get_running_loop().create_future() + + def write_completion(count): + future.get_loop().call_soon_threadsafe(future.set_result, count) + + self.__mic_device.write_frames(frames, completion=write_completion) + + await future + + async def send_wav_file(self, file_name): + await self.__start_event.wait() + + if self.__app_error: + print(f"Unable to send WAV file!") + return + + try: + wav = wave.open(file_name, "rb") + + sent_frames = 0 + total_frames = wav.getnframes() + sample_rate = wav.getframerate() + while sent_frames < total_frames: + # Read 100ms worth of audio frames. + frames = wav.readframes(int(sample_rate / 10)) + if len(frames) > 0: + await self.write_frames(frames) + sent_frames += sample_rate / 10 + except asyncio.CancelledError: + pass + + +async def sig_handler(app): + print("Ctrl-C detected. Exiting!") + await app.leave() + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-m", "--meeting", required=True, help="Meeting URL") + parser.add_argument("-t", "--token", required=True, help="Meeting token") + parser.add_argument("-i", "--input", required=True, help="WAV input file") + parser.add_argument( + "-c", "--channels", type=int, default=NUM_CHANNELS, help="Number of channels" + ) + parser.add_argument("-r", "--rate", type=int, default=SAMPLE_RATE, help="Sample rate") + + args = parser.parse_args() + + Daily.init() + + app = AsyncSendWavApp(args.input, args.rate, args.channels) + + loop = asyncio.get_running_loop() + + loop.add_signal_handler(signal.SIGINT, lambda *args: asyncio.create_task(sig_handler(app))) + + await app.run(args.meeting, args.token) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/flows/assets/hold_music/hold_music.wav b/examples/flows/assets/hold_music/hold_music.wav new file mode 100644 index 00000000000..a3d02ff11d0 Binary files /dev/null and b/examples/flows/assets/hold_music/hold_music.wav differ diff --git a/examples/flows/food_ordering.py b/examples/flows/food_ordering.py new file mode 100644 index 00000000000..bf9d832c938 --- /dev/null +++ b/examples/flows/food_ordering.py @@ -0,0 +1,369 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""A food ordering flow example for Pipecat Flows. + +This example demonstrates a food ordering system using flows where +conversation paths are determined at runtime. The flow handles: + +1. Initial greeting and food type selection (pizza or sushi) +2. Order details collection based on food type +3. Order confirmation and revision +4. Order completion + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- CARTESIA_API_KEY (for TTS) +- DEEPGRAM_API_KEY (for STT) +- DAILY_API_KEY (for transport) +- LLM API key (varies by provider - see env.example) +""" + +import os +from datetime import datetime, timedelta +from typing import TypedDict + +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import FlowManager, NodeConfig +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# Type definitions +class PizzaOrderResult(TypedDict): + size: str + type: str + price: float + + +class SushiOrderResult(TypedDict): + count: int + type: str + price: float + + +class DeliveryEstimateResult(TypedDict): + time: str + + +# Pre-action handlers +async def check_kitchen_status(action: dict, flow_manager: FlowManager) -> None: + """Check if kitchen is open and log status.""" + logger.info("Checking kitchen status") + + +# Functions for Initial Node +async def choose_pizza(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """ + User wants to order pizza. Let's get that order started. + """ + return None, create_pizza_node() + + +async def choose_sushi(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """ + User wants to order sushi. Let's get that order started. + """ + return None, create_sushi_node() + + +# Functions for Pizza Node +async def select_pizza_order( + flow_manager: FlowManager, size: str, pizza_type: str +) -> tuple[PizzaOrderResult, NodeConfig]: + """ + Record the pizza order details. + + Args: + size (str): Size of the pizza. Must be one of "small", "medium", or "large". + pizza_type (str): Type of pizza. Must be one of "pepperoni", "cheese", "supreme", or "vegetarian". + """ + # Simple pricing + base_price = {"small": 10.00, "medium": 15.00, "large": 20.00} + price = base_price[size] + + result = PizzaOrderResult(size=size, type=pizza_type, price=price) + + # Store order details in flow state + flow_manager.state["order"] = { + "type": "pizza", + "size": size, + "pizza_type": pizza_type, + "price": price, + } + + return result, create_confirmation_node() + + +# Functions for Sushi Node +async def select_sushi_order( + flow_manager: FlowManager, count: int, roll_type: str +) -> tuple[SushiOrderResult, NodeConfig]: + """ + Record the sushi order details. + + Args: + count (int): Number of sushi rolls to order. Must be between 1 and 10. + roll_type (str): Type of sushi roll. Must be one of "california", "spicy tuna", "rainbow", or "dragon". + """ + # Simple pricing: $8 per roll + price = count * 8.00 + + result = SushiOrderResult(count=count, type=roll_type, price=price) + + # Store order details in flow state + flow_manager.state["order"] = { + "type": "sushi", + "count": count, + "roll_type": roll_type, + "price": price, + } + + return result, create_confirmation_node() + + +# Functions for Confirmation Node +async def complete_order(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """ + User confirms the order is correct. + """ + return None, create_end_node() + + +async def revise_order(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """ + User wants to make changes to their order. + """ + return None, create_initial_node() + + +# Node creation functions +def create_initial_node() -> NodeConfig: + """Create the initial node for food type selection.""" + return NodeConfig( + name="initial", + role_message="You are an order-taking assistant. You must ALWAYS use the available functions to progress the conversation. This is a phone conversation and your responses will be converted to audio. Keep the conversation friendly, casual, and polite. Avoid outputting special characters and emojis.", + task_messages=[ + { + "role": "developer", + "content": "For this step, ask the user if they want pizza or sushi, and wait for them to use a function to choose. Start off by greeting them. Be friendly and casual; you're taking an order for food over the phone.", + } + ], + pre_actions=[ + { + "type": "function", + "handler": check_kitchen_status, + }, + ], + functions=[choose_pizza, choose_sushi], + ) + + +def create_pizza_node() -> NodeConfig: + """Create the pizza ordering node.""" + return NodeConfig( + name="choose_pizza", + task_messages=[ + { + "role": "developer", + "content": """You are handling a pizza order. Use the available functions: +- Use select_pizza_order when the user specifies both size AND type + +Pricing: +- Small: $10 +- Medium: $15 +- Large: $20 + +Remember to be friendly and casual.""", + } + ], + functions=[select_pizza_order], + ) + + +def create_sushi_node() -> NodeConfig: + """Create the sushi ordering node.""" + return NodeConfig( + name="choose_sushi", + task_messages=[ + { + "role": "developer", + "content": """You are handling a sushi order. Use the available functions: +- Use select_sushi_order when the user specifies both count AND type + +Pricing: +- $8 per roll + +Remember to be friendly and casual.""", + } + ], + functions=[select_sushi_order], + ) + + +def create_confirmation_node() -> NodeConfig: + """Create the order confirmation node.""" + return NodeConfig( + name="confirm", + task_messages=[ + { + "role": "developer", + "content": """Read back the complete order details to the user and ask if they want anything else or if they want to make changes. Use the available functions: +- Use complete_order when the user confirms that the order is correct and no changes are needed +- Use revise_order if they want to change something + +Be friendly and clear when reading back the order details.""", + } + ], + functions=[complete_order, revise_order], + ) + + +def create_end_node() -> NodeConfig: + """Create the final node.""" + return NodeConfig( + name="end", + task_messages=[ + { + "role": "developer", + "content": "Thank the user for their order and end the conversation politely and concisely.", + } + ], + post_actions=[{"type": "end_conversation"}], + ) + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """Run the food ordering bot.""" + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="820a3788-2b37-4d21-847a-b65d8a68c99a", # Salesman + ), + ) + # LLM service is created using the create_llm function from utils.py + # Default is OpenAI; can be changed by setting LLM_PROVIDER environment variable + llm = create_llm() + + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Define "global" functions available at every node + async def get_delivery_estimate( + flow_manager: FlowManager, + ) -> tuple[DeliveryEstimateResult, None]: + """Provide delivery estimate information.""" + delivery_time = datetime.now() + timedelta(minutes=30) + return DeliveryEstimateResult( + time=f"{delivery_time}", + ), None + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + global_functions=[get_delivery_estimate], + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + # Kick off the conversation with the initial node + await flow_manager.initialize(create_initial_node()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/food_ordering_advanced_functionschema.py b/examples/flows/food_ordering_advanced_functionschema.py new file mode 100644 index 00000000000..54039724034 --- /dev/null +++ b/examples/flows/food_ordering_advanced_functionschema.py @@ -0,0 +1,455 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""An "advanced" food ordering flow example using FlowsFunctionSchema. + +This is the FlowsFunctionSchema counterpart to the standard food_ordering.py +(which uses direct functions). Direct functions are the recommended way to +define a node's functions: their schema is derived from the function signature +and docstring. Reach for a FlowsFunctionSchema when you need property control +the direct-function generator can't give you — for example a strict ``enum`` +constraint or a numeric ``minimum``/``maximum`` (both used below, on the pizza +size and type and the sushi count and type), which a direct function can only +hint at in prose in its docstring. + +The flow handles: + +1. Initial greeting and food type selection (pizza or sushi) +2. Order details collection based on food type +3. Order confirmation and revision +4. Order completion + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- CARTESIA_API_KEY (for TTS) +- DEEPGRAM_API_KEY (for STT) +- DAILY_API_KEY (for transport) +- LLM API key (varies by provider - see env.example) +""" + +import os +from datetime import datetime, timedelta +from typing import TypedDict + +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import ( + FlowArgs, + FlowManager, + FlowsFunctionSchema, + NodeConfig, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# Type definitions +class PizzaOrderResult(TypedDict): + size: str + type: str + price: float + + +class SushiOrderResult(TypedDict): + count: int + type: str + price: float + + +class DeliveryEstimateResult(TypedDict): + time: str + + +# Pre-action handlers +async def check_kitchen_status(action: dict, flow_manager: FlowManager) -> None: + """Check if kitchen is open and log status.""" + logger.info("Checking kitchen status") + + +# Node creation functions +def create_initial_node() -> NodeConfig: + """Create the initial node for food type selection.""" + + async def choose_pizza(args: FlowArgs, flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Transition to pizza order selection.""" + return None, create_pizza_node() + + async def choose_sushi(args: FlowArgs, flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Transition to sushi order selection.""" + return None, create_sushi_node() + + choose_pizza_func = FlowsFunctionSchema( + name="choose_pizza", + handler=choose_pizza, + description="User wants to order pizza. Let's get that order started.", + properties={}, + required=[], + ) + + choose_sushi_func = FlowsFunctionSchema( + name="choose_sushi", + handler=choose_sushi, + description="User wants to order sushi. Let's get that order started.", + properties={}, + required=[], + ) + + return NodeConfig( + name="initial", + role_message="You are an order-taking assistant. You must ALWAYS use the available functions to progress the conversation. This is a phone conversation and your responses will be converted to audio. Keep the conversation friendly, casual, and polite. Avoid outputting special characters and emojis.", + task_messages=[ + { + "role": "developer", + "content": "For this step, ask the user if they want pizza or sushi, and wait for them to use a function to choose. Start off by greeting them. Be friendly and casual; you're taking an order for food over the phone.", + } + ], + pre_actions=[ + { + "type": "function", + "handler": check_kitchen_status, + }, + ], + functions=[choose_pizza_func, choose_sushi_func], + ) + + +def create_pizza_node() -> NodeConfig: + """Create the pizza ordering node.""" + + async def select_pizza_order( + args: FlowArgs, flow_manager: FlowManager + ) -> tuple[PizzaOrderResult, NodeConfig]: + """Handle pizza size and type selection.""" + size = args["size"] + pizza_type = args["type"] + + # Simple pricing + base_price = {"small": 10.00, "medium": 15.00, "large": 20.00} + price = base_price[size] + + result = PizzaOrderResult(size=size, type=pizza_type, price=price) + + # Store order details in flow state + flow_manager.state["order"] = { + "type": "pizza", + "size": size, + "pizza_type": pizza_type, + "price": price, + } + + return result, create_confirmation_node() + + # Spelling the schema out explicitly gives precise control over the + # parameters — here, strict ``enum`` constraints on size and type (and, in + # the sushi node, a numeric ``minimum``/``maximum`` on the roll count) — + # that a direct function could only describe in prose. + select_pizza_func = FlowsFunctionSchema( + name="select_pizza_order", + handler=select_pizza_order, + description="Record the pizza order details", + properties={ + "size": { + "type": "string", + "enum": ["small", "medium", "large"], + "description": "Size of the pizza", + }, + "type": { + "type": "string", + "enum": ["pepperoni", "cheese", "supreme", "vegetarian"], + "description": "Type of pizza", + }, + }, + required=["size", "type"], + ) + + return NodeConfig( + name="choose_pizza", + task_messages=[ + { + "role": "developer", + "content": """You are handling a pizza order. + +As soon as the user has given both a size AND a type, immediately call +select_pizza_order to record it. Do not acknowledge the order conversationally, +ask whether they want anything else, or wait for further confirmation first — the +confirmation step handles all of that. If the size or the type is still missing, +ask only for the missing detail. + +Pricing: +- Small: $10 +- Medium: $15 +- Large: $20 + +Remember to be friendly and casual.""", + } + ], + functions=[select_pizza_func], + ) + + +def create_sushi_node() -> NodeConfig: + """Create the sushi ordering node.""" + + async def select_sushi_order( + args: FlowArgs, flow_manager: FlowManager + ) -> tuple[SushiOrderResult, NodeConfig]: + """Handle sushi roll count and type selection.""" + count = args["count"] + roll_type = args["type"] + + # Simple pricing: $8 per roll + price = count * 8.00 + + result = SushiOrderResult(count=count, type=roll_type, price=price) + + # Store order details in flow state + flow_manager.state["order"] = { + "type": "sushi", + "count": count, + "roll_type": roll_type, + "price": price, + } + + return result, create_confirmation_node() + + select_sushi_func = FlowsFunctionSchema( + name="select_sushi_order", + handler=select_sushi_order, + description="Record the sushi order details", + properties={ + "count": { + "type": "integer", + "minimum": 1, + "maximum": 10, + "description": "Number of rolls to order", + }, + "type": { + "type": "string", + "enum": ["california", "spicy tuna", "rainbow", "dragon"], + "description": "Type of sushi roll", + }, + }, + required=["count", "type"], + ) + + return NodeConfig( + name="choose_sushi", + task_messages=[ + { + "role": "developer", + "content": """You are handling a sushi order. + +As soon as the user has given both a roll count AND a roll type, immediately call +select_sushi_order to record it. Do not acknowledge the order conversationally, +ask whether they want anything else, or wait for further confirmation first — the +confirmation step handles all of that. If the count or the type is still missing, +ask only for the missing detail. + +Pricing: +- $8 per roll + +Remember to be friendly and casual.""", + } + ], + functions=[select_sushi_func], + ) + + +def create_confirmation_node() -> NodeConfig: + """Create the order confirmation node.""" + + async def complete_order(args: FlowArgs, flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Transition to end state.""" + return None, create_end_node() + + async def revise_order(args: FlowArgs, flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Transition to start for order revision.""" + return None, create_initial_node() + + complete_order_func = FlowsFunctionSchema( + name="complete_order", + handler=complete_order, + description="User confirms the order is correct", + properties={}, + required=[], + ) + + revise_order_func = FlowsFunctionSchema( + name="revise_order", + handler=revise_order, + description="User wants to make changes to their order", + properties={}, + required=[], + ) + + return NodeConfig( + name="confirm", + task_messages=[ + { + "role": "developer", + "content": """Read back the complete order details to the user and ask if they want anything else or if they want to make changes. Use the available functions: +- Use complete_order when the user confirms that the order is correct and no changes are needed +- Use revise_order if they want to change something + +Be friendly and clear when reading back the order details.""", + } + ], + functions=[complete_order_func, revise_order_func], + ) + + +def create_end_node() -> NodeConfig: + """Create the final node.""" + return NodeConfig( + name="end", + task_messages=[ + { + "role": "developer", + "content": "Thank the user for their order and end the conversation politely and concisely.", + } + ], + post_actions=[{"type": "end_conversation"}], + ) + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """Run the food ordering bot.""" + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="820a3788-2b37-4d21-847a-b65d8a68c99a", # Salesman + ), + ) + # LLM service is created using the create_llm function from utils.py + # Default is OpenAI; can be changed by setting LLM_PROVIDER environment variable + llm = create_llm() + + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Define "global" functions available at every node + async def get_delivery_estimate( + args: FlowArgs, flow_manager: FlowManager + ) -> tuple[DeliveryEstimateResult, None]: + """Provide delivery estimate information.""" + delivery_time = datetime.now() + timedelta(minutes=30) + return DeliveryEstimateResult( + time=f"{delivery_time}", + ), None + + get_delivery_estimate_func = FlowsFunctionSchema( + name="get_delivery_estimate", + handler=get_delivery_estimate, + description="Get a delivery estimate for the current order", + properties={}, + required=[], + ) + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + global_functions=[get_delivery_estimate_func], + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + # Kick off the conversation with the initial node + await flow_manager.initialize(create_initial_node()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/hello_world.py b/examples/flows/hello_world.py new file mode 100644 index 00000000000..08b9878aec3 --- /dev/null +++ b/examples/flows/hello_world.py @@ -0,0 +1,189 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License + +"""A 'Hello-World' introduction to Pipecat Flows. + +Requirements: +- CARTESIA_API_KEY +- GOOGLE_API_KEY + +Run the example: +uv run hello_world.py +""" + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import FlowManager, NodeConfig +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.stt import CartesiaSTTService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# Flow nodes +def create_initial_node() -> NodeConfig: + """Create the initial node of the flow. + + Define the bot's role and task for the node, plus the function it can call. + The function records the result and transitions to the next node. + """ + return NodeConfig( + name="initial", + role_message="You are an inquisitive child. Use very simple language. Ask simple questions. You must ALWAYS use one of the available functions to progress the conversation. Your responses will be converted to audio. Avoid outputting special characters and emojis.", + task_messages=[ + { + "role": "developer", + "content": "Say 'Hello world' and ask what is the user's favorite color.", + } + ], + functions=[record_favorite_color], + ) + + +async def record_favorite_color(flow_manager: FlowManager, color: str) -> tuple[str, NodeConfig]: + """Record the color the user said is their favorite. + + Here "record" means print to the console, but any logic could go here: + write to a database, make an API call, etc. + + Args: + color: The user's favorite color. + """ + print(f"Your favorite color is: {color}") + return color, create_end_node() + + +def create_end_node() -> NodeConfig: + """End the conversation. + + Flows transitions to this node when the user has answered the question. + It thanks the user and ends the conversation using the `end_conversation` + post-action. + """ + return NodeConfig( + name="create_end_node", + task_messages=[ + { + "role": "developer", + "content": "Thank the user for answering and end the conversation", + } + ], + post_actions=[{"type": "end_conversation"}], + ) + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + stt = CartesiaSTTService(api_key=os.getenv("CARTESIA_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="32b3f3c5-7171-46aa-abe7-b598964aa793", + ), + ) + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY", "")) + + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await flow_manager.initialize(create_initial_node()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/insurance_quote.py b/examples/flows/insurance_quote.py new file mode 100644 index 00000000000..7fee7ccfef9 --- /dev/null +++ b/examples/flows/insurance_quote.py @@ -0,0 +1,377 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +"""Insurance Quote Example using Pipecat Flows. + +This example demonstrates how to create a conversational insurance quote bot using: +- Flow management for flexible conversation paths +- Node configurations for different conversation states +- Pre/post actions for user feedback +- Transition logic based on user responses + +The flow allows users to: +1. Provide their age +2. Specify marital status +3. Get an insurance quote +4. Adjust coverage options +5. Complete the quote process + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- CARTESIA_API_KEY (for TTS) +- DEEPGRAM_API_KEY (for STT) +- DAILY_API_KEY (for transport) +- LLM API key (varies by provider - see env.example) +""" + +import os +from typing import Any, TypedDict + +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import FlowManager, NodeConfig +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# Type definitions +class InsuranceQuote(TypedDict): + monthly_premium: float + coverage_amount: int + deductible: int + + +class AgeCollectionResult(TypedDict): + age: int + + +class MaritalStatusResult(TypedDict): + marital_status: str + + +class QuoteCalculationResult(InsuranceQuote): + pass + + +class CoverageUpdateResult(InsuranceQuote): + pass + + +# Simulated insurance data +INSURANCE_RATES = { + "young_single": {"base_rate": 150, "risk_multiplier": 1.5}, + "young_married": {"base_rate": 130, "risk_multiplier": 1.3}, + "adult_single": {"base_rate": 100, "risk_multiplier": 1.0}, + "adult_married": {"base_rate": 90, "risk_multiplier": 0.9}, +} + + +# Functions +async def collect_age( + flow_manager: FlowManager, age: int +) -> tuple[AgeCollectionResult, NodeConfig]: + """Record customer's age. + + Args: + age (int): The customer's age. + """ + logger.debug(f"collect_age handler executing with age: {age}") + + flow_manager.state["age"] = age + result = AgeCollectionResult(age=age) + + next_node = create_marital_status_node() + + return result, next_node + + +async def collect_marital_status( + flow_manager: FlowManager, marital_status: str +) -> tuple[MaritalStatusResult, NodeConfig]: + """Record marital status after customer provides it. + + Args: + marital_status (str): The customer's marital status. Must be one of "single", "married". + """ + logger.debug(f"collect_marital_status handler executing with status: {marital_status}") + + result = MaritalStatusResult(marital_status=marital_status) + + next_node = create_quote_calculation_node(flow_manager.state["age"], marital_status) + + return result, next_node + + +async def calculate_quote( + flow_manager: FlowManager, age: int, marital_status: str +) -> tuple[QuoteCalculationResult, NodeConfig]: + """Calculate initial insurance quote. + + Args: + age (int): The customer's age. + marital_status (str): The customer's marital status. Must be one of "single", "married". + """ + logger.debug(f"calculate_quote handler executing with age: {age}, status: {marital_status}") + + # Determine rate category + age_category = "young" if age < 25 else "adult" + rate_key = f"{age_category}_{marital_status}" + rates = INSURANCE_RATES.get(rate_key, INSURANCE_RATES["adult_single"]) + + # Calculate quote + monthly_premium = rates["base_rate"] * rates["risk_multiplier"] + + result = QuoteCalculationResult( + monthly_premium=monthly_premium, + coverage_amount=250000, + deductible=1000, + ) + next_node = create_quote_results_node(result) + return result, next_node + + +async def update_coverage( + flow_manager: FlowManager, coverage_amount: int, deductible: int +) -> tuple[CoverageUpdateResult, NodeConfig]: + """Recalculate quote with new coverage options. + + Args: + coverage_amount (int): The desired coverage amount in dollars. + deductible (int): The desired deductible amount in dollars. + """ + logger.debug( + f"update_coverage handler executing with amount: {coverage_amount}, deductible: {deductible}" + ) + + # Calculate adjusted quote + monthly_premium = (coverage_amount / 250000) * 100 + if deductible > 1000: + monthly_premium *= 0.9 # 10% discount for higher deductible + + result = CoverageUpdateResult( + monthly_premium=monthly_premium, + coverage_amount=coverage_amount, + deductible=deductible, + ) + next_node = create_quote_results_node(result) + return result, next_node + + +async def end_quote(flow_manager: FlowManager) -> tuple[Any, NodeConfig]: + """Complete the quote process when customer is satisfied.""" + logger.debug("end_quote handler executing") + return {"status": "completed"}, create_end_node() + + +# Node configurations +def create_initial_node() -> NodeConfig: + """Create the initial node asking for age.""" + return NodeConfig( + name="initial", + role_message="You are a friendly insurance agent. Your responses will be converted to audio, so avoid special characters. Always use the available functions to progress the conversation naturally.", + task_messages=[ + { + "role": "developer", + "content": "Start by asking for the customer's age.", + } + ], + functions=[collect_age], + ) + + +def create_marital_status_node() -> NodeConfig: + """Create node for collecting marital status.""" + return NodeConfig( + name="marital_status", + task_messages=[ + { + "role": "developer", + "content": "Ask about the customer's marital status for premium calculation.", + } + ], + functions=[collect_marital_status], + ) + + +def create_quote_calculation_node(age: int, marital_status: str) -> NodeConfig: + """Create node for calculating initial quote.""" + return NodeConfig( + name="quote_calculation", + task_messages=[ + { + "role": "developer", + "content": ( + f"Calculate a quote for {age} year old {marital_status} customer. " + "First, call calculate_quote with their information. " + "Then explain the quote details and ask if they'd like to adjust coverage." + ), + } + ], + functions=[calculate_quote], + ) + + +def create_quote_results_node( + quote: QuoteCalculationResult | CoverageUpdateResult, +) -> NodeConfig: + """Create node for showing quote and adjustment options.""" + return NodeConfig( + name="quote_results", + task_messages=[ + { + "role": "developer", + "content": ( + f"Quote details:\n" + f"Monthly Premium: ${quote['monthly_premium']:.2f}\n" + f"Coverage Amount: ${quote['coverage_amount']:,}\n" + f"Deductible: ${quote['deductible']:,}\n\n" + "Explain these quote details to the customer. When they request changes, " + "use update_coverage to recalculate their quote. Explain how their " + "changes affected the premium and compare it to their previous quote. " + "Ask if they'd like to make any other adjustments or if they're ready " + "to end the quote process." + ), + } + ], + functions=[update_coverage, end_quote], + ) + + +def create_end_node() -> NodeConfig: + """Create the final node.""" + return NodeConfig( + name="end", + task_messages=[ + { + "role": "developer", + "content": ( + "Thank the customer for their time and end the conversation. " + "Mention that a representative will contact them about the quote." + ), + } + ], + post_actions=[{"type": "end_conversation"}], + ) + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """Run the insurance quote bot.""" + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ), + ) + # LLM service is created using the create_llm function from utils.py + # Default is OpenAI; can be changed by setting LLM_PROVIDER environment variable + llm = create_llm() + + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + # Kick off the conversation with the initial node + await flow_manager.initialize(create_initial_node()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/llm_switching.py b/examples/flows/llm_switching.py new file mode 100644 index 00000000000..78dea7d864f --- /dev/null +++ b/examples/flows/llm_switching.py @@ -0,0 +1,268 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""A LLM switching flow example for Pipecat Flows. + +This example demonstrates how to dynamically switch between different LLM providers +during a conversation using Pipecat's LLMSwitcher. + +Multi-LLM Support: +This example requires API keys for all supported LLM providers: +- OpenAI (default), Google, Anthropic, and AWS Bedrock +- Users can switch between providers in real-time during conversation + +Requirements: +- CARTESIA_API_KEY (required for TTS) +- DEEPGRAM_API_KEY (required for STT) +- DAILY_API_KEY (optional for transport) +- OPENAI_API_KEY (for OpenAI LLM) +- GOOGLE_API_KEY (for Google LLM) +- ANTHROPIC_API_KEY (for Anthropic LLM) +- AWS credentials configured (for AWS Bedrock LLM) +""" + +import os +import sys +from typing import TypedDict + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import FlowManager, NodeConfig +from pipecat.flows.types import ContextStrategy, ContextStrategyConfig +from pipecat.frames.frames import ManuallySwitchServiceFrame +from pipecat.pipeline.llm_switcher import LLMSwitcher +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.service_switcher import ServiceSwitcherStrategyManual +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.anthropic.llm import AnthropicLLMService +from pipecat.services.aws.llm import AWSBedrockLLMService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +class SwitchLLMResult(TypedDict): + """Result of switching the LLM service.""" + + status: str + message: str + + +async def switch_llm(flow_manager: FlowManager, llm: str) -> tuple[SwitchLLMResult, None]: + """Switch the current LLM service. + + Args: + llm: The name of the LLM service to switch to. Must be one of "OpenAI", "Google", "Anthropic", or "AWS". + """ + if llm == "OpenAI": + new_llm = llm_openai + elif llm == "Google": + new_llm = llm_google + elif llm == "Anthropic": + new_llm = llm_anthropic + elif llm == "AWS": + new_llm = llm_aws + + if llm_switcher.active_llm == new_llm: + return SwitchLLMResult(status="success", message=f"Already using {llm} LLM service."), None + + # Typically, you would just switch LLMs like this: + # await flow_manager.worker.queue_frames([ManuallySwitchServiceFrame(service=new_llm)]) + + # But because we're in a tool call, and tool calls result in upstream + # updates from the assistant context aggregator, we're pushing the + # LLM-switching frame upstream from the aggregator to guarantee that the + # switch happens before the LLM is run with the tool call result. + await context_aggregator.assistant().push_frame( + ManuallySwitchServiceFrame(service=new_llm), FrameDirection.UPSTREAM + ) + + return SwitchLLMResult(status="success", message=f"Switched to {llm} LLM service."), None + + +class WeatherResult(TypedDict): + """Result of getting the current weather.""" + + status: str + conditions: str + temperature: int + + +async def get_current_weather( + flow_manager: FlowManager, location: str, format: str +) -> tuple[WeatherResult, None]: + """Get the current weather. + + Args: + location: The city and state, e.g. "San Francisco, CA". + format: The temperature unit to use. Must be either "celsius" or "fahrenheit". Infer this from the user's location. + """ + # This is a placeholder for the actual implementation + # In a real scenario, you would call an API to get the weather data + return WeatherResult( + status="success", conditions="sunny", temperature=75 if format == "fahrenheit" else 24 + ), None + + +async def summarize_conversation(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Summarize the conversation so far.""" + return None, create_main_node(summarize=True) + + +def create_main_node(summarize: bool = False) -> NodeConfig: + return NodeConfig( + name="main", + role_message="You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, + summary_prompt="Summarize the conversation so far in a concise way.", + ) + if summarize + else ContextStrategyConfig(strategy=ContextStrategy.APPEND), + task_messages=[ + { + "role": "developer", + "content": "Say the conversation summary, which was already retrieved (do not invoke the summarize_conversation function again)." + if summarize + else "Say a brief hello.", + } + ], + functions=[switch_llm, get_current_weather, summarize_conversation], + ) + + +# Main setup +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """Run the LLM switching bot.""" + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ), + ) + + # Shared context and aggregators for LLM services + context = LLMContext() + global context_aggregator + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + # LLM services + global llm_openai, llm_google, llm_anthropic, llm_aws, llm_switcher + llm_openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm_google = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY", "")) + llm_anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY", "")) + llm_aws = AWSBedrockLLMService( + aws_region="us-west-2", + model="us.anthropic.claude-3-5-haiku-20241022-v1:0", + params=AWSBedrockLLMService.InputParams(temperature=0.8, latency="optimized"), + ) + llm_switcher = LLMSwitcher( + llms=[llm_openai, llm_google, llm_anthropic, llm_aws], + strategy_type=ServiceSwitcherStrategyManual, + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm_switcher, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm_switcher, + context_aggregator=context_aggregator, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, participant): + logger.debug("Initializing flow manager") + await flow_manager.initialize(create_main_node()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/multi_worker_handoff.py b/examples/flows/multi_worker_handoff.py new file mode 100644 index 00000000000..18527e123c3 --- /dev/null +++ b/examples/flows/multi_worker_handoff.py @@ -0,0 +1,473 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Multi-worker handoff: a free-form LLM router and a structured Flows worker. + +This example demonstrates how Pipecat Flows composes with Pipecat's +multi-worker framework. Three workers share a single bus: + +- A *main* worker owns the transport (STT, TTS) and the shared conversation + context. It does not run an LLM itself; instead it bridges user/assistant + frames onto the bus so other workers can take turns speaking to the user. +- A *router* worker (a plain ``LLMWorker``) handles open-ended chit-chat and + general questions about the restaurant. When the user wants to book a table + it hands off to the reservation worker. +- A *reservation* worker (``build_reservation_worker``) drives a structured + Pipecat Flows conversation: party size, then time, then an availability + check, then confirmation. When it's done — or if the user changes their + mind — it hands control back to the router. + +Only one worker is active at a time. Hand-offs are seamless: the user never +hears that they've been transferred. + +The reservation worker is built as a plain ``PipelineWorker`` (no subclass), +the same way the sensor-controller example builds its worker. A ``FlowManager`` +is wired onto the worker and the flow is (re)initialized from the worker's +``on_activated`` event handler each time control is handed to it. The shared +``LLMContextAggregatorPair`` is owned by the main worker, so every worker +speaks into the same conversation history. + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- CARTESIA_API_KEY (for TTS) +- DEEPGRAM_API_KEY (for STT) +- DAILY_API_KEY (for transport) +- LLM API key (varies by provider - see env.example) +""" + +import asyncio +import os +from typing import Any, TypedDict + +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.bus import BusBridgeProcessor +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import FlowManager, FlowResult, NodeConfig +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.llm import LLMWorker, LLMWorkerActivationArgs, tool +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +MAIN_NAME = "restaurant" +ROUTER_NAME = "router" +RESERVATION_NAME = "reservation" + + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# ============================================================================= +# Mock reservation backend. +# ============================================================================= + + +class MockReservationSystem: + """Simulates a restaurant reservation API.""" + + booked_times = {"7:00 PM", "8:00 PM"} + + async def check_availability(self, party_size: int, time: str) -> tuple[bool, list[str]]: + """Return whether a time is open and, if not, some alternatives.""" + await asyncio.sleep(0.5) # Simulate a network call. + is_available = time not in self.booked_times + alternatives: list[str] = [] + if not is_available: + all_times = ["5:00 PM", "6:00 PM", "7:00 PM", "8:00 PM", "9:00 PM", "10:00 PM"] + alternatives = [t for t in all_times if t not in self.booked_times] + return is_available, alternatives + + +# ============================================================================= +# Reservation worker: a structured Pipecat Flows conversation. +# ============================================================================= + + +class PartySizeResult(TypedDict): + """Result of recording the party size.""" + + size: int + + +class AvailabilityResult(TypedDict): + """Result of an availability check.""" + + time: str + available: bool + + +def build_reservation_worker( + *, + llm: Any, + context_aggregator: LLMContextAggregatorPair, + reservation_system: MockReservationSystem, +) -> PipelineWorker: + """Build the reservation worker: a Flows conversation as a ``PipelineWorker``. + + The worker's pipeline is just the LLM. ``bridged=()`` wraps it with bus + edge processors so user frames arrive from the main worker and generated + frames are sent back the same way. A ``FlowManager`` drives the + conversation; it shares the main worker's ``context_aggregator`` so the + whole session uses a single conversation history. + + The worker starts inactive (``active=False``) and stays quiet until the + router hands it control. The ``on_activated`` event handler initializes the + flow the first time and resumes it on subsequent hand-offs. + """ + worker = PipelineWorker( + Pipeline([llm]), + name=RESERVATION_NAME, + active=False, + bridged=(), + ) + + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + ) + + # --- Nodes ------------------------------------------------------------- + + def party_size_node() -> NodeConfig: + return NodeConfig( + name="party_size", + role_message=( + "You are a reservation assistant for La Maison, an upscale French " + "restaurant. Be casual and friendly. This is a voice conversation, " + "so avoid special characters and emojis." + ), + task_messages=[ + {"role": "developer", "content": "Ask how many people are in their party."} + ], + functions=[collect_party_size, transfer_to_router], + ) + + def time_selection_node() -> NodeConfig: + return NodeConfig( + name="get_time", + task_messages=[ + { + "role": "developer", + "content": ( + "Ask what time they would like to dine. The restaurant is " + "open from 5 PM to 10 PM." + ), + } + ], + functions=[check_availability, transfer_to_router], + ) + + def confirmation_node() -> NodeConfig: + return NodeConfig( + name="confirm", + task_messages=[ + { + "role": "developer", + "content": "Confirm the reservation details and ask if there is anything else.", + } + ], + functions=[end_reservation, transfer_to_router], + ) + + def end_node() -> NodeConfig: + return NodeConfig( + name="end", + task_messages=[ + { + "role": "developer", + "content": "Thank them for their reservation and say goodbye.", + } + ], + post_actions=[{"type": "end_conversation"}], + ) + + # --- Flow functions ---------------------------------------------------- + + async def collect_party_size( + flow_manager: FlowManager, size: int + ) -> tuple[PartySizeResult, NodeConfig]: + """Record the number of people in the party. + + Args: + size (int): Number of people in the party. Must be between 1 and 12. + """ + flow_manager.state["party_size"] = size + return PartySizeResult(size=size), time_selection_node() + + async def check_availability( + flow_manager: FlowManager, time: str + ) -> tuple[AvailabilityResult, NodeConfig]: + """Check availability for the requested time. + + Args: + time (str): Reservation time (e.g., '6:00 PM'). + """ + party_size = flow_manager.state.get("party_size", 2) + is_available, alternatives = await reservation_system.check_availability(party_size, time) + + if is_available: + flow_manager.state["time"] = time + return AvailabilityResult(time=time, available=True), confirmation_node() + + times_list = ", ".join(alternatives) + no_availability = NodeConfig( + name="no_availability", + task_messages=[ + { + "role": "developer", + "content": ( + f"Apologize that {time} is not available. " + f"Suggest these alternative times: {times_list}." + ), + } + ], + functions=[check_availability, transfer_to_router], + ) + return AvailabilityResult(time=time, available=False), no_availability + + async def end_reservation(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Confirm and end the reservation.""" + return None, end_node() + + async def transfer_to_router( + flow_manager: FlowManager, reason: str + ) -> tuple[FlowResult, NodeConfig]: + """Hand the conversation back to the general assistant. + + Call this when the user no longer wants to make a reservation, or asks + a general question unrelated to booking a table. + + Args: + reason (str): Why control is being handed back (e.g. 'user changed + their mind', 'user asked about the menu'). + """ + logger.info(f"Worker '{RESERVATION_NAME}': handing back to '{ROUTER_NAME}' ({reason})") + await worker.activate_worker( + ROUTER_NAME, + args=LLMWorkerActivationArgs( + messages=[{"role": "developer", "content": reason}], + ), + deactivate_self=True, + ) + return {"status": "transferred"}, party_size_node() + + # --- Activation: start or resume the flow ------------------------------ + + async def end_conversation_action(action: dict) -> None: + await worker.end(reason=action.get("reason")) + + flow_manager.register_action("end_conversation", end_conversation_action) + + initialized = {"done": False} + + @worker.event_handler("on_activated") + async def on_activated(worker, args): + if not initialized["done"]: + initialized["done"] = True + await flow_manager.initialize(party_size_node()) + else: + # Control was handed back to us; restart the reservation flow. + await flow_manager.set_node_from_config(party_size_node()) + + return worker + + +# ============================================================================= +# Router worker: free-form LLM that routes to the reservation flow. +# ============================================================================= + + +class RouterWorker(LLMWorker): + """Open-ended assistant that transfers to the reservation worker.""" + + @tool(cancel_on_interruption=False) + async def transfer_to_reservation(self, params: FunctionCallParams, reason: str): + """Transfer the user to the reservation assistant. + + Call this as soon as the user wants to book, change, or ask about + making a table reservation. + + Args: + reason (str): Why the user is being transferred. + """ + logger.info(f"Worker '{self.name}': transferring to '{RESERVATION_NAME}' ({reason})") + await self.activate_worker( + RESERVATION_NAME, + args=LLMWorkerActivationArgs( + messages=[{"role": "developer", "content": reason}], + ), + deactivate_self=True, + result_callback=params.result_callback, + ) + + @tool + async def end_conversation(self, params: FunctionCallParams, reason: str): + """End the conversation when the user says goodbye. + + Args: + reason (str): Why the conversation is ending. + """ + logger.info(f"Worker '{self.name}': ending conversation ({reason})") + await self.end( + reason=reason, + messages=[{"role": "developer", "content": reason}], + result_callback=params.result_callback, + ) + + +def build_router(llm: Any) -> RouterWorker: + """Build the free-form router worker.""" + return RouterWorker(ROUTER_NAME, llm=llm, bridged=()) + + +# ============================================================================= +# Bot setup. +# ============================================================================= + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """Wire up the transport, the shared context, and the three workers.""" + logger.info("Starting multi-worker handoff bot") + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="9626c31c-bec5-4cca-baa8-f8ba9e84c8bc", # Jacqueline + ), + ) + + # The shared conversation context lives in the main worker. Both the router + # and the reservation worker speak into this same history via the bus. + context = LLMContext() + aggregators = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + # The main bridge sends user-side context to the active worker and brings + # its generated frames back so the TTS can speak them. + bridge = BusBridgeProcessor( + bus=runner.bus, + worker_name=MAIN_NAME, + name=f"{MAIN_NAME}::BusBridge", + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + aggregators.user(), + bridge, + tts, + transport.output(), + aggregators.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + name=MAIN_NAME, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Each LLM worker gets its own LLM service instance. + router = build_router(create_llm()) + reservation = build_reservation_worker( + llm=create_llm(), + context_aggregator=aggregators, + reservation_system=MockReservationSystem(), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + # Start the conversation with the router. + await worker.activate_worker( + ROUTER_NAME, + args=LLMWorkerActivationArgs( + messages=[ + { + "role": "developer", + "content": ( + "You are a friendly assistant for La Maison restaurant. Greet the " + "user, mention you can answer questions or book a table, and ask how " + "you can help. When the user wants to make a reservation, call the " + "transfer_to_reservation tool. If the user says goodbye, call the " + "end_conversation tool. Do not mention transferring, just do it " + "seamlessly. Keep responses brief, this is a voice conversation." + ), + } + ], + ), + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info("Client disconnected") + await runner.cancel() + + await runner.add_workers(router, reservation, worker) + + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/patient_intake.py b/examples/flows/patient_intake.py new file mode 100644 index 00000000000..de068185fdd --- /dev/null +++ b/examples/flows/patient_intake.py @@ -0,0 +1,422 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""A patient intake flow example for Pipecat Flows. + +This example demonstrates a medical intake system using flows with direct +functions where conversation paths are determined at runtime. The flow handles: + +1. Patient identity verification through birthday +2. Prescription collection +3. Allergy information gathering +4. Medical conditions collection +5. Visit reason documentation +6. Information verification and confirmation + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- CARTESIA_API_KEY (for TTS) +- DEEPGRAM_API_KEY (for STT) +- DAILY_API_KEY (optionalfor transport) +- LLM API key (varies by provider - see env.example) +""" + +import os +from typing import TypedDict + +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import ( + ContextStrategy, + ContextStrategyConfig, + FlowManager, + NodeConfig, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# Type definitions +class Prescription(TypedDict): + medication: str + dosage: str + + +class Allergy(TypedDict): + name: str + + +class Condition(TypedDict): + name: str + + +class VisitReason(TypedDict): + name: str + + +# Result types for each handler +class BirthdayVerificationResult(TypedDict): + verified: bool + + +class PrescriptionRecordResult(TypedDict): + count: int + + +class AllergyRecordResult(TypedDict): + count: int + + +class ConditionRecordResult(TypedDict): + count: int + + +class VisitReasonRecordResult(TypedDict): + count: int + + +# Functions +async def verify_birthday( + flow_manager: FlowManager, birthday: str +) -> tuple[BirthdayVerificationResult, NodeConfig]: + """Verify the user has provided their correct birthday. Once confirmed, the next step is to record the user's prescriptions. + + Args: + birthday (str): The user's birthdate (convert to YYYY-MM-DD format). + """ + # In a real app, this would verify against patient records + is_valid = birthday == "1983-01-01" + + # Store verification result in flow state + flow_manager.state["birthday_verified"] = is_valid + flow_manager.state["birthday"] = birthday + + return BirthdayVerificationResult(verified=is_valid), create_prescriptions_node() + + +async def record_prescriptions( + flow_manager: FlowManager, prescriptions: list[dict] +) -> tuple[PrescriptionRecordResult, NodeConfig]: + """Record the user's prescriptions. Once confirmed, the next step is to collect allergy information. + + Args: + prescriptions (list[dict]): List of prescription objects, each with "medication" (str, the medication's name) and "dosage" (str, the prescription's dosage). + """ + # Store prescriptions in flow state + flow_manager.state["prescriptions"] = prescriptions + + # In a real app, this would store in patient records + return PrescriptionRecordResult(count=len(prescriptions)), create_allergies_node() + + +async def record_allergies( + flow_manager: FlowManager, allergies: list[dict] +) -> tuple[AllergyRecordResult, NodeConfig]: + """Record the user's allergies. Once confirmed, then next step is to collect medical conditions. + + Args: + allergies (list[dict]): List of allergy objects, each with "name" (str, what the user is allergic to). + """ + # Store allergies in flow state + flow_manager.state["allergies"] = allergies + + # In a real app, this would store in patient records + return AllergyRecordResult(count=len(allergies)), create_conditions_node() + + +async def record_conditions( + flow_manager: FlowManager, conditions: list[dict] +) -> tuple[ConditionRecordResult, NodeConfig]: + """Record the user's medical conditions. Once confirmed, the next step is to collect visit reasons. + + Args: + conditions (list[dict]): List of condition objects, each with "name" (str, the user's medical condition). + """ + # Store conditions in flow state + flow_manager.state["conditions"] = conditions + + # In a real app, this would store in patient records + return ConditionRecordResult(count=len(conditions)), create_visit_reasons_node() + + +async def record_visit_reasons( + flow_manager: FlowManager, visit_reasons: list[dict] +) -> tuple[VisitReasonRecordResult, NodeConfig]: + """Record the reasons for their visit. Once confirmed, the next step is to verify all information. + + Args: + visit_reasons (list[dict]): List of visit reason objects, each with "name" (str, the user's reason for visiting). + """ + # Store visit reasons in flow state + flow_manager.state["visit_reasons"] = visit_reasons + + # In a real app, this would store in patient records + return VisitReasonRecordResult(count=len(visit_reasons)), create_verification_node() + + +async def revise_information(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Return to prescriptions to revise information.""" + return None, create_prescriptions_node() + + +async def confirm_information(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Proceed with confirmed information.""" + return None, create_confirmation_node() + + +async def complete_intake(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Complete the intake process.""" + return None, create_end_node() + + +# Node creation functions +def create_initial_node() -> NodeConfig: + """Create the initial node for patient identity verification.""" + return NodeConfig( + name="start", + role_message="You are Jessica, an agent for Tri-County Health Services. You must ALWAYS use one of the available functions to progress the conversation. Be professional but friendly.", + task_messages=[ + { + "role": "developer", + "content": "Start by introducing yourself to Chad Bailey, then ask for their date of birth, including the year. Once they provide their birthday, use verify_birthday to check it. If verified (1983-01-01), proceed to prescriptions.", + } + ], + functions=[verify_birthday], + ) + + +def create_prescriptions_node() -> NodeConfig: + """Create the prescriptions collection node.""" + return NodeConfig( + name="get_prescriptions", + role_message="You are Jessica, an agent for Tri-County Health Services. You must ALWAYS use one of the available functions to progress the conversation. Be professional but friendly.", + task_messages=[ + { + "role": "developer", + "content": "This step is for collecting prescriptions. Ask them what prescriptions they're taking, including the dosage. Get to the point by saying 'Thanks for confirming that. First up, what prescriptions are you currently taking, including the dosage for each medication?'. After recording prescriptions (or confirming none), proceed to allergies.", + } + ], + context_strategy=ContextStrategyConfig(strategy=ContextStrategy.RESET), + functions=[record_prescriptions], + ) + + +def create_allergies_node() -> NodeConfig: + """Create the allergies collection node.""" + return NodeConfig( + name="get_allergies", + task_messages=[ + { + "role": "developer", + "content": "Collect allergy information. Ask about any allergies they have. After recording allergies (or confirming none), proceed to medical conditions.", + } + ], + functions=[record_allergies], + ) + + +def create_conditions_node() -> NodeConfig: + """Create the medical conditions collection node.""" + return NodeConfig( + name="get_conditions", + task_messages=[ + { + "role": "developer", + "content": "Collect medical condition information. Ask about any medical conditions they have. After recording conditions (or confirming none), proceed to visit reasons.", + } + ], + functions=[record_conditions], + ) + + +def create_visit_reasons_node() -> NodeConfig: + """Create the visit reasons collection node.""" + return NodeConfig( + name="get_visit_reasons", + task_messages=[ + { + "role": "developer", + "content": "Collect information about the reason for their visit. Ask what brings them to the doctor today. After recording their reasons, proceed to verification.", + } + ], + functions=[record_visit_reasons], + ) + + +def create_verification_node() -> NodeConfig: + """Create the information verification node with context reset and summary.""" + return NodeConfig( + name="verify", + task_messages=[ + { + "role": "developer", + "content": """Review all collected information with the patient. Follow these steps: +1. Summarize their prescriptions, allergies, conditions, and visit reasons +2. Ask if everything is correct +3. Use the appropriate function based on their response + +Be thorough in reviewing all details and wait for explicit confirmation.""", + } + ], + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, + summary_prompt=( + "Summarize the patient intake conversation, including their birthday, " + "prescriptions, allergies, medical conditions, and reasons for visiting. " + "Focus on the specific medical information provided." + ), + ), + functions=[revise_information, confirm_information], + ) + + +def create_confirmation_node() -> NodeConfig: + """Create the final confirmation node.""" + return NodeConfig( + name="confirm", + task_messages=[ + { + "role": "developer", + "content": "Once confirmed, thank them, then use the complete_intake function to end the conversation.", + } + ], + functions=[complete_intake], + ) + + +def create_end_node() -> NodeConfig: + """Create the final end node.""" + return NodeConfig( + name="end", + task_messages=[ + { + "role": "developer", + "content": "Thank them for their time and end the conversation.", + } + ], + post_actions=[{"type": "end_conversation"}], + ) + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """Run the patient intake bot.""" + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ), + ) + # LLM service is created using the create_llm function from utils.py + # Default is OpenAI; can be changed by setting LLM_PROVIDER environment variable + llm = create_llm() + + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + # Kick off the conversation with the initial node + await flow_manager.initialize(create_initial_node()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/podcast_interview.py b/examples/flows/podcast_interview.py new file mode 100644 index 00000000000..e72896c1023 --- /dev/null +++ b/examples/flows/podcast_interview.py @@ -0,0 +1,281 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Pipecat Podcast Interview Example. + +This example demonstrates a podcast interview flow using Pipecat Flows. + +The flow handles: +1. Introduction and guest introduction +2. Topic selection +3. Interview with multiple questions and follow-ups +4. Conclusion and wrap-up +5. Final thank you + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- CARTESIA_API_KEY (for TTS) +- DEEPGRAM_API_KEY (for STT) +- DAILY_API_KEY (for transport) +- LLM API key (varies by provider - see env.example) +""" + +import os +from typing import TypedDict + +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import FlowManager, NodeConfig +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# Type definitions +class ProceedToTopicResult(TypedDict): + """Result type for proceed_to_topic function""" + + guest_summary: str + + +class StartInterviewResult(TypedDict): + """Result type for start_interview function""" + + topic: str + + +# Functions for Introduction Node +async def proceed_to_topic( + flow_manager: FlowManager, guest_summary: str +) -> tuple[ProceedToTopicResult | None, NodeConfig]: + """Use after the guest has introduced themselves. + + Args: + guest_summary (str): A quick summary of who the guest is (name, role, area of expertise, etc.). + """ + return ProceedToTopicResult(guest_summary=guest_summary), create_topic_node() + + +# Functions for Topic Node +async def start_interview( + flow_manager: FlowManager, topic: str +) -> tuple[StartInterviewResult | None, NodeConfig]: + """Use this when the guest has shared a clear topic they want to explore. + + Args: + topic (str): The topic the guest wants to discuss. + """ + return StartInterviewResult(topic=topic), create_interview_node() + + +# Functions for Interview Node +async def next_question(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Use this after you've thoroughly explored the current aspect with multiple questions and follow-ups.""" + return None, create_interview_node() + + +async def wrap_up(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Use this when you've gathered substantial insights and are ready to wrap up.""" + return None, create_conclusion_node() + + +# Functions for Conclusion Node +async def end_interview(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Use this after the guest has shared their final thoughts.""" + return None, create_final_node() + + +def create_introduction_node() -> NodeConfig: + """Create the Introduction node.""" + return NodeConfig( + name="introduction", + role_message="You are a warm, engaging podcast host with a natural conversational style. You're genuinely curious about your guests and skilled at making them feel comfortable while drawing out interesting insights. Your questions flow naturally, and you listen actively, building on what your guest shares.", + task_messages=[ + { + "role": "developer", + "content": "Welcome the guest warmly and enthusiastically. Focus this exchange on getting to know who they are. Invite them to briefly introduce themselves—name, role, current focus, or anything fun they'd like to share. Ask one follow-up question if it helps clarify or highlight something interesting about them. Once you feel you have a clear introduction, use the proceed_to_topic function to move into topic selection.", + } + ], + functions=[proceed_to_topic], + ) + + +def create_topic_node() -> NodeConfig: + """Create the Topic Selection node.""" + return NodeConfig( + name="topic", + task_messages=[ + { + "role": "developer", + "content": "Now that you know who the guest is, help them choose the topic they'd like to explore. Refer back to their introduction to personalize the transition. Ask what topic, story, or challenge they're excited to discuss today. Show genuine interest and, if needed, ask a clarifying question to make sure you understand the angle they want to take. Once the topic feels clear and specific enough to dive into, use the start_interview function.", + } + ], + functions=[start_interview], + ) + + +def create_interview_node() -> NodeConfig: + """Create the Interview node.""" + return NodeConfig( + name="interview", + task_messages=[ + { + "role": "developer", + "content": "You're now in the heart of the interview. Start by introducing the topic with enthusiasm, then dive deep into one key aspect at a time. Ask open-ended, thoughtful questions that invite storytelling and personal insights. Listen actively to responses and ask natural follow-up questions that build on what your guest shares—dig deeper into interesting points, ask for examples, or explore the 'why' behind their answers. Keep the conversation flowing naturally, like a genuine dialogue between friends. Once you've thoroughly explored an aspect (typically after 3-5 exchanges), use the next_question function to smoothly transition to the next key aspect. After covering 3 key aspects of the topic, use the wrap_up function to conclude the interview.", + } + ], + functions=[next_question, wrap_up], + ) + + +def create_conclusion_node() -> NodeConfig: + """Create the Conclusion node.""" + return NodeConfig( + name="conclusion", + task_messages=[ + { + "role": "developer", + "content": "Express genuine appreciation for the conversation and the insights your guest shared. Summarize 2-3 key takeaways or memorable points from your discussion in a warm, conversational way—this helps reinforce the value of the conversation. Then, ask your guest if they have any final thoughts, a last word, or anything else they'd like to add. Wait for their response, then use the end_interview function to wrap up.", + } + ], + functions=[end_interview], + ) + + +def create_final_node() -> NodeConfig: + """Create the Final node.""" + return NodeConfig( + name="final", + task_messages=[ + { + "role": "developer", + "content": "Thank the guest one final time for joining you and for sharing their insights. End the conversation on a positive, warm note.", + } + ], + post_actions=[{"type": "end_conversation"}], + ) + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ), + ) + + # LLM service is created using the create_llm function from utils.py + # Default is OpenAI; can be changed by setting LLM_PROVIDER environment variable + llm = create_llm() + + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await flow_manager.initialize(create_introduction_node()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point for the bot starter.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/flows/restaurant_reservation.py b/examples/flows/restaurant_reservation.py new file mode 100644 index 00000000000..9e0047df1b1 --- /dev/null +++ b/examples/flows/restaurant_reservation.py @@ -0,0 +1,364 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""A restaurant reservation flow example for Pipecat Flows. + +This example demonstrates a restaurant reservation system using flows where +conversation paths are determined at runtime. The flow handles: + +1. Greeting and party size collection +2. Time preference gathering with availability checking +3. Alternative time suggestions when unavailable +4. Reservation confirmation + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- CARTESIA_API_KEY (for TTS) +- DEEPGRAM_API_KEY (for STT) +- DAILY_API_KEY (for transport) +- LLM API key (varies by provider - see env.example) +""" + +import asyncio +import os +import sys +from typing import TypedDict + +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.evals.transport import EvalTransportParams +from pipecat.flows import FlowManager, NodeConfig +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + # Behavioral evals: run with `-t eval` to drive this bot via `pipecat eval`. + "eval": lambda: EvalTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +# Mock reservation system +class MockReservationSystem: + """Simulates a restaurant reservation system API.""" + + def __init__(self): + # Mock data: Times that are "fully booked" + self.booked_times = {"7:00 PM", "8:00 PM"} # Changed to AM/PM format + + async def check_availability( + self, party_size: int, requested_time: str + ) -> tuple[bool, list[str]]: + """Check if a table is available for the given party size and time.""" + # Simulate API call delay + await asyncio.sleep(0.5) + + # Check if time is booked + is_available = requested_time not in self.booked_times + + # If not available, suggest alternative times + alternatives = [] + if not is_available: + base_times = ["5:00 PM", "6:00 PM", "7:00 PM", "8:00 PM", "9:00 PM", "10:00 PM"] + alternatives = [t for t in base_times if t not in self.booked_times] + + return is_available, alternatives + + +# Initialize mock system +reservation_system = MockReservationSystem() + + +# Type definitions for function results +class PartySizeResult(TypedDict): + size: int + status: str + + +class TimeResult(TypedDict): + status: str + time: str + available: bool + alternative_times: list[str] + + +# Function handlers +async def collect_party_size( + flow_manager: FlowManager, size: int +) -> tuple[PartySizeResult, NodeConfig]: + """ + Record the number of people in the party. + + Args: + size (int): Number of people in the party. Must be between 1 and 12. + """ + # Result: the recorded party size + result = PartySizeResult(size=size, status="success") + + # Next node: time selection + next_node = create_time_selection_node() + + return result, next_node + + +async def check_availability( + flow_manager: FlowManager, time: str, party_size: int +) -> tuple[TimeResult, NodeConfig]: + """ + Check availability for requested time. + + Args: + time (str): Requested reservation time in "HH:MM AM/PM" format. Must be between 5 PM and 10 PM. + party_size (int): Number of people in the party. + """ + # Check availability with mock API + is_available, alternative_times = await reservation_system.check_availability(party_size, time) + + # Result: availability status and alternative times, if any + result = TimeResult( + status="success", time=time, available=is_available, alternative_times=alternative_times + ) + + # Next node: confirmation or no availability + if is_available: + next_node = create_confirmation_node() + else: + next_node = create_no_availability_node(alternative_times) + + return result, next_node + + +async def end_conversation(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """End the conversation.""" + return None, create_end_node() + + +# Node configurations +def create_initial_node(wait_for_user: bool) -> NodeConfig: + """Create initial node for party size collection.""" + return NodeConfig( + name="initial", + role_message="You are a restaurant reservation assistant for La Maison, an upscale French restaurant. Be casual and friendly. This is a voice conversation, so avoid special characters and emojis.", + task_messages=[ + { + "role": "developer", + "content": "Warmly greet the customer and ask how many people are in their party. This is your only job for now; if the customer asks for something else, politely remind them you can't do it.", + } + ], + functions=[collect_party_size], + respond_immediately=not wait_for_user, + ) + + +def create_time_selection_node() -> NodeConfig: + """Create node for time selection and availability check.""" + logger.debug("Creating time selection node") + return NodeConfig( + name="get_time", + task_messages=[ + { + "role": "developer", + "content": "Ask what time they'd like to dine. Restaurant is open 5 PM to 10 PM.", + } + ], + functions=[check_availability], + ) + + +def create_confirmation_node() -> NodeConfig: + """Create confirmation node for successful reservations.""" + return NodeConfig( + name="confirm", + task_messages=[ + { + "role": "developer", + "content": ( + "Confirm the reservation details and ask if they need anything else. " + "When the customer says they're all set or have nothing else, call the " + "end_conversation function to wrap up. If they still need something, help " + "them and then ask again whether there's anything else." + ), + } + ], + functions=[end_conversation], + ) + + +def create_no_availability_node(alternative_times: list[str]) -> NodeConfig: + """Create node for handling no availability.""" + times_list = ", ".join(alternative_times) + return NodeConfig( + name="no_availability", + task_messages=[ + { + "role": "developer", + "content": ( + f"Apologize that the requested time is not available. " + f"Suggest these alternative times: {times_list}. " + "Ask if they'd like to try one of these times. If they pick a time, check " + "its availability. If they'd rather not book after all, call the " + "end_conversation function to wrap up." + ), + } + ], + functions=[check_availability, end_conversation], + ) + + +def create_end_node() -> NodeConfig: + """Create the final node.""" + return NodeConfig( + name="end", + task_messages=[ + { + "role": "developer", + "content": "Thank them and end the conversation.", + } + ], + functions=[], + post_actions=[{"type": "end_conversation"}], + ) + + +async def run_bot( + transport: BaseTransport, runner_args: RunnerArguments, wait_for_user: bool = False +): + """Run the restaurant reservation bot.""" + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ), + ) + # LLM service is created using the create_llm function from utils.py + # Default is OpenAI; can be changed by setting LLM_PROVIDER environment variable + llm = create_llm() + + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + # Kick off the conversation with the initial node + await flow_manager.initialize(create_initial_node(wait_for_user)) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await worker.cancel() + + runner = WorkerRunner(handle_sigint=runner_args.handle_sigint) + await runner.add_workers(worker) + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + # Use the global flag if available, otherwise default to False + wait_for_user = globals().get("WAIT_FOR_USER", False) + + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args, wait_for_user) + + +if __name__ == "__main__": + import argparse + import sys + + # Parse our custom argument first + parser = argparse.ArgumentParser(description="Restaurant reservation bot") + parser.add_argument( + "--wait-for-user", + action="store_true", + help="If set, the bot will wait for the user to speak first", + ) + + # Parse only our known args, leave the rest for the runner + args, remaining = parser.parse_known_args() + + # Store the flag globally so bot() can access it + WAIT_FOR_USER = args.wait_for_user + + # Remove our custom arg from sys.argv and let the runner handle the rest + if "--wait-for-user" in sys.argv: + sys.argv.remove("--wait-for-user") + + # Now run the standard runner + from pipecat.runner.run import main + + main() diff --git a/examples/flows/utils.py b/examples/flows/utils.py new file mode 100644 index 00000000000..9907001235a --- /dev/null +++ b/examples/flows/utils.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Utility functions for Pipecat Flows examples. + +This module provides helper functions to reduce boilerplate and keep examples +focused on the core flow concepts. +""" + +import os +from typing import Any + + +def create_llm(provider: str | None = None, model: str | None = None) -> Any: + """Create an LLM service instance based on environment configuration. + + Args: + provider: LLM provider name. If None, uses LLM_PROVIDER env var (defaults to 'openai') + model: Model name. If None, uses provider's default model + + Returns: + Configured LLM service instance + + Raises: + ValueError: If provider is unsupported or required API keys are missing + + Supported Providers: + - openai: Requires OPENAI_API_KEY + - openai_responses: Requires OPENAI_API_KEY + - anthropic: Requires ANTHROPIC_API_KEY + - google: Requires GOOGLE_API_KEY + - aws: Uses AWS default credential chain (SSO, environment variables, or IAM roles) + Optionally set AWS_REGION (defaults to us-west-2) + + Usage: + # Use default provider (from LLM_PROVIDER env var, defaults to OpenAI) + llm = create_llm() + + # Use specific provider + llm = create_llm("anthropic") + + # Use specific provider and model + llm = create_llm("openai", "gpt-4o-mini") + + # Use AWS Bedrock (requires AWS credentials via SSO, env vars, or IAM) + llm = create_llm("aws") + """ + if provider is None: + provider = os.getenv("LLM_PROVIDER", "openai_responses").lower() + else: + provider = provider.lower() + + # Provider configurations + configs = { + "openai": { + "service": "pipecat.services.openai.llm.OpenAILLMService", + "api_key_env": "OPENAI_API_KEY", + "default_model": "gpt-4.1", + }, + "openai_responses": { + "service": "pipecat.services.openai.responses.llm.OpenAIResponsesLLMService", + "api_key_env": "OPENAI_API_KEY", + "default_model": "gpt-4.1", + }, + "anthropic": { + "service": "pipecat.services.anthropic.llm.AnthropicLLMService", + "api_key_env": "ANTHROPIC_API_KEY", + "default_model": "claude-sonnet-4-6", + }, + "google": { + "service": "pipecat.services.google.llm.GoogleLLMService", + "api_key_env": "GOOGLE_API_KEY", + "default_model": "gemini-2.5-flash", + }, + "aws": { + "service": "pipecat.services.aws.llm.AWSBedrockLLMService", + "api_key_env": None, # AWS uses default credential chain + "default_model": "us.anthropic.claude-sonnet-4-6", + "region": "us-west-2", + }, + } + + config = configs.get(provider) + if not config: + available = ", ".join(configs.keys()) + raise ValueError(f"Unsupported LLM provider: {provider}. Available: {available}") + + # Dynamic import of the LLM service + module_path, class_name = config["service"].rsplit(".", 1) + module = __import__(module_path, fromlist=[class_name]) + service_class = getattr(module, class_name) + + # Get API key (skip for AWS which uses default credential chain) + if provider == "aws" or config["api_key_env"] is None: + api_key = None # AWS uses default credential chain + else: + api_key = os.getenv(config["api_key_env"]) + if not api_key: + raise ValueError(f"Missing API key: {config['api_key_env']} for provider: {provider}") + + # Use provided model or default + selected_model = model or config["default_model"] + + # Build settings + settings_kwargs = {"model": selected_model} + if provider == "aws": + settings_kwargs["temperature"] = 0.8 + settings = service_class.Settings(**settings_kwargs) + + # Build constructor kwargs + kwargs = {"settings": settings} + if api_key is not None: + kwargs["api_key"] = api_key + if provider == "aws": + kwargs["aws_region"] = os.getenv("AWS_REGION", config["region"]) + + return service_class(**kwargs) diff --git a/examples/flows/warm_transfer.py b/examples/flows/warm_transfer.py new file mode 100644 index 00000000000..bb9f2452be1 --- /dev/null +++ b/examples/flows/warm_transfer.py @@ -0,0 +1,713 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""'Warm Handoff' Example using Pipecat Flows. + +This example demonstrates how to create a bot that transfers a customer to a human agent when the bot is unable to fulfill the customers's request. +This example uses: +- Pipecat Flows for conversation management +- LLM selection (OpenAI, Anthropic, Google, AWS Bedrock) +- Daily as the transport service + +The bot asks how they could be of assistance, and offers to provide information about store location and hours of operation, or begin placing an order. +If the customer says they'd like to do the former, the bot provides an answer. +If the customer says they'd like to do the latter, the bot tries, fails, and transfers the customer to a human agent. +The bot then brings the agent up to speed on the customer's issue before connecting them to the customer and dropping out of the call. + +The various parties join with the following Daily meeting token properties: +- bot: + - owner: true +- customer: + - user_id: customer +- human agent: + - user_id: agent + +The bot joins with a token with the following properties: +- owner: true + +Multi-LLM Support: +Set LLM_PROVIDER environment variable to choose your LLM provider. +Supported: openai_responses (default), openai, anthropic, google, aws + +Requirements: +- Daily room URL +- Daily API key +- LLM API key (varies by provider - see env.example) +- Deepgram API key +- Cartesia API key +""" + +import asyncio +import atexit +import os +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any, TypedDict + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from utils import create_llm + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.flows import ContextStrategyConfig, FlowManager, NodeConfig +from pipecat.flows.types import ActionConfig, ContextStrategy +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.daily import configure +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.daily.transport import DailyParams, DailyTransport +from pipecat.transports.daily.utils import ( + DailyMeetingTokenParams, + DailyMeetingTokenProperties, + DailyRESTHelper, +) +from pipecat.workers.runner import WorkerRunner + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +# Flow nodes: +# +# 1. initial_customer_interaction +# The initial node, where the bot interacts with the customer and tries to help with their requests. +# Functions: +# - check_store_location_and_hours_of_operation (always succeeds) +# - start_order (always fails) +# - end_customer_conversation +# Transitions to either: +# - continued_customer_interaction +# - transferring_to_human_agent +# +# 2. continued_customer_interaction +# The bot has already helped the customer with something. Now they're helping them with something else. +# Functions: +# - check_store_location_and_hours_of_operation (always succeeds) +# - start_order (always fails) +# - end_customer_conversation +# Transitions to either: +# - continued_customer_interaction +# - transferring_to_human_agent +# +# 3. transferring_to_human_agent +# The customer is asked to please hold while the bot transfers them to a human agent. Hold music plays while the customer waits. +# Transition: +# - As soon as the agent connects to the room, we transition to human_agent_interaction. +# +# 4. human_agent_interaction +# The bot fills in the human agent about what the customer was trying to accomplish that the bot was unable to help with, and what went wrong. +# The customer continues to hear hold music. +# Functions: +# - connect_human_agent_and_customer +# +# 5a. end_customer_conversation +# The bot says goodbye to the customer and ends the conversation. +# This is how a conversation ends when a human agent did not need to be brought in. +# +# 5b. end_human_agent_conversation +# The bot tells the agent that they're being patched through to the customer and ends the conversation (leaving the customer and agent in the room talking to each other). + + +# Type definitions +class StoreLocationAndHoursOfOperationResult(TypedDict): + status: str + store_location: str + hours_of_operation: str + + +class StartOrderResult(TypedDict): + status: str + + +# Tool functions +async def check_store_location_and_hours_of_operation( + flow_manager: FlowManager, +) -> tuple[StoreLocationAndHoursOfOperationResult, NodeConfig]: + """Check store location and hours of operation.""" + result = StoreLocationAndHoursOfOperationResult( + status="success", + store_location="123 Main St, Anytown, USA", + hours_of_operation="9am to 5pm, Monday through Friday", + ) + next_node = next_node_after_customer_task(result) + return result, next_node + + +async def start_order(flow_manager: FlowManager) -> tuple[StartOrderResult, NodeConfig]: + """Start placing an order.""" + result = StartOrderResult(status="error") + next_node = next_node_after_customer_task(result) + return result, next_node + + +# Action handlers +async def mute_customer(action: dict, flow_manager: FlowManager): + """Mute the customer. + + Do it by revoking their canSnd permission, which both mutes them and ensures that they can't unmute. + """ + assert isinstance(flow_manager.transport, DailyTransport) + transport: DailyTransport = flow_manager.transport + customer_participant_id = get_customer_participant_id(transport=transport) + + if customer_participant_id: + await transport.update_remote_participants( + remote_participants={ + customer_participant_id: { + "permissions": { + "canSend": [], + } + } + } + ) + + +async def start_hold_music(action: dict, flow_manager: FlowManager): + hold_music_args = flow_manager.state["hold_music_args"] + flow_manager.state["hold_music_process"] = await asyncio.create_subprocess_exec( + sys.executable, + str(hold_music_args["script_path"]), + "-m", + hold_music_args["room_url"], + "-t", + hold_music_args["token"], + "-i", + hold_music_args["wav_file_path"], + ) + + +async def make_customer_hear_only_hold_music(action: dict, flow_manager: FlowManager): + """Make it so the customer only hears hold music. + + We don't want them hearing the bot and the human agent talking. + """ + assert isinstance(flow_manager.transport, DailyTransport) + transport: DailyTransport = flow_manager.transport + customer_participant_id = get_customer_participant_id(transport=transport) + + if customer_participant_id: + await transport.update_remote_participants( + remote_participants={ + customer_participant_id: { + "permissions": {"canReceive": {"byUserId": {"hold-music": True}}} + } + } + ) + + +async def print_human_agent_join_url(action: dict, flow_manager: FlowManager): + """Print the URL for joining as a human agent.""" + logger.info(f"\n\nJOIN AS AGENT:\n{flow_manager.state['human_agent_join_url']}\n") + + +async def unmute_customer_and_make_humans_hear_each_other(action: dict, flow_manager: FlowManager): + """Unmute the customer and make it so the customer and human agent can hear each other.""" + assert isinstance(flow_manager.transport, DailyTransport) + transport: DailyTransport = flow_manager.transport + customer_participant_id = get_customer_participant_id(transport=transport) + agent_participant_id = get_human_agent_participant_id(transport=transport) + + if customer_participant_id and agent_participant_id: + await transport.update_remote_participants( + remote_participants={ + customer_participant_id: { + "permissions": { + "canSend": ["microphone"], + "canReceive": {"byUserId": {"agent": True}}, + }, + "inputsEnabled": {"microphone": True}, + }, + agent_participant_id: { + "permissions": {"canReceive": {"byUserId": {"customer": True}}} + }, + } + ) + + +async def end_customer_conversation(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """End the conversation.""" + return None, create_end_customer_conversation_node() + + +async def connect_human_agent_and_customer(flow_manager: FlowManager) -> tuple[None, NodeConfig]: + """Connect the human agent to the customer.""" + return None, create_end_human_agent_conversation_node() + + +# Helpers +def next_node_after_customer_task(result: Mapping[str, Any]) -> NodeConfig: + """Transition to either the "continued_customer_interaction" node or "transferring_to_human_agent" node, depending on the outcome of the previous customer task""" + if result.get("status") == "success": + return create_continued_customer_interaction_node() + else: + return create_transferring_to_human_agent_node() + + +# Transitions +async def start_human_agent_interaction(flow_manager: FlowManager): + """Transition to the "human_agent_interaction" node.""" + await flow_manager.set_node_from_config(create_human_agent_interaction_node()) + + +# Node configuration +def create_initial_customer_interaction_node() -> NodeConfig: + """Create the "initial_customer_interaction" node. + This is the initial node where the bot interacts with the customer and tries to help with their requests. + """ + return NodeConfig( + name="customer_interaction", + role_message="You are an assistant for ABC Widget Company. You must ALWAYS use the available functions to progress the conversation. This is a phone conversation and your responses will be converted to audio. Keep the conversation friendly, casual, and polite. Avoid outputting special characters and emojis.", + task_messages=[ + { + "role": "developer", + "content": """Start off by greeting the customer. Then ask how you could help, offering two choices of what you could help with: you could provide store location and hours of operation, or begin placing an order. Be friendly and casual. + + To help the customer: + - Use the check_store_location_and_hours_of_operation function to check store location and hours of operation to provide to the customer + - Use the start_order function to begin placing an order on the customer's behalf + + If the customer wants to end the conversation, call the end_customer_conversation function. + """, + } + ], + functions=[ + check_store_location_and_hours_of_operation, + start_order, + end_customer_conversation, + ], + ) + + +def create_continued_customer_interaction_node() -> NodeConfig: + """Create the "continued_customer_interaction" node. + This is a node where the bot interacts with the customer and tries to help with their requests. + It assumes that the bot has already previously helped the customer with something. + """ + return NodeConfig( + name="continued_customer_interaction", + task_messages=[ + { + "role": "developer", + "content": """Ask the customer there's anything else you could help them with today, or if they'd like to end the conversation. If they need more help, re-offer the two choices you offered before: you could provide store location and hours of operation, or begin placing an order. + + To help the customer: + - Use the check_store_location_and_hours_of_operation function to check store location and hours of operation to provide to the customer + - Use the start_order function to begin placing an order on the customer's behalf + + If the customer wants to end the conversation, call the end_customer_conversation function. + """, + } + ], + functions=[ + check_store_location_and_hours_of_operation, + start_order, + end_customer_conversation, + ], + ) + + +def create_transferring_to_human_agent_node() -> NodeConfig: + """Create the "transferring_to_human_agent" node. + This is the node where the customer is asked to please hold while the bot transfers them to a human agent. Hold music plays while the customer waits. + """ + return NodeConfig( + name="transferring_to_human_agent", + task_messages=[ + { + "role": "developer", + "content": "Start by apologizing to the customer that there was an issue fulfilling their last request, then inform them that they are being transferred to a human agent. Tell them to please hold while you connect them, and thank them for their patience.", + } + ], + pre_actions=[ + ActionConfig(type="function", handler=mute_customer), + ], + post_actions=[ + ActionConfig(type="function", handler=start_hold_music), + ActionConfig(type="function", handler=make_customer_hear_only_hold_music), + ActionConfig(type="function", handler=print_human_agent_join_url), + ], + ) + + +def create_human_agent_interaction_node() -> NodeConfig: + """Create the "human_agent_interaction" node. + This is the node where the bot fills in the human agent about what the customer was trying to accomplish that the bot was unable to help with, and what went wrong. + The customer continues to hear hold music. + """ + return NodeConfig( + name="human_agent_interaction", + task_messages=[ + { + "role": "developer", + "content": """You're now talking to an agent who has just joined the call. Assume that the customer you were helping up until this point can no longer hear you. Your job is to be as helpful as you can and bring the agent up to speed so that they can assist the customer. Start by greeting the agent politely and explaining what the customer was trying to do that you were unable to help with, and any relevant error details. Ask the agent if they have any questions or whether they're ready to connect to the customer. + + Once the agent tells you they're ready to connect to the customer, call the connect_human_agent_and_customer function. + """, + } + ], + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, + summary_prompt=( + "Summarize the conversation with the customer, including what they were trying to accomplish and what, if anything, went wrong while trying to fulfill their requests. Include specific error details." + ), + ), + functions=[ + connect_human_agent_and_customer, + ], + ) + + +def create_end_customer_conversation_node() -> NodeConfig: + """Create the "end_customer_conversation" node. + This is the node where the bot says goodbye to the customer and ends the conversation. + This is how a conversation ends when a human agent did not need to be brought in. + """ + return NodeConfig( + name="end_customer_conversation", + task_messages=[ + { + "role": "developer", + "content": "Thank the customer warmly and mention they can call back anytime if they need more help.", + } + ], + post_actions=[ActionConfig(type="end_conversation")], + ) + + +def create_end_human_agent_conversation_node() -> NodeConfig: + """Create the "end_human_agent_conversation" node. + This is the node where the bot tells the agent that they're being patched through to the customer and ends the conversation (leaving the customer and agent in the room talking to each other). + """ + return NodeConfig( + name="end_human_agent_conversation", + task_messages=[ + { + "role": "developer", + "content": "Tell the agent that you're patching them through to the customer right now.", + }, + ], + post_actions=[ + ActionConfig(type="function", handler=unmute_customer_and_make_humans_hear_each_other), + ActionConfig(type="end_conversation"), + ], + ) + + +# Helpers +def get_customer_participant_id(transport: DailyTransport) -> str | None: + return next( + ( + p["id"] + for p in transport.participants().values() + if not p["info"]["isLocal"] and p["info"].get("userId") == "customer" + ), + None, + ) + + +def get_human_agent_participant_id(transport: DailyTransport) -> str | None: + return next( + ( + p["id"] + for p in transport.participants().values() + if not p["info"]["isLocal"] and p["info"].get("userId") == "agent" + ), + None, + ) + + +async def get_bot_token(daily_rest_helper: DailyRESTHelper, room_url: str) -> str: + """Gets a Daily token for the bot, configured with properties: + { + user_id: "bot", + user_name: "Bot", + owner: true, + permissions: { + canReceive: { + base: false, + byUserId: { + customer: true, + agent: true + } + } + } + } + We only need the bot to be able to hear the customer and the human agent; + it shouldn't hear the hold music. + """ + return await get_token( + user_id="bot", + permissions={"canReceive": {"base": False, "byUserId": {"customer": True, "agent": True}}}, + daily_rest_helper=daily_rest_helper, + room_url=room_url, + user_name="Bot", + owner=True, + ) + + +async def get_customer_token(daily_rest_helper: DailyRESTHelper, room_url: str) -> str: + """Gets a Daily token for the customer, configured with properties: + { + user_id: "customer", + user_name: "Customer", + permissions: { + canReceive: { + base: false, + byUserId: { + bot: true + } + } + } + } + At join time we only need the customer to be able to hear the bot. + """ + return await get_token( + user_id="customer", + permissions={ + "canReceive": { + "base": False, + "byUserId": { + "bot": True, + }, + } + }, + daily_rest_helper=daily_rest_helper, + room_url=room_url, + user_name="Customer", + owner=False, + ) + + +async def get_human_agent_token(daily_rest_helper: DailyRESTHelper, room_url: str) -> str: + """Gets a Daily token for the human agent, configured with properties: + { + user_id: "agent", + user_name: "Agent", + permissions: { + canReceive: { + base: false, + byUserId: { + bot: true + } + } + } + } + At join time we only need the human agent to be able to hear the bot. + """ + return await get_token( + user_id="agent", + permissions={ + "canReceive": { + "base": False, + "byUserId": { + "bot": True, + }, + } + }, + daily_rest_helper=daily_rest_helper, + room_url=room_url, + user_name="Agent", + owner=False, + ) + + +async def get_hold_music_player_token(daily_rest_helper: DailyRESTHelper, room_url: str) -> str: + """Gets a Daily token for the hold music player, configured with properrties: + { + user_id: "hold-music", + user_name: "Hold music" + } + """ + return await get_token( + user_id="hold-music", + permissions={}, + daily_rest_helper=daily_rest_helper, + room_url=room_url, + user_name="Hold music", + owner=False, + ) + + +async def get_token( + user_id: str, + permissions: dict, + daily_rest_helper: DailyRESTHelper, + room_url: str, + user_name: str, + owner: bool, +) -> str: + return await daily_rest_helper.get_token( + room_url=room_url, + owner=owner, + params=DailyMeetingTokenParams( + properties=DailyMeetingTokenProperties( + user_id=user_id, user_name=user_name, permissions=permissions + ) + ), + ) + + +async def main(): + """Main function to set up and run the bot.""" + async with aiohttp.ClientSession() as session: + daily_rest_helper = DailyRESTHelper( + daily_api_key=os.getenv("DAILY_API_KEY", ""), + daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), + aiohttp_session=session, + ) + + # Get room URL and bot token + (room_url, _) = await configure(session) + bot_token = await get_bot_token(daily_rest_helper=daily_rest_helper, room_url=room_url) + + # Initialize services + transport = DailyTransport( + room_url=room_url, + token=bot_token, + bot_name="ABC Widget Company Bot", + params=DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY", "")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY", ""), + settings=CartesiaTTSService.Settings( + voice="d46abd1d-2d02-43e8-819f-51fb652c1c61", # Newsman + ), + ) + llm = create_llm() + + # Initialize context + context = LLMContext() + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + filter_incomplete_user_turns=True, + ), + ) + + # Create pipeline + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + # Initialize flow manager + flow_manager = FlowManager( + worker=worker, + llm=llm, + context_aggregator=context_aggregator, + transport=transport, + ) + + # Set up event handlers + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined( + transport: DailyTransport, participant: dict[str, Any] + ): + """Start the flow. + We're assuming the first participant is the customer and not the human agent. + """ + await transport.capture_participant_transcription(participant["id"]) + # Initialize flow + await flow_manager.initialize(create_initial_customer_interaction_node()) + + @transport.event_handler("on_participant_joined") + async def on_participant_joined(transport: DailyTransport, participant: dict[str, Any]): + """Handle the human agent maybe having joined the call: + - If the participant who joined is the human agent and we're currently in the "transferring_to_human_agent" node, go to the "human_agent_interaction" node. + - Otherwise...nothing, for the purposes of this demo. We're assuming the human agent won't join while the conversation flow is any other node. + """ + user_id = participant.get("info", {}).get("userId") + if user_id == "agent" and flow_manager.current_node == "transferring_to_human_agent": + await start_human_agent_interaction(flow_manager=flow_manager) + + @transport.event_handler("on_participant_left") + async def on_participant_left( + transport: DailyTransport, participant: dict[str, Any], reason: str + ): + # NOTE: an opportunity for refinement here is to handle the customer leaving while on + # hold, informing the human agent if needed + """If all human participants have left, stop the bot""" + human_participants = { + k: v + for k, v in transport.participants().items() + if v.get("info", {}).get("userId") in {"agent", "customer"} + } + if not human_participants: + await worker.cancel() + + # Print URL for joining as customer, and store URL for joining as human agent, to be printed later + customer_token = await get_customer_token( + daily_rest_helper=daily_rest_helper, room_url=room_url + ) + human_agent_token = await get_human_agent_token( + daily_rest_helper=daily_rest_helper, room_url=room_url + ) + logger.info( + f"\n\nJOIN AS CUSTOMER:\n{room_url}{'?' if '?' not in room_url else '&'}t={customer_token}\n" + ) + flow_manager.state["human_agent_join_url"] = ( + f"{room_url}{'?' if '?' not in room_url else '&'}t={human_agent_token}" + ) + + # Prepare hold music args + flow_manager.state["hold_music_args"] = { + "script_path": Path(__file__).parent / "assets" / "hold_music" / "hold_music.py", + "wav_file_path": Path(__file__).parent / "assets" / "hold_music" / "hold_music.wav", + "room_url": room_url, + "token": await get_hold_music_player_token( + daily_rest_helper=daily_rest_helper, room_url=room_url + ), + } + + # Clean up hold music process at exit, if needed + def cleanup_hold_music_process(): + hold_music_process = flow_manager.state.get("hold_music_process") + if hold_music_process: + try: + hold_music_process.terminate() + except: + # Exception if process already done; we don't care, it didn't hurt to try + pass + + atexit.register(cleanup_hold_music_process) + + # Run the pipeline + runner = WorkerRunner() + await runner.add_workers(worker) + await runner.run() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/deprecations/deprecations.json b/scripts/deprecations/deprecations.json index e629e625743..e687859ecab 100644 --- a/scripts/deprecations/deprecations.json +++ b/scripts/deprecations/deprecations.json @@ -101,6 +101,83 @@ "message": "Use :class:`AICQuailVADAnalyzer` instead. Will be removed in 1.6.0.", "location": "pipecat/audio/vad/aic_vad.py" }, + { + "subject": "FlowManager.task", + "module": "pipecat.flows.manager", + "kind": "parameter", + "deprecated_in": "1.5.0", + "removed_in": "2.0.0", + "relation": "use_existing", + "replacement": "worker", + "message": "Use ``worker`` instead. Will be removed in 2.0.0.", + "location": "pipecat/flows/manager.py" + }, + { + "subject": "FlowManager.task", + "module": "pipecat.flows.manager", + "kind": "property", + "deprecated_in": "1.5.0", + "removed_in": "2.0.0", + "relation": "use_existing", + "replacement": "FlowManager.worker", + "message": "`FlowManager.task` is deprecated since 1.5.0 and will be removed in 2.0.0. Use `FlowManager.worker` instead.", + "location": "pipecat/flows/manager.py" + }, + { + "subject": "ContextStrategy.RESET_WITH_SUMMARY", + "module": "pipecat.flows.types", + "kind": "parameter", + "deprecated_in": "1.5.0", + "removed_in": "2.0.0", + "relation": "use_existing", + "replacement": "LLMSummarizeContextFrame", + "message": "Use :class:`LLMSummarizeContextFrame` instead — push it in a pre-action to trigger on-demand summarization during a node transition. See https://docs.pipecat.ai/guides/fundamentals/context-summarization. Will be removed in 2.0.0.", + "location": "pipecat/flows/types.py" + }, + { + "subject": "ContextStrategyConfig.summary_prompt", + "module": "pipecat.flows.types", + "kind": "parameter", + "deprecated_in": "1.5.0", + "removed_in": "2.0.0", + "relation": "use_existing", + "replacement": "LLMContextSummaryConfig.summarization_prompt", + "message": "Use ``LLMContextSummaryConfig.summarization_prompt`` instead. Deprecated together with ``RESET_WITH_SUMMARY``. Will be removed in 2.0.0.", + "location": "pipecat/flows/types.py" + }, + { + "subject": "FlowResult", + "module": "pipecat.flows.types", + "kind": "class", + "deprecated_in": "1.5.0", + "removed_in": "2.0.0", + "relation": "none", + "replacement": null, + "message": "`FlowResult` is deprecated since 1.5.0 and will be removed in 2.0.0. No replacement.", + "location": "pipecat/flows/types.py" + }, + { + "subject": "NodeConfig.role_messages", + "module": "pipecat.flows.types", + "kind": "parameter", + "deprecated_in": "1.5.0", + "removed_in": "2.0.0", + "relation": "use_existing", + "replacement": "role_message", + "message": "Use ``role_message`` (str) instead. Will be removed in 2.0.0.", + "location": "pipecat/flows/types.py" + }, + { + "subject": "flows_direct_function", + "module": "pipecat.flows.types", + "kind": "function", + "deprecated_in": "1.5.0", + "removed_in": "2.0.0", + "relation": "rename", + "replacement": "flows_tool_options", + "message": "`flows_direct_function` is deprecated since 1.5.0 and will be removed in 2.0.0. Use `flows_tool_options` instead.", + "location": "pipecat/flows/types.py" + }, { "subject": "CancelTaskFrame", "module": "pipecat.frames.frames", diff --git a/src/pipecat/__init__.py b/src/pipecat/__init__.py index 6a0b5d9e4ba..3cf640dbd33 100644 --- a/src/pipecat/__init__.py +++ b/src/pipecat/__init__.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import importlib.util import os import sys from importlib.metadata import version as lib_version @@ -24,8 +25,35 @@ def _should_log_version_banner() -> bool: return program not in ("pipecat", "pc") +def _warn_if_standalone_flows_installed() -> None: + """Flag the deprecated standalone ``pipecat-ai-flows`` package if it is also installed. + + Pipecat Flows now ships inside ``pipecat-ai`` as ``pipecat.flows``. Older + ``pipecat-ai-flows`` releases allow ``pipecat-ai<2``, so they can end up + installed next to a Pipecat that already includes Flows — a redundant, easily + confused setup. The check lives here, in the top-level package init that runs + on any ``pipecat`` import, rather than in ``pipecat.flows`` — so it also fires + for apps still importing the standalone ``pipecat_flows`` (which pulls in core + Pipecat but never ``pipecat.flows``). Detection uses ``find_spec`` to avoid + importing the standalone package. + """ + try: + installed = importlib.util.find_spec("pipecat_flows") is not None + except (ImportError, ValueError): + installed = False + if installed: + logger.error( + "The separate `pipecat-ai-flows` package is installed alongside a version " + "of Pipecat that already includes Pipecat Flows as `pipecat.flows`. You do " + "not need both — uninstall `pipecat-ai-flows` and import Flows from " + "`pipecat.flows`." + ) + + if _should_log_version_banner(): logger.info(f"ᓚᘏᗢ Pipecat {__version__} (Python {sys.version}) ᓚᘏᗢ") + # Gated like the banner: skip the redundant-package check for the pipecat/pc CLI. + _warn_if_standalone_flows_installed() def version() -> str: diff --git a/src/pipecat/flows/__init__.py b/src/pipecat/flows/__init__.py new file mode 100644 index 00000000000..e9cd377fe6c --- /dev/null +++ b/src/pipecat/flows/__init__.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +"""Pipecat Flows - Structured conversation framework for Pipecat. + +This package provides a framework for building structured conversations in Pipecat. +The FlowManager handles conversation flows with support for state management, +function calling, and cross-provider compatibility. + +Pipecat Flows determines conversation structure at runtime, supporting function +calling, action execution, and seamless transitions between conversation states. +""" + +from .exceptions import ( + ActionError, + FlowError, + FlowInitializationError, + FlowTransitionError, + InvalidFunctionError, +) +from .manager import FlowManager +from .types import ( + ActionConfig, + ConsolidatedFunctionResult, + ContextStrategy, + ContextStrategyConfig, + FlowArgs, + FlowFunctionHandler, + FlowResult, + FlowsDirectFunction, + FlowsFunctionSchema, + LegacyFunctionHandler, + NodeConfig, + ZeroArgFunctionHandler, + flows_direct_function, + flows_tool_options, +) + +# NOTE: In Pipecat, we typically don't do this sort of re-exporting of nested +# modules, but this is how we did it in the previous standalone pipecat-flows +# package, which was subsequently moved here as pipecat.flows. Asking users to +# import from pipecat.flows instead of pipecat_flows was a simple enough +# breaking change, but asking them to also look through all their Flows-related +# imports and find the right submodule felt like a much bigger ask. +__all__ = [ + # Flow Manager + "FlowManager", + # Types + "ActionConfig", + "ContextStrategy", + "ContextStrategyConfig", + "FlowArgs", + "FlowFunctionHandler", + "FlowResult", + "ConsolidatedFunctionResult", + "FlowsFunctionSchema", + "LegacyFunctionHandler", + "FlowsDirectFunction", + "NodeConfig", + "ZeroArgFunctionHandler", + "flows_tool_options", + "flows_direct_function", + # Exceptions + "FlowError", + "FlowInitializationError", + "FlowTransitionError", + "InvalidFunctionError", + "ActionError", +] diff --git a/src/pipecat/flows/actions.py b/src/pipecat/flows/actions.py new file mode 100644 index 00000000000..172447a7a16 --- /dev/null +++ b/src/pipecat/flows/actions.py @@ -0,0 +1,400 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Action management system for conversation flows. + +This module provides the ActionManager class which handles execution of actions +during conversation state transitions. It supports: + +- Built-in actions (TTS, conversation ending) +- Custom action registration +- Synchronous and asynchronous handlers +- Pre and post-transition actions +- Error handling and validation + +Actions are used to perform side effects during conversations, such as: + +- Text-to-speech output +- Database updates +- External API calls +- Custom integrations +""" + +import asyncio +import inspect +import warnings +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from loguru import logger + +from pipecat.flows.exceptions import ActionError +from pipecat.flows.types import ActionConfig, FlowActionHandler +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + ControlFrame, + EndFrame, + TTSSpeakFrame, +) +from pipecat.pipeline.worker import PipelineWorker + +if TYPE_CHECKING: + from pipecat.flows.manager import FlowManager + + +@dataclass +class FunctionActionFrame(ControlFrame): + """Frame containing a function action to be executed. + + Parameters: + action: Action configuration dictionary. + function: Function handler to execute. + """ + + action: dict + function: FlowActionHandler + + +@dataclass +class ActionFinishedFrame(ControlFrame): + """Frame indicating that an action has completed execution.""" + + pass + + +class ActionManager: + """Manages the registration and execution of flow actions. + + Actions are executed during state transitions and can include: + + - Text-to-speech output + - Database updates + - External API calls + - Custom user-defined actions + + Built-in actions: + + - tts_say: Speak text using TTS + - end_conversation: End the current conversation + - function: Execute inline functions in the pipeline + + Custom actions can be registered using register_action(). + """ + + def __init__(self, worker: PipelineWorker, flow_manager: "FlowManager"): + """Initialize the action manager. + + Args: + worker: PipelineWorker instance used to queue frames. + flow_manager: FlowManager instance that this ActionManager is part of. + """ + self._action_handlers: dict[str, Callable] = {} + self._worker = worker + self._flow_manager = flow_manager + self._ongoing_actions_count = 0 + self._ongoing_actions_finished_event = asyncio.Event() + self._deferred_post_actions: list[ActionConfig] = [] + self._showed_deprecation_warning_for_legacy_action_handler = False + + # Register built-in actions + self._register_action("tts_say", self._handle_tts_action) + self._register_action("end_conversation", self._handle_end_action) + self._register_action("function", self._handle_function_action) + + # Add pipeline observation + worker.set_reached_downstream_filter( + (ActionFinishedFrame, FunctionActionFrame, BotStoppedSpeakingFrame) + ) + + @worker.event_handler("on_frame_reached_downstream") + async def on_frame_reached_downstream(worker, frame): + if isinstance(frame, FunctionActionFrame): + # Run function action + await frame.function(frame.action, flow_manager) + self._decrement_ongoing_actions_count() + elif isinstance(frame, BotStoppedSpeakingFrame): + # Execute deferred post-actions if the bot's turn is over. + # A BotStoppedSpeakingFrame only indicates that the bot's turn is over if there are + # no ongoing actions (otherwise one of those actions may have been responsible for it). + if self._ongoing_actions_count == 0: + await self._execute_deferred_post_actions() + elif isinstance(frame, ActionFinishedFrame): + # Handle action finished + self._decrement_ongoing_actions_count() + + def _register_action(self, action_type: str, handler: Callable) -> None: + """Register a handler for a specific action type. + + Args: + action_type: String identifier for the action (e.g., "tts_say"). + handler: Async or sync function that handles the action. + + Raises: + ValueError: If handler is not callable. + """ + if not callable(handler): + raise ValueError("Action handler must be callable") + self._action_handlers[action_type] = handler + logger.debug(f"Registered handler for action type: {action_type}") + + async def execute_actions(self, actions: list[ActionConfig] | None) -> None: + """Execute a list of actions. + + Args: + actions: List of action configurations to execute. + + Raises: + ActionError: If action execution fails. + + Note: + Each action must have a 'type' field matching a registered handler. + """ + if not actions: + return + + previous_action_type = None + for action in actions: + action_type = action.get("type") + if not action_type: + raise ActionError("Action missing required 'type' field") + + handler = self._action_handlers.get(action_type) + if not handler: + raise ActionError(f"No handler registered for action type: {action_type}") + + ongoing_actions_count = self._ongoing_actions_count + try: + # Based on the type of the previous action and the one coming up, we can determine + # if we need to wait for ongoing actions to finish before proceeding with this next + # one + await self._maybe_wait_for_ongoing_actions_to_finish( + previous_action_type, action_type + ) + + # Determine if handler can accept flow_manager argument by inspecting its signature + # Handlers can either take (action) or (action, flow_manager) + try: + sig = inspect.signature(handler) + can_handle_flow_manager_arg = len(sig.parameters) > 1 + except (ValueError, TypeError): + logger.warning( + f"Unable to determine handler signature for action type '{action_type}', " + "falling back to legacy single-parameter call" + ) + can_handle_flow_manager_arg = False + + # Invoke handler appropriately, with async and flow_manager arg as needed + if can_handle_flow_manager_arg: + if asyncio.iscoroutinefunction(handler): + await handler(action, self._flow_manager) + else: + handler(action, self._flow_manager) + else: + if not self._showed_deprecation_warning_for_legacy_action_handler: + self._showed_deprecation_warning_for_legacy_action_handler = True + warnings.warn( + "Single-argument (legacy) action handlers are deprecated " + "and will be removed in 2.0.0. Update handlers to accept " + "(action: dict, flow_manager: FlowManager) instead.", + DeprecationWarning, + stacklevel=2, + ) + if asyncio.iscoroutinefunction(handler): + await handler(action) + else: + handler(action) + + # Record the type of the action we just executed + previous_action_type = action_type + logger.debug(f"Successfully executed action: {action_type}") + + # If action was end_conversation, break + # (If we didn't, we could end up waiting for the next actions to finish, and...they + # never would) + if action_type == "end_conversation": + break + except Exception as e: + # Undo any increment of ongoing actions count that happened during this action + if self._ongoing_actions_count > ongoing_actions_count: + self._decrement_ongoing_actions_count() # Assumption: on increment per action + raise ActionError(f"Failed to execute action {action_type}: {str(e)}") from e + + # Based on the type of the last action, we may need to wait for ongoing actions to finish + # before considering this set of actions complete. + await self._maybe_wait_for_ongoing_actions_to_finish(previous_action_type, None) + + def schedule_deferred_post_actions(self, post_actions: list[ActionConfig]) -> None: + """Schedule "deferred" post-actions to be executed after next LLM completion. + + Args: + post_actions: List of actions to execute after LLM response. + """ + self._deferred_post_actions = post_actions + + def clear_deferred_post_actions(self) -> None: + """Clear any scheduled deferred post-actions.""" + self._deferred_post_actions = [] + + async def _execute_deferred_post_actions(self) -> None: + """Execute deferred post-actions.""" + actions = self._deferred_post_actions + self._deferred_post_actions = [] + if actions: + await self.execute_actions(actions) + + async def _maybe_wait_for_ongoing_actions_to_finish( + self, previous_action_type: str | None, upcoming_action_type: str | None + ) -> None: + """Wait for ongoing actions to finish before executing the next action if needed. + + This method determines whether to wait based on the types of the previous + and upcoming actions to avoid the upcoming action having an effect before + the previous one is done. + + Args: + previous_action_type: Type of the previously executed action, or None + if this is the start of the action sequence. + upcoming_action_type: Type of the next action to execute, or None if + this is the end of the action sequence. + """ + needs_wait = False + if previous_action_type == "tts_say": + # "tts_say" enqueues a TTSSpeakFrame, which has an effect when it hits the TTS node in + # the pipeline. + # As long as the upcoming action enqueues a frame with an effect at the same point or + # later in the pipeline, we don't need to wait. + # If the upcoming action is: + # - "tts_say": no need to wait (effect happens at the same point) + # - "end_conversation": no need to wait (effect happens at the end of the pipeline) + # - "function": no need to wait (effect happens at the end of the pipeline) + # - None: wait (we're done with this set of actions; the next thing to occur may be a + # node change/LLM context update, which has an effect earlier in the pipeline) + # - custom action: wait (we don't know what it will do) + if upcoming_action_type not in ["tts_say", "end_conversation", "function"]: + needs_wait = True # None or custom action + elif previous_action_type == "function": + # "function" enqueues a FunctionActionFrame, which has an effect at the end of the + # pipeline. + # Functions can take some time to execute (and don't hold up the pipeline as they're + # doing so), so we need to wait for them to finish before proceeding with the next + # action or moving on from the current set of actions. + needs_wait = True + else: + # Either previous action was: + # - None (the upcoming action is the first one), so there's nothing to wait for. + # - A fully custom action, where we don't wait, like we've always done. Note that we + # could, in the future, add new API affordances for users to tell us to wait for the + # the action to finish before moving on to the next one along with a way for them to + # tell us when the action is done. But let's hold off on doing that since we're + # de-emphasizing custom actions in favor of "function" actions, which should meet most + # needs. + # Note that it should not be possible for the previous action to be "end_conversation", + # since we stop processing actions after that one. + pass + + if needs_wait: + await self._ongoing_actions_finished_event.wait() + + async def _handle_tts_action(self, action: dict) -> None: + """Built-in handler for TTS actions. + + Args: + action: Action configuration dictionary. Required 'text' key with + the text to speak. Optional 'append_text_to_context' key (bool) + controlling whether the spoken text is appended to the LLM + context. Defaults to True. + """ + text = action.get("text") + if not text: + logger.error("TTS action missing 'text' field") + return + + try: + # Mark that we're starting the action + self._increment_ongoing_actions_count() + + # Queue the action frame. Default to appending the spoken text to the + # context; callers opt out with append_text_to_context=False. + await self._worker.queue_frame( + TTSSpeakFrame( + text=text, append_to_context=action.get("append_text_to_context", True) + ) + ) + + # Queue a frame marking the end of the action + await self._worker.queue_frame(ActionFinishedFrame()) + except Exception as e: + self._decrement_ongoing_actions_count() + logger.error(f"TTS error: {e}") + + async def _handle_end_action(self, action: dict) -> None: + """Built-in handler for ending the conversation. + + This handler queues an EndFrame to terminate the conversation. If the action + includes a 'text' key, it will queue that text to be spoken before ending. + + Args: + action: Action configuration dictionary. Optional 'text' key for a + goodbye message. Optional 'append_text_to_context' key (bool) + controlling whether that goodbye text is appended to the LLM + context. Defaults to True. + """ + # Mark that we're starting the action + self._increment_ongoing_actions_count() + + # Queue the action frames + if action.get("text"): # Optional goodbye message + # Default to appending the goodbye text to the context; callers opt + # out with append_text_to_context=False. + await self._worker.queue_frame( + TTSSpeakFrame( + text=action["text"], + append_to_context=action.get("append_text_to_context", True), + ) + ) + await self._worker.queue_frame(EndFrame()) + + # NOTE: there's no point queueing an ActionFinishedFrame here, since the previously-queued + # EndFrame ensures that it'll never get delivered to our observer + + async def _handle_function_action(self, action: dict) -> None: + """Built-in handler for queuing functions to run inline in the pipeline. + + This handler queues a FunctionActionFrame to be executed when the pipeline + is done with all the work queued before it. It expects a 'handler' key in + the action containing the function to execute. + + Args: + action: Action configuration dictionary. Required 'handler' key + containing the function to execute. + """ + handler = action.get("handler") + if not handler: + logger.error("Function action missing 'handler' field") + return + + # Mark that we're starting the action + self._increment_ongoing_actions_count() + + # Queue the action frame (we're queueing rather than running it here to ensure it happens + # at the appropriate time in the pipeline, like when the bot's turn is over, for example). + await self._worker.queue_frame(FunctionActionFrame(action=action, function=handler)) + + # NOTE: we do NOT queue an ActionFinishedFrame here; instead, we will decrement the ongoing + # actions count when the function has finished executing (the function may take some time) + + def _increment_ongoing_actions_count(self) -> None: + """Increment the count of ongoing actions and reset the finished event if this is the first action.""" + self._ongoing_actions_count += 1 + if self._ongoing_actions_count == 1: + self._ongoing_actions_finished_event.clear() + + def _decrement_ongoing_actions_count(self) -> None: + """Decrement the count of ongoing actions and set the finished event if this was the last action.""" + self._ongoing_actions_count = max(0, self._ongoing_actions_count - 1) + if self._ongoing_actions_count == 0: + self._ongoing_actions_finished_event.set() diff --git a/src/pipecat/flows/adapters.py b/src/pipecat/flows/adapters.py new file mode 100644 index 00000000000..93871e7fec5 --- /dev/null +++ b/src/pipecat/flows/adapters.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""LLM adapter for conversation-summary generation and formatting. + +This module provides the LLMAdapter class used by the flow manager to: + +- Format a generated summary as a context message +- Generate a summary via out-of-band LLM inference +""" + +from typing import Any + +from loguru import logger + +from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage + + +class LLMAdapter: + """Helpers for generating and formatting conversation summaries.""" + + def format_summary_message(self, summary: str) -> dict: + """Format a summary as a developer message. + + Summary messages use the LLMContextMessage format (OpenAI-style), + as summarization triggers an LLMMessagesUpdateFrame. + + Args: + summary: The generated summary text. + + Returns: + A developer message containing the summary. + """ + return {"role": "developer", "content": f"Here's a summary of the conversation:\n{summary}"} + + async def generate_summary( + self, llm: Any, summary_prompt: str, context: LLMContext + ) -> str | None: + """Generate a summary by running a direct one-shot, out-of-band inference with the LLM. + + Args: + llm: LLM service instance containing client/credentials. + summary_prompt: Prompt text to guide summary generation. + context: Context object containing conversation history for the summary. + + Returns: + Generated summary text, or None if generation fails. + """ + try: + messages = context.get_messages() + + prompt_messages: list[LLMContextMessage] = [ + { + "role": "developer", + "content": f"Conversation history: {messages}", + }, + ] + + summary_context = LLMContext(messages=prompt_messages) + + return await llm.run_inference(summary_context, system_instruction=summary_prompt) + + except Exception as e: + logger.error(f"Summary generation failed: {e}", exc_info=True) + return None diff --git a/src/pipecat/flows/exceptions.py b/src/pipecat/flows/exceptions.py new file mode 100644 index 00000000000..254cae56fcc --- /dev/null +++ b/src/pipecat/flows/exceptions.py @@ -0,0 +1,62 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Custom exceptions for the conversation flow system. + +This module defines the exception hierarchy used throughout the flow system +for better error handling and debugging. All exceptions inherit from FlowError +to provide a common base for flow-related errors. +""" + + +class FlowError(Exception): + """Base exception for all flow-related errors. + + This is the parent class for all flow system exceptions. Use this + for generic flow errors or when a more specific exception doesn't apply. + """ + + pass + + +class FlowInitializationError(FlowError): + """Raised when flow initialization fails. + + This exception occurs during flow manager setup, typically due to + invalid configuration, missing dependencies, or initialization errors. + """ + + pass + + +class FlowTransitionError(FlowError): + """Raised when a state transition fails. + + This exception occurs when transitioning between nodes fails due to + invalid node configurations, missing target nodes, or transition errors. + """ + + pass + + +class InvalidFunctionError(FlowError): + """Raised when an invalid or unavailable function is called. + + This exception occurs when attempting to call functions that are not + properly registered, have invalid signatures, or cannot be found. + """ + + pass + + +class ActionError(FlowError): + """Raised when an action execution fails. + + This exception occurs during action execution, including built-in actions + like TTS or custom actions, due to invalid configuration or execution errors. + """ + + pass diff --git a/src/pipecat/flows/manager.py b/src/pipecat/flows/manager.py new file mode 100644 index 00000000000..8c40cc61c1b --- /dev/null +++ b/src/pipecat/flows/manager.py @@ -0,0 +1,901 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Core conversation flow management system. + +This module provides the FlowManager class which orchestrates +conversations across different LLM providers. It supports: + +- Flows with runtime-determined transitions +- State management and transitions +- Function registration and execution +- Action handling +- Cross-provider compatibility + +The flow manager coordinates all aspects of a conversation, including: + +- LLM context management +- Function registration +- State transitions +- Action execution +- Error handling +""" + +import asyncio +import inspect +import warnings +from collections.abc import Callable +from typing import Any, cast + +from loguru import logger + +from pipecat.adapters.schemas.direct_function import tool_options +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.flows.actions import ActionError, ActionManager +from pipecat.flows.adapters import LLMAdapter +from pipecat.flows.exceptions import ( + FlowError, + FlowInitializationError, + FlowTransitionError, + InvalidFunctionError, +) +from pipecat.flows.types import ( + ActionConfig, + ConsolidatedFunctionResult, + ContextStrategy, + ContextStrategyConfig, + FlowArgs, + FlowFunctionHandler, + FlowsDirectFunction, + FlowsDirectFunctionWrapper, + FlowsFunctionSchema, + FunctionHandler, + LegacyFunctionHandler, + NodeConfig, + ZeroArgFunctionHandler, + get_or_generate_node_name, +) +from pipecat.frames.frames import ( + FunctionCallResultProperties, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMRunFrame, + LLMSetToolsFrame, + LLMUpdateSettingsFrame, +) +from pipecat.pipeline.llm_switcher import LLMSwitcher +from pipecat.pipeline.worker import PipelineWorker +from pipecat.processors.aggregators.llm_context import NOT_GIVEN, LLMContext, NotGiven +from pipecat.services.llm_service import FunctionCallParams, LLMService +from pipecat.services.settings import LLMSettings +from pipecat.transports.base_transport import BaseTransport +from pipecat.utils.deprecation import deprecated + + +class FlowManager: + """Manages conversation flows. + + The FlowManager orchestrates conversation flows by managing state transitions, + function registration, and message handling across different LLM providers, + with comprehensive action handling and error management. + + The manager coordinates all aspects of a conversation including LLM context + management, function registration, state transitions, and action execution. + """ + + def __init__( + self, + *, + llm: LLMService | LLMSwitcher, + context_aggregator: Any, + worker: PipelineWorker | None = None, + task: PipelineWorker | None = None, + context_strategy: ContextStrategyConfig | None = None, + transport: BaseTransport | None = None, + global_functions: list[FlowsFunctionSchema | FlowsDirectFunction] | None = None, + ): + """Initialize the flow manager. + + Args: + llm: LLM service or LLMSwitcher. + context_aggregator: Context aggregator for updating user context. + worker: PipelineWorker instance for queueing frames. + task: PipelineWorker instance for queueing frames. + + .. deprecated:: 1.5.0 + Use ``worker`` instead. Will be removed in 2.0.0. + + context_strategy: Context strategy configuration for managing conversation + context during transitions. + transport: Transport instance for communication. + global_functions: Optional list of FlowsFunctionSchemas or FlowsDirectFunctions + that will be available at every node. These functions are registered once + during initialization and automatically included alongside node-specific + functions. + """ + if worker is not None and task is not None: + raise ValueError("Pass either 'worker' or 'task' (deprecated), not both.") + if task is not None: + warnings.warn( + "The 'task' parameter is deprecated since 1.5.0 and will be removed " + "in 2.0.0. Use 'worker' instead.", + DeprecationWarning, + stacklevel=2, + ) + worker = task + if worker is None: + raise ValueError("FlowManager requires a 'worker' (PipelineWorker).") + + self._worker = worker + self._llm = llm + self._action_manager = ActionManager(worker, flow_manager=self) + self._adapter = LLMAdapter() + self._initialized = False + self._context_aggregator = context_aggregator + self._pending_transition: dict[str, Any] | None = None + self._context_strategy = context_strategy or ContextStrategyConfig( + strategy=ContextStrategy.APPEND + ) + self._transport = transport + self._global_functions = global_functions or [] + + self._state: dict[str, Any] = {} # Internal state storage + self._current_functions: set[str] = set() # Track registered functions + self._current_node: str | None = None + + self._showed_deprecation_warning_for_role_messages = False + self._showed_deprecation_warning_for_reset_with_summary = False + self._showed_deprecation_warning_for_zero_arg_handler = False + self._showed_deprecation_warning_for_legacy_handler = False + + @property + def state(self) -> dict[str, Any]: + """Access the shared state dictionary across nodes. + + This property provides access to a persistent dictionary that maintains + data across node transitions. It can be used to store and retrieve + conversation state, user preferences, or any other data that needs + to persist throughout the flow. + + Returns: + Dict[str, Any]: The shared state dictionary that can be used for + reading and writing state data. + + Examples: + Setting state:: + + flow_manager.state["user_name"] = "Alice" + flow_manager.state["age"] = 25 + + Getting state:: + + name = flow_manager.state.get("user_name", "Unknown") + age = flow_manager.state["age"] + + Checking for state:: + + if "user_preferences" in flow_manager.state: + preferences = flow_manager.state["user_preferences"] + """ + return self._state + + @property + def transport(self) -> BaseTransport | None: + """Access the transport instance used for communication. + + This property provides access to the transport instance that handles + communication with the client (e.g., DailyTransport for Daily rooms). + The transport can be used to interact with participants, manage + audio/video settings, or access platform-specific features. + + Returns: + Optional[BaseTransport]: The transport instance if provided during + initialization, None otherwise. + + Examples: + Accessing transport in action handlers:: + + async def mute_participant(action: dict, flow_manager: FlowManager): + transport = flow_manager.transport + if transport and hasattr(transport, 'update_participant'): + await transport.update_participant(participant_id, {"canSnd": False}) + + Working with Daily transport features:: + + async def get_room_info(action: dict, flow_manager: FlowManager): + transport = flow_manager.transport + if isinstance(transport, DailyTransport): + participants = transport.participants() + return {"participant_count": len(participants)} + """ + return self._transport + + @property + def current_node(self) -> str | None: + """Access the identifier of the currently active conversation node. + + This property provides access to the current node name/identifier in the + conversation flow. It can be used to make decisions based on the current + state of the conversation, implement conditional logic, or for debugging + and logging purposes. + + Returns: + Optional[str]: The identifier of the current node if a node is active, + None if no node has been set or before initialization. + + Examples: + Conditional logic based on current node:: + + async def participant_joined(action: dict, flow_manager: FlowManager): + current = flow_manager.current_node + if current == "transferring_to_human_agent": + await start_human_agent_interaction(flow_manager) + elif current == "collecting_payment": + await setup_secure_session(flow_manager) + + Logging and debugging:: + + async def log_conversation_state(action: dict, flow_manager: FlowManager): + node = flow_manager.current_node + logger.info(f"Current conversation node: {node}") + return {"current_node": node} + """ + return self._current_node + + @property + def worker(self) -> PipelineWorker: + """Access the pipeline worker instance for frame queueing. + + This property provides access to the PipelineWorker instance used by the + FlowManager. The worker can be used to queue custom frames directly into + the pipeline, enabling advanced flow control and custom frame injection. + + Returns: + PipelineWorker: The pipeline worker instance used for frame processing + and queueing operations. + + Examples: + Queueing frames in handlers:: + + async def send_custom_notification(action: dict, flow_manager: FlowManager): + from pipecat.frames.frames import TTSUpdateSettingsFrame + + # Queue a TTS settings update frame + await flow_manager.worker.queue_frame( + TTSUpdateSettingsFrame(settings={"voice": "your-new-voice-id"}) + ) + """ + return self._worker + + @property + @deprecated( + "`FlowManager.task` is deprecated since 1.5.0 and will be removed in 2.0.0. " + "Use `FlowManager.worker` instead." + ) + def task(self) -> PipelineWorker: + """Access the pipeline worker instance for frame queueing. + + .. deprecated:: 1.5.0 + Use :attr:`worker` instead. Will be removed in 2.0.0. + + Returns: + PipelineWorker: The pipeline worker instance used for frame processing + and queueing operations. + """ + return self._worker + + async def initialize(self, initial_node: NodeConfig | None = None) -> None: + """Initialize the flow manager. + + Args: + initial_node: Optional initial node configuration. If provided, + the flow will start at this node immediately. + + Raises: + FlowInitializationError: If initialization fails. + + Examples: + Initialize with an initial node:: + + flow_manager = FlowManager( + ... # Initialization parameters + ) + await flow_manager.initialize(create_initial_node()) + + Initialize without an initial node (set later via set_node_from_config):: + + flow_manager = FlowManager( + ... # Initialization parameters + ) + await flow_manager.initialize() + """ + if self._initialized: + logger.warning(f"{self.__class__.__name__} already initialized") + return + + try: + self._initialized = True + logger.debug(f"Initialized {self.__class__.__name__}") + + # Set initial node if provided (otherwise initial node + # will be set later via set_node_from_config()) + if initial_node: + node_name = get_or_generate_node_name(initial_node) + logger.debug(f"Setting initial node: {node_name}") + await self._set_node(node_name, initial_node) + + except Exception as e: + self._initialized = False + raise FlowInitializationError(f"Failed to initialize flow: {str(e)}") from e + + def get_current_context(self) -> list[dict]: + """Get the current conversation context. + + Returns: + List of messages in the current context, including system messages, + user messages, and assistant responses. + + Raises: + FlowError: If context aggregator is not available. + """ + if not self._context_aggregator: + raise FlowError("No context aggregator available") + + context = self._context_aggregator.user()._context + + return context.get_messages() + + def register_action(self, action_type: str, handler: Callable) -> None: + """Register a handler for a specific action type. + + Args: + action_type: String identifier for the action (e.g., "tts_say"). + handler: Async or sync function that handles the action. + + Example:: + + async def custom_notification(action: dict): + text = action.get("text", "") + await notify_user(text) + + flow_manager.register_action("notify", custom_notification) + """ + self._action_manager._register_action(action_type, handler) + + def _register_action_from_config(self, action: ActionConfig) -> None: + """Register an action handler from action configuration. + + Args: + action: Action configuration dictionary containing type and optional handler. + + Raises: + ActionError: If action type is not registered and no valid handler provided. + """ + action_type = action.get("type") + handler = action.get("handler") + + # Register action if not already registered + if action_type and action_type not in self._action_manager._action_handlers: + # Register handler if provided + if handler and callable(handler): + self.register_action(action_type, handler) + logger.debug(f"Registered action handler from config: {action_type}") + else: + raise ActionError( + f"Action '{action_type}' not registered. " + "Provide handler in action config or register manually." + ) + + async def _call_handler( + self, handler: FunctionHandler, args: FlowArgs + ) -> Any | ConsolidatedFunctionResult: + """Call handler with appropriate parameters based on its signature. + + Detects whether the handler can accept a flow_manager parameter and + calls it accordingly to maintain backward compatibility with legacy handlers. + + Args: + handler: The function handler to call (either legacy or modern format). + args: Arguments dictionary from the function call. + + Returns: + The result returned by the handler. + """ + # Get the function signature + sig = inspect.signature(handler) + + # Calculate effective parameter count + effective_param_count = len(sig.parameters) + + # Handle different function signatures. inspect.signature has already + # proven the shape, so each cast narrows the union to the branch we know + # we're in. + if effective_param_count == 0: + if not self._showed_deprecation_warning_for_zero_arg_handler: + self._showed_deprecation_warning_for_zero_arg_handler = True + warnings.warn( + "Zero-argument function handlers are deprecated and will be " + "removed in 2.0.0. Update handlers to accept " + "(args: FlowArgs, flow_manager: FlowManager) instead.", + DeprecationWarning, + stacklevel=2, + ) + return await cast(ZeroArgFunctionHandler, handler)() + elif effective_param_count == 1: + if not self._showed_deprecation_warning_for_legacy_handler: + self._showed_deprecation_warning_for_legacy_handler = True + warnings.warn( + "Single-argument (legacy) function handlers are deprecated " + "and will be removed in 2.0.0. Update handlers to accept " + "(args: FlowArgs, flow_manager: FlowManager) instead.", + DeprecationWarning, + stacklevel=2, + ) + return await cast(LegacyFunctionHandler, handler)(args) + else: + return await cast(FlowFunctionHandler, handler)(args, self) + + async def _create_transition_func( + self, + name: str, + handler: Callable | FlowsDirectFunctionWrapper, + ) -> Callable: + """Create a transition function for the given name and handler. + + Args: + name: Name of the function being registered. + handler: Function to process the call: a Flows function handler or a + direct-function wrapper. + + Returns: + Async function that handles the tool invocation. + """ + + async def transition_func(params: FunctionCallParams) -> None: + """Inner function that handles the actual tool invocation.""" + try: + logger.debug(f"Function called: {name}") + + is_transition_only_function = False + acknowledged_result = {"status": "acknowledged"} + + # Invoke the handler with the provided arguments + if isinstance(handler, FlowsDirectFunctionWrapper): + handler_response = await handler.invoke(params.arguments, self) + else: + # Convert Pipecat's Mapping to a fresh dict so handlers may + # mutate without touching Pipecat's internal state. (In 2.0.0 + # FlowArgs is planned to widen to Mapping; this conversion + # can go away then.) + handler_response = await self._call_handler(handler, dict(params.arguments)) + # Support both "consolidated" handlers that return (result, next_node) and handlers + # that return just the result. + if isinstance(handler_response, tuple): + result, next_node = handler_response + if result is None: + result = acknowledged_result + is_transition_only_function = True + else: + result = handler_response + next_node = None + # FlowsDirectFunctions should always be "consolidated" functions that return a tuple + if isinstance(handler, FlowsDirectFunctionWrapper): + raise InvalidFunctionError( + f"Direct function {name} expected to return a tuple (result, next_node) but got {type(result)}" + ) + + logger.debug( + f"{'Transition-only function called for' if is_transition_only_function else 'Function handler completed for'} {name}" + ) + + # Determine if this is an edge function + is_edge_function = bool(next_node) + + if is_edge_function: + # Store transition info for coordinated execution + transition_info = { + "next_node": next_node, + "function_name": name, + "arguments": params.arguments, + "result": result, + } + self._pending_transition = transition_info + + properties = FunctionCallResultProperties( + run_llm=False, # Don't run LLM until transition completes + on_context_updated=self._check_and_execute_transition, + ) + else: + # Node function - run LLM immediately + properties = FunctionCallResultProperties( + run_llm=True, + on_context_updated=None, + ) + + await params.result_callback(result, properties=properties) + + except Exception as e: + logger.error(f"Error in transition function {name}: {str(e)}") + error_result = {"status": "error", "error": str(e)} + await params.result_callback(error_result) + + return transition_func + + async def _check_and_execute_transition(self) -> None: + """Check if all functions are complete and execute transition if so.""" + if not self._pending_transition: + return + + # Check if all function calls are complete using Pipecat's state + assistant_aggregator = self._context_aggregator.assistant() + if not assistant_aggregator.has_function_calls_in_progress: + # All functions complete, execute transition + transition_info = self._pending_transition + self._pending_transition = None + + await self._execute_transition(transition_info) + + async def _execute_transition(self, transition_info: dict[str, Any]) -> None: + """Execute the stored transition.""" + next_node = transition_info.get("next_node") + + try: + if next_node: + node_name = get_or_generate_node_name(next_node) + logger.debug(f"Transition to function-returned node: {node_name}") + await self._set_node(node_name, next_node) + except Exception as e: + logger.error(f"Error executing transition: {str(e)}") + raise + + async def _create_function_schema( + self, tool: FlowsFunctionSchema | FlowsDirectFunctionWrapper + ) -> FunctionSchema: + """Build a FunctionSchema that carries the handler the LLM service will run. + + Flows wraps each tool's handler in a "transition function" that runs the + tool's work and coordinates any node transition. + + Args: + tool: The node's function, as a ``FlowsFunctionSchema`` or a wrapped + direct function. + + Returns: + A ``FunctionSchema`` describing the tool and carrying its handler. + """ + # For a direct function the wrapper itself is the handler; a + # FlowsFunctionSchema carries its handler separately. + handler = tool if isinstance(tool, FlowsDirectFunctionWrapper) else tool.handler + # Stamp the resolved call options onto the handler so the LLM service + # applies them when it registers the advertised tool. This is base + # Pipecat's ``tool_options`` (not ``flows_tool_options``): the handler + # rides on a base ``FunctionSchema``, and ``tool`` already carries Flows' + # resolved option values. + transition_func = tool_options( + cancel_on_interruption=tool.cancel_on_interruption, + timeout_secs=tool.timeout_secs, + )(await self._create_transition_func(tool.name, handler)) + base = tool.to_function_schema() + return FunctionSchema( + name=base.name, + description=base.description, + properties=base.properties, + required=base.required, + handler=transition_func, + ) + + async def set_node_from_config(self, node_config: NodeConfig) -> None: + """Set up a new conversation node and transition to it. + + Used to manually transition between nodes in a flow. + + Args: + node_config: Configuration for the new node. + + Raises: + FlowTransitionError: If manager not initialized. + FlowError: If node setup fails. + """ + await self._set_node(get_or_generate_node_name(node_config), node_config) + + async def _set_node(self, node_id: str, node_config: NodeConfig) -> None: + """Set up a new conversation node and transition to it. + + Handles the complete node transition process in the following order: + 1. Execute pre-actions (if any) + 2. Set up messages (role and task) + 3. Register node functions + 4. Update LLM context with messages and tools + 5. Update state (current node and functions) + 6. Trigger LLM completion with new context + 7. Execute post-actions (if any) + + Args: + node_id: Identifier for the new node. + node_config: Complete configuration for the node. + + Raises: + FlowTransitionError: If manager not initialized. + FlowError: If node setup fails. + """ + if not self._initialized: + raise FlowTransitionError(f"{self.__class__.__name__} must be initialized first") + + try: + # Clear any pending transition state when starting a new node + # This ensures clean state regardless of how we arrived here: + # - Normal transition flow (already cleared in _check_and_execute_transition) + # - Direct calls to set_node/set_node_from_config + self._pending_transition = None + + self._validate_node_config(node_id, node_config) + logger.debug(f"Setting node: {node_id}") + + # Clear any deferred post-actions from previous node + self._action_manager.clear_deferred_post_actions() + + # Register action handlers from config + for action_list in [ + node_config.get("pre_actions", []), + node_config.get("post_actions", []), + ]: + for action in action_list: + self._register_action_from_config(action) + + # Execute pre-actions if any + if pre_actions := node_config.get("pre_actions"): + await self._execute_actions(pre_actions=pre_actions) + + # Build the node's function schemas (carrying handlers) + new_functions: set[str] = set() + + # Mix in global functions that should be available at every node + functions_list = self._global_functions + node_config.get("functions", []) + + standard_functions: list[FunctionSchema] = [] + for func_config in functions_list: + if callable(func_config): + tool = FlowsDirectFunctionWrapper(function=func_config) + elif isinstance(func_config, FlowsFunctionSchema): + tool = func_config + else: + raise InvalidFunctionError( + f"Invalid function format in node '{node_id}'. " + "Use FlowsFunctionSchema or direct functions." + ) + standard_functions.append(await self._create_function_schema(tool)) + new_functions.add(tool.name) + + formatted_tools = ( + ToolsSchema(standard_tools=standard_functions) if standard_functions else NOT_GIVEN + ) + + role_message = node_config.get("role_message") + role_messages = node_config.get("role_messages") + + if role_message and role_messages: + logger.warning( + "Both 'role_message' and 'role_messages' specified; using 'role_message'" + ) + + if role_messages and not role_message: + if not self._showed_deprecation_warning_for_role_messages: + self._showed_deprecation_warning_for_role_messages = True + warnings.warn( + "'role_messages' is deprecated and will be removed in 2.0.0. " + "Use 'role_message' (singular, str) instead.", + DeprecationWarning, + stacklevel=2, + ) + + # Update LLM context + await self._update_llm_context( + role_message=role_message, + role_messages=role_messages if not role_message else None, + task_messages=node_config["task_messages"], + functions=formatted_tools, + strategy=node_config.get("context_strategy"), + ) + logger.debug("Updated LLM context") + + # Update state + self._current_node = node_id + self._current_functions = new_functions + + # Trigger completion with new context + respond_immediately = node_config.get("respond_immediately", True) + if respond_immediately: + await self._worker.queue_frames([LLMRunFrame()]) + + # Execute post-actions if any + if post_actions := node_config.get("post_actions"): + if respond_immediately: + await self._execute_actions(post_actions=post_actions) + else: + # Schedule post-actions for execution after first LLM response in this node + self._schedule_deferred_post_actions(post_actions=post_actions) + + logger.debug(f"Successfully set node: {node_id}") + + except Exception as e: + logger.error(f"Error setting node {node_id}: {str(e)}") + raise FlowError(f"Failed to set node {node_id}: {str(e)}") from e + + def _schedule_deferred_post_actions(self, post_actions: list[ActionConfig]) -> None: + self._action_manager.schedule_deferred_post_actions(post_actions=post_actions) + + async def _create_conversation_summary( + self, summary_prompt: str, context: LLMContext + ) -> str | None: + """Generate a conversation summary from a given context.""" + return await self._adapter.generate_summary(self._llm, summary_prompt, context) + + async def _update_llm_context( + self, + role_message: str | None, + role_messages: list[dict] | None, + task_messages: list[dict], + functions: ToolsSchema | NotGiven, + strategy: ContextStrategyConfig | None = None, + ) -> None: + """Update LLM context with new messages and functions. + + If ``role_message`` is provided, it is sent as an + ``LLMUpdateSettingsFrame`` (system instruction on the LLM itself). + + If ``role_messages`` (deprecated) is provided, the messages are + prepended to the conversation context alongside ``task_messages``. + + Args: + role_message: Optional role/personality string sent as the LLM + system instruction via ``LLMUpdateSettingsFrame``. + role_messages: Deprecated list-of-dicts prepended to context + messages for backward compatibility. + task_messages: Task messages to add to context. + functions: New functions to make available. + strategy: Optional context update configuration. + + Raises: + FlowError: If context update fails. + """ + try: + frames = [] + + # New path: role_message as LLM system instruction (persists until changed) + if role_message: + frames.append( + LLMUpdateSettingsFrame(delta=LLMSettings(system_instruction=role_message)) + ) + + messages = [] + + # Legacy path: role_messages prepended to context messages + if role_messages: + messages.extend(role_messages) + + update_config = strategy or self._context_strategy + + if update_config.strategy == ContextStrategy.RESET_WITH_SUMMARY: + if not self._showed_deprecation_warning_for_reset_with_summary: + self._showed_deprecation_warning_for_reset_with_summary = True + warnings.warn( + "RESET_WITH_SUMMARY is deprecated and will be removed in 2.0.0. " + "Use Pipecat's native context summarization instead. To trigger " + "on-demand summarization during a node transition, push an " + "LLMSummarizeContextFrame in a pre-action. See " + "https://docs.pipecat.ai/guides/fundamentals/context-summarization", + DeprecationWarning, + stacklevel=2, + ) + + if ( + update_config.strategy == ContextStrategy.RESET_WITH_SUMMARY + and self._context_aggregator + and self._context_aggregator.user()._context + ): + # We know summary_prompt exists because of __post_init__ validation in ContextStrategyConfig + summary_prompt = cast(str, update_config.summary_prompt) + try: + # Try to get summary with 5 second timeout + summary = await asyncio.wait_for( + self._create_conversation_summary( + summary_prompt, + self._context_aggregator.user()._context, + ), + timeout=5.0, + ) + + if summary: + summary_message = self._adapter.format_summary_message(summary) + messages.append(summary_message) + logger.debug(f"Added conversation summary to context: {summary_message}") + else: + # Fall back to APPEND strategy if summary fails + logger.warning( + "Failed to generate summary, falling back to APPEND strategy" + ) + update_config.strategy = ContextStrategy.APPEND + + except TimeoutError: + logger.warning("Summary generation timed out, falling back to APPEND strategy") + update_config.strategy = ContextStrategy.APPEND + + # Add task messages + messages.extend(task_messages) + + # Use an "update" (replace) frame for the RESET/RESET_WITH_SUMMARY + # strategies; otherwise append. (Note that even the first node follows + # the same rule: appending ensures any prior context contributions, + # such as by tts_say pre-actions, is preserved rather than replaced). + frame_type = ( + LLMMessagesUpdateFrame + if update_config.strategy + in [ContextStrategy.RESET, ContextStrategy.RESET_WITH_SUMMARY] + else LLMMessagesAppendFrame + ) + + frames.append(frame_type(messages=messages)) + frames.append(LLMSetToolsFrame(tools=functions)) + + await self._worker.queue_frames(frames) + + logger.debug( + f"Updated LLM context using {frame_type.__name__} with strategy {update_config.strategy}" + ) + + except Exception as e: + logger.error(f"Failed to update LLM context: {str(e)}") + raise FlowError(f"Context update failed: {str(e)}") from e + + async def _execute_actions( + self, + pre_actions: list[ActionConfig] | None = None, + post_actions: list[ActionConfig] | None = None, + ) -> None: + """Execute pre and post actions. + + Args: + pre_actions: Actions to execute before context update. + post_actions: Actions to execute after context update. + """ + if pre_actions: + await self._action_manager.execute_actions(pre_actions) + if post_actions: + await self._action_manager.execute_actions(post_actions) + + def _validate_node_config(self, node_id: str, config: NodeConfig) -> None: + """Validate the configuration of a conversation node. + + This method ensures that: + 1. Required fields (task_messages) are present. + 2. Each function is either a ``FlowsFunctionSchema`` or a valid direct + function. + + Args: + node_id: Identifier for the node being validated. + config: Complete node configuration to validate. + + Raises: + FlowError: If required fields are missing. + InvalidFunctionError: If function format is invalid. + """ + # Check required fields + if "task_messages" not in config: + raise FlowError(f"Node '{node_id}' missing required 'task_messages' field") + + # Get functions list with default empty list if not provided + functions_list = config.get("functions", []) + + # Validate each function configuration if there are any + for func in functions_list: + if callable(func): + FlowsDirectFunctionWrapper.validate_function(func) + elif not isinstance(func, FlowsFunctionSchema): + raise InvalidFunctionError( + f"Invalid function format in node '{node_id}'. " + "Use FlowsFunctionSchema or direct functions." + ) diff --git a/src/pipecat/flows/types.py b/src/pipecat/flows/types.py new file mode 100644 index 00000000000..ba106fa5533 --- /dev/null +++ b/src/pipecat/flows/types.py @@ -0,0 +1,495 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Type definitions for the conversation flow system. + +This module defines the core types used throughout the flow system: + +- FlowResult: Function return type +- FlowArgs: Function argument type +- NodeConfig: Node configuration type +- FlowsFunctionSchema: A uniform schema for function calls in flows + +These types provide structure and validation for flow configurations +and function interactions. +""" + +import uuid +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Required, + TypedDict, +) + +from pipecat.adapters.schemas.direct_function import BaseDirectFunctionWrapper, tool_options +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.flows.exceptions import InvalidFunctionError +from pipecat.utils.deprecation import deprecated + +if TYPE_CHECKING: + from pipecat.flows.manager import FlowManager + + +@deprecated("`FlowResult` is deprecated since 1.5.0 and will be removed in 2.0.0. No replacement.") +class FlowResult(TypedDict, total=False): + """Optional convention TypedDict for ``status``/``error`` results. + + .. deprecated:: 1.5.0 + No replacement. FlowResult is no longer required or referenced by any + handler type, and Pipecat's upstream function-call-result contract is + Any — define your own TypedDict or return any JSON-serializable value. + Will be removed in 2.0.0. + + Parameters: + status: Status of the function execution. + error: Optional error message if execution failed. + """ + + status: str + error: str + + +FlowArgs = dict[str, Any] +"""Type alias for function handler arguments. + +Each invocation gets its own dict, so handlers may mutate it freely without +affecting Pipecat's internal state. + +.. note:: + + In 2.0.0 this alias is planned to widen to ``Mapping[str, Any]`` to align + with Pipecat's typing of ``FunctionCallParams.arguments``. Handlers that + only read args will be unaffected; handlers that mutate args will need to + keep the annotation as ``dict[str, Any]`` explicitly. + +Example:: + + { + "user_name": "John", + "age": 25, + "preferences": {"color": "blue"} + } +""" + + +LegacyActionHandler = Callable[[dict[str, Any]], Awaitable[None]] +"""Legacy action handler type that only receives the action dictionary. + +.. deprecated:: 1.5.0 + Use :data:`FlowActionHandler` (``(action, flow_manager)``) instead. Will be + removed in 2.0.0. + +Args: + action: Dictionary containing action configuration and parameters. + +Example:: + + async def simple_handler(action: dict): + await notify(action["text"]) +""" + +FlowActionHandler = Callable[[dict[str, Any], "FlowManager"], Awaitable[None]] +"""Modern action handler type that receives both action and flow_manager. + +Args: + action: Dictionary containing action configuration and parameters. + flow_manager: Reference to the FlowManager instance. + +Example:: + + async def advanced_handler(action: dict, flow_manager: FlowManager): + await flow_manager.transport.notify(action["text"]) +""" + + +class ActionConfig(TypedDict, total=False): + """Configuration for an action. + + Parameters: + type: Action type identifier (e.g. "tts_say", "notify_slack"). + handler: Callable to handle the action. + text: Text to speak for the "tts_say" action, or the optional goodbye + message for the "end_conversation" action. + append_text_to_context: For the built-in TTS actions ("tts_say" and + "end_conversation"), whether the spoken ``text`` is appended to the + LLM context. Defaults to True. + + Note: + Additional fields are allowed and passed to the handler. + """ + + type: Required[str] + handler: LegacyActionHandler | FlowActionHandler + text: str + append_text_to_context: bool + + +class ContextStrategy(Enum): + """Strategy for managing context during node transitions. + + Parameters: + APPEND: Append new messages to existing context (default). + RESET: Reset context with new messages only. + RESET_WITH_SUMMARY: Reset context but include an LLM-generated summary. + + .. deprecated:: 1.5.0 + Use :class:`LLMSummarizeContextFrame` instead — push it in a + pre-action to trigger on-demand summarization during a node + transition. See + https://docs.pipecat.ai/guides/fundamentals/context-summarization. + Will be removed in 2.0.0. + """ + + APPEND = "append" + RESET = "reset" + RESET_WITH_SUMMARY = "reset_with_summary" + + +@dataclass +class ContextStrategyConfig: + """Configuration for context management. + + Parameters: + strategy: Strategy to use for context management. + summary_prompt: Required prompt text when using RESET_WITH_SUMMARY. + + .. deprecated:: 1.5.0 + Use ``LLMContextSummaryConfig.summarization_prompt`` instead. + Deprecated together with ``RESET_WITH_SUMMARY``. Will be removed + in 2.0.0. + """ + + strategy: ContextStrategy + summary_prompt: str | None = None + + def __post_init__(self): + """Validate configuration. + + Raises: + ValueError: If summary_prompt is missing when using RESET_WITH_SUMMARY. + """ + if self.strategy == ContextStrategy.RESET_WITH_SUMMARY and not self.summary_prompt: + raise ValueError("summary_prompt is required when using RESET_WITH_SUMMARY strategy") + + +class NodeConfig(TypedDict, total=False): + """Configuration for a single node in the flow. + + Parameters: + task_messages: List of message dicts defining the current node's objectives. + name: Name of the node, useful for debug logging when returning a next node + from a "consolidated" function. + role_message: The bot's role/personality as a plain string, sent as the + LLM's system instruction via ``LLMUpdateSettingsFrame``. When + provided, the system instruction persists across node transitions + until a new node explicitly sets ``role_message`` again. + role_messages: Deprecated list-of-dicts format for the bot's role/personality. + + .. deprecated:: 1.5.0 + Use ``role_message`` (str) instead. Will be removed in 2.0.0. + + functions: List of FlowsFunctionSchema definitions or direct functions + whose definitions are automatically extracted from their signatures. + pre_actions: Actions to execute before LLM inference. + post_actions: Actions to execute after LLM inference. + context_strategy: Strategy for updating context during transitions. + respond_immediately: Whether to run LLM inference as soon as the node is + set (default: True). + + Example:: + + { + "role_message": "You are a helpful assistant...", + "task_messages": [ + { + "role": "developer", + "content": "Ask the user for their name..." + } + ], + "functions": [...], + "pre_actions": [...], + "post_actions": [...], + "context_strategy": ContextStrategyConfig(strategy=ContextStrategy.APPEND), + "respond_immediately": true, + } + """ + + task_messages: Required[list[dict]] + name: str + role_message: str + role_messages: list[dict[str, Any]] + # ``FlowsFunctionSchema`` and ``FlowsDirectFunction`` are defined below + # (see the note above ``ConsolidatedFunctionResult``); string forward + # references keep ``NodeConfig`` definable here without re-introducing the + # cross-module forward reference that ``ConsolidatedFunctionResult`` used + # to require. + functions: "list[FlowsFunctionSchema | FlowsDirectFunction]" + pre_actions: list[ActionConfig] + post_actions: list[ActionConfig] + context_strategy: ContextStrategyConfig + respond_immediately: bool + + +# ``ConsolidatedFunctionResult`` is the public return-type alias for "direct" +# functions. It must be defined **after** ``NodeConfig`` and without a string +# forward reference: ``get_type_hints()`` on a user-defined direct function +# resolves names against the user's module globals, not this module's, so a +# ``"NodeConfig"`` forward reference here would fail unless the user happened +# to import ``NodeConfig`` themselves. +ConsolidatedFunctionResult = tuple[Any, NodeConfig | None] +"""Return type for "consolidated" functions. + +Return type for "consolidated" functions that do either or both of: +- doing some work +- specifying the next node to transition to after the work is done + +The first tuple element is the function-call result delivered to the LLM. +Any JSON-serializable value is accepted (matching Pipecat's upstream +``FunctionCallResultCallback`` contract). Pass ``None`` to signal a +transition-only handler; FlowManager substitutes an acknowledgement result. +""" + + +ZeroArgFunctionHandler = Callable[[], Awaitable[Any]] +"""Function handler that takes no arguments. + +.. deprecated:: 1.5.0 + Use :data:`FlowFunctionHandler` (``(args, flow_manager)``) instead. Will be + removed in 2.0.0. + +Returns: + Any JSON-serializable value, or a :data:`ConsolidatedFunctionResult` + tuple to also specify the next node. +""" + +LegacyFunctionHandler = Callable[[FlowArgs], Awaitable[Any]] +"""Legacy function handler that only receives arguments. + +.. deprecated:: 1.5.0 + Use :data:`FlowFunctionHandler` (``(args, flow_manager)``) instead. Will be + removed in 2.0.0. + +Args: + args: Dictionary of arguments from the function call. + +Returns: + Any JSON-serializable value, or a :data:`ConsolidatedFunctionResult` + tuple to also specify the next node. +""" + +FlowFunctionHandler = Callable[[FlowArgs, "FlowManager"], Awaitable[Any]] +"""Modern function handler that receives both arguments and flow_manager. + +Args: + args: Dictionary of arguments from the function call. + flow_manager: Reference to the FlowManager instance. + +Returns: + Any JSON-serializable value, or a :data:`ConsolidatedFunctionResult` + tuple to also specify the next node. +""" + + +FunctionHandler = ZeroArgFunctionHandler | LegacyFunctionHandler | FlowFunctionHandler +"""Union type for function handlers supporting 0-arg, legacy, and modern patterns.""" + + +FlowsDirectFunction = Callable[..., Awaitable[ConsolidatedFunctionResult]] +"""Type alias for "direct" functions with automatic metadata extraction. + +"Direct" functions have their definition automatically extracted from the +function signature and docstring. This can be used in :data:`NodeConfig` +directly, in lieu of a :class:`FlowsFunctionSchema` or function definition +dict. + +Expected shape: + +.. code-block:: python + + async def f(flow_manager: FlowManager, **params) -> ConsolidatedFunctionResult: + ... + +where ``**params`` are any named parameters described by the function's +docstring. + +This is defined as ``Callable[..., ...]`` rather than a Protocol because +Python's Protocol system cannot express "any concrete named-parameter list" +against ``**kwargs: Any`` — a function with named params like ``llm: str`` +is not structurally compatible with a ``**kwargs: Any`` protocol signature. +Runtime validation of the expected shape happens in +:meth:`FlowsDirectFunctionWrapper.validate_function`. +""" + + +@dataclass +class FlowsFunctionSchema: + """Function schema with Flows-specific properties. + + This class extends a standard function schema with the Flows-specific + ``handler`` that runs when the function is called, plus its call options. + + Parameters: + name: Name of the function. + description: Description of the function. + properties: Dictionary defining parameter types and descriptions. + required: List of required parameter names. + handler: Function handler to process the function call. + cancel_on_interruption: Whether to cancel this function call when an + interruption occurs. Defaults to False. + timeout_secs: Optional per-tool timeout in seconds, overriding the global + ``function_call_timeout_secs``. Defaults to None (use global timeout). + """ + + name: str + description: str + properties: dict[str, Any] + required: list[str] + handler: FunctionHandler + cancel_on_interruption: bool = False + timeout_secs: float | None = None + + def to_function_schema(self) -> FunctionSchema: + """Convert to a standard FunctionSchema for use with LLMs. + + Returns: + FunctionSchema without flow-specific fields. + """ + return FunctionSchema( + name=self.name, + description=self.description, + properties=self.properties, + required=self.required, + ) + + +def flows_tool_options( + *, cancel_on_interruption: bool = False, timeout_secs: float | None = None +) -> Callable[[Callable], Callable]: + """Configure a Flows direct function's call options. + + This decorator is optional; use it to override the defaults for a Flows + direct function (an async function whose first parameter is ``flow_manager``). + + Args: + cancel_on_interruption: Whether to cancel the function call when the user + interrupts. Defaults to False. + timeout_secs: Optional per-tool timeout in seconds, overriding the global + ``function_call_timeout_secs``. Defaults to None (use global timeout). + + Returns: + A decorator that attaches the metadata to the function. + + Example:: + + @flows_tool_options(cancel_on_interruption=False, timeout_secs=30) + async def long_running_task(flow_manager: FlowManager, query: str): + '''Perform a long-running task that should not be cancelled on interruption.''' + # ... implementation + return {"status": "complete"}, None + """ + return tool_options(cancel_on_interruption=cancel_on_interruption, timeout_secs=timeout_secs) + + +@deprecated( + "`flows_direct_function` is deprecated since 1.5.0 and will be removed in 2.0.0. " + "Use `flows_tool_options` instead." +) +def flows_direct_function( + *, cancel_on_interruption: bool = False, timeout_secs: float | None = None +) -> Callable[[Callable], Callable]: + """Configure a Flows direct function's call options. + + .. deprecated:: 1.5.0 + Renamed to :func:`flows_tool_options` to align with Pipecat's + ``@tool_options`` and make clearer that it configures call options. + Will be removed in 2.0.0. + + Args: + cancel_on_interruption: Whether to cancel the function call when the user + interrupts. Defaults to False. + timeout_secs: Optional per-tool timeout in seconds, overriding the global + ``function_call_timeout_secs``. Defaults to None (use global timeout). + + Returns: + A decorator that attaches the metadata to the function. + """ + return flows_tool_options( + cancel_on_interruption=cancel_on_interruption, timeout_secs=timeout_secs + ) + + +class FlowsDirectFunctionWrapper(BaseDirectFunctionWrapper): + """Wrapper around a FlowsDirectFunction for metadata extraction and invocation. + + The wrapper: + + - extracts metadata from the function signature and docstring + - generates a corresponding FunctionSchema + - helps with function invocation + """ + + @classmethod + def special_first_param_name(cls) -> str: + """Get the special first parameter name for Flows direct functions. + + Returns: + The string "flow_manager" which is expected as the first parameter. + """ + return "flow_manager" + + @classmethod + def validate_function(cls, function: Callable) -> None: + """Validate the function signature and docstring. + + Args: + function: The function to validate. + + Raises: + InvalidFunctionError: If the function does not meet the requirements. + """ + try: + super().validate_function(function) + except Exception as e: + raise InvalidFunctionError(str(e)) from e + + def _initialize_metadata(self): + """Initialize metadata from function signature, docstring, and decorator.""" + super()._initialize_metadata() + # Read the call options attached by @flows_tool_options (built on Pipecat's + # @tool_options, which stores them under the _pipecat_* attributes). Fall + # back to Flows' defaults when the function is undecorated. + self.cancel_on_interruption = getattr( + self.function, "_pipecat_cancel_on_interruption", False + ) + self.timeout_secs = getattr(self.function, "_pipecat_timeout_secs", None) + + async def invoke(self, args: Mapping[str, Any], flow_manager: "FlowManager"): + """Invoke the wrapped function with the provided arguments. + + Args: + args: Arguments to pass to the function. + flow_manager: FlowManager instance for function execution context. + + Returns: + The result of the function call. + """ + return await self.function(flow_manager=flow_manager, **args) + + +def get_or_generate_node_name(node_config: NodeConfig) -> str: + """Get the node name from configuration or generate a UUID if not set. + + Args: + node_config: Node configuration dictionary. + + Returns: + Node name from config or generated UUID string. + """ + return node_config.get("name", str(uuid.uuid4())) diff --git a/tests/flows_test_helpers.py b/tests/flows_test_helpers.py new file mode 100644 index 00000000000..40e96093a6f --- /dev/null +++ b/tests/flows_test_helpers.py @@ -0,0 +1,96 @@ +from unittest.mock import AsyncMock, Mock + + +def assert_tts_speak_frames_queued(mock_task, expected_texts): + """Assert that TTSSpeakFrames with expected texts were queued.""" + from pipecat.frames.frames import TTSSpeakFrame + + tts_calls = [ + call + for call in mock_task.queue_frame.call_args_list + if isinstance(call[0][0], TTSSpeakFrame) + ] + assert len(tts_calls) == len(expected_texts), ( + f"Expected {len(expected_texts)} TTS calls, got {len(tts_calls)}" + ) + for text in expected_texts: + assert any(text in getattr(call[0][0], "text", "") for call in tts_calls), ( + f"{text} TTS call not found" + ) + + +def get_queued_tts_speak_frames(mock_task): + """Return the TTSSpeakFrames queued on the mock task, in order.""" + from pipecat.frames.frames import TTSSpeakFrame + + return [ + call[0][0] + for call in mock_task.queue_frame.call_args_list + if isinstance(call[0][0], TTSSpeakFrame) + ] + + +def assert_end_frame_queued(mock_task): + """Assert that an EndFrame was queued.""" + from pipecat.frames.frames import EndFrame + + end_calls = [ + call for call in mock_task.queue_frame.call_args_list if isinstance(call[0][0], EndFrame) + ] + assert len(end_calls) == 1, "EndFrame not queued" + + +def get_advertised_tools(mock_task): + """Return the tools from the most recent LLMSetToolsFrame queued (or NOT_GIVEN). + + FlowManager advertises a node's tools via an LLMSetToolsFrame; the LLM service + registers the handlers they carry when it sees them. + """ + from pipecat.frames.frames import LLMSetToolsFrame + from pipecat.processors.aggregators.llm_context import NOT_GIVEN + + set_tools_frames = [ + frame + for call in mock_task.queue_frames.call_args_list + for frame in call[0][0] + if isinstance(frame, LLMSetToolsFrame) + ] + return set_tools_frames[-1].tools if set_tools_frames else NOT_GIVEN + + +def get_advertised_tool_handlers(mock_task): + """Return {name: handler} from the most recent LLMSetToolsFrame queued.""" + from pipecat.processors.aggregators.llm_context import NOT_GIVEN + + tools = get_advertised_tools(mock_task) + if tools is NOT_GIVEN: + return {} + return {schema.name: schema.handler for schema in tools.standard_tools} + + +def make_mock_task(): + """Create a mock PipelineTask wired up so that actions don't hang.""" + mock_task = AsyncMock() + + # Mock queue_frame method that simulates queued frames reaching all the way downstream. + # This is necessary for action execution to not hang, waiting. + async def queue_frame(frame): + handler = getattr(mock_task, "on_frame_reached_downstream", None) + if handler: + await handler(mock_task, frame) + + mock_task.queue_frame = AsyncMock(side_effect=queue_frame) + + # Mock stuff necessary for registering on_frame_reached_downstream handler. + mock_task.set_reached_downstream_filter = Mock() + + def mock_event_handler(event_name): + def decorator(func): + setattr(mock_task, event_name, func) + return func + + return decorator + + mock_task.event_handler = mock_event_handler + + return mock_task diff --git a/tests/test_flows_actions.py b/tests/test_flows_actions.py new file mode 100644 index 00000000000..ca3225d68c4 --- /dev/null +++ b/tests/test_flows_actions.py @@ -0,0 +1,290 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Test suite for ActionManager functionality. + +This module tests the ActionManager class which handles execution of actions +during conversation flows. Tests cover: +- Built-in actions (TTS, end conversation) +- Custom action registration and execution +- Error handling and validation +- Action sequencing +- TTS service integration +- Frame queueing + +The tests use unittest.IsolatedAsyncioTestCase for async support and include +mocked dependencies for PipelineTask. +""" + +import asyncio +import unittest +import warnings +from typing import Any +from unittest.mock import AsyncMock, patch + +from pipecat.flows.actions import ActionManager +from pipecat.flows.exceptions import ActionError +from tests.flows_test_helpers import ( + assert_end_frame_queued, + assert_tts_speak_frames_queued, + get_queued_tts_speak_frames, + make_mock_task, +) + + +class TestActionManager(unittest.IsolatedAsyncioTestCase): + """Test suite for ActionManager class. + + Tests functionality of ActionManager including: + - Built-in action handlers: + - TTS speech synthesis + - Conversation ending + - Custom action registration + - Action execution sequencing + - Error handling: + - Missing TTS service + - Invalid actions + - Failed handlers + - Multiple action execution + - Frame queueing validation + + Each test uses mocked dependencies to verify: + - Correct frame generation + - Proper service calls + - Error handling behavior + - Action sequencing + """ + + def setUp(self): + """Set up test fixtures before each test. + + Creates: + - Mock PipelineTask for frame queueing + - ActionManager instance with mocked dependencies + """ + self.mock_task = make_mock_task() + self.mock_flow_manager = AsyncMock() + self.action_manager = ActionManager(self.mock_task, self.mock_flow_manager) + + async def test_initialization(self): + """Test ActionManager initialization and default handlers.""" + # Verify built-in action handlers are registered + self.assertIn("tts_say", self.action_manager._action_handlers) + self.assertIn("end_conversation", self.action_manager._action_handlers) + + async def test_tts_action(self): + """Test basic TTS action execution.""" + action = {"type": "tts_say", "text": "Hello"} + await self.action_manager.execute_actions([action]) + assert_tts_speak_frames_queued(self.mock_task, ["Hello"]) + + async def test_end_conversation_action(self): + """Test basic end conversation action.""" + action = {"type": "end_conversation"} + await self.action_manager.execute_actions([action]) + + # Verify EndFrame was queued + assert_end_frame_queued(self.mock_task) + + async def test_end_conversation_with_goodbye(self): + """Test end conversation action with goodbye message.""" + action = {"type": "end_conversation", "text": "Goodbye!"} + await self.action_manager.execute_actions([action]) + + # Verify TTSSpeakFrame + assert_tts_speak_frames_queued(self.mock_task, ["Goodbye!"]) + + # Verify EndFrame + assert_end_frame_queued(self.mock_task) + + async def test_tts_action_append_text_to_context(self): + """Test that tts_say maps append_text_to_context onto the TTSSpeakFrame.""" + # Explicitly True + await self.action_manager.execute_actions( + [{"type": "tts_say", "text": "Hello", "append_text_to_context": True}] + ) + frames = get_queued_tts_speak_frames(self.mock_task) + self.assertEqual(len(frames), 1) + self.assertIs(frames[0].append_to_context, True) + + # Explicitly False + self.mock_task.queue_frame.reset_mock() + await self.action_manager.execute_actions( + [{"type": "tts_say", "text": "Hello", "append_text_to_context": False}] + ) + frames = get_queued_tts_speak_frames(self.mock_task) + self.assertEqual(len(frames), 1) + self.assertIs(frames[0].append_to_context, False) + + # Omitted: Flows applies its own default of True (and never passes None, + # so no append_to_context deprecation warning fires). + self.mock_task.queue_frame.reset_mock() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await self.action_manager.execute_actions([{"type": "tts_say", "text": "Hello"}]) + frames = get_queued_tts_speak_frames(self.mock_task) + self.assertEqual(len(frames), 1) + self.assertIs(frames[0].append_to_context, True) + self.assertEqual( + [w for w in caught if "append_to_context" in str(w.message)], + [], + "Flows must not pass None to TTSSpeakFrame", + ) + + async def test_end_conversation_append_text_to_context(self): + """Test that end_conversation maps append_text_to_context onto its goodbye frame.""" + # Explicitly False + await self.action_manager.execute_actions( + [{"type": "end_conversation", "text": "Goodbye!", "append_text_to_context": False}] + ) + frames = get_queued_tts_speak_frames(self.mock_task) + self.assertEqual(len(frames), 1) + self.assertIs(frames[0].append_to_context, False) + assert_end_frame_queued(self.mock_task) + + # Explicitly True + self.mock_task.queue_frame.reset_mock() + await self.action_manager.execute_actions( + [{"type": "end_conversation", "text": "Goodbye!", "append_text_to_context": True}] + ) + frames = get_queued_tts_speak_frames(self.mock_task) + self.assertEqual(len(frames), 1) + self.assertIs(frames[0].append_to_context, True) + + # Omitted: Flows applies its own default of True (and never passes None, + # so no append_to_context deprecation warning fires). + self.mock_task.queue_frame.reset_mock() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await self.action_manager.execute_actions( + [{"type": "end_conversation", "text": "Goodbye!"}] + ) + frames = get_queued_tts_speak_frames(self.mock_task) + self.assertEqual(len(frames), 1) + self.assertIs(frames[0].append_to_context, True) + self.assertEqual( + [w for w in caught if "append_to_context" in str(w.message)], + [], + "Flows must not pass None to TTSSpeakFrame", + ) + + async def test_function_actions(self): + """Test executing function actions.""" + results = [] + + async def first_function(action, flow_manager): + results.append("first_start") + await asyncio.sleep(0.25) + results.append("first_end") + + async def second_function(action, flow_manager): + results.append("second_start") + results.append("second_end") + + actions = [ + {"type": "function", "handler": first_function}, + {"type": "function", "handler": second_function}, + ] + + await self.action_manager.execute_actions(actions) + + # Validate the order + self.assertEqual( + results, + ["first_start", "first_end", "second_start", "second_end"], + ) + + async def test_action_handler_signatures(self): + """Test both legacy and modern action handler signatures.""" + + # Test legacy single-parameter handler + async def legacy_handler(action: dict): + self.assertEqual(action["data"], "legacy") + + self.action_manager._register_action("legacy", legacy_handler) + await self.action_manager.execute_actions([{"type": "legacy", "data": "legacy"}]) + + # Test modern two-parameter handler + async def modern_handler(action: dict, flow_manager: Any): + self.assertEqual(action["data"], "modern") + self.assertEqual(flow_manager, self.mock_flow_manager) + + self.action_manager._register_action("modern", modern_handler) + await self.action_manager.execute_actions([{"type": "modern", "data": "modern"}]) + + async def test_invalid_action(self): + """Test handling invalid actions.""" + # Test missing type + with self.assertRaises(ActionError) as context: + await self.action_manager.execute_actions([{}]) + self.assertIn("missing required 'type' field", str(context.exception)) + + # Test unknown action type + with self.assertRaises(ActionError) as context: + await self.action_manager.execute_actions([{"type": "invalid"}]) + self.assertIn("No handler registered", str(context.exception)) + + async def test_multiple_actions(self): + """Test executing multiple actions in sequence.""" + actions = [ + {"type": "tts_say", "text": "First"}, + {"type": "tts_say", "text": "Second"}, + ] + await self.action_manager.execute_actions(actions) + + # Verify TTS was called twice in correct order + assert_tts_speak_frames_queued(self.mock_task, ["First", "Second"]) + + def test_register_invalid_handler(self): + """Test registering invalid action handlers.""" + # Test non-callable handler + with self.assertRaises(ValueError) as context: + self.action_manager._register_action("invalid", "not_callable") + self.assertIn("must be callable", str(context.exception)) + + # Test None handler + with self.assertRaises(ValueError) as context: + self.action_manager._register_action("invalid", None) + self.assertIn("must be callable", str(context.exception)) + + async def test_none_or_empty_actions(self): + """Test handling None or empty action lists.""" + # Test None actions + await self.action_manager.execute_actions(None) + self.mock_task.queue_frame.assert_not_called() + + # Test empty list + await self.action_manager.execute_actions([]) + self.mock_task.queue_frame.assert_not_called() + + @patch("loguru.logger.error") + async def test_action_error_handling(self, mock_logger): + """Test error handling during action execution.""" + # Configure task mock to raise an error + self.mock_task.queue_frame = AsyncMock(side_effect=Exception("Frame error")) + + action = {"type": "tts_say", "text": "Hello"} + await self.action_manager.execute_actions([action]) + + # Verify error was logged + mock_logger.assert_called_with("TTS error: Frame error") + + async def test_action_execution_error_handling(self): + """Test error handling during action execution.""" + action_manager = ActionManager(self.mock_task, self.mock_flow_manager) + + # Test action with missing handler + with self.assertRaises(ActionError): + await action_manager.execute_actions([{"type": "nonexistent_action"}]) + + # Test action handler that raises an exception + async def failing_handler(action): + raise Exception("Handler error") + + action_manager._register_action("failing_action", failing_handler) + + with self.assertRaises(ActionError): + await action_manager.execute_actions([{"type": "failing_action"}]) diff --git a/tests/test_flows_adapters.py b/tests/test_flows_adapters.py new file mode 100644 index 00000000000..4589128b2cc --- /dev/null +++ b/tests/test_flows_adapters.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Tests for the LLM adapter. + +This module tests the LLMAdapter class used by the flow manager for +conversation-summary generation and formatting. + +Tests: + - Summary message formatting +""" + +import pytest + +from pipecat.flows.adapters import LLMAdapter + + +@pytest.fixture +def adapter(): + return LLMAdapter() + + +def test_format_summary_message(adapter): + """Test summary message formatting.""" + message = adapter.format_summary_message("Test summary") + assert message == { + "role": "developer", + "content": "Here's a summary of the conversation:\nTest summary", + } diff --git a/tests/test_flows_context_strategies.py b/tests/test_flows_context_strategies.py new file mode 100644 index 00000000000..4cac71beeae --- /dev/null +++ b/tests/test_flows_context_strategies.py @@ -0,0 +1,412 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Test suite for context management strategies. + +This module contains tests for the context management features of Pipecat Flows, +focusing on: +- Context strategy configuration +- Strategy behavior (APPEND, RESET, RESET_WITH_SUMMARY) +- Provider-specific message formatting +- Summary generation and integration +""" + +import unittest +import warnings +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +from pipecat.flows.exceptions import FlowError +from pipecat.flows.manager import FlowManager +from pipecat.flows.types import ContextStrategy, ContextStrategyConfig, NodeConfig +from pipecat.frames.frames import ( + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMUpdateSettingsFrame, +) +from pipecat.services.anthropic.llm import AnthropicLLMService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.openai.llm import OpenAILLMService + + +class TestContextStrategies(unittest.IsolatedAsyncioTestCase): + """Test suite for context management strategies. + + Tests functionality including: + - Strategy configuration and validation + - Strategy behavior and message handling + - Provider-specific adaptations + - Summary generation and integration + """ + + async def asyncSetUp(self): + """Set up test fixtures before each test.""" + self.mock_task = AsyncMock() + self.mock_task.event_handler = Mock() + self.mock_task.set_reached_downstream_filter = Mock() + + # Set up mock LLM with client + self.mock_llm = OpenAILLMService(api_key="test-key") + self.mock_llm.run_inference = AsyncMock() + + self.mock_tts = AsyncMock() + + # Create mock context aggregator with messages + self.mock_context = MagicMock() + self.mock_context.messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + ] + self.mock_context.get_messages.return_value = self.mock_context.messages + + self.mock_context_aggregator = MagicMock() + self.mock_context_aggregator.user = MagicMock() + self.mock_context_aggregator.user.return_value = MagicMock() + self.mock_context_aggregator.user.return_value._context = self.mock_context + + # Sample node configuration + self.sample_node: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Test task."}], + "functions": [], + } + + async def test_context_strategy_config_validation(self): + """Test ContextStrategyConfig validation.""" + # Valid configurations + ContextStrategyConfig(strategy=ContextStrategy.APPEND) + ContextStrategyConfig(strategy=ContextStrategy.RESET) + ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, summary_prompt="Summarize the conversation" + ) + + # Invalid configuration - missing prompt + with self.assertRaises(ValueError): + ContextStrategyConfig(strategy=ContextStrategy.RESET_WITH_SUMMARY) + + async def test_reset_with_summary_deprecation_warning(self): + """Test that RESET_WITH_SUMMARY emits a DeprecationWarning at runtime.""" + mock_summary = "Conversation summary" + self.mock_llm.run_inference.return_value = mock_summary + + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, + summary_prompt="Summarize the conversation", + ), + ) + await flow_manager.initialize() + + # First node using RESET_WITH_SUMMARY should trigger the deprecation warning + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + await flow_manager._set_node("first", self.sample_node) + + deprecation_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + self.assertTrue(len(deprecation_warnings) >= 1) + self.assertIn("RESET_WITH_SUMMARY is deprecated", str(deprecation_warnings[0].message)) + + # Second node should NOT trigger a second warning (once-only) + self.mock_task.queue_frames.reset_mock() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + await flow_manager._set_node("second", self.sample_node) + + deprecation_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + self.assertEqual(len(deprecation_warnings), 0) + + async def test_default_strategy(self): + """Test default context strategy (APPEND).""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Under the default (APPEND) strategy the first node appends, keeping any + # context already present. + await flow_manager._set_node("first", self.sample_node) + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + self.assertTrue(any(isinstance(f, LLMMessagesAppendFrame) for f in first_frames)) + self.assertFalse(any(isinstance(f, LLMMessagesUpdateFrame) for f in first_frames)) + + # Reset mock + self.mock_task.queue_frames.reset_mock() + + # Subsequent node should use AppendFrame with default strategy + await flow_manager._set_node("second", self.sample_node) + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + self.assertTrue(any(isinstance(f, LLMMessagesAppendFrame) for f in second_frames)) + + async def test_reset_strategy(self): + """Test RESET strategy behavior.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig(strategy=ContextStrategy.RESET), + ) + await flow_manager.initialize() + + # First node should use UpdateFrame under the RESET strategy + await flow_manager._set_node("first", self.sample_node) + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + self.assertTrue(any(isinstance(f, LLMMessagesUpdateFrame) for f in first_frames)) + self.mock_task.queue_frames.reset_mock() + + # Second node should use UpdateFrame with RESET strategy + await flow_manager._set_node("second", self.sample_node) + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + self.assertTrue(any(isinstance(f, LLMMessagesUpdateFrame) for f in second_frames)) + + async def test_reset_with_summary_success(self): + """Test successful RESET_WITH_SUMMARY strategy.""" + # Mock successful summary generation + mock_summary = "Conversation summary" + self.mock_llm.run_inference.return_value = mock_summary + + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, + summary_prompt="Summarize the conversation", + ), + ) + await flow_manager.initialize() + + # Set nodes and verify summary inclusion + await flow_manager._set_node("first", self.sample_node) + self.mock_task.queue_frames.reset_mock() + + await flow_manager._set_node("second", self.sample_node) + + # Verify summary was included in context update + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + update_frame = next(f for f in second_frames if isinstance(f, LLMMessagesUpdateFrame)) + self.assertTrue(any(mock_summary in str(m) for m in update_frame.messages)) + + async def test_reset_with_summary_timeout(self): + """Test RESET_WITH_SUMMARY fallback to APPEND on timeout.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, + summary_prompt="Summarize the conversation", + ), + ) + await flow_manager.initialize() + + # Mock timeout + self.mock_llm.run_inference.side_effect = AsyncMock(side_effect=TimeoutError) + + # Set nodes and verify fallback to APPEND + await flow_manager._set_node("first", self.sample_node) + self.mock_task.queue_frames.reset_mock() + + await flow_manager._set_node("second", self.sample_node) + + # Verify UpdateFrame was used (APPEND behavior) + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + self.assertTrue(any(isinstance(f, LLMMessagesAppendFrame) for f in second_frames)) + + async def test_provider_specific_summary_formatting(self): + """Test summary formatting for different LLM providers.""" + summary = "Test summary" + + # Test OpenAI format + flow_manager = FlowManager( + worker=self.mock_task, + llm=OpenAILLMService(api_key="test-key"), + context_aggregator=self.mock_context_aggregator, + ) + openai_message = flow_manager._adapter.format_summary_message(summary) + self.assertEqual(openai_message["role"], "developer") + + # Test Anthropic format + flow_manager = FlowManager( + worker=self.mock_task, + llm=AnthropicLLMService(api_key="test-key"), + context_aggregator=self.mock_context_aggregator, + ) + anthropic_message = flow_manager._adapter.format_summary_message(summary) + self.assertEqual(anthropic_message["role"], "developer") + + # Test Gemini format + flow_manager = FlowManager( + worker=self.mock_task, + llm=GoogleLLMService(api_key=" "), # dummy key (GoogleLLMService rejects empty string) + context_aggregator=self.mock_context_aggregator, + ) + gemini_message = flow_manager._adapter.format_summary_message(summary) + self.assertEqual(gemini_message["role"], "developer") + + async def test_node_level_strategy_override(self): + """Test that node-level strategy overrides global strategy.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig(strategy=ContextStrategy.APPEND), + ) + await flow_manager.initialize() + + # Create node with RESET strategy + node_with_strategy = { + **self.sample_node, + "context_strategy": ContextStrategyConfig(strategy=ContextStrategy.RESET), + } + + # Set nodes and verify strategy override + await flow_manager._set_node("first", self.sample_node) + self.mock_task.queue_frames.reset_mock() + + await flow_manager._set_node("second", node_with_strategy) + + # Verify UpdateFrame was used (RESET behavior) despite global APPEND + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + self.assertTrue(any(isinstance(f, LLMMessagesUpdateFrame) for f in second_frames)) + + async def test_summary_generation_content(self): + """Test that summary generation uses correct prompt and context.""" + mock_summary = "Generated summary" + self.mock_llm.run_inference.return_value = mock_summary + + summary_prompt = "Create a detailed summary" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, summary_prompt=summary_prompt + ), + ) + await flow_manager.initialize() + + # Set nodes to trigger summary generation + await flow_manager._set_node("first", self.sample_node) + await flow_manager._set_node("second", self.sample_node) + + # Verify summary generation call + run_inference_call = self.mock_llm.run_inference.call_args + run_inference_args = run_inference_call[0] + run_inference_kwargs = run_inference_call[1] + + # Verify summary prompt was passed as system_instruction kwarg + self.assertEqual(run_inference_kwargs["system_instruction"], summary_prompt) + + # Verify conversation history was included in context messages + context = run_inference_args[0] + self.assertTrue( + any( + str(self.mock_context.messages[0]["content"]) in str(m) + for m in context.get_messages() + ) + ) + + async def test_context_structure_after_summary(self): + """Test the structure of context after summary generation.""" + mock_summary = "Generated summary" + self.mock_llm.run_inference.return_value = mock_summary + + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, summary_prompt="Summarize" + ), + ) + await flow_manager.initialize() + + # Set nodes to trigger summary generation + await flow_manager._set_node("first", self.sample_node) + self.mock_task.queue_frames.reset_mock() + + # Node with new task messages + new_node = { + "task_messages": [{"role": "developer", "content": "New task."}], + "functions": [], + } + await flow_manager._set_node("second", new_node) + + # Verify context structure + update_call = self.mock_task.queue_frames.call_args_list[0] + update_frames = update_call[0][0] + messages_frame = next(f for f in update_frames if isinstance(f, LLMMessagesUpdateFrame)) + + # Verify order: summary message, then new task messages + self.assertTrue(mock_summary in str(messages_frame.messages[0])) + self.assertEqual( + messages_frame.messages[1]["content"], new_node["task_messages"][0]["content"] + ) + + async def test_reset_with_summary_and_role_messages(self): + """Test that LLMUpdateSettingsFrame and summary coexist correctly.""" + mock_summary = "Conversation summary" + self.mock_llm.run_inference.return_value = mock_summary + + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig( + strategy=ContextStrategy.RESET_WITH_SUMMARY, + summary_prompt="Summarize the conversation", + ), + ) + await flow_manager.initialize() + + # Set first node (with role_message) + first_node = { + "role_message": "You are a helpful assistant.", + "task_messages": [{"role": "developer", "content": "First task."}], + "functions": [], + } + await flow_manager._set_node("first", first_node) + self.mock_task.queue_frames.reset_mock() + + # Set second node with role_message — triggers summary + settings update + second_node = { + "role_message": "You are now a different assistant.", + "task_messages": [{"role": "developer", "content": "Second task."}], + "functions": [], + } + await flow_manager._set_node("second", second_node) + + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + + # Verify LLMUpdateSettingsFrame is present with new system instruction + settings_frames = [f for f in second_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 1) + self.assertEqual( + settings_frames[0].delta.system_instruction, "You are now a different assistant." + ) + + # Verify UpdateFrame contains summary + task_messages (not role_messages) + update_frames = [f for f in second_frames if isinstance(f, LLMMessagesUpdateFrame)] + self.assertEqual(len(update_frames), 1) + messages = update_frames[0].messages + self.assertTrue(mock_summary in str(messages[0])) + self.assertEqual(messages[1]["content"], "Second task.") + + # Verify frame ordering: LLMUpdateSettingsFrame before LLMMessagesUpdateFrame + settings_idx = second_frames.index(settings_frames[0]) + update_idx = second_frames.index(update_frames[0]) + self.assertLess(settings_idx, update_idx) diff --git a/tests/test_flows_direct_functions.py b/tests/test_flows_direct_functions.py new file mode 100644 index 00000000000..c62b6e7a5b8 --- /dev/null +++ b/tests/test_flows_direct_functions.py @@ -0,0 +1,422 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import unittest +from typing import Optional, TypedDict, Union + +from pipecat.flows.exceptions import InvalidFunctionError +from pipecat.flows.manager import FlowManager +from pipecat.flows.types import ( + ConsolidatedFunctionResult, + FlowsDirectFunctionWrapper, + flows_direct_function, + flows_tool_options, +) + +"""Tests for FlowsDirectFunction class.""" + + +class TestFlowsDirectFunction(unittest.TestCase): + def test_name_is_set_from_function(self): + """Test that FlowsDirectFunction extracts the name from the function.""" + + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function)) + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertEqual(func.name, "my_function") + + def test_description_is_set_from_function(self): + """Test that FlowsDirectFunction extracts the description from the function.""" + + async def my_function_short_description(flow_manager: FlowManager): + """This is a test function.""" + return {"status": "success"}, None + + self.assertIsNone( + FlowsDirectFunctionWrapper.validate_function(my_function_short_description) + ) + func = FlowsDirectFunctionWrapper(function=my_function_short_description) + self.assertEqual(func.description, "This is a test function.") + + async def my_function_long_description(flow_manager: FlowManager): + """ + This is a test function. + + It does some really cool stuff. + + Trust me, you'll want to use it. + """ + return {"status": "success"}, None + + self.assertIsNone( + FlowsDirectFunctionWrapper.validate_function(my_function_long_description) + ) + func = FlowsDirectFunctionWrapper(function=my_function_long_description) + self.assertEqual( + func.description, + "This is a test function.\n\nIt does some really cool stuff.\n\nTrust me, you'll want to use it.", + ) + + def test_properties_are_set_from_function(self): + """Test that FlowsDirectFunction extracts the properties from the function.""" + + async def my_function_no_params(flow_manager: FlowManager): + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function_no_params)) + func = FlowsDirectFunctionWrapper(function=my_function_no_params) + self.assertEqual(func.properties, {}) + + async def my_function_simple_params( + flow_manager: FlowManager, name: str, age: int, height: float | None + ): + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function_simple_params)) + func = FlowsDirectFunctionWrapper(function=my_function_simple_params) + self.assertEqual( + func.properties, + { + "name": {"type": "string"}, + "age": {"type": "integer"}, + "height": {"anyOf": [{"type": "number"}, {"type": "null"}]}, + }, + ) + + async def my_function_complex_params( + flow_manager: FlowManager, + address_lines: list[str], + nickname: str | int | float, + extra: dict[str, str] | None, + ): + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function_complex_params)) + func = FlowsDirectFunctionWrapper(function=my_function_complex_params) + self.assertEqual( + func.properties, + { + "address_lines": {"type": "array", "items": {"type": "string"}}, + "nickname": { + "anyOf": [{"type": "string"}, {"type": "integer"}, {"type": "number"}] + }, + "extra": { + "anyOf": [ + {"type": "object", "additionalProperties": {"type": "string"}}, + {"type": "null"}, + ] + }, + }, + ) + + class MyInfo1(TypedDict): + name: str + age: int + + class MyInfo2(TypedDict, total=False): + name: str + age: int + + async def my_function_complex_type_params( + flow_manager: FlowManager, info1: MyInfo1, info2: MyInfo2 + ): + return {"status": "success"}, None + + self.assertIsNone( + FlowsDirectFunctionWrapper.validate_function(my_function_complex_type_params) + ) + func = FlowsDirectFunctionWrapper(function=my_function_complex_type_params) + self.assertEqual( + func.properties, + { + "info1": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + "required": ["name", "age"], + }, + "info2": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + }, + }, + ) + + def test_required_is_set_from_function(self): + """Test that FlowsDirectFunction extracts the required properties from the function.""" + + async def my_function_no_params(flow_manager: FlowManager): + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function_no_params)) + func = FlowsDirectFunctionWrapper(function=my_function_no_params) + self.assertEqual(func.required, []) + + async def my_function_simple_params( + flow_manager: FlowManager, name: str, age: int, height: float | None = None + ): + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function_simple_params)) + func = FlowsDirectFunctionWrapper(function=my_function_simple_params) + self.assertEqual(func.required, ["name", "age"]) + + async def my_function_complex_params( + flow_manager: FlowManager, + address_lines: list[str] | None, + nickname: str | int = "Bud", + extra: dict[str, str] | None = None, + ): + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function_complex_params)) + func = FlowsDirectFunctionWrapper(function=my_function_complex_params) + self.assertEqual(func.required, ["address_lines"]) + + def test_property_descriptions_are_set_from_function(self): + """Test that FlowsDirectFunction extracts the property descriptions from the function.""" + + async def my_function(flow_manager: FlowManager, name: str, age: int, height: float | None): + """ + This is a test function. + + Args: + name (str): The name of the person. + age (int): The age of the person. + height (float | None): The height of the person in meters. Defaults to None. + """ + return {"status": "success"}, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_function)) + func = FlowsDirectFunctionWrapper(function=my_function) + + # Validate that the function description is still set correctly even with the longer docstring + self.assertEqual(func.description, "This is a test function.") + + # Validate that the property descriptions are set correctly + self.assertEqual( + func.properties, + { + "name": {"type": "string", "description": "The name of the person."}, + "age": {"type": "integer", "description": "The age of the person."}, + "height": { + "anyOf": [{"type": "number"}, {"type": "null"}], + "description": "The height of the person in meters. Defaults to None.", + }, + }, + ) + + def test_invalid_functions_fail_validation(self): + """Test that invalid functions fail FlowsDirectFunction validation.""" + + def my_function_non_async(flow_manager: FlowManager): + return {"status": "success"}, None + + with self.assertRaises(InvalidFunctionError): + FlowsDirectFunctionWrapper.validate_function(my_function_non_async) + + async def my_function_missing_flow_manager(): + return {"status": "success"}, None + + with self.assertRaises(InvalidFunctionError): + FlowsDirectFunctionWrapper.validate_function(my_function_missing_flow_manager) + + async def my_function_misplaced_flow_manager(foo: str, flow_manager: FlowManager): + return {"status": "success"}, None + + with self.assertRaises(InvalidFunctionError): + FlowsDirectFunctionWrapper.validate_function(my_function_misplaced_flow_manager) + + def test_invoke_calls_function_with_args_and_flow_manager(self): + """Test that FlowsDirectFunction.invoke calls the function with correct args and flow_manager.""" + + called = {} + + class DummyFlowManager: + pass + + async def my_function(flow_manager: DummyFlowManager, name: str, age: int): + called["flow_manager"] = flow_manager + called["name"] = name + called["age"] = age + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_function) + flow_manager = DummyFlowManager() + args = {"name": "Alice", "age": 30} + + result = asyncio.run(func.invoke(args=args, flow_manager=flow_manager)) + self.assertEqual(result, ({"status": "success"}, None)) + self.assertIs(called["flow_manager"], flow_manager) + self.assertEqual(called["name"], "Alice") + self.assertEqual(called["age"], 30) + + +class TestFlowsDirectFunctionDecorator(unittest.TestCase): + def test_cancel_on_interruption_defaults_to_false(self): + """Test that cancel_on_interruption defaults to False for non-decorated functions.""" + + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertFalse(func.cancel_on_interruption) + + def test_cancel_on_interruption_can_be_set_to_false(self): + """Test that cancel_on_interruption can be set to False via decorator.""" + + @flows_tool_options(cancel_on_interruption=False) + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertFalse(func.cancel_on_interruption) + + def test_cancel_on_interruption_can_be_explicitly_set_to_true(self): + """Test that cancel_on_interruption can be explicitly set to True via decorator.""" + + @flows_tool_options(cancel_on_interruption=True) + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertTrue(func.cancel_on_interruption) + + def test_decorator_preserves_function_metadata(self): + """Test that the decorator preserves function name and docstring.""" + + @flows_tool_options(cancel_on_interruption=False) + async def my_decorated_function(flow_manager: FlowManager, name: str): + """This is a decorated function. + + Args: + name: The name to use. + """ + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_decorated_function) + self.assertEqual(func.name, "my_decorated_function") + self.assertEqual(func.description, "This is a decorated function.") + self.assertEqual( + func.properties, + {"name": {"type": "string", "description": "The name to use."}}, + ) + self.assertFalse(func.cancel_on_interruption) + + def test_timeout_secs_defaults_to_none(self): + """Test that timeout_secs defaults to None for non-decorated functions.""" + + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertIsNone(func.timeout_secs) + + def test_timeout_secs_can_be_set(self): + """Test that timeout_secs can be set via decorator.""" + + @flows_tool_options(timeout_secs=30) + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertEqual(func.timeout_secs, 30) + + def test_decorator_preserves_function_metadata_with_timeout(self): + """Test that the decorator preserves function name and docstring with timeout_secs.""" + + @flows_tool_options(cancel_on_interruption=False, timeout_secs=15.5) + async def my_decorated_function(flow_manager: FlowManager, name: str): + """This is a decorated function. + + Args: + name: The name to use. + """ + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_decorated_function) + self.assertEqual(func.name, "my_decorated_function") + self.assertEqual(func.description, "This is a decorated function.") + self.assertEqual( + func.properties, + {"name": {"type": "string", "description": "The name to use."}}, + ) + self.assertFalse(func.cancel_on_interruption) + self.assertEqual(func.timeout_secs, 15.5) + + +class TestFlowsDirectFunctionDeprecatedAlias(unittest.TestCase): + """@flows_direct_function is a deprecated alias of @flows_tool_options.""" + + def test_alias_warns_but_still_attaches_options(self): + with self.assertWarns(DeprecationWarning): + + @flows_direct_function(cancel_on_interruption=True, timeout_secs=42) + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + # The deprecated alias still configures the same options. + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertTrue(func.cancel_on_interruption) + self.assertEqual(func.timeout_secs, 42) + + def test_tool_options_does_not_warn(self): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + @flows_tool_options(cancel_on_interruption=False, timeout_secs=10) + async def my_function(flow_manager: FlowManager): + return {"status": "success"}, None + + func = FlowsDirectFunctionWrapper(function=my_function) + self.assertEqual(func.timeout_secs, 10) + + +class TestConsolidatedFunctionResult(unittest.TestCase): + """Regression tests for ``ConsolidatedFunctionResult`` resolvability.""" + + def test_get_type_hints_resolves_without_nodeconfig_import(self): + """Annotating a function with ``ConsolidatedFunctionResult`` should not require importing ``NodeConfig``. + + Previously the alias was defined as ``tuple[Any, "NodeConfig | None"]`` + with a string forward reference, which ``get_type_hints()`` resolves + against the user's module globals. Users who did not separately import + ``NodeConfig`` got ``NameError: name 'NodeConfig' is not defined`` — + which then surfaced from ``FlowManager._set_node`` and stalled the + flow. See https://github.com/pipecat-ai/pipecat-flows/issues/271. + """ + from typing import get_type_hints + + async def my_tool(flow_manager: FlowManager) -> ConsolidatedFunctionResult: + """Do something and optionally transition.""" + return None, None + + hints = get_type_hints(my_tool) + self.assertIn("return", hints) + + def test_direct_function_wrapper_accepts_consolidated_return_type(self): + """``FlowsDirectFunctionWrapper`` introspects via ``get_type_hints`` internally.""" + + async def my_tool(flow_manager: FlowManager) -> ConsolidatedFunctionResult: + """Do something and optionally transition.""" + return None, None + + self.assertIsNone(FlowsDirectFunctionWrapper.validate_function(my_tool)) + FlowsDirectFunctionWrapper(function=my_tool) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_flows_manager.py b/tests/test_flows_manager.py new file mode 100644 index 00000000000..eb922aa1e5b --- /dev/null +++ b/tests/test_flows_manager.py @@ -0,0 +1,1346 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Test suite for FlowManager functionality. + +This module contains tests for the FlowManager class, which handles conversation +flow management across different LLM providers. Tests cover: +- Flow initialization +- State transitions and validation +- Function registration and execution +- Action handling +- Error cases + +The tests use unittest.IsolatedAsyncioTestCase for async support and +include mocked dependencies for PipelineTask and LLM services. +""" + +import unittest +from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch + +from pipecat.flows.exceptions import FlowError, FlowTransitionError +from pipecat.flows.manager import FlowManager, NodeConfig +from pipecat.flows.types import FlowArgs, FlowResult, FlowsFunctionSchema, flows_tool_options +from pipecat.frames.frames import ( + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMUpdateSettingsFrame, +) +from pipecat.services.llm_service import FunctionCallParams +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.settings import LLMSettings +from tests.flows_test_helpers import ( + assert_tts_speak_frames_queued, + get_advertised_tool_handlers, + get_advertised_tools, + make_mock_task, +) + + +class TestFlowManager(unittest.IsolatedAsyncioTestCase): + """Test suite for FlowManager class. + + Tests functionality of FlowManager including: + - Flow initialization + - State transitions + - Function registration + - Action execution + - Error handling + - Node validation + """ + + async def asyncSetUp(self): + """Set up test fixtures before each test.""" + self.mock_task = make_mock_task() + self.mock_llm = OpenAILLMService(api_key="test-key") + + # Create mock assistant aggregator with public property only + self.mock_assistant_aggregator = MagicMock() + type(self.mock_assistant_aggregator).has_function_calls_in_progress = PropertyMock( + return_value=False # Default to no functions in progress + ) + + # Create mock context aggregator + self.mock_context_aggregator = MagicMock() + self.mock_context_aggregator.user = MagicMock() + self.mock_context_aggregator.user.return_value = MagicMock() + + self.mock_context_aggregator.assistant = MagicMock( + return_value=self.mock_assistant_aggregator + ) + + self.mock_result_callback = AsyncMock() + + # Sample node configurations + self.sample_node: NodeConfig = { + "role_message": "You are a helpful test assistant.", + "task_messages": [{"role": "developer", "content": "Complete the test task."}], + "functions": [ + FlowsFunctionSchema( + name="test_function", + description="Test function", + properties={}, + required=[], + handler=AsyncMock(return_value={"status": "success"}), + ), + ], + } + + async def test_worker_and_task_arguments(self): + """Test the worker argument and the deprecated task argument.""" + # worker= is the canonical argument + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + self.assertIs(flow_manager.worker, self.mock_task) + + # task= still works but is deprecated + with self.assertWarns(DeprecationWarning): + flow_manager = FlowManager( + task=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + self.assertIs(flow_manager.worker, self.mock_task) + + # The task property still resolves to the worker, but is deprecated + with self.assertWarns(DeprecationWarning): + self.assertIs(flow_manager.task, self.mock_task) + + # Passing both is an error + with self.assertRaises(ValueError): + FlowManager( + worker=self.mock_task, + task=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + + # Passing neither is an error + with self.assertRaises(ValueError): + FlowManager( + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + + async def test_flow_initialization(self): + """Test initialization of flow.""" + # Create mock transition callback + mock_function = AsyncMock() + + # Initialize flow manager + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + + # Create test node + test_node: NodeConfig = { + "name": "test", + "task_messages": [{"role": "developer", "content": "Test message"}], + "functions": [ + FlowsFunctionSchema( + name="test_function", + description="Test function", + properties={}, + required=[], + handler=mock_function, + ), + ], + } + + # Initialize and set node + await flow_manager.initialize() + await flow_manager.set_node_from_config(test_node) + + self.assertFalse(mock_function.called) # Shouldn't be called until function is used + self.assertEqual(flow_manager._current_node, "test") + + async def test_node_validation(self): + """Test node configuration validation.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Test missing task_messages + invalid_config = {"functions": []} + with self.assertRaises(FlowError) as context: + await flow_manager.set_node_from_config(invalid_config) + self.assertIn("missing required 'task_messages' field", str(context.exception)) + + # Test valid config + valid_config = {"name": "test", "task_messages": []} + await flow_manager.set_node_from_config(valid_config) + + self.assertEqual(flow_manager._current_node, "test") + self.assertEqual(flow_manager._current_functions, set()) + + async def test_function_registration(self): + """Test that a node's functions are advertised with a handler for auto-registration.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Reset mock to clear initialization calls + self.mock_task.queue_frames.reset_mock() + + # Set node with function + await flow_manager.set_node_from_config(self.sample_node) + + # The tool is advertised carrying its handler, which the LLM service + # registers when it sees the advertised tools. + handlers = get_advertised_tool_handlers(self.mock_task) + self.assertEqual(set(handlers), {"test_function"}) + self.assertTrue(callable(handlers["test_function"])) + + async def test_action_execution(self): + """Test execution of pre and post actions.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Create node config with actions + node_with_actions: NodeConfig = { + "role_message": self.sample_node["role_message"], + "task_messages": self.sample_node["task_messages"], + "functions": self.sample_node["functions"], + "pre_actions": [{"type": "tts_say", "text": "Pre action"}], + "post_actions": [{"type": "tts_say", "text": "Post action"}], + } + + # Reset mock to clear initialization calls + self.mock_task.queue_frame.reset_mock() + + # Set node with actions + await flow_manager.set_node_from_config(node_with_actions) + + assert_tts_speak_frames_queued(self.mock_task, ["Pre action", "Post action"]) + + async def test_error_handling(self): + """Test error handling in flow manager. + + Verifies: + 1. Cannot set node before initialization + 2. Initialization fails properly when task queue fails + 3. Node setting fails when task queue fails + """ + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + + # Test setting node before initialization + with self.assertRaises(FlowTransitionError): + await flow_manager.set_node_from_config(self.sample_node) + + # Initialize normally + await flow_manager.initialize() + self.assertTrue(flow_manager._initialized) + + # Test node setting error + self.mock_task.queue_frames.side_effect = Exception("Queue error") + with self.assertRaises(FlowError): + await flow_manager.set_node_from_config(self.sample_node) + + # Verify flow manager remains initialized despite error + self.assertTrue(flow_manager._initialized) + + async def test_state_management(self): + """Test state management across nodes.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Set state data + test_value = "test_value" + flow_manager.state["test_key"] = test_value + + # Reset mock to clear initialization calls + self.mock_task.queue_frames.reset_mock() + + # Verify state persists across node transitions + await flow_manager.set_node_from_config(self.sample_node) + self.assertEqual(flow_manager.state["test_key"], test_value) + + async def test_multiple_function_registration(self): + """Test registration of multiple functions.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Create node config with multiple functions + node_config: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Test"}], + "functions": [ + FlowsFunctionSchema( + name=f"func_{i}", + description=f"Function {i}", + properties={}, + required=[], + handler=AsyncMock(return_value={"status": "success"}), + ) + for i in range(3) + ], + } + + await flow_manager.set_node_from_config(node_config) + + # Verify all functions were advertised (each carrying a handler) and tracked + handlers = get_advertised_tool_handlers(self.mock_task) + self.assertEqual(set(handlers), {"func_0", "func_1", "func_2"}) + self.assertEqual(len(flow_manager._current_functions), 3) + + async def test_advertised_handlers_register_with_node_call_options(self): + """Advertised handlers register with each tool's resolved call options. + + The wrapped handler carries the tool's call options (via @tool_options), + so the LLM service resolves cancel_on_interruption to Flows' default of + False — not the service's own default of True — and honors explicit + overrides on both FlowsFunctionSchemas and direct functions. + """ + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + async def handler(args, flow_manager): + return {"ok": True}, None + + @flows_tool_options(cancel_on_interruption=True, timeout_secs=7) + async def direct_tool(flow_manager, city: str): + """Do a thing. + + Args: + city: A city. + """ + return {"ok": True}, None + + await flow_manager.set_node_from_config( + { + "task_messages": [{"role": "developer", "content": "Test"}], + "functions": [ + FlowsFunctionSchema( + name="defaults", + description="Uses default call options", + properties={}, + required=[], + handler=handler, + ), + FlowsFunctionSchema( + name="overrides", + description="Overrides call options", + properties={}, + required=[], + handler=handler, + cancel_on_interruption=True, + timeout_secs=12.5, + ), + direct_tool, + ], + } + ) + + # Register the advertised tools the way the LLM service does on inference. + self.mock_llm._sync_registered_tool_handlers(get_advertised_tools(self.mock_task)) + + # FlowsFunctionSchema default: Flows' False default survives (not the service's True). + defaults = self.mock_llm._functions["defaults"] + self.assertFalse(defaults.cancel_on_interruption) + self.assertIsNone(defaults.timeout_secs) + + # FlowsFunctionSchema explicit overrides are honored. + overrides = self.mock_llm._functions["overrides"] + self.assertTrue(overrides.cancel_on_interruption) + self.assertEqual(overrides.timeout_secs, 12.5) + + # A direct function's @flows_tool_options values are honored. + direct = self.mock_llm._functions["direct_tool"] + self.assertTrue(direct.cancel_on_interruption) + self.assertEqual(direct.timeout_secs, 7) + + async def test_redeclared_function_rebinds_new_handler(self): + """Regression: redeclaring a function in a new node must bind the new handler. + + Two adjacent nodes declare ``go`` with different handlers returning + different next nodes. The handler advertised for ``go`` must reflect + the latest node's handler, not the first one's. + + See https://github.com/pipecat-ai/pipecat-flows/issues/269. + """ + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + handler_a_calls = [] + handler_b_calls = [] + + async def handler_a(args, flow_manager): + handler_a_calls.append(args) + return {"from": "A"}, { + "task_messages": [{"role": "developer", "content": "menu"}], + "functions": [], + } + + async def handler_b(args, flow_manager): + handler_b_calls.append(args) + return {"from": "B"}, { + "task_messages": [{"role": "developer", "content": "home"}], + "functions": [], + } + + await flow_manager.set_node_from_config( + { + "task_messages": [{"role": "developer", "content": "A"}], + "functions": [ + FlowsFunctionSchema( + name="go", + description="A's go", + properties={}, + required=[], + handler=handler_a, + ), + ], + } + ) + await flow_manager.set_node_from_config( + { + "task_messages": [{"role": "developer", "content": "B"}], + "functions": [ + FlowsFunctionSchema( + name="go", + description="B's go", + properties={}, + required=[], + handler=handler_b, + ), + ], + } + ) + + # After node B, the advertised ``go`` handler must be node B's, not node A's. + latest_go = get_advertised_tool_handlers(self.mock_task)["go"] + + async def result_callback(result, *, properties=None): + pass + + params = FunctionCallParams( + function_name="go", + tool_call_id="t1", + arguments={}, + llm=None, + pipeline_worker=self.mock_task, + context=None, + result_callback=result_callback, + ) + await latest_go(params) + + self.assertEqual(len(handler_b_calls), 1, "handler_b should have run") + self.assertEqual(len(handler_a_calls), 0, "handler_a should NOT have run") + + async def test_initialize_already_initialized(self): + """Test initializing an already initialized flow manager.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Try to initialize again + with patch("loguru.logger.warning") as mock_logger: + await flow_manager.initialize() + mock_logger.assert_called_once() + + async def test_register_action(self): + """Test registering custom actions.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + + async def custom_action(action): + pass + + flow_manager.register_action("custom", custom_action) + self.assertIn("custom", flow_manager._action_manager._action_handlers) + + async def test_call_handler_variations(self): + """Test different handler signature variations.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Test handler with args + async def handler_with_args(args): + return {"status": "success", "args": args} + + result = await flow_manager._call_handler(handler_with_args, {"test": "value"}) + self.assertEqual(result["args"]["test"], "value") + + # Test handler without args + async def handler_no_args(): + return {"status": "success"} + + result = await flow_manager._call_handler(handler_no_args, {}) + self.assertEqual(result["status"], "success") + + # Test handler with FlowManager parameter (2+ parameters) + async def handler_with_flow_manager(args, flow_manager_param): + return { + "status": "success", + "has_flow_manager": True, + "flow_manager": flow_manager_param, # Return for verification + "args": args, + } + + result = await flow_manager._call_handler(handler_with_flow_manager, {"test": "value"}) + self.assertEqual(result["status"], "success") + self.assertTrue(result["has_flow_manager"]) + self.assertIs(result["flow_manager"], flow_manager) # Verify it's the same instance + self.assertTrue(isinstance(result["flow_manager"], FlowManager)) + self.assertEqual(result["args"]["test"], "value") + + # Test instance method handler + class TestHandlerClass: + def __init__(self): + self.instance_data = "test_instance" + + async def instance_method_handler(self, args): + return {"status": "success", "instance_data": self.instance_data, "args": args} + + async def instance_method_with_flow_manager(self, args, flow_manager_param): + return { + "status": "success", + "has_flow_manager": True, + "flow_manager": flow_manager_param, # Return for verification + "instance_data": self.instance_data, + "args": args, + } + + @classmethod + async def class_method_handler(cls, args): + return {"status": "success", "class_data": "test_class", "args": args} + + @classmethod + async def class_method_with_flow_manager(cls, args, flow_manager_param): + return { + "status": "success", + "has_flow_manager": True, + "flow_manager": flow_manager_param, # Return for verification + "class_data": "test_class", + "args": args, + } + + test_instance = TestHandlerClass() + + # Test instance method (1 parameter after self) + result = await flow_manager._call_handler( + test_instance.instance_method_handler, {"test": "value"} + ) + self.assertEqual(result["status"], "success") + self.assertEqual(result["instance_data"], "test_instance") + self.assertEqual(result["args"]["test"], "value") + + # Test instance method with FlowManager (2+ parameters after self) + result = await flow_manager._call_handler( + test_instance.instance_method_with_flow_manager, {"test": "value"} + ) + self.assertEqual(result["status"], "success") + self.assertTrue(result["has_flow_manager"]) + self.assertIs(result["flow_manager"], flow_manager) # Verify it's the same instance + self.assertEqual(result["instance_data"], "test_instance") + self.assertEqual(result["args"]["test"], "value") + + # Test classmethod (1 parameter after cls) + result = await flow_manager._call_handler( + TestHandlerClass.class_method_handler, {"test": "value"} + ) + self.assertEqual(result["status"], "success") + self.assertEqual(result["class_data"], "test_class") + self.assertEqual(result["args"]["test"], "value") + + # Test classmethod with FlowManager (2+ parameters after cls) + result = await flow_manager._call_handler( + TestHandlerClass.class_method_with_flow_manager, {"test": "value"} + ) + self.assertEqual(result["status"], "success") + self.assertTrue(result["has_flow_manager"]) + self.assertIs(result["flow_manager"], flow_manager) # Verify it's the same instance + self.assertEqual(result["class_data"], "test_class") + self.assertEqual(result["args"]["test"], "value") + + async def test_transition_func_error_handling(self): + """Test error handling in transition functions.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + async def error_handler(args): + raise ValueError("Test error") + + transition_func = await flow_manager._create_transition_func("test", error_handler) + + # Mock result callback + callback_called = False + + async def result_callback(result): + nonlocal callback_called + callback_called = True + self.assertIn("error", result) + self.assertEqual(result["status"], "error") + self.assertIn("Test error", result["error"]) + + # The transition function should catch the error and pass it to the callback + params = FunctionCallParams( + function_name="test", + tool_call_id="id", + arguments={}, + llm=None, + pipeline_worker=self.mock_task, + context=None, + result_callback=result_callback, + ) + await transition_func(params) + self.assertTrue(callback_called, "Result callback was not called") + + async def test_node_validation_edge_cases(self): + """Test edge cases in node validation.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Test invalid function format (dict instead of FlowsFunctionSchema) + invalid_config = { + "task_messages": [{"role": "developer", "content": "Test"}], + "functions": [{"type": "function"}], + } + with self.assertRaises(FlowError) as context: + await flow_manager.set_node_from_config(invalid_config) + self.assertIn("Invalid function format", str(context.exception)) + + # A FlowsFunctionSchema requires a handler: omitting it is a construction-time error. + with self.assertRaises(TypeError): + FlowsFunctionSchema( + name="test_func", + description="Test", + properties={}, + required=[], + ) + + async def test_action_execution_error_handling(self): + """Test error handling in action execution.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Create node config with actions that will fail + node_config: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Test"}], + "functions": [], + "pre_actions": [{"type": "invalid_action"}], + "post_actions": [{"type": "another_invalid_action"}], + } + + # Should raise FlowError due to invalid actions + with self.assertRaises(FlowError): + await flow_manager.set_node_from_config(node_config) + + # Verify error handling for pre and post actions separately + with self.assertRaises(FlowError): + await flow_manager._execute_actions(pre_actions=[{"type": "invalid_action"}]) + + with self.assertRaises(FlowError): + await flow_manager._execute_actions(post_actions=[{"type": "invalid_action"}]) + + async def test_update_llm_context_error_handling(self): + """Test error handling in LLM context updates.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Mock worker to raise error on queue_frames + flow_manager._worker.queue_frames.side_effect = Exception("Queue error") + + with self.assertRaises(FlowError): + await flow_manager._update_llm_context( + role_message=None, + role_messages=None, + task_messages=[{"role": "developer", "content": "Test"}], + functions=[], + ) + + async def test_function_declarations_processing(self): + """Test processing of function declarations format.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + async def test_handler(args): + return {"status": "success"} + + # Create node config with multiple FlowsFunctionSchema functions + node_config: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Test"}], + "functions": [ + FlowsFunctionSchema( + name="test1", + description="Test function 1", + properties={}, + required=[], + handler=test_handler, + ), + FlowsFunctionSchema( + name="test2", + description="Test function 2", + properties={}, + required=[], + handler=test_handler, + ), + ], + } + + # Set node and verify function registration + await flow_manager.set_node_from_config(node_config) + + # Verify both functions were registered + self.assertIn("test1", flow_manager._current_functions) + self.assertIn("test2", flow_manager._current_functions) + + async def test_role_message_inheritance(self): + """Test that role_message is sent as LLMUpdateSettingsFrame.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # First node with role_message (singular) + first_node: NodeConfig = { + "role_message": "You are a helpful assistant.", + "task_messages": [{"role": "developer", "content": "First task."}], + "functions": [], + } + + # Second node without role messages + second_node: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Second task."}], + "functions": [], + } + + # Set first node + await flow_manager.set_node_from_config(first_node) + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + + # Verify LLMUpdateSettingsFrame with system_instruction + settings_frames = [f for f in first_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 1) + self.assertEqual( + settings_frames[0].delta.system_instruction, "You are a helpful assistant." + ) + + # Verify AppendFrame contains only task_messages (not role_messages) + append_frames = [f for f in first_frames if isinstance(f, LLMMessagesAppendFrame)] + self.assertEqual(len(append_frames), 1) + self.assertEqual(append_frames[0].messages, first_node["task_messages"]) + + # Verify frame ordering: LLMUpdateSettingsFrame before LLMMessagesAppendFrame + settings_idx = first_frames.index(settings_frames[0]) + append_idx = first_frames.index(append_frames[0]) + self.assertLess(settings_idx, append_idx) + + # Reset mock and set second node + self.mock_task.queue_frames.reset_mock() + await flow_manager.set_node_from_config(second_node) + + # Verify no LLMUpdateSettingsFrame for second node (no role_messages) + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + settings_frames = [f for f in second_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 0) + + # Verify AppendFrame with only task messages + append_frames = [f for f in second_frames if isinstance(f, LLMMessagesAppendFrame)] + self.assertEqual(len(append_frames), 1) + self.assertEqual(append_frames[0].messages, second_node["task_messages"]) + + async def test_frame_type_selection(self): + """Test that the context-update frame type follows the context strategy. + + Under the default (APPEND) strategy, the context update appends for + every node. + """ + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + test_node: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Test task."}], + "functions": [], + } + + # Under the default strategy the first node appends. + await flow_manager.set_node_from_config(test_node) + first_call = self.mock_task.queue_frames.call_args_list[0] # Get first call + first_frames = first_call[0][0] + self.assertTrue( + any(isinstance(f, LLMMessagesAppendFrame) for f in first_frames), + "First node should use AppendFrame under the default strategy", + ) + self.assertFalse( + any(isinstance(f, LLMMessagesUpdateFrame) for f in first_frames), + "First node should not use UpdateFrame under the default strategy", + ) + + # Reset mock + self.mock_task.queue_frames.reset_mock() + + # Subsequent node should also use AppendFrame + await flow_manager.set_node_from_config(test_node) + first_call = self.mock_task.queue_frames.call_args_list[0] # Get first call + second_frames = first_call[0][0] + self.assertTrue( + any(isinstance(f, LLMMessagesAppendFrame) for f in second_frames), + "Subsequent nodes should use AppendFrame", + ) + self.assertFalse( + any(isinstance(f, LLMMessagesUpdateFrame) for f in second_frames), + "Subsequent nodes should not use UpdateFrame", + ) + + async def test_edge_vs_node_function_behavior(self): + """Test different completion behavior for edge and node functions.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Create test functions + async def test_handler(args): + return {"status": "success"} + + async def consolidated_test_handler_1(args): + next_node = { + "task_messages": [{"role": "developer", "content": "Next"}], + "functions": [], + } + return {"status": "success"}, next_node + + async def consolidated_test_handler_2(args): + next_node = { + "task_messages": [{"role": "developer", "content": "Next"}], + "functions": [], + } + return {"status": "success"}, next_node + + # Create node with both types of functions + node_config: NodeConfig = { + "name": "test", + "task_messages": [{"role": "developer", "content": "Test"}], + "functions": [ + FlowsFunctionSchema( + name="node_function", + description="Node function", + properties={}, + required=[], + handler=test_handler, + ), + FlowsFunctionSchema( + name="edge_function_1", + description="Edge function", + properties={}, + required=[], + handler=consolidated_test_handler_1, + ), + FlowsFunctionSchema( + name="edge_function_2", + description="Edge function", + properties={}, + required=[], + handler=consolidated_test_handler_2, + ), + ], + } + + await flow_manager.set_node_from_config(node_config) + + # Get the advertised handlers (which the LLM service auto-registers) + handlers = get_advertised_tool_handlers(self.mock_task) + node_func = handlers["node_function"] + edge_func_1 = handlers["edge_function_1"] + edge_func_2 = handlers["edge_function_2"] + + # Test node function + self.mock_task.queue_frames.reset_mock() + node_result = None + node_properties = None + + async def node_callback(result, *, properties=None): + nonlocal node_result, node_properties + node_result = result + node_properties = properties + + params_1 = FunctionCallParams( + function_name="node_function", + tool_call_id="id1", + arguments={}, + llm=None, + pipeline_worker=self.mock_task, + context=None, + result_callback=node_callback, + ) + + await node_func(params_1) + # Node function should not set run_llm=False + self.assertTrue(node_properties is None or node_properties.run_llm is not False) + + # Test edge function 1 + self.mock_task.queue_frames.reset_mock() + edge_result_1 = None + edge_properties_1 = None + + async def edge_callback_1(result, *, properties=None): + nonlocal edge_result_1, edge_properties_1 + edge_result_1 = result + edge_properties_1 = properties + + params_1 = FunctionCallParams( + function_name="edge_function_1", + tool_call_id="id2", + arguments={}, + llm=None, + pipeline_worker=self.mock_task, + context=None, + result_callback=edge_callback_1, + ) + + await edge_func_1(params_1) + # Edge functions should set run_llm=False + self.assertTrue(edge_properties_1 is not None and edge_properties_1.run_llm is False) + + # Test edge function 2 + self.mock_task.queue_frames.reset_mock() + edge_result_2 = None + edge_properties_2 = None + + async def edge_callback_2(result, *, properties=None): + nonlocal edge_result_2, edge_properties_2 + edge_result_2 = result + edge_properties_2 = properties + + params_2 = FunctionCallParams( + function_name="edge_function_2", + tool_call_id="id3", + arguments={}, + llm=None, + pipeline_worker=self.mock_task, + context=None, + result_callback=edge_callback_2, + ) + + await edge_func_2(params_2) + # Edge functions should set run_llm=False + self.assertTrue(edge_properties_2 is not None and edge_properties_2.run_llm is False) + + @patch("pipecat.flows.manager.LLMRunFrame") + async def test_completion_timing(self, mock_llm_run_frame): + """Test that completions occur at the right time.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Test initial node setup + self.mock_task.queue_frames.reset_mock() + mock_llm_run_frame.reset_mock() + + await flow_manager.set_node_from_config( + { + "task_messages": [{"role": "developer", "content": "Test"}], + "functions": [], + }, + ) + + # Should see context update and completion trigger + # First call is for updating context + self.assertTrue(self.mock_task.queue_frames.called) + + # Verify that LLM completion was triggered by checking LLMRunFrame instantiation + mock_llm_run_frame.assert_called_once() + + # Test node transition by directly setting next node + next_node: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Next test"}], + "functions": [], + } + + self.mock_task.queue_frames.reset_mock() + mock_llm_run_frame.reset_mock() + + await flow_manager.set_node_from_config(next_node) + + # Should see context update and completion trigger again + self.assertTrue(self.mock_task.queue_frames.called) + mock_llm_run_frame.assert_called_once() + + async def test_get_current_context(self): + """Test getting current conversation context.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Mock the context messages + mock_messages = [{"role": "developer", "content": "Test message"}] + self.mock_context_aggregator.user()._context.get_messages.return_value = mock_messages + + # Test getting context + context = flow_manager.get_current_context() + self.assertEqual(context, mock_messages) + + # Test error when context aggregator is not available + flow_manager._context_aggregator = None + with self.assertRaises(FlowError) as context: + flow_manager.get_current_context() + self.assertIn("No context aggregator available", str(context.exception)) + + async def test_handler_with_flow_manager(self): + """Test function handler that receives both args and flow_manager.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + handler_called = False + correct_flow_manager = False + + async def modern_handler(args: FlowArgs, flow_mgr: FlowManager) -> FlowResult: + nonlocal handler_called, correct_flow_manager + handler_called = True + correct_flow_manager = flow_mgr is flow_manager + return {"status": "success", "args_received": args, "has_flow_manager": True} + + result = await flow_manager._call_handler(modern_handler, {"test": "value"}) + + self.assertTrue(handler_called) + self.assertTrue(correct_flow_manager) + self.assertEqual(result["args_received"]["test"], "value") + self.assertTrue(result["has_flow_manager"]) + + async def test_node_without_functions(self): + """Test node configuration without functions field.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Create node config without functions field + node_config: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Test task without functions."}], + } + + # Set node and verify it works without error + await flow_manager.set_node_from_config(node_config) + + # Verify current_functions is empty set + self.assertEqual(flow_manager._current_functions, set()) + + # Verify LLM tools were still set (with empty or placeholder functions) + tools_frames_call = [ + call + for call in self.mock_task.queue_frames.call_args_list + if any(isinstance(frame, LLMSetToolsFrame) for frame in call[0][0]) + ] + self.assertTrue(len(tools_frames_call) > 0, "Should have called LLMSetToolsFrame") + + async def test_node_with_empty_functions(self): + """Test node configuration with empty functions list.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + # Create node config with empty functions list + node_config: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Test task with empty functions."}], + "functions": [], + } + + # Set node and verify it works without error + await flow_manager.set_node_from_config(node_config) + + # Verify current_functions is empty set + self.assertEqual(flow_manager._current_functions, set()) + + # Verify LLM tools were still set (with empty or placeholder functions) + tools_frames_call = [ + call + for call in self.mock_task.queue_frames.call_args_list + if any(isinstance(frame, LLMSetToolsFrame) for frame in call[0][0]) + ] + self.assertTrue(len(tools_frames_call) > 0, "Should have called LLMSetToolsFrame") + + async def test_role_message_singular(self): + """Test that plain string role_message (singular) works correctly.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + node: NodeConfig = { + "role_message": "You are a helpful assistant.", + "task_messages": [{"role": "developer", "content": "Do the task."}], + "functions": [], + } + + await flow_manager.set_node_from_config(node) + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + + # Verify LLMUpdateSettingsFrame with correct system_instruction + settings_frames = [f for f in first_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 1) + self.assertEqual( + settings_frames[0].delta.system_instruction, "You are a helpful assistant." + ) + + # Verify messages frame contains only task_messages + append_frames = [f for f in first_frames if isinstance(f, LLMMessagesAppendFrame)] + self.assertEqual(len(append_frames), 1) + self.assertEqual(append_frames[0].messages, node["task_messages"]) + + async def test_role_messages_persist_across_reset(self): + """Test that system instruction persists when a RESET node omits role_message.""" + from pipecat.flows.types import ContextStrategy, ContextStrategyConfig + + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + context_strategy=ContextStrategyConfig(strategy=ContextStrategy.RESET), + ) + await flow_manager.initialize() + + # First node sets role_message + first_node: NodeConfig = { + "role_message": "You are a helpful assistant.", + "task_messages": [{"role": "developer", "content": "First task."}], + "functions": [], + } + + await flow_manager.set_node_from_config(first_node) + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + + # Verify first node sends LLMUpdateSettingsFrame + settings_frames = [f for f in first_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 1) + self.assertEqual( + settings_frames[0].delta.system_instruction, "You are a helpful assistant." + ) + + # Second node with RESET strategy but no role_messages + self.mock_task.queue_frames.reset_mock() + second_node: NodeConfig = { + "task_messages": [{"role": "developer", "content": "Second task."}], + "functions": [], + } + + await flow_manager.set_node_from_config(second_node) + second_call = self.mock_task.queue_frames.call_args_list[0] + second_frames = second_call[0][0] + + # No LLMUpdateSettingsFrame since no role_message — system instruction + # persists in LLM settings from the first node + settings_frames = [f for f in second_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 0) + + # Verify RESET still uses UpdateFrame for context messages + update_frames = [f for f in second_frames if isinstance(f, LLMMessagesUpdateFrame)] + self.assertEqual(len(update_frames), 1) + self.assertEqual(update_frames[0].messages, second_node["task_messages"]) + + async def test_role_messages_deprecated_warning(self): + """Test that using role_messages (plural) emits a DeprecationWarning.""" + import warnings + + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + node: NodeConfig = { + "role_messages": [{"role": "developer", "content": "You are a helpful assistant."}], + "task_messages": [{"role": "developer", "content": "Do the task."}], + "functions": [], + } + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + await flow_manager.set_node_from_config(node) + + deprecation_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + self.assertEqual(len(deprecation_warnings), 1) + self.assertIn("role_messages", str(deprecation_warnings[0].message)) + self.assertIn("role_message", str(deprecation_warnings[0].message)) + + # Verify the node still works correctly despite the warning — + # legacy role_messages go into context messages, not LLMUpdateSettingsFrame + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + settings_frames = [f for f in first_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 0) + + append_frames = [f for f in first_frames if isinstance(f, LLMMessagesAppendFrame)] + self.assertEqual(len(append_frames), 1) + self.assertEqual( + append_frames[0].messages[0], + {"role": "developer", "content": "You are a helpful assistant."}, + ) + + # Verify the warning is only emitted once + self.mock_task.queue_frames.reset_mock() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + await flow_manager.set_node_from_config(node) + deprecation_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + self.assertEqual(len(deprecation_warnings), 0) + + async def test_role_message_and_role_messages_both_specified(self): + """Test that role_message takes precedence when both are specified.""" + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + node: NodeConfig = { + "role_message": "I am the preferred role.", + "role_messages": [{"role": "developer", "content": "I am the deprecated role."}], + "task_messages": [{"role": "developer", "content": "Do the task."}], + "functions": [], + } + + with patch("pipecat.flows.manager.logger") as mock_logger: + await flow_manager.set_node_from_config(node) + mock_logger.warning.assert_any_call( + "Both 'role_message' and 'role_messages' specified; using 'role_message'" + ) + + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + settings_frames = [f for f in first_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 1) + self.assertEqual(settings_frames[0].delta.system_instruction, "I am the preferred role.") + + async def test_role_messages_list_format_still_works(self): + """Test that legacy list-of-dicts role_messages are prepended to context messages.""" + import warnings + + flow_manager = FlowManager( + worker=self.mock_task, + llm=self.mock_llm, + context_aggregator=self.mock_context_aggregator, + ) + await flow_manager.initialize() + + node: NodeConfig = { + "role_messages": [ + {"role": "developer", "content": "You are a helpful assistant."}, + {"role": "developer", "content": "Be concise."}, + ], + "task_messages": [{"role": "developer", "content": "Do the task."}], + "functions": [], + } + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + await flow_manager.set_node_from_config(node) + # Should emit deprecation warning for role_messages + deprecation_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + self.assertEqual(len(deprecation_warnings), 1) + + first_call = self.mock_task.queue_frames.call_args_list[0] + first_frames = first_call[0][0] + + # Legacy role_messages should NOT produce LLMUpdateSettingsFrame + settings_frames = [f for f in first_frames if isinstance(f, LLMUpdateSettingsFrame)] + self.assertEqual(len(settings_frames), 0) + + # Legacy role_messages should be prepended to context messages + append_frames = [f for f in first_frames if isinstance(f, LLMMessagesAppendFrame)] + self.assertEqual(len(append_frames), 1) + messages = append_frames[0].messages + self.assertEqual( + messages[0], {"role": "developer", "content": "You are a helpful assistant."} + ) + self.assertEqual(messages[1], {"role": "developer", "content": "Be concise."}) + self.assertEqual(messages[2], {"role": "developer", "content": "Do the task."})