diff --git a/.github/workflows/broken-links.yml b/.github/workflows/broken-links.yml index 71c6e4611..70bc31aac 100644 --- a/.github/workflows/broken-links.yml +++ b/.github/workflows/broken-links.yml @@ -24,7 +24,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: "20" + node-version-file: .nvmrc - name: Install Mintlify CLI run: npm install -g mint diff --git a/.gitignore b/.gitignore index b120acdbe..646ac519e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .DS_Store -.nvmrc +node_modules/ diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100644 index 000000000..2312dc587 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1 @@ +npx lint-staged diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..60ade1ae0 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24.19.0 diff --git a/CLAUDE.md b/CLAUDE.md index dc9bc1ed3..cb6b9dbae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,33 +2,39 @@ ## Project overview -This is the documentation site for [Pipecat](https://github.com/pipecat-ai/pipecat), hosted at [docs.pipecat.ai](https://docs.pipecat.ai). It's built with [Mintlify](https://mintlify.com/) and contains ~314 MDX files covering guides, API references, and deployment docs. +This is the documentation site for [Pipecat](https://github.com/pipecat-ai/pipecat), hosted at [docs.pipecat.ai](https://docs.pipecat.ai). It's built with [Mintlify](https://mintlify.com/) and contains several hundred MDX files covering guides, API references, and deployment docs. ## Development commands ```bash +# Install the Node version in .nvmrc, then dependencies and Git hooks +nvm install +npm install + # Start local dev server -mint dev +npx mint dev # Check for broken links (also runs in CI) -mint broken-links +npx mint broken-links -# Format files with Prettier -npx prettier --write . +# Format the whole site with Prettier +npm run format ``` ## Project structure +The content directories correspond one-to-one with the navigation tabs in `docs.json`: + ``` docs.json # Site config: navigation, tabs, theme, metadata -getting-started/ # Intro, quickstart, ecosystem overview -guides/ # Learning guides, feature how-tos -server/ # Server-side framework reference (pipelines, services, utilities) -client/ # Client SDK docs (JS, React, React Native, etc.) -cli/ # Pipecat CLI reference -deployment/ # Pipecat Cloud deployment docs +overview/ # Intro and ecosystem overview +pipecat/ # Pipecat framework docs (fundamentals, learn, features, telephony, deployment) +client/ # Client SDK docs (concepts, guides) +pipecat-flows/ # Pipecat Flows docs +pipecat-cloud/ # Pipecat Cloud docs (fundamentals, guides, security) +api-reference/ # Reference for server, client, CLI, Flows, and Cloud REST snippets/ # Reusable MDX snippets (shared across pages) -images/ # Static images +images/ logo/ videos/ # Static assets ``` ## Content conventions @@ -46,7 +52,7 @@ description: "Short description for SEO and navigation." ### Adding pages to navigation -All pages must be registered in `docs.json` under `navigation.tabs[].groups[].pages`. The path is relative to the repo root without the `.mdx` extension (e.g., `"guides/learn/overview"`). +All pages must be registered in `docs.json` under `navigation.tabs[].groups[].pages`. The path is relative to the repo root without the `.mdx` extension (e.g., `"overview/introduction"`). ### Mintlify components @@ -68,6 +74,9 @@ Prettier is configured via `.prettierrc`: - Double quotes - Semicolons enabled +A husky pre-commit hook runs lint-staged, which formats staged files. The whole +site is Prettier-clean, so `npm run format` should be a no-op on a clean tree. + ## CI/CD A GitHub Actions workflow (`.github/workflows/broken-links.yml`) runs `mint broken-links` on PRs and pushes to `main`. It comments on PRs if broken links are detected. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d25772d48..270e5fc15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,7 @@ This project follows the [Contributor Covenant v2.1](https://www.contributor-cov ### Prerequisites -- [Node.js](https://nodejs.org/) 20+ -- [Mintlify CLI](https://www.npmjs.com/package/mint) (`npm i -g mint`) +[nvm](https://github.com/nvm-sh/nvm), or another way to install the Node version pinned in `.nvmrc`. ### Setup @@ -22,17 +21,29 @@ This project follows the [Contributor Covenant v2.1](https://www.contributor-cov cd docs ``` -2. Start the local dev server: +2. Install and switch to the Node version this repo targets (see `.nvmrc`): ```bash - mint dev + nvm install ``` -3. Open `https://localhost:3000` in your browser to preview changes. +3. Install dependencies. This also installs the Git hooks that format your changes on commit: + + ```bash + npm install + ``` + +4. Start the local dev server: + + ```bash + npx mint dev + ``` + +5. Open `https://localhost:3000` in your browser to preview changes. ### Troubleshooting -- **Mintlify dev isn't running** — Run `mint update` to get the latest version. +- **Mintlify dev isn't running** — Run `npx mint update` to get the latest version. - **Page loads as a 404** — Make sure you are running in a folder with `docs.json`. ## Making a Contribution @@ -45,25 +56,20 @@ This project follows the [Contributor Covenant v2.1](https://www.contributor-cov 2. **Make your edits.** See the [Content Guidelines](#content-guidelines) below. -3. **Format with Prettier** to match the repo style: +3. **Check for broken links:** ```bash - npx prettier --write . + npx mint broken-links ``` -4. **Check for broken links:** - - ```bash - mint broken-links - ``` - -5. **Commit your changes** with a meaningful message: +4. **Commit your changes** with a meaningful message. A pre-commit hook formats + the files you staged: ```bash git commit -m "Description of your changes" ``` -6. **Push your branch** and open a Pull Request against `main`: +5. **Push your branch** and open a Pull Request against `main`: ```bash git push origin your-branch-name @@ -89,21 +95,23 @@ description: "Short description for SEO and navigation." All pages must be registered in `docs.json` under `navigation.tabs[].groups[].pages`. The path is relative to the repo root without the `.mdx` extension: ``` -"guides/learn/overview" +"overview/introduction" ``` ### Project Structure +The content directories correspond one-to-one with the navigation tabs in `docs.json`: + ``` docs.json # Site config: navigation, tabs, theme, metadata -getting-started/ # Intro, quickstart, ecosystem overview -guides/ # Learning guides, feature how-tos -server/ # Server-side framework reference (pipelines, services, utilities) -client/ # Client SDK docs (JS, React, React Native, etc.) -cli/ # Pipecat CLI reference -deployment/ # Pipecat Cloud deployment docs +overview/ # Intro and ecosystem overview +pipecat/ # Pipecat framework docs (fundamentals, learn, features, telephony, deployment) +client/ # Client SDK docs (concepts, guides) +pipecat-flows/ # Pipecat Flows docs +pipecat-cloud/ # Pipecat Cloud docs (fundamentals, guides, security) +api-reference/ # Reference for server, client, CLI, Flows, and Cloud REST snippets/ # Reusable MDX snippets (shared across pages) -images/ # Static images +images/ logo/ videos/ # Static assets ``` ### Mintlify Components @@ -128,14 +136,19 @@ Prettier is configured via `.prettierrc`: - Double quotes - Semicolons enabled -Run `npx prettier --write .` before committing to ensure consistent formatting. +A pre-commit hook (husky + lint-staged) formats staged files, so formatting is +usually taken care of for you. To format the whole site by hand: + +```bash +npm run format +``` ## Continuous Integration A GitHub Actions workflow runs `mint broken-links` on every PR and push to `main`. If broken links are detected, the workflow will fail and post a comment on your PR. You can run the same check locally: ```bash -mint broken-links +npx mint broken-links ``` ## Getting Help diff --git a/api-reference/cli/cloud/auth.mdx b/api-reference/cli/cloud/auth.mdx index d5df0f693..8ebd0f34a 100644 --- a/api-reference/cli/cloud/auth.mdx +++ b/api-reference/cli/cloud/auth.mdx @@ -63,8 +63,8 @@ pipecat cloud auth use-pat You can also set the `PIPECAT_TOKEN` environment variable instead of storing - the token locally. See the [PAT guide](/pipecat-cloud/guides/personal-access-tokens) - for details. + the token locally. See the [PAT + guide](/pipecat-cloud/guides/personal-access-tokens) for details. ## whoami diff --git a/api-reference/cli/cloud/build.mdx b/api-reference/cli/cloud/build.mdx index c3f46daf1..2c7714acd 100644 --- a/api-reference/cli/cloud/build.mdx +++ b/api-reference/cli/cloud/build.mdx @@ -20,7 +20,8 @@ pipecat cloud build logs [OPTIONS] BUILD_ID **Arguments:** - The ID of the build to get logs for. You can find build IDs using `pipecat cloud build list`. + The ID of the build to get logs for. You can find build IDs using `pipecat + cloud build list`. **Options:** @@ -30,7 +31,8 @@ pipecat cloud build logs [OPTIONS] BUILD_ID - Organization to use. If not provided, uses the current organization from your configuration. + Organization to use. If not provided, uses the current organization from your + configuration. ## status @@ -52,7 +54,8 @@ pipecat cloud build status [OPTIONS] BUILD_ID **Options:** - Organization to use. If not provided, uses the current organization from your configuration. + Organization to use. If not provided, uses the current organization from your + configuration. The status command displays detailed build information including: @@ -80,7 +83,8 @@ pipecat cloud build list [OPTIONS] - Filter by build status. Valid values: `pending`, `building`, `success`, `failed`, `timeout`. + Filter by build status. Valid values: `pending`, `building`, `success`, + `failed`, `timeout`. @@ -88,7 +92,8 @@ pipecat cloud build list [OPTIONS] - Organization to use. If not provided, uses the current organization from your configuration. + Organization to use. If not provided, uses the current organization from your + configuration. ## Examples diff --git a/api-reference/cli/init.mdx b/api-reference/cli/init.mdx index 898d5f485..f35a51908 100644 --- a/api-reference/cli/init.mdx +++ b/api-reference/cli/init.mdx @@ -34,8 +34,8 @@ pipecat init [TARGET_DIR] [OPTIONS] **Guide options:** - Overwrite existing `AGENTS.md`, `CLAUDE.md`, and `GETTING_STARTED.md` files. By - default existing guide files are kept, so your edits are never clobbered. + Overwrite existing `AGENTS.md`, `CLAUDE.md`, and `GETTING_STARTED.md` files. + By default existing guide files are kept, so your edits are never clobbered. **Scaffold options:** Passing any of these (or `--config`) scaffolds a bot non-interactively, in-place in `TARGET_DIR`. diff --git a/api-reference/client/android/overview.mdx b/api-reference/client/android/overview.mdx index 996bc692c..e464126d7 100644 --- a/api-reference/client/android/overview.mdx +++ b/api-reference/client/android/overview.mdx @@ -59,7 +59,11 @@ client.startBotAndConnect(startBotParams).withCallback { ## Documentation - + SDK API documentation Transports of this type connect directly to OpenAI's API from the client, which exposes your API key. This is designed primarily for development and - testing. For production applications, proxy through a server component to - keep credentials secure. + testing. For production applications, proxy through a server component to keep + credentials secure. ## Installation @@ -73,26 +73,26 @@ client.connect( ### OpenAIServiceOptions -| Parameter | Type | Description | -|---|---|---| -| `apiKey` | `String` | Your OpenAI API key | -| `sessionConfig` | `OpenAIRealtimeSessionConfig` | Session configuration | -| `model` | `String?` | Model name (default: `"gpt-realtime"`) | -| `initialMessages` | `List` | Messages to inject at session start | +| Parameter | Type | Description | +| ----------------- | ----------------------------- | -------------------------------------- | +| `apiKey` | `String` | Your OpenAI API key | +| `sessionConfig` | `OpenAIRealtimeSessionConfig` | Session configuration | +| `model` | `String?` | Model name (default: `"gpt-realtime"`) | +| `initialMessages` | `List` | Messages to inject at session start | ### OpenAIRealtimeSessionConfig -| Parameter | Type | Description | -|---|---|---| -| `modalities` | `List?` | Output modalities (e.g. `["audio", "text"]`), sent to the API as `output_modalities` | -| `instructions` | `String?` | System instructions for the model | -| `voice` | `String?` | Voice name (e.g. `"alloy"`, `"ballad"`) | -| `turnDetection` | `Value?` | Turn detection config | -| `inputAudioNoiseReduction` | `Value?` | Noise reduction config | -| `inputAudioTranscription` | `Value?` | Transcription model config | -| `tools` | `Value?` | Tool/function definitions | -| `toolChoice` | `String?` | Tool choice strategy | -| `temperature` | `Float?` | Deprecated — not supported by the GA Realtime API, this value is ignored | +| Parameter | Type | Description | +| -------------------------- | --------------- | ------------------------------------------------------------------------------------ | +| `modalities` | `List?` | Output modalities (e.g. `["audio", "text"]`), sent to the API as `output_modalities` | +| `instructions` | `String?` | System instructions for the model | +| `voice` | `String?` | Voice name (e.g. `"alloy"`, `"ballad"`) | +| `turnDetection` | `Value?` | Turn detection config | +| `inputAudioNoiseReduction` | `Value?` | Noise reduction config | +| `inputAudioTranscription` | `Value?` | Transcription model config | +| `tools` | `Value?` | Tool/function definitions | +| `toolChoice` | `String?` | Tool choice strategy | +| `temperature` | `Float?` | Deprecated — not supported by the GA Realtime API, this value is ignored | ### Audio devices diff --git a/api-reference/client/ios/overview.mdx b/api-reference/client/ios/overview.mdx index 885f42d61..ccf5b598c 100644 --- a/api-reference/client/ios/overview.mdx +++ b/api-reference/client/ios/overview.mdx @@ -63,7 +63,11 @@ self.pipecatClientIOS?.startBotAndConnect(startBotParams: startBotParams) { (res ## Documentation - + SDK API documentation { await client.startBotAndConnect({ - endpoint: `${process.env.PIPECAT_API_URL || "/api"}/connect` + endpoint: `${process.env.PIPECAT_API_URL || "/api"}/connect`, }); }; @@ -89,7 +89,11 @@ function VoiceBot() { ## Explore the SDK - + Ready-to-use components for audio, video, and visualization diff --git a/api-reference/pipecat-cloud/rest-reference/endpoint/build-create.mdx b/api-reference/pipecat-cloud/rest-reference/endpoint/build-create.mdx index ad806080b..736ca5f49 100644 --- a/api-reference/pipecat-cloud/rest-reference/endpoint/build-create.mdx +++ b/api-reference/pipecat-cloud/rest-reference/endpoint/build-create.mdx @@ -74,10 +74,10 @@ fi ## Build Statuses -| Status | Description | -|--------|-------------| -| `pending` | Build created, waiting to start | -| `building` | Build is in progress | -| `success` | Build completed successfully, `imageUri` is available | -| `failed` | Build failed, check `errorMessage` for details | -| `timeout` | Build exceeded the time limit | +| Status | Description | +| ---------- | ----------------------------------------------------- | +| `pending` | Build created, waiting to start | +| `building` | Build is in progress | +| `success` | Build completed successfully, `imageUri` is available | +| `failed` | Build failed, check `errorMessage` for details | +| `timeout` | Build exceeded the time limit | diff --git a/api-reference/pipecat-cloud/rest-reference/endpoint/build-get-logs.mdx b/api-reference/pipecat-cloud/rest-reference/endpoint/build-get-logs.mdx index 61e8c6ed8..d7f558b4e 100644 --- a/api-reference/pipecat-cloud/rest-reference/endpoint/build-get-logs.mdx +++ b/api-reference/pipecat-cloud/rest-reference/endpoint/build-get-logs.mdx @@ -34,12 +34,13 @@ done Use the `limit` query parameter to control how many log lines are returned: -| Parameter | Default | Min | Max | Description | -|-----------|---------|-----|-----|-------------| -| `limit` | 500 | 1 | 10,000 | Number of log events to return | +| Parameter | Default | Min | Max | Description | +| --------- | ------- | --- | ------ | ------------------------------ | +| `limit` | 500 | 1 | 10,000 | Number of log events to return | -Logs may be empty if the build was just created and hasn't started executing yet. Continue polling until logs appear or the build completes. + Logs may be empty if the build was just created and hasn't started executing + yet. Continue polling until logs appear or the build completes. ## Debugging Failed Builds @@ -55,6 +56,7 @@ curl -s "https://api.pipecat.daily.co/v1/builds/$BUILD_ID/logs?limit=10000" \ ``` Common issues visible in build logs include: + - Missing dependencies in `requirements.txt` - Dockerfile syntax errors - Failed `pip install` commands diff --git a/api-reference/pipecat-cloud/rest-reference/endpoint/build-get.mdx b/api-reference/pipecat-cloud/rest-reference/endpoint/build-get.mdx index 10200509b..8e77954d4 100644 --- a/api-reference/pipecat-cloud/rest-reference/endpoint/build-get.mdx +++ b/api-reference/pipecat-cloud/rest-reference/endpoint/build-get.mdx @@ -41,5 +41,6 @@ done ``` -Once your build succeeds, use the Pipecat CLI to deploy your agent. The CLI will automatically use the built image. + Once your build succeeds, use the Pipecat CLI to deploy your agent. The CLI + will automatically use the built image. diff --git a/api-reference/pipecat-cloud/rest-reference/endpoint/build-upload-url.mdx b/api-reference/pipecat-cloud/rest-reference/endpoint/build-upload-url.mdx index 4336b9158..01d07328d 100644 --- a/api-reference/pipecat-cloud/rest-reference/endpoint/build-upload-url.mdx +++ b/api-reference/pipecat-cloud/rest-reference/endpoint/build-upload-url.mdx @@ -37,11 +37,15 @@ echo "Upload ID: $UPLOAD_ID" ``` -The context archive must be a gzipped tar file (`.tar.gz`). The upload URL validates both the content type and file size. + The context archive must be a gzipped tar file (`.tar.gz`). The upload URL + validates both the content type and file size. -**Field names are case-sensitive.** When uploading to S3, you must use the exact field names returned in `uploadFields` (e.g., `X-Amz-Algorithm`, not `x-amz-algorithm`). Using incorrect casing will result in authentication errors. + **Field names are case-sensitive.** When uploading to S3, you must use the + exact field names returned in `uploadFields` (e.g., `X-Amz-Algorithm`, not + `x-amz-algorithm`). Using incorrect casing will result in authentication + errors. ## Creating the Context Archive @@ -59,5 +63,6 @@ tar -czvf context.tar.gz \ ``` -The maximum context size is **500MB**. Use a `.dockerignore` file to exclude unnecessary files and keep your context small for faster uploads and builds. + The maximum context size is **500MB**. Use a `.dockerignore` file to exclude + unnecessary files and keep your context small for faster uploads and builds. diff --git a/api-reference/pipecat-cloud/rest-reference/endpoint/session-proxy.mdx b/api-reference/pipecat-cloud/rest-reference/endpoint/session-proxy.mdx index fe838f0ab..a69824352 100644 --- a/api-reference/pipecat-cloud/rest-reference/endpoint/session-proxy.mdx +++ b/api-reference/pipecat-cloud/rest-reference/endpoint/session-proxy.mdx @@ -16,6 +16,5 @@ Headers are forwarded to your bot with these exceptions: Requires base image version `0.1.2` or later. See the [Session API - guide](/pipecat-cloud/guides/session-api) for setup instructions - and examples. + guide](/pipecat-cloud/guides/session-api) for setup instructions and examples. diff --git a/api-reference/pipecat-flows/types.mdx b/api-reference/pipecat-flows/types.mdx index 8f8b30c20..cec454ab2 100644 --- a/api-reference/pipecat-flows/types.mdx +++ b/api-reference/pipecat-flows/types.mdx @@ -372,7 +372,10 @@ For functions that transition to a new node, use `respond_immediately: False` in ### FlowResult - **Deprecated.** No replacement. `FlowResult` is no longer required or referenced by any handler type, and Pipecat's upstream function-call-result contract is `Any` — define your own `TypedDict` or return any JSON-serializable value. Will be removed in 2.0.0. + **Deprecated.** No replacement. `FlowResult` is no longer required or + referenced by any handler type, and Pipecat's upstream function-call-result + contract is `Any` — define your own `TypedDict` or return any + JSON-serializable value. Will be removed in 2.0.0. ```python diff --git a/api-reference/server/frames/control-frames.mdx b/api-reference/server/frames/control-frames.mdx index 0313fbeea..795c04422 100644 --- a/api-reference/server/frames/control-frames.mdx +++ b/api-reference/server/frames/control-frames.mdx @@ -232,10 +232,10 @@ Signals the beginning of a TTS audio response. - Whether the spoken text for this response will be appended to the LLM - context. When `True`, the assistant aggregator will track this output as an - assistant turn, firing `on_assistant_turn_started` and - `on_assistant_turn_stopped` events. + Whether the spoken text for this response will be appended to the LLM context. + When `True`, the assistant aggregator will track this output as an assistant + turn, firing `on_assistant_turn_started` and `on_assistant_turn_stopped` + events. ### TTSStoppedFrame diff --git a/api-reference/server/frames/data-frames.mdx b/api-reference/server/frames/data-frames.mdx index bfb83cb06..a37910c4d 100644 --- a/api-reference/server/frames/data-frames.mdx +++ b/api-reference/server/frames/data-frames.mdx @@ -329,7 +329,11 @@ Frames for configuring LLM function calling behavior and output settings at runt Changes the set of tools advertised to the LLM mid-conversation. - + The tools to advertise. May be a `ToolsSchema`, a plain list of direct functions and/or `FunctionSchema` objects, a list of provider-specific tool dicts, or `NOT_GIVEN` to clear all tools. Direct functions and diff --git a/api-reference/server/frames/llm-frames.mdx b/api-reference/server/frames/llm-frames.mdx index 1b3807f49..741d9a880 100644 --- a/api-reference/server/frames/llm-frames.mdx +++ b/api-reference/server/frames/llm-frames.mdx @@ -56,8 +56,8 @@ Configures how a function call result is handled after execution. - Whether this is the final result for the function call. Set to `False` to - send an intermediate update from an async function call registered with + Whether this is the final result for the function call. Set to `False` to send + an intermediate update from an async function call registered with `cancel_on_interruption=False`. diff --git a/api-reference/server/introduction.mdx b/api-reference/server/introduction.mdx index 672971832..73d9da9de 100644 --- a/api-reference/server/introduction.mdx +++ b/api-reference/server/introduction.mdx @@ -24,7 +24,11 @@ This is the API reference for the server-side Pipecat Python framework. It cover > Browse the full list of AI service integrations and their install commands - + Understand the data, control, system, and LLM frames that flow through pipelines diff --git a/api-reference/server/pipeline/pipeline-worker.mdx b/api-reference/server/pipeline/pipeline-worker.mdx index c0f0b3908..b004128dd 100644 --- a/api-reference/server/pipeline/pipeline-worker.mdx +++ b/api-reference/server/pipeline/pipeline-worker.mdx @@ -225,15 +225,15 @@ await worker.queue_frames(frames, direction=FrameDirection.UPSTREAM) PipelineWorker provides event handlers for monitoring pipeline lifecycle and frame flow. Register handlers using the `@event_handler` decorator. -| Event | Description | -| ----------------------------- | -------------------------------------------------------------------- | -| `on_pipeline_started` | Pipeline has started processing | -| `on_pipeline_finished` | Pipeline reached a terminal state | -| `on_pipeline_error` | An error frame reached the pipeline worker | -| `on_frame_reached_upstream` | A filtered frame type reached the pipeline source | -| `on_frame_reached_downstream` | A filtered frame type reached the pipeline sink | -| `on_heartbeat_timeout` | No heartbeat received within the monitor timeout (pipeline may stall)| -| `on_idle_timeout` | No activity detected within the idle timeout period | +| Event | Description | +| ----------------------------- | --------------------------------------------------------------------- | +| `on_pipeline_started` | Pipeline has started processing | +| `on_pipeline_finished` | Pipeline reached a terminal state | +| `on_pipeline_error` | An error frame reached the pipeline worker | +| `on_frame_reached_upstream` | A filtered frame type reached the pipeline source | +| `on_frame_reached_downstream` | A filtered frame type reached the pipeline sink | +| `on_heartbeat_timeout` | No heartbeat received within the monitor timeout (pipeline may stall) | +| `on_idle_timeout` | No activity detected within the idle timeout period | ### on_pipeline_started @@ -362,7 +362,11 @@ async def on_heartbeat_timeout(worker): | `worker` | `PipelineWorker` | The pipeline worker instance | - Heartbeat monitoring must be enabled by setting `enable_heartbeats=True` in `PipelineParams`. The timeout period is controlled by `heartbeats_monitor_secs` (default: 5 seconds). See [PipelineParams](/api-reference/server/pipeline/pipeline-params) for configuration details. + Heartbeat monitoring must be enabled by setting `enable_heartbeats=True` in + `PipelineParams`. The timeout period is controlled by + `heartbeats_monitor_secs` (default: 5 seconds). See + [PipelineParams](/api-reference/server/pipeline/pipeline-params) for + configuration details. ### on_idle_timeout diff --git a/api-reference/server/rtvi/rtvi-observer.mdx b/api-reference/server/rtvi/rtvi-observer.mdx index 8a163fa58..4c620dcfb 100644 --- a/api-reference/server/rtvi/rtvi-observer.mdx +++ b/api-reference/server/rtvi/rtvi-observer.mdx @@ -60,7 +60,9 @@ worker = PipelineWorker( - Indicates if raw VAD user started/stopped speaking messages should be sent. These reflect the VAD signal directly, independent of turn finalization (unlike `user_speaking_enabled`, which a turn strategy may gate or defer). + Indicates if raw VAD user started/stopped speaking messages should be sent. + These reflect the VAD signal directly, independent of turn finalization + (unlike `user_speaking_enabled`, which a turn strategy may gate or defer). diff --git a/api-reference/server/services/llm/baseten.mdx b/api-reference/server/services/llm/baseten.mdx index 244162612..2708b2e06 100644 --- a/api-reference/server/services/llm/baseten.mdx +++ b/api-reference/server/services/llm/baseten.mdx @@ -68,7 +68,11 @@ Before using Baseten LLM services, you need: Baseten API key for authentication. - + Base URL for Baseten API endpoint. Use the default for serverless Model APIs, or set to your dedicated deployment's `/sync/v1` URL to use a model running on your own GPUs. diff --git a/api-reference/server/services/llm/crusoe.mdx b/api-reference/server/services/llm/crusoe.mdx index 3e0b6ba22..1c477caa1 100644 --- a/api-reference/server/services/llm/crusoe.mdx +++ b/api-reference/server/services/llm/crusoe.mdx @@ -25,11 +25,7 @@ description: "Large Language Model services using Crusoe Cloud's Managed Inferen Official Crusoe documentation - + Access models and manage API keys @@ -78,14 +74,14 @@ Before using Crusoe LLM services, you need: Runtime-configurable settings passed via the `settings` constructor argument using `CrusoeLLMService.Settings(...)`. These can be updated mid-conversation with `LLMUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ------------------- | ------- | -------------- | -------------------------------------------------------------------------------------- | +| Parameter | Type | Default | Description | +| ------------------- | ------- | --------------- | -------------------------------------------------------------------------------------- | | `model` | `str` | `"zai/GLM-5.2"` | Crusoe model identifier. Check Crusoe Cloud for available models. | -| `temperature` | `float` | `NOT_GIVEN` | Sampling temperature (0.0 to 2.0). Lower values are more focused, higher are creative. | -| `max_tokens` | `int` | `NOT_GIVEN` | Maximum tokens to generate. | -| `top_p` | `float` | `NOT_GIVEN` | Top-p (nucleus) sampling (0.0 to 1.0). Controls diversity of output. | -| `frequency_penalty` | `float` | `NOT_GIVEN` | Penalty for frequent tokens (-2.0 to 2.0). Positive values discourage repetition. | -| `presence_penalty` | `float` | `NOT_GIVEN` | Penalty for new topics (-2.0 to 2.0). Positive values encourage new topics. | +| `temperature` | `float` | `NOT_GIVEN` | Sampling temperature (0.0 to 2.0). Lower values are more focused, higher are creative. | +| `max_tokens` | `int` | `NOT_GIVEN` | Maximum tokens to generate. | +| `top_p` | `float` | `NOT_GIVEN` | Top-p (nucleus) sampling (0.0 to 1.0). Controls diversity of output. | +| `frequency_penalty` | `float` | `NOT_GIVEN` | Penalty for frequent tokens (-2.0 to 2.0). Positive values discourage repetition. | +| `presence_penalty` | `float` | `NOT_GIVEN` | Penalty for new topics (-2.0 to 2.0). Positive values encourage new topics. | `NOT_GIVEN` values are omitted from the API request entirely, letting the diff --git a/api-reference/server/services/llm/floe.mdx b/api-reference/server/services/llm/floe.mdx index 7178a847c..2f630a69b 100644 --- a/api-reference/server/services/llm/floe.mdx +++ b/api-reference/server/services/llm/floe.mdx @@ -78,7 +78,11 @@ Before using the Floe LLM service, you need: is rejected by Floe. - + Floe OpenAI-compatible base URL. diff --git a/api-reference/server/services/llm/inception.mdx b/api-reference/server/services/llm/inception.mdx index 49441cbee..ff7a121af 100644 --- a/api-reference/server/services/llm/inception.mdx +++ b/api-reference/server/services/llm/inception.mdx @@ -59,7 +59,11 @@ Before using Inception LLM services, you need: Inception API key for authentication. - + Base URL for Inception API endpoint. @@ -74,15 +78,23 @@ Runtime-configurable settings passed via the `settings` constructor argument usi This service extends `OpenAILLMService.Settings` with Inception-specific parameters: - Model identifier to use. Defaults to "mercury-2", Inception's diffusion-based reasoning model. + Model identifier to use. Defaults to "mercury-2", Inception's diffusion-based + reasoning model. - - Controls how much reasoning the model applies. Options are "instant", "low", "medium", or "high". When unset, the parameter is omitted and Inception's server-side default applies. + + Controls how much reasoning the model applies. Options are "instant", "low", + "medium", or "high". When unset, the parameter is omitted and Inception's + server-side default applies. - When True, reduces time to first diffusion block (TTFT) for faster initial response times. + When True, reduces time to first diffusion block (TTFT) for faster initial + response times. For additional settings inherited from OpenAI, see [OpenAI LLM Settings](/api-reference/server/services/llm/openai#settings). diff --git a/api-reference/server/services/llm/nvidia.mdx b/api-reference/server/services/llm/nvidia.mdx index 8fa6fb590..f68e92855 100644 --- a/api-reference/server/services/llm/nvidia.mdx +++ b/api-reference/server/services/llm/nvidia.mdx @@ -64,7 +64,8 @@ Before using NVIDIA NIM LLM services, you need: ## Configuration - NVIDIA API key for authentication. Required when using the cloud endpoint (`https://integrate.api.nvidia.com/v1`). Not needed for local NIM deployments. + NVIDIA API key for authentication. Required when using the cloud endpoint + (`https://integrate.api.nvidia.com/v1`). Not needed for local NIM deployments. - Base URL for NIM API endpoint. Defaults to NVIDIA's cloud endpoint. For local deployments, pass the local address (e.g., `http://localhost:8000/v1`). + Base URL for NIM API endpoint. Defaults to NVIDIA's cloud endpoint. For local + deployments, pass the local address (e.g., `http://localhost:8000/v1`). ...` tags (e.g., DeepSeek-R1, some Nemotron models) - + Reasoning frames are accessible to observers and logging but are not sent to TTS, keeping the spoken output clean while preserving visibility into the model's thought process. diff --git a/api-reference/server/services/llm/sarvam.mdx b/api-reference/server/services/llm/sarvam.mdx index 52d6775bc..5245e3b6f 100644 --- a/api-reference/server/services/llm/sarvam.mdx +++ b/api-reference/server/services/llm/sarvam.mdx @@ -78,16 +78,16 @@ Before using Sarvam LLM services, you need: Runtime-configurable settings passed via the `settings` constructor argument using `SarvamLLMService.Settings(...)`. These can be updated mid-conversation with `LLMUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ------------------- | ---------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------ | -| `model` | `str` | `"sarvam-105b"` | Sarvam model identifier. Only `sarvam-105b` is currently supported. | -| `wiki_grounding` | `bool` | `None` | Enable or disable wiki grounding feature. Sarvam-specific parameter. | -| `reasoning_effort` | `Literal["low", "medium", "high"]` | `None` | Set reasoning effort level. Sarvam-specific parameter. | -| `temperature` | `float` | `NOT_GIVEN` | Sampling temperature (0.0 to 2.0). Lower values are more focused, higher values are more creative. | -| `max_tokens` | `int` | `NOT_GIVEN` | Maximum tokens to generate. | -| `top_p` | `float` | `NOT_GIVEN` | Top-p (nucleus) sampling (0.0 to 1.0). Controls diversity of output. | -| `frequency_penalty` | `float` | `NOT_GIVEN` | Penalty for frequent tokens (-2.0 to 2.0). Positive values discourage repetition. | -| `presence_penalty` | `float` | `NOT_GIVEN` | Penalty for new topics (-2.0 to 2.0). Positive values encourage the model to talk about new topics. | +| Parameter | Type | Default | Description | +| ------------------- | ---------------------------------- | --------------- | --------------------------------------------------------------------------------------------------- | +| `model` | `str` | `"sarvam-105b"` | Sarvam model identifier. Only `sarvam-105b` is currently supported. | +| `wiki_grounding` | `bool` | `None` | Enable or disable wiki grounding feature. Sarvam-specific parameter. | +| `reasoning_effort` | `Literal["low", "medium", "high"]` | `None` | Set reasoning effort level. Sarvam-specific parameter. | +| `temperature` | `float` | `NOT_GIVEN` | Sampling temperature (0.0 to 2.0). Lower values are more focused, higher values are more creative. | +| `max_tokens` | `int` | `NOT_GIVEN` | Maximum tokens to generate. | +| `top_p` | `float` | `NOT_GIVEN` | Top-p (nucleus) sampling (0.0 to 1.0). Controls diversity of output. | +| `frequency_penalty` | `float` | `NOT_GIVEN` | Penalty for frequent tokens (-2.0 to 2.0). Positive values discourage repetition. | +| `presence_penalty` | `float` | `NOT_GIVEN` | Penalty for new topics (-2.0 to 2.0). Positive values encourage the model to talk about new topics. | `NOT_GIVEN` values are omitted from the API request entirely, letting the diff --git a/api-reference/server/services/s2s/aws.mdx b/api-reference/server/services/s2s/aws.mdx index 2fbd4d0bf..dd1e0d5c3 100644 --- a/api-reference/server/services/s2s/aws.mdx +++ b/api-reference/server/services/s2s/aws.mdx @@ -140,12 +140,20 @@ _Deprecated in v0.0.105. Use `settings=AWSNovaSonicLLMService.Settings(system_in - + Available tools for the model: a `ToolsSchema`, or a plain list of direct functions and/or `FunctionSchema` objects. - + Configuration for automatic session continuation. When enabled (the default), sessions are seamlessly rotated before the AWS time limit (~8 minutes) with no user-perceptible interruption. See @@ -190,20 +198,20 @@ Audio configuration passed via the `audio_config` constructor argument. Configuration for automatic session continuation, passed via the `session_continuation` constructor argument. Nova Sonic sessions have an AWS-imposed time limit (~8 minutes). When enabled, session continuation proactively creates a new session in the background before the limit is reached, buffers user audio during the transition, and seamlessly hands off — preserving conversation context with no user-perceptible gap. -| Parameter | Type | Default | Description | -| --------------------------------- | ------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | `bool` | `True` | Whether automatic session continuation is enabled. | -| `transition_threshold_seconds` | `float` | `360.0` | How many seconds into a session to begin monitoring for a transition opportunity. The transition will occur when the assistant next starts speaking after this threshold. | -| `audio_buffer_duration_seconds` | `float` | `3.0` | Duration of the rolling audio buffer (in seconds) that captures user audio during the transition window. This audio is replayed into the new session so no user input is lost. | -| `audio_start_timeout_seconds` | `float` | `80.0` | Maximum time to wait for the assistant to start speaking after the threshold is reached. If no assistant audio arrives within this window, the transition is forced. Set to `0` to disable the timeout (wait indefinitely). | +| Parameter | Type | Default | Description | +| ------------------------------- | ------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | `bool` | `True` | Whether automatic session continuation is enabled. | +| `transition_threshold_seconds` | `float` | `360.0` | How many seconds into a session to begin monitoring for a transition opportunity. The transition will occur when the assistant next starts speaking after this threshold. | +| `audio_buffer_duration_seconds` | `float` | `3.0` | Duration of the rolling audio buffer (in seconds) that captures user audio during the transition window. This audio is replayed into the new session so no user input is lost. | +| `audio_start_timeout_seconds` | `float` | `80.0` | Maximum time to wait for the assistant to start speaking after the threshold is reached. If no assistant audio arrives within this window, the transition is forced. Set to `0` to disable the timeout (wait indefinitely). | ## Usage - Pair this service with - `LLMContextAggregatorPair(context, realtime_service_mode=True)`. Realtime mode - keeps context-writing correct for speech-to-speech services and adapts turn - handling to the service. See [Realtime (Speech-to-Speech) + Pair this service with `LLMContextAggregatorPair(context, + realtime_service_mode=True)`. Realtime mode keeps context-writing correct for + speech-to-speech services and adapts turn handling to the service. See + [Realtime (Speech-to-Speech) Services](/api-reference/server/utilities/turn-management/external-turn-management#realtime-speech-to-speech-services). diff --git a/api-reference/server/services/s2s/gemini-live-vertex.mdx b/api-reference/server/services/s2s/gemini-live-vertex.mdx index ee543d141..f9d0feff0 100644 --- a/api-reference/server/services/s2s/gemini-live-vertex.mdx +++ b/api-reference/server/services/s2s/gemini-live-vertex.mdx @@ -124,10 +124,14 @@ _Deprecated in v0.0.105. Use `settings=GeminiLiveVertexLLMService.Settings(voice System prompt for the model. Can also be provided via the LLM context. - - Tools available to the model: a `ToolsSchema`, a plain list of direct functions - and/or `FunctionSchema` objects, or a list of provider-native tool dicts. Can - also be provided via the LLM context. + + Tools available to the model: a `ToolsSchema`, a plain list of direct + functions and/or `FunctionSchema` objects, or a list of provider-native tool + dicts. Can also be provided via the LLM context. @@ -176,10 +180,10 @@ The Vertex AI variant uses the same Settings as the base Gemini Live service. Se ## Usage - Pair this service with - `LLMContextAggregatorPair(context, realtime_service_mode=True)`. Realtime mode - keeps context-writing correct for speech-to-speech services and adapts turn - handling to the service. See [Realtime (Speech-to-Speech) + Pair this service with `LLMContextAggregatorPair(context, + realtime_service_mode=True)`. Realtime mode keeps context-writing correct for + speech-to-speech services and adapts turn handling to the service. See + [Realtime (Speech-to-Speech) Services](/api-reference/server/utilities/turn-management/external-turn-management#realtime-speech-to-speech-services). diff --git a/api-reference/server/services/s2s/gemini-live.mdx b/api-reference/server/services/s2s/gemini-live.mdx index 4fadc18fa..587c44be1 100644 --- a/api-reference/server/services/s2s/gemini-live.mdx +++ b/api-reference/server/services/s2s/gemini-live.mdx @@ -105,10 +105,14 @@ _Deprecated in v0.0.105. Use `settings=GeminiLiveLLMService.Settings(voice=...)` System prompt for the model. Can also be provided via the LLM context. - - Tools available to the model: a `ToolsSchema`, a plain list of direct functions - and/or `FunctionSchema` objects, or a list of provider-native tool dicts. Can - also be provided via the LLM context. + + Tools available to the model: a `ToolsSchema`, a plain list of direct + functions and/or `FunctionSchema` objects, or a list of provider-native tool + dicts. Can also be provided via the LLM context. diff --git a/api-reference/server/services/s2s/grok.mdx b/api-reference/server/services/s2s/grok.mdx index 2829d7784..a30883892 100644 --- a/api-reference/server/services/s2s/grok.mdx +++ b/api-reference/server/services/s2s/grok.mdx @@ -123,13 +123,13 @@ Runtime-configurable settings passed via the `settings` constructor argument usi ### SessionProperties -| Parameter | Type | Default | Description | -| ---------------- | ------------------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `instructions` | `str` | `None` | System instructions for the assistant. | -| `voice` | `str` | `"eve"` | Voice the model uses to respond. Accepts any built-in voice ID (see [xAI's voice catalogue](https://docs.x.ai/docs/guides/voice/agent)) or a custom voice ID from the Custom Voices API. Case-insensitive. | -| `turn_detection` | `TurnDetection` | `TurnDetection(type="server_vad")` | Turn detection configuration. Set to `None` for manual turn detection. | -| `audio` | `AudioConfiguration` | `None` | Configuration for input and output audio formats. | -| `tools` | `List[GrokTool]` | `None` | Available tools: `web_search`, `x_search`, `file_search`, or custom `function` tools. | +| Parameter | Type | Default | Description | +| ---------------- | -------------------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `instructions` | `str` | `None` | System instructions for the assistant. | +| `voice` | `str` | `"eve"` | Voice the model uses to respond. Accepts any built-in voice ID (see [xAI's voice catalogue](https://docs.x.ai/docs/guides/voice/agent)) or a custom voice ID from the Custom Voices API. Case-insensitive. | +| `turn_detection` | `TurnDetection` | `TurnDetection(type="server_vad")` | Turn detection configuration. Set to `None` for manual turn detection. | +| `audio` | `AudioConfiguration` | `None` | Configuration for input and output audio formats. | +| `tools` | `List[GrokTool]` | `None` | Available tools: `web_search`, `x_search`, `file_search`, or custom `function` tools. | ### AudioConfiguration @@ -162,10 +162,10 @@ Grok provides several built-in tools in addition to custom function tools: ## Usage - Pair this service with - `LLMContextAggregatorPair(context, realtime_service_mode=True)`. Realtime mode - keeps context-writing correct for speech-to-speech services and adapts turn - handling to the service. See [Realtime (Speech-to-Speech) + Pair this service with `LLMContextAggregatorPair(context, + realtime_service_mode=True)`. Realtime mode keeps context-writing correct for + speech-to-speech services and adapts turn handling to the service. See + [Realtime (Speech-to-Speech) Services](/api-reference/server/utilities/turn-management/external-turn-management#realtime-speech-to-speech-services). diff --git a/api-reference/server/services/s2s/inworld.mdx b/api-reference/server/services/s2s/inworld.mdx index fdf5e5716..2b1b813e8 100644 --- a/api-reference/server/services/s2s/inworld.mdx +++ b/api-reference/server/services/s2s/inworld.mdx @@ -193,10 +193,10 @@ Inworld PCM audio supports sample rates: 8000, 16000, 24000, 32000, 44100, and 4 ## Usage - Pair this service with - `LLMContextAggregatorPair(context, realtime_service_mode=True)`. Realtime mode - keeps context-writing correct for speech-to-speech services and adapts turn - handling to the service. See [Realtime (Speech-to-Speech) + Pair this service with `LLMContextAggregatorPair(context, + realtime_service_mode=True)`. Realtime mode keeps context-writing correct for + speech-to-speech services and adapts turn handling to the service. See + [Realtime (Speech-to-Speech) Services](/api-reference/server/utilities/turn-management/external-turn-management#realtime-speech-to-speech-services). diff --git a/api-reference/server/services/s2s/openai.mdx b/api-reference/server/services/s2s/openai.mdx index 4dcf75c1d..f6e895d5f 100644 --- a/api-reference/server/services/s2s/openai.mdx +++ b/api-reference/server/services/s2s/openai.mdx @@ -227,10 +227,10 @@ Reasoning configuration for reasoning-capable Realtime models (e.g. `gpt-realtim ## Usage - Pair this service with - `LLMContextAggregatorPair(context, realtime_service_mode=True)`. Realtime mode - keeps context-writing correct for speech-to-speech services and adapts turn - handling to the service. See [Realtime (Speech-to-Speech) + Pair this service with `LLMContextAggregatorPair(context, + realtime_service_mode=True)`. Realtime mode keeps context-writing correct for + speech-to-speech services and adapts turn handling to the service. See + [Realtime (Speech-to-Speech) Services](/api-reference/server/utilities/turn-management/external-turn-management#realtime-speech-to-speech-services). diff --git a/api-reference/server/services/s2s/ultravox.mdx b/api-reference/server/services/s2s/ultravox.mdx index 29b7871de..4f5780f26 100644 --- a/api-reference/server/services/s2s/ultravox.mdx +++ b/api-reference/server/services/s2s/ultravox.mdx @@ -91,7 +91,11 @@ Before using Ultravox Realtime services, you need: Types](#input-parameter-types) below. - + Tools to use with a one-shot call: a `ToolsSchema`, or a plain list of direct functions and/or `FunctionSchema` objects. May only be set when using `OneShotInputParams`. @@ -162,10 +166,10 @@ Join an existing Ultravox call using a join URL. ## Usage - Pair this service with - `LLMContextAggregatorPair(context, realtime_service_mode=True)`. Realtime mode - keeps context-writing correct for speech-to-speech services and adapts turn - handling to the service. See [Realtime (Speech-to-Speech) + Pair this service with `LLMContextAggregatorPair(context, + realtime_service_mode=True)`. Realtime mode keeps context-writing correct for + speech-to-speech services and adapts turn handling to the service. See + [Realtime (Speech-to-Speech) Services](/api-reference/server/utilities/turn-management/external-turn-management#realtime-speech-to-speech-services). diff --git a/api-reference/server/services/serializers/exotel.mdx b/api-reference/server/services/serializers/exotel.mdx index 4a1502d79..d317f506b 100644 --- a/api-reference/server/services/serializers/exotel.mdx +++ b/api-reference/server/services/serializers/exotel.mdx @@ -79,12 +79,12 @@ Before using ExotelFrameSerializer, you need: ### InputParams -| Parameter | Type | Default | Description | -| ----------------------------- | -------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `exotel_sample_rate` | `int` | `8000` | Sample rate used by Exotel (Hz). | -| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | -| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | -| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | +| Parameter | Type | Default | Description | +| ---------------------------- | --------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exotel_sample_rate` | `int` | `8000` | Sample rate used by Exotel (Hz). | +| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | +| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | +| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | ## Usage diff --git a/api-reference/server/services/serializers/genesys.mdx b/api-reference/server/services/serializers/genesys.mdx index 7997a33d1..876b89f74 100644 --- a/api-reference/server/services/serializers/genesys.mdx +++ b/api-reference/server/services/serializers/genesys.mdx @@ -62,18 +62,18 @@ Before using GenesysAudioHookSerializer, you need: ### InputParams -| Parameter | Type | Default | Description | -| ---------------------------- | ---------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `genesys_sample_rate` | `int` | `8000` | Sample rate used by Genesys (Hz). | -| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | -| `channel` | `AudioHookChannel` | `"external"` | Which audio channels to process: `"external"` (customer), `"internal"` (agent), or `"both"` (stereo). | -| `media_format` | `AudioHookMediaFormat` | `"PCMU"` | Audio format: `"PCMU"` (mu-law) or `"L16"` (16-bit linear PCM). | -| `process_external` | `bool` | `True` | Whether to process external (customer) audio. | -| `process_internal` | `bool` | `False` | Whether to process internal (agent) audio. | -| `supported_languages` | `list[str]` | `None` | List of language codes the bot supports (e.g., `["en-US", "es-ES"]`). | -| `selected_language` | `str` | `None` | Default language code to use. | -| `start_paused` | `bool` | `False` | Whether to start the session in paused state. | -| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | +| Parameter | Type | Default | Description | +| ---------------------------- | ---------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `genesys_sample_rate` | `int` | `8000` | Sample rate used by Genesys (Hz). | +| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | +| `channel` | `AudioHookChannel` | `"external"` | Which audio channels to process: `"external"` (customer), `"internal"` (agent), or `"both"` (stereo). | +| `media_format` | `AudioHookMediaFormat` | `"PCMU"` | Audio format: `"PCMU"` (mu-law) or `"L16"` (16-bit linear PCM). | +| `process_external` | `bool` | `True` | Whether to process external (customer) audio. | +| `process_internal` | `bool` | `False` | Whether to process internal (agent) audio. | +| `supported_languages` | `list[str]` | `None` | List of language codes the bot supports (e.g., `["en-US", "es-ES"]`). | +| `selected_language` | `str` | `None` | Default language code to use. | +| `start_paused` | `bool` | `False` | Whether to start the session in paused state. | +| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | | `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | ## Usage diff --git a/api-reference/server/services/serializers/plivo.mdx b/api-reference/server/services/serializers/plivo.mdx index e92cbff4e..aa5fad4cd 100644 --- a/api-reference/server/services/serializers/plivo.mdx +++ b/api-reference/server/services/serializers/plivo.mdx @@ -99,13 +99,13 @@ Before using PlivoFrameSerializer, you need: ### InputParams -| Parameter | Type | Default | Description | -| ----------------------------- | -------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `plivo_sample_rate` | `int` | `8000` | Sample rate used by Plivo (Hz). | -| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | -| `auto_hang_up` | `bool` | `True` | Whether to automatically terminate the call on `EndFrame` or `CancelFrame`. | -| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | -| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | +| Parameter | Type | Default | Description | +| ---------------------------- | --------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `plivo_sample_rate` | `int` | `8000` | Sample rate used by Plivo (Hz). | +| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | +| `auto_hang_up` | `bool` | `True` | Whether to automatically terminate the call on `EndFrame` or `CancelFrame`. | +| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | +| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | ## Usage diff --git a/api-reference/server/services/serializers/telnyx.mdx b/api-reference/server/services/serializers/telnyx.mdx index b5af8dad7..fefe8246e 100644 --- a/api-reference/server/services/serializers/telnyx.mdx +++ b/api-reference/server/services/serializers/telnyx.mdx @@ -104,14 +104,14 @@ Before using TelnyxFrameSerializer, you need: ### InputParams -| Parameter | Type | Default | Description | -| ----------------------------- | -------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `telnyx_sample_rate` | `int` | `8000` | Sample rate used by Telnyx (Hz). | -| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | -| `inbound_encoding` | `str` | `"PCMU"` | Audio encoding for data sent to Telnyx. | -| `outbound_encoding` | `str` | `"PCMU"` | Audio encoding for data received from Telnyx. | -| `auto_hang_up` | `bool` | `True` | Whether to automatically terminate the call on `EndFrame` or `CancelFrame`. | -| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | +| Parameter | Type | Default | Description | +| ---------------------------- | --------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `telnyx_sample_rate` | `int` | `8000` | Sample rate used by Telnyx (Hz). | +| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | +| `inbound_encoding` | `str` | `"PCMU"` | Audio encoding for data sent to Telnyx. | +| `outbound_encoding` | `str` | `"PCMU"` | Audio encoding for data received from Telnyx. | +| `auto_hang_up` | `bool` | `True` | Whether to automatically terminate the call on `EndFrame` or `CancelFrame`. | +| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | ## Usage diff --git a/api-reference/server/services/serializers/twilio.mdx b/api-reference/server/services/serializers/twilio.mdx index 8cd0ef251..738c1d560 100644 --- a/api-reference/server/services/serializers/twilio.mdx +++ b/api-reference/server/services/serializers/twilio.mdx @@ -93,8 +93,8 @@ Before using TwilioFrameSerializer, you need: - Twilio region (e.g., `"au1"`, `"ie1"`). Must be specified together with - `edge` when `base_url` is not provided. + Twilio region (e.g., `"au1"`, `"ie1"`). Must be specified together with `edge` + when `base_url` is not provided. @@ -115,13 +115,13 @@ Before using TwilioFrameSerializer, you need: ### InputParams -| Parameter | Type | Default | Description | -| ----------------------------- | -------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `twilio_sample_rate` | `int` | `8000` | Sample rate used by Twilio (Hz). | -| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | -| `auto_hang_up` | `bool` | `True` | Whether to automatically terminate the call on `EndFrame` or `CancelFrame`. | -| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | -| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | +| Parameter | Type | Default | Description | +| ---------------------------- | --------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `twilio_sample_rate` | `int` | `8000` | Sample rate used by Twilio (Hz). | +| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | +| `auto_hang_up` | `bool` | `True` | Whether to automatically terminate the call on `EndFrame` or `CancelFrame`. | +| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | +| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | ## Usage diff --git a/api-reference/server/services/serializers/vonage.mdx b/api-reference/server/services/serializers/vonage.mdx index d21be7b12..3ac37f79f 100644 --- a/api-reference/server/services/serializers/vonage.mdx +++ b/api-reference/server/services/serializers/vonage.mdx @@ -84,12 +84,12 @@ Before using VonageFrameSerializer, you need: ### InputParams -| Parameter | Type | Default | Description | -| ----------------------------- | -------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `vonage_sample_rate` | `int` | `16000` | Sample rate used by Vonage (Hz). Common values: 8000, 16000, 24000. | -| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | -| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | -| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | +| Parameter | Type | Default | Description | +| ---------------------------- | --------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vonage_sample_rate` | `int` | `16000` | Sample rate used by Vonage (Hz). Common values: 8000, 16000, 24000. | +| `sample_rate` | `int` | `None` | Optional override for pipeline input sample rate. When `None`, uses the pipeline's configured rate. | +| `ignore_rtvi_messages` | `bool` | `True` | Whether to ignore RTVI protocol messages during serialization. | +| `resampler_clear_after_secs` | `float \| None` | `0.2` | Seconds of inactivity after which the stream resampler clears its internal history to avoid audio artefacts. Set to `None` to never clear (recommended for telephony providers with irregular gaps). | ## Usage diff --git a/api-reference/server/services/stt/assemblyai.mdx b/api-reference/server/services/stt/assemblyai.mdx index 51737e2fe..eb9c9ccb0 100644 --- a/api-reference/server/services/stt/assemblyai.mdx +++ b/api-reference/server/services/stt/assemblyai.mdx @@ -144,29 +144,29 @@ Before using AssemblyAI STT services, you need: Runtime-configurable settings passed via the `settings` constructor argument using `AssemblyAISTTService.Settings(...)`. These can be updated mid-conversation with `STTUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------------------------------- | ---------------------------------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `model` | `str` | `"universal-3-5-pro"` | STT model identifier. Set to `"universal-3-5-pro"`. _(Inherited from base STT settings.)_ | -| `language` | `Language \| str` | `Language.EN` | Language for speech recognition. _(Inherited from base STT settings.)_ | -| `formatted_finals` | `bool` | `True` | Whether to enable transcript formatting. | -| `word_finalization_max_wait_time` | `int` | `None` | Maximum time to wait for word finalization in milliseconds. | -| `end_of_turn_confidence_threshold` | `float` | `None` | Confidence threshold for end-of-turn detection. | -| `min_turn_silence` | `int` | `None` | Silence duration (ms) before a speculative end-of-turn check. If terminal punctuation is found, the turn ends; otherwise a partial is emitted and the turn continues. Clamped to 50–10000 ms. Server default is mode-dependent (set by the `mode` preset). | -| `max_turn_silence` | `int` | `None` | Maximum silence (ms) before the turn is forced to end, regardless of punctuation. | -| `keyterms_prompt` | `List[str]` | `None` | A list of words and phrases to improve recognition accuracy for. Maximum 100 terms. May be combined with `prompt` on U3 Pro models. | -| `prompt` | `str` | `None` | A contextual prompt describing what the audio is about — its domain, scenario, or conversation details — so the model better recognizes likely vocabulary. Carries context about your audio, not transcription instructions; formatting or behavioral commands are not supported. Maximum ~1500 characters. Only applicable to U3 Pro models; may be combined with `keyterms_prompt` on those models. | -| `language_code` | `str` | `None` | Customer-declared audio language as an ISO code (e.g. `"en"`, `"es"`, `"fr"`). On U3 Pro models, a tier-1 code (`"en"`/`"es"`/`"fr"`/`"de"`/`"it"`/`"pt"`) steers transcription toward that language; other supported codes are `"tr"`, `"nl"`, `"sv"`, `"no"`, `"da"`, `"fi"`, `"hi"`, `"vi"`, `"ar"`, `"he"`, `"ja"`, `"zh"`. This is one of the names AssemblyAI accepts for its declared-language parameter, alongside `language_codes`, which covers the same languages as `Language` enums and is bound in preference to this one when both are set. Prefer `language_codes`. Distinct from `language_detection`, which controls whether the detected language is reported. | +| Parameter | Type | Default | Description | +| ---------------------------------- | ---------------------------------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | `str` | `"universal-3-5-pro"` | STT model identifier. Set to `"universal-3-5-pro"`. _(Inherited from base STT settings.)_ | +| `language` | `Language \| str` | `Language.EN` | Language for speech recognition. _(Inherited from base STT settings.)_ | +| `formatted_finals` | `bool` | `True` | Whether to enable transcript formatting. | +| `word_finalization_max_wait_time` | `int` | `None` | Maximum time to wait for word finalization in milliseconds. | +| `end_of_turn_confidence_threshold` | `float` | `None` | Confidence threshold for end-of-turn detection. | +| `min_turn_silence` | `int` | `None` | Silence duration (ms) before a speculative end-of-turn check. If terminal punctuation is found, the turn ends; otherwise a partial is emitted and the turn continues. Clamped to 50–10000 ms. Server default is mode-dependent (set by the `mode` preset). | +| `max_turn_silence` | `int` | `None` | Maximum silence (ms) before the turn is forced to end, regardless of punctuation. | +| `keyterms_prompt` | `List[str]` | `None` | A list of words and phrases to improve recognition accuracy for. Maximum 100 terms. May be combined with `prompt` on U3 Pro models. | +| `prompt` | `str` | `None` | A contextual prompt describing what the audio is about — its domain, scenario, or conversation details — so the model better recognizes likely vocabulary. Carries context about your audio, not transcription instructions; formatting or behavioral commands are not supported. Maximum ~1500 characters. Only applicable to U3 Pro models; may be combined with `keyterms_prompt` on those models. | +| `language_code` | `str` | `None` | Customer-declared audio language as an ISO code (e.g. `"en"`, `"es"`, `"fr"`). On U3 Pro models, a tier-1 code (`"en"`/`"es"`/`"fr"`/`"de"`/`"it"`/`"pt"`) steers transcription toward that language; other supported codes are `"tr"`, `"nl"`, `"sv"`, `"no"`, `"da"`, `"fi"`, `"hi"`, `"vi"`, `"ar"`, `"he"`, `"ja"`, `"zh"`. This is one of the names AssemblyAI accepts for its declared-language parameter, alongside `language_codes`, which covers the same languages as `Language` enums and is bound in preference to this one when both are set. Prefer `language_codes`. Distinct from `language_detection`, which controls whether the detected language is reported. | | `language_codes` | `List[Language]` | `None` | Customer-declared audio languages. A single language (e.g. `[Language.ES]`) pins transcription to that language; several (e.g. `[Language.EN, Language.ES]`) steer toward that subset while keeping code-switching among them. Order is significant — the steering prompt follows the declared order. Regional variants resolve to their base code, so at most 10 distinct languages. Steering is prompt-based, so it applies to U3 Pro models only and is not sent for other models — including `universal-streaming-multilingual`, which transcribes multilingual audio without steering. Unlike most settings, a change applies to a live session without reconnecting; pass an empty list to clear steering back to the model default. | -| `speaker_labels` | `bool` | `None` | Whether to enable streaming speaker diarization. When enabled, each turn includes a `speaker_label` and each final word includes a `speaker` field for word-level attribution. | -| `vad_threshold` | `float` | `None` | Confidence threshold (0.0–1.0) for classifying audio frames as silence. Frames with VAD confidence below this value are considered silent; increase for noisy environments to reduce false speech detection. Server default is mode-dependent (set by the `mode` preset). | -| `domain` | `str` | `None` | Enable a domain-specific model for specialized terminology. Set to `"medical-v1"` for Medical Mode (improved accuracy on medications, procedures, conditions, and dosages). Supported languages: `en`, `es`, `de`, `fr`. | -| `continuous_partials` | `bool` | `True` | Whether to emit additional partial transcripts during long turns at a steady ~3 second cadence. When enabled, partials covering the full turn transcript are emitted about every 3 seconds while speech continues; when disabled, only one early partial is emitted near turn start. The first partial (at 750 ms) is unaffected. | -| `interruption_delay` | `int` | `None` | How soon the first partial is emitted, in milliseconds (0–1000). Useful for tuning barge-in responsiveness or emitting earlier partials for LLM inference; larger values are more confident on interruptions, smaller values give faster time to first partial. The server adds a fixed 256 ms, so `0` yields an effective 256 ms and `500` yields 756 ms. Server default is mode-dependent (set by the `mode` preset). | -| `agent_context` | `str` | `None` | Your voice agent's spoken text (TTS reply), used as context for the next user turn — improves accuracy on short or ambiguous replies and spelled-out entities like emails or IDs. Set at connection to seed the agent's greeting and/or update after each reply; each update replaces the previous value. Maximum ~1500 characters. | -| `previous_context_n_turns` | `int` | `None` | Advanced. Maximum number of prior conversation entries (user transcripts and any `agent_context` values) carried forward as context. Range 0–100; set to `0` to disable automatic carryover. Most integrations should leave this unset. | -| `voice_focus` | `Literal["near-field", "far-field"]` | `None` | Enable Voice Focus to isolate the primary voice and suppress background noise before transcription. Set to `"near-field"` for close-talking mics (headsets, phones) or `"far-field"` for distant mics (conference rooms). Off when unset. | -| `voice_focus_threshold` | `float` | `None` | Controls how aggressively Voice Focus suppresses background audio, from `0.0` (least) to `1.0` (most). Requires `voice_focus` to be set, otherwise a validation error is returned. | -| `mode` | `Literal["min_latency", "balanced", "max_accuracy"]` | `None` | Latency and accuracy preset controlling turn-detection and partial-emission defaults. `max_accuracy` favors quality, `min_latency` favors speed, and `balanced` trades off between them. When omitted, the server applies its own preset, which sets the defaults for mode-dependent fields such as `interruption_delay`, `min_turn_silence`, `vad_threshold`, `previous_context_n_turns`, and `continuous_partials`. | +| `speaker_labels` | `bool` | `None` | Whether to enable streaming speaker diarization. When enabled, each turn includes a `speaker_label` and each final word includes a `speaker` field for word-level attribution. | +| `vad_threshold` | `float` | `None` | Confidence threshold (0.0–1.0) for classifying audio frames as silence. Frames with VAD confidence below this value are considered silent; increase for noisy environments to reduce false speech detection. Server default is mode-dependent (set by the `mode` preset). | +| `domain` | `str` | `None` | Enable a domain-specific model for specialized terminology. Set to `"medical-v1"` for Medical Mode (improved accuracy on medications, procedures, conditions, and dosages). Supported languages: `en`, `es`, `de`, `fr`. | +| `continuous_partials` | `bool` | `True` | Whether to emit additional partial transcripts during long turns at a steady ~3 second cadence. When enabled, partials covering the full turn transcript are emitted about every 3 seconds while speech continues; when disabled, only one early partial is emitted near turn start. The first partial (at 750 ms) is unaffected. | +| `interruption_delay` | `int` | `None` | How soon the first partial is emitted, in milliseconds (0–1000). Useful for tuning barge-in responsiveness or emitting earlier partials for LLM inference; larger values are more confident on interruptions, smaller values give faster time to first partial. The server adds a fixed 256 ms, so `0` yields an effective 256 ms and `500` yields 756 ms. Server default is mode-dependent (set by the `mode` preset). | +| `agent_context` | `str` | `None` | Your voice agent's spoken text (TTS reply), used as context for the next user turn — improves accuracy on short or ambiguous replies and spelled-out entities like emails or IDs. Set at connection to seed the agent's greeting and/or update after each reply; each update replaces the previous value. Maximum ~1500 characters. | +| `previous_context_n_turns` | `int` | `None` | Advanced. Maximum number of prior conversation entries (user transcripts and any `agent_context` values) carried forward as context. Range 0–100; set to `0` to disable automatic carryover. Most integrations should leave this unset. | +| `voice_focus` | `Literal["near-field", "far-field"]` | `None` | Enable Voice Focus to isolate the primary voice and suppress background noise before transcription. Set to `"near-field"` for close-talking mics (headsets, phones) or `"far-field"` for distant mics (conference rooms). Off when unset. | +| `voice_focus_threshold` | `float` | `None` | Controls how aggressively Voice Focus suppresses background audio, from `0.0` (least) to `1.0` (most). Requires `voice_focus` to be set, otherwise a validation error is returned. | +| `mode` | `Literal["min_latency", "balanced", "max_accuracy"]` | `None` | Latency and accuracy preset controlling turn-detection and partial-emission defaults. `max_accuracy` favors quality, `min_latency` favors speed, and `balanced` trades off between them. When omitted, the server applies its own preset, which sets the defaults for mode-dependent fields such as `interruption_delay`, `min_turn_silence`, `vad_threshold`, `previous_context_n_turns`, and `continuous_partials`. | ## Usage diff --git a/api-reference/server/services/stt/azure.mdx b/api-reference/server/services/stt/azure.mdx index 2af69a937..edb2be4e0 100644 --- a/api-reference/server/services/stt/azure.mdx +++ b/api-reference/server/services/stt/azure.mdx @@ -104,11 +104,11 @@ Before using Azure STT services, you need: Runtime-configurable settings passed via the `settings` constructor argument using `AzureSTTService.Settings(...)`. These can be updated mid-conversation with `STTUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ----------- | ------------------------------------ | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `model` | `str` | `None` | STT model identifier. _(Inherited from base STT settings.)_ | -| `language` | `Language \| str` | `Language.EN_US` | Language for speech recognition. _(Inherited from base STT settings.)_ | -| `profanity` | `"raw" \| "masked" \| "removed"` | `None` | How Azure handles profanity in transcripts. `"raw"` returns text as recognized with no masking, `"masked"` replaces profane words with `****` (Azure default), `"removed"` drops profane words. Default `None` keeps Azure SDK default (`"masked"`). Use `"raw"` for non-English deployments where Azure's profanity filter over-eagerly masks ordinary words. | +| Parameter | Type | Default | Description | +| ----------- | -------------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | `str` | `None` | STT model identifier. _(Inherited from base STT settings.)_ | +| `language` | `Language \| str` | `Language.EN_US` | Language for speech recognition. _(Inherited from base STT settings.)_ | +| `profanity` | `"raw" \| "masked" \| "removed"` | `None` | How Azure handles profanity in transcripts. `"raw"` returns text as recognized with no masking, `"masked"` replaces profane words with `****` (Azure default), `"removed"` drops profane words. Default `None` keeps Azure SDK default (`"masked"`). Use `"raw"` for non-English deployments where Azure's profanity filter over-eagerly masks ordinary words. | ## Usage diff --git a/api-reference/server/services/stt/cartesia.mdx b/api-reference/server/services/stt/cartesia.mdx index cefc23ef5..8fb838d6f 100644 --- a/api-reference/server/services/stt/cartesia.mdx +++ b/api-reference/server/services/stt/cartesia.mdx @@ -117,11 +117,11 @@ Before using Cartesia STT services, you need: Runtime-configurable settings passed via the `settings` constructor argument using `CartesiaSTTService.Settings(...)`. These can be updated mid-conversation with `STTUpdateSettingsFrame`, which triggers an automatic reconnection with the new parameters. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------- | ----------------- | --------------- | ------------------------------------------------------------------------ | -| `model` | `str` | `"ink-whisper"` | The transcription model to use. _(Inherited from base STT settings.)_ | -| `language` | `Language \| str` | `"en"` | Target language for transcription. _(Inherited from base STT settings.)_ | -| `keyterm` | `list[str] \| None` | `None` | Key terms or phrases to bias transcription towards (e.g., product names, jargon). Only honored by `ink-2` models; ignored with a warning for other models. Cartesia binds keyterms to a connection, so updating this setting via `STTUpdateSettingsFrame` triggers a reconnect. Lists longer than Cartesia's limit of 100 keyterms or 1200 total characters are truncated with a warning. | +| Parameter | Type | Default | Description | +| ---------- | ------------------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | `str` | `"ink-whisper"` | The transcription model to use. _(Inherited from base STT settings.)_ | +| `language` | `Language \| str` | `"en"` | Target language for transcription. _(Inherited from base STT settings.)_ | +| `keyterm` | `list[str] \| None` | `None` | Key terms or phrases to bias transcription towards (e.g., product names, jargon). Only honored by `ink-2` models; ignored with a warning for other models. Cartesia binds keyterms to a connection, so updating this setting via `STTUpdateSettingsFrame` triggers a reconnect. Lists longer than Cartesia's limit of 100 keyterms or 1200 total characters are truncated with a warning. | ### Usage @@ -201,7 +201,11 @@ The server drives turn boundaries with the `ink-2` model, pushing structured eve Cartesia API key for authentication. - + WebSocket URL for the Cartesia Streaming ASR v2 endpoint. @@ -210,18 +214,25 @@ The server drives turn boundaries with the `ink-2` model, pushing structured eve - Whether to broadcast an interruption when the server signals the start of a new turn. + Whether to broadcast an interruption when the server signals the start of a + new turn. - Minimum idle timeout (in seconds) before sending silence to prevent dangling turns. The actual threshold is `max(chunk_duration * 2, watchdog_min_timeout)`. + Minimum idle timeout (in seconds) before sending silence to prevent dangling + turns. The actual threshold is `max(chunk_duration * 2, + watchdog_min_timeout)`. Optional additional HTTP headers to send with the WebSocket handshake. - + Runtime-updatable settings. See [Settings](#settings-2) below. @@ -229,11 +240,11 @@ The server drives turn boundaries with the `ink-2` model, pushing structured eve Runtime-configurable settings passed via the `settings` constructor argument using `CartesiaTurnsSTTService.Settings(...)`. The ink-2 model family is English-only and does not support runtime model or language switching. Attempts to update these fields will be reported as unhandled. -| Parameter | Type | Default | Description | -| ---------- | ----------------- | --------- | --------------------------------------------------------------------- | -| `model` | `str` | `"ink-2"` | The transcription model to use. _(Inherited from base STT settings.)_ | -| `language` | `Language \| str` | `None` | Target language (fixed to English). _(Inherited from base STT settings.)_ | -| `keyterm` | `list[str] \| None` | `None` | Key terms or phrases to bias transcription towards (e.g., product names, jargon). Cartesia binds keyterms to a connection, so updating this setting via `STTUpdateSettingsFrame` triggers a reconnect. Lists longer than Cartesia's limit of 100 keyterms or 1200 total characters are truncated with a warning. | +| Parameter | Type | Default | Description | +| ---------- | ------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | `str` | `"ink-2"` | The transcription model to use. _(Inherited from base STT settings.)_ | +| `language` | `Language \| str` | `None` | Target language (fixed to English). _(Inherited from base STT settings.)_ | +| `keyterm` | `list[str] \| None` | `None` | Key terms or phrases to bias transcription towards (e.g., product names, jargon). Cartesia binds keyterms to a connection, so updating this setting via `STTUpdateSettingsFrame` triggers a reconnect. Lists longer than Cartesia's limit of 100 keyterms or 1200 total characters are truncated with a warning. | ### Usage @@ -319,16 +330,16 @@ Transcripts are cumulative per turn. There is no `is_final` flag and no `finaliz Cartesia Turns STT supports the following event handlers: -| Event | Handler Signature | Description | -| --------------------- | ----------------------------------------- | ---------------------------------------------------------- | -| `on_connected` | `async def(service)` | Connected to Cartesia WebSocket | -| `on_disconnected` | `async def(service)` | Disconnected from Cartesia WebSocket | -| `on_connection_error` | `async def(service, error_msg)` | Connection error occurred | -| `on_turn_start` | `async def(service, transcript: str)` | Server detected start of a turn | -| `on_turn_update` | `async def(service, transcript: str)` | Incremental transcript update | -| `on_turn_eager_end` | `async def(service, transcript: str)` | Server eagerly predicted end of turn | -| `on_turn_resume` | `async def(service)` | User resumed speaking after an eager end | -| `on_turn_end` | `async def(service, transcript: str)` | Final transcript for the completed turn | +| Event | Handler Signature | Description | +| --------------------- | ------------------------------------- | ---------------------------------------- | +| `on_connected` | `async def(service)` | Connected to Cartesia WebSocket | +| `on_disconnected` | `async def(service)` | Disconnected from Cartesia WebSocket | +| `on_connection_error` | `async def(service, error_msg)` | Connection error occurred | +| `on_turn_start` | `async def(service, transcript: str)` | Server detected start of a turn | +| `on_turn_update` | `async def(service, transcript: str)` | Incremental transcript update | +| `on_turn_eager_end` | `async def(service, transcript: str)` | Server eagerly predicted end of turn | +| `on_turn_resume` | `async def(service)` | User resumed speaking after an eager end | +| `on_turn_end` | `async def(service, transcript: str)` | Final transcript for the completed turn | Example: diff --git a/api-reference/server/services/stt/deepgram.mdx b/api-reference/server/services/stt/deepgram.mdx index 533ba70b4..7a1ba4d74 100644 --- a/api-reference/server/services/stt/deepgram.mdx +++ b/api-reference/server/services/stt/deepgram.mdx @@ -238,8 +238,9 @@ Supports the standard [service connection events](/api-reference/server/events/s Deepgram Flux provides its own user turn start and end detection and automatically requests `ExternalUserTurnStrategies` at start, so you don't - need to configure turn strategies manually. Pass your own `user_turn_strategies` - only to override the service's recommendation. See [User Turn + need to configure turn strategies manually. Pass your own + `user_turn_strategies` only to override the service's recommendation. See + [User Turn Strategies](/api-reference/server/utilities/turn-management/user-turn-strategies) for more details. @@ -518,8 +519,9 @@ Supports the standard [service connection events](/api-reference/server/events/s Deepgram Flux provides its own user turn start and end detection and automatically requests `ExternalUserTurnStrategies` at start, so you don't - need to configure turn strategies manually. Pass your own `user_turn_strategies` - only to override the service's recommendation. See [User Turn + need to configure turn strategies manually. Pass your own + `user_turn_strategies` only to override the service's recommendation. See + [User Turn Strategies](/api-reference/server/utilities/turn-management/user-turn-strategies) for more details. diff --git a/api-reference/server/services/stt/floe.mdx b/api-reference/server/services/stt/floe.mdx index 3b4c8c01d..03aa226f7 100644 --- a/api-reference/server/services/stt/floe.mdx +++ b/api-reference/server/services/stt/floe.mdx @@ -96,7 +96,11 @@ Before using the Floe STT service, you need: BCP-47 language hint. - + Floe streaming-STT WebSocket URL. diff --git a/api-reference/server/services/stt/funasr.mdx b/api-reference/server/services/stt/funasr.mdx index f7c8d6b1f..a9b1e031b 100644 --- a/api-reference/server/services/stt/funasr.mdx +++ b/api-reference/server/services/stt/funasr.mdx @@ -62,7 +62,8 @@ Before using FunASR STT service, you need: - **No API Key**: Runs entirely locally for complete privacy - No API keys required - FunASR runs entirely locally for complete privacy. Can run on CPU or GPU. + No API keys required - FunASR runs entirely locally for complete privacy. Can + run on CPU or GPU. ## Configuration @@ -72,18 +73,19 @@ Before using FunASR STT service, you need: - Runtime-configurable settings for the STT service. See [FunASRSTTSettings](#funasrsttsettings) below. + Runtime-configurable settings for the STT service. See + [FunASRSTTSettings](#funasrsttsettings) below. ## FunASRSTTSettings Runtime-configurable settings passed via the `settings` constructor argument using `FunASRSTTService.Settings(...)`. These can be updated mid-conversation with `STTUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------- | ----------------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `model` | `str` | `"iic/SenseVoiceSmall"` | FunASR model identifier. Default is SenseVoiceSmall. _(Inherited from base STT settings.)_ | -| `language` | `Language \| str` | `Language.EN` | Language for transcription. Natively supports Chinese (`zh`), Cantonese (`yue`), English (`en`), Japanese (`ja`), and Korean (`ko`). Falls back to auto-detection for other languages. _(Inherited from base STT settings.)_ | -| `use_itn` | `bool` | `True` | Apply inverse text normalization (e.g., converts spoken numbers like "nine" to numerals "9"). | +| Parameter | Type | Default | Description | +| ---------- | ----------------- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | `str` | `"iic/SenseVoiceSmall"` | FunASR model identifier. Default is SenseVoiceSmall. _(Inherited from base STT settings.)_ | +| `language` | `Language \| str` | `Language.EN` | Language for transcription. Natively supports Chinese (`zh`), Cantonese (`yue`), English (`en`), Japanese (`ja`), and Korean (`ko`). Falls back to auto-detection for other languages. _(Inherited from base STT settings.)_ | +| `use_itn` | `bool` | `True` | Apply inverse text normalization (e.g., converts spoken numbers like "nine" to numerals "9"). | ## Usage diff --git a/api-reference/server/services/stt/groq.mdx b/api-reference/server/services/stt/groq.mdx index d0cd9f87a..d17156ce8 100644 --- a/api-reference/server/services/stt/groq.mdx +++ b/api-reference/server/services/stt/groq.mdx @@ -76,7 +76,11 @@ Before using Groq STT services, you need: - Custom `httpx.AsyncClient` for API requests, e.g. one with a longer request timeout. Prefer `openai.DefaultAsyncHttpxClient`, which keeps the SDK's connection limits and redirect handling; a bare `httpx.AsyncClient` uses httpx's defaults instead. Defaults to `None`, which lets the SDK build its own client. + Custom `httpx.AsyncClient` for API requests, e.g. one with a longer request + timeout. Prefer `openai.DefaultAsyncHttpxClient`, which keeps the SDK's + connection limits and redirect handling; a bare `httpx.AsyncClient` uses + httpx's defaults instead. Defaults to `None`, which lets the SDK build its own + client. diff --git a/api-reference/server/services/stt/hakim.mdx b/api-reference/server/services/stt/hakim.mdx index 065a9457e..c8b160e92 100644 --- a/api-reference/server/services/stt/hakim.mdx +++ b/api-reference/server/services/stt/hakim.mdx @@ -105,16 +105,15 @@ Before using the Hakim STT service, you need: Runtime-updatable configuration passed via the `settings` constructor argument using `HakimSTTService.Settings(...)`. -| Parameter | Type | Default | Description | -| ------------ | ------ | ------------- | --------------------------------------------------------------------------------------------------------- | -| `timestamps` | `str` | `"segment"` | One of `"word"`, `"segment"`, or `"none"`. | -| `diarize` | `bool` | `False` | Speaker diarization. Only useful for stereo audio with one speaker per channel (e.g. call recordings) -- mono `diarize=True` transcribes without speaker labels. | -| `partials` | `bool` | `True` | Whether to emit interim (non-final) transcripts. | +| Parameter | Type | Default | Description | +| ------------ | ------ | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `timestamps` | `str` | `"segment"` | One of `"word"`, `"segment"`, or `"none"`. | +| `diarize` | `bool` | `False` | Speaker diarization. Only useful for stereo audio with one speaker per channel (e.g. call recordings) -- mono `diarize=True` transcribes without speaker labels. | +| `partials` | `bool` | `True` | Whether to emit interim (non-final) transcripts. | - See the [source - repository](https://github.com/tryHakimAI/hakim-pipecat) for the - authoritative, up-to-date configuration options. + See the [source repository](https://github.com/tryHakimAI/hakim-pipecat) for + the authoritative, up-to-date configuration options. ## Usage diff --git a/api-reference/server/services/stt/mistral.mdx b/api-reference/server/services/stt/mistral.mdx index 7f81f21b4..7d13c0bfb 100644 --- a/api-reference/server/services/stt/mistral.mdx +++ b/api-reference/server/services/stt/mistral.mdx @@ -8,6 +8,7 @@ description: "Speech-to-text service using Mistral's Voxtral Realtime transcript `MistralSTTService` provides real-time speech recognition using Mistral's Voxtral Realtime API. It uses the Mistral SDK's `RealtimeConnection` to stream audio and receive transcription events over WebSocket. Key features include: + - Streaming transcription with interim results - Automatic language detection - VAD-driven utterance lifecycle management @@ -98,10 +99,10 @@ Before using `MistralSTTService`, you need: Runtime-configurable settings passed via the `settings` constructor argument using `MistralSTTService.Settings(...)`. These can be updated mid-conversation with `STTUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------- | ----------------- | ---------------------------------------- | ------------------------------------------------------------ | -| `model` | `str` | `"voxtral-mini-transcribe-realtime-2602"` | Mistral STT model to use. _(Inherited from base STT settings.)_ | -| `language` | `Language \| str` | `None` | Language hint for transcription. _(Inherited from base STT settings.)_ | +| Parameter | Type | Default | Description | +| ---------- | ----------------- | ----------------------------------------- | ---------------------------------------------------------------------- | +| `model` | `str` | `"voxtral-mini-transcribe-realtime-2602"` | Mistral STT model to use. _(Inherited from base STT settings.)_ | +| `language` | `Language \| str` | `None` | Language hint for transcription. _(Inherited from base STT settings.)_ | ## Usage @@ -142,11 +143,11 @@ stt = MistralSTTService( Supports the standard [service connection events](/api-reference/server/events/service-events): -| Event | Description | -| --------------------- | --------------------------------- | -| `on_connected` | Transcription session created | -| `on_disconnected` | Connection closed | -| `on_connection_error` | Transcription error occurred | +| Event | Description | +| --------------------- | ----------------------------- | +| `on_connected` | Transcription session created | +| `on_disconnected` | Connection closed | +| `on_connection_error` | Transcription error occurred | ```python @stt.event_handler("on_connected") diff --git a/api-reference/server/services/stt/moonshine.mdx b/api-reference/server/services/stt/moonshine.mdx index 879b752ff..c2d8280f7 100644 --- a/api-reference/server/services/stt/moonshine.mdx +++ b/api-reference/server/services/stt/moonshine.mdx @@ -61,23 +61,25 @@ Before using Moonshine STT service, you need: - **No API Key**: Runs entirely locally for complete privacy - No API keys or GPU required - Moonshine runs efficiently on CPU for complete privacy. + No API keys or GPU required - Moonshine runs efficiently on CPU for complete + privacy. ## Configuration - Runtime-configurable settings for the STT service. See [MoonshineSTTService Settings](#moonshinesttsettings) below. + Runtime-configurable settings for the STT service. See [MoonshineSTTService + Settings](#moonshinesttsettings) below. ## MoonshineSTTSettings Runtime-configurable settings passed via the `settings` constructor argument using `MoonshineSTTService.Settings(...)`. These can be updated mid-conversation with `STTUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------- | ----------------- | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `model` | `str \| Model` | `Model.SMALL_STREAMING` | Moonshine model architecture. Available models: `TINY`, `BASE`, `TINY_STREAMING`, `BASE_STREAMING`, `SMALL_STREAMING` (default), `MEDIUM_STREAMING`. | -| `language` | `Language \| str` | `Language.EN` | Language for transcription. Moonshine supports English, Spanish, and other languages. The base language code is used (e.g., "en" from "en-US"). _(Inherited from base STT settings.)_ | +| Parameter | Type | Default | Description | +| ---------- | ----------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | `str \| Model` | `Model.SMALL_STREAMING` | Moonshine model architecture. Available models: `TINY`, `BASE`, `TINY_STREAMING`, `BASE_STREAMING`, `SMALL_STREAMING` (default), `MEDIUM_STREAMING`. | +| `language` | `Language \| str` | `Language.EN` | Language for transcription. Moonshine supports English, Spanish, and other languages. The base language code is used (e.g., "en" from "en-US"). _(Inherited from base STT settings.)_ | ## Usage diff --git a/api-reference/server/services/stt/openai.mdx b/api-reference/server/services/stt/openai.mdx index a89113229..f2d166bca 100644 --- a/api-reference/server/services/stt/openai.mdx +++ b/api-reference/server/services/stt/openai.mdx @@ -82,7 +82,11 @@ Uses VAD-based audio segmentation with HTTP transcription requests. Records spee - Custom `httpx.AsyncClient` for API requests, e.g. one with a longer request timeout. Prefer `openai.DefaultAsyncHttpxClient`, which keeps the SDK's connection limits and redirect handling; a bare `httpx.AsyncClient` uses httpx's defaults instead. Defaults to `None`, which lets the SDK build its own client. + Custom `httpx.AsyncClient` for API requests, e.g. one with a longer request + timeout. Prefer `openai.DefaultAsyncHttpxClient`, which keeps the SDK's + connection limits and redirect handling; a bare `httpx.AsyncClient` uses + httpx's defaults instead. Defaults to `None`, which lets the SDK build its own + client. @@ -160,9 +164,9 @@ Real-time streaming speech-to-text using OpenAI's Realtime API WebSocket transcr Transcription model. For low-latency streaming transcription, use - `"gpt-realtime-whisper"`. Other supported models include - `"gpt-4o-transcribe"` and `"gpt-4o-mini-transcribe"`. *Deprecated in v0.0.105. - Use `settings=OpenAIRealtimeSTTService.Settings(...)` instead.* + `"gpt-realtime-whisper"`. Other supported models include `"gpt-4o-transcribe"` + and `"gpt-4o-mini-transcribe"`. *Deprecated in v0.0.105. Use + `settings=OpenAIRealtimeSTTService.Settings(...)` instead.* Speechmatics provides its own user turn start and end detection. When using `TurnDetectionMode.ADAPTIVE` or `TurnDetectionMode.SMART_TURN`, the service - automatically requests `ExternalUserTurnStrategies` at start, so you don't need - to configure turn strategies manually. Pass your own `user_turn_strategies` only - to override the service's recommendation. See [User Turn + automatically requests `ExternalUserTurnStrategies` at start, so you don't + need to configure turn strategies manually. Pass your own + `user_turn_strategies` only to override the service's recommendation. See + [User Turn Strategies](/api-reference/server/utilities/turn-management/user-turn-strategies) for more details. A VAD in the transport (such as `SileroVADAnalyzer`) is - optional when Speechmatics drives turn detection; include it if you want useful - STT metrics. + optional when Speechmatics drives turn detection; include it if you want + useful STT metrics. diff --git a/api-reference/server/services/stt/xai.mdx b/api-reference/server/services/stt/xai.mdx index c5986a706..41aa722e6 100644 --- a/api-reference/server/services/stt/xai.mdx +++ b/api-reference/server/services/stt/xai.mdx @@ -92,23 +92,22 @@ Before using xAI STT services, you need: P99 latency from speech end to final transcript in seconds. Override for your - deployment. See - [stt-benchmark](https://github.com/pipecat-ai/stt-benchmark). + deployment. See [stt-benchmark](https://github.com/pipecat-ai/stt-benchmark). ### Settings Runtime-configurable settings passed via the `settings` constructor argument using `XAISTTService.Settings(...)`. These can be updated mid-conversation with `STTUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------------- | ----------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| `model` | `str` | `None` | Not applicable for xAI STT. _(Inherited from base STT settings.)_ | -| `language` | `Language \| str` | `Language.EN` | Recognition language. Supports: AR, BN, DE, EN, ES, FR, HI, ID, IT, JA, KO, PT, RU, TR, VI, ZH. _(Inherited from base STT settings.)_ | -| `interim_results` | `bool` | `True` | When True, partial transcripts are emitted approximately every 500ms. | -| `endpointing` | `int \| None` | `None` | Silence duration in milliseconds that triggers a speech-final event. Range 0-5000. Server default is 10ms. | -| `multichannel` | `bool \| None` | `None` | When True, transcribes each interleaved channel independently. Requires `channels` >= 2. | -| `channels` | `int \| None` | `None` | Number of interleaved channels (2-8). Required when `multichannel` is True. | -| `diarize` | `bool \| None` | `None` | When True, the server attaches a `speaker` field to each word identifying the detected speaker. | +| Parameter | Type | Default | Description | +| ----------------- | ----------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | `str` | `None` | Not applicable for xAI STT. _(Inherited from base STT settings.)_ | +| `language` | `Language \| str` | `Language.EN` | Recognition language. Supports: AR, BN, DE, EN, ES, FR, HI, ID, IT, JA, KO, PT, RU, TR, VI, ZH. _(Inherited from base STT settings.)_ | +| `interim_results` | `bool` | `True` | When True, partial transcripts are emitted approximately every 500ms. | +| `endpointing` | `int \| None` | `None` | Silence duration in milliseconds that triggers a speech-final event. Range 0-5000. Server default is 10ms. | +| `multichannel` | `bool \| None` | `None` | When True, transcribes each interleaved channel independently. Requires `channels` >= 2. | +| `channels` | `int \| None` | `None` | Number of interleaved channels (2-8). Required when `multichannel` is True. | +| `diarize` | `bool \| None` | `None` | When True, the server attaches a `speaker` field to each word identifying the detected speaker. | ## Usage @@ -168,9 +167,9 @@ stt = XAISTTService( xAI STT supports the standard [service connection events](/api-reference/server/events/service-events): -| Event | Description | -| ----------------- | ----------------------------- | -| `on_connected` | Connected to xAI WebSocket | +| Event | Description | +| ----------------- | ------------------------------- | +| `on_connected` | Connected to xAI WebSocket | | `on_disconnected` | Disconnected from xAI WebSocket | ```python diff --git a/api-reference/server/services/transport/lemonslice.mdx b/api-reference/server/services/transport/lemonslice.mdx index a4ced10ba..e0ed89b95 100644 --- a/api-reference/server/services/transport/lemonslice.mdx +++ b/api-reference/server/services/transport/lemonslice.mdx @@ -103,18 +103,18 @@ Before using LemonSlice video services, you need: Configuration for creating a new LemonSlice session. - URL to an agent image. Provide exactly one of `agent_image_url`, `agent_id`, or - `agent_image`. + URL to an agent image. Provide exactly one of `agent_image_url`, `agent_id`, + or `agent_image`. - ID of a LemonSlice agent. Provide exactly one of `agent_image_url`, `agent_id`, - or `agent_image`. + ID of a LemonSlice agent. Provide exactly one of `agent_image_url`, + `agent_id`, or `agent_image`. - PIL image uploaded as the agent image. Provide exactly one of `agent_image_url`, - `agent_id`, or `agent_image`. + PIL image uploaded as the agent image. Provide exactly one of + `agent_image_url`, `agent_id`, or `agent_image`. diff --git a/api-reference/server/services/transport/livekit.mdx b/api-reference/server/services/transport/livekit.mdx index e07adc927..60646bd85 100644 --- a/api-reference/server/services/transport/livekit.mdx +++ b/api-reference/server/services/transport/livekit.mdx @@ -117,22 +117,22 @@ LiveKitTransport provides event handlers for room lifecycle, participant managem ### Events Summary -| Event | Description | -| ----------------------------- | -------------------------------------- | -| `on_connected` | Connected to the room | -| `on_disconnected` | Disconnected from the room | -| `on_before_disconnect` | About to disconnect (sync) | -| `on_call_state_updated` | Call state changed | -| `on_first_participant_joined` | First participant joined | -| `on_participant_connected` | A participant connected | -| `on_participant_disconnected` | A participant disconnected | -| `on_participant_left` | A participant left | -| `on_audio_track_subscribed` | Audio track subscribed | -| `on_audio_track_unsubscribed` | Audio track unsubscribed | -| `on_video_track_subscribed` | Video track subscribed | -| `on_video_track_unsubscribed` | Video track unsubscribed | -| `on_data_received` | Data message received | -| `on_dtmf_event` | SIP DTMF tone received (telephony) | +| Event | Description | +| ----------------------------- | ---------------------------------- | +| `on_connected` | Connected to the room | +| `on_disconnected` | Disconnected from the room | +| `on_before_disconnect` | About to disconnect (sync) | +| `on_call_state_updated` | Call state changed | +| `on_first_participant_joined` | First participant joined | +| `on_participant_connected` | A participant connected | +| `on_participant_disconnected` | A participant disconnected | +| `on_participant_left` | A participant left | +| `on_audio_track_subscribed` | Audio track subscribed | +| `on_audio_track_unsubscribed` | Audio track unsubscribed | +| `on_video_track_subscribed` | Video track subscribed | +| `on_video_track_unsubscribed` | Video track unsubscribed | +| `on_data_received` | Data message received | +| `on_dtmf_event` | SIP DTMF tone received (telephony) | ### Room Lifecycle @@ -335,13 +335,15 @@ async def on_dtmf_event(transport, data): **Parameters:** -| Parameter | Type | Description | -| ----------- | ------------------ | ------------------------------------------------------------------------------------- | -| `transport` | `LiveKitTransport` | The transport instance | -| `data` | `dict` | DTMF data with keys `tone`/`digit` (str), `code` (int), and `participant_id` (str) | +| Parameter | Type | Description | +| ----------- | ------------------ | ---------------------------------------------------------------------------------- | +| `transport` | `LiveKitTransport` | The transport instance | +| `data` | `dict` | DTMF data with keys `tone`/`digit` (str), `code` (int), and `participant_id` (str) | - The transport pushes `InputDTMFFrame` automatically, so you only need this handler for custom DTMF logic. Use `DTMFAggregator` to collect digits into sequences. + The transport pushes `InputDTMFFrame` automatically, so you only need this + handler for custom DTMF logic. Use `DTMFAggregator` to collect digits into + sequences. ## Additional Resources diff --git a/api-reference/server/services/transport/small-webrtc.mdx b/api-reference/server/services/transport/small-webrtc.mdx index dcbd1d048..01abe8d8d 100644 --- a/api-reference/server/services/transport/small-webrtc.mdx +++ b/api-reference/server/services/transport/small-webrtc.mdx @@ -57,11 +57,15 @@ uv add "pipecat-ai[webrtc-video]" ``` - The `webrtc` extra provides audio and basic video support via `aiortc`. The `webrtc-video` extra adds OpenCV support for using `SmallWebRTCTransport` with video. + The `webrtc` extra provides audio and basic video support via `aiortc`. The + `webrtc-video` extra adds OpenCV support for using `SmallWebRTCTransport` with + video. - `opencv-python-headless` will be removed from the `webrtc` extra in 2.0.0. Video pipelines using `SmallWebRTCTransport` should start installing the new `webrtc-video` extra (`pipecat-ai[webrtc-video]`) instead. + `opencv-python-headless` will be removed from the `webrtc` extra in 2.0.0. + Video pipelines using `SmallWebRTCTransport` should start installing the new + `webrtc-video` extra (`pipecat-ai[webrtc-video]`) instead. ## Prerequisites diff --git a/api-reference/server/services/transport/tavus.mdx b/api-reference/server/services/transport/tavus.mdx index 374ee2ddc..e3fcf1cb2 100644 --- a/api-reference/server/services/transport/tavus.mdx +++ b/api-reference/server/services/transport/tavus.mdx @@ -165,8 +165,8 @@ async def on_connected(transport, data): ``` - For local development, set the `TAVUS_SAMPLE_ROOM_URL` environment variable - to skip the Tavus API call and use a pre-existing Daily room URL instead. + For local development, set the `TAVUS_SAMPLE_ROOM_URL` environment variable to + skip the Tavus API call and use a pre-existing Daily room URL instead. See the [complete example](https://github.com/pipecat-ai/pipecat/blob/main/examples/video-avatar/video-avatar-tavus-video-service.py) for a full implementation including: diff --git a/api-reference/server/services/transport/websocket-server.mdx b/api-reference/server/services/transport/websocket-server.mdx index 71864a94c..4be5bac7f 100644 --- a/api-reference/server/services/transport/websocket-server.mdx +++ b/api-reference/server/services/transport/websocket-server.mdx @@ -121,10 +121,10 @@ Inherits from `TransportParams` with additional WebSocket-specific parameters. - List of allowed origins for WebSocket connections. Empty list allows all origins. - When set, connections with a missing or disallowed `Origin` header are rejected - before the WebSocket handshake completes. Defaults to `PIPECAT_ALLOWED_ORIGINS` - environment variable (comma-separated). + List of allowed origins for WebSocket connections. Empty list allows all + origins. When set, connections with a missing or disallowed `Origin` header + are rejected before the WebSocket handshake completes. Defaults to + `PIPECAT_ALLOWED_ORIGINS` environment variable (comma-separated). ### WebsocketClientTransport diff --git a/api-reference/server/services/transport/whatsapp.mdx b/api-reference/server/services/transport/whatsapp.mdx index a0a4e9cec..0165e5a94 100644 --- a/api-reference/server/services/transport/whatsapp.mdx +++ b/api-reference/server/services/transport/whatsapp.mdx @@ -126,12 +126,13 @@ async def connection_callback( caller_number = call.From # Caller's phone number call_id = call.id direction = call.direction # "inbound" or "outbound" - + # Spawn your bot with the connection await spawn_bot(connection, call) ``` The `WhatsAppConnectCall` object contains: + - `From`: Caller's phone number - `To`: Called phone number - `id`: Unique call identifier @@ -139,7 +140,10 @@ The `WhatsAppConnectCall` object contains: - `timestamp`: Call timestamp -The single-argument `connection_callback(connection)` signature is deprecated as of Pipecat 1.4.0. Update your callbacks to accept both `connection` and `call` arguments to receive caller metadata. The old signature still works but emits a `DeprecationWarning`. + The single-argument `connection_callback(connection)` signature is deprecated + as of Pipecat 1.4.0. Update your callbacks to accept both `connection` and + `call` arguments to receive caller metadata. The old signature still works but + emits a `DeprecationWarning`. See the [complete example](https://github.com/pipecat-ai/pipecat-examples/tree/main/whatsapp) for a full implementation including: diff --git a/api-reference/server/services/tts/azure.mdx b/api-reference/server/services/tts/azure.mdx index ccef0c85d..a0883d3e7 100644 --- a/api-reference/server/services/tts/azure.mdx +++ b/api-reference/server/services/tts/azure.mdx @@ -156,19 +156,19 @@ The HTTP service accepts the same parameters as the streaming service except `te Runtime-configurable settings passed via the `settings` constructor argument using `AzureTTSService.Settings(...)`. These can be updated mid-conversation with `TTSUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| -------------- | ----------------- | ----------- | --------------------------------------------------------------------------------------------------------------- | -| `model` | `str` | `None` | Model identifier. _(Inherited.)_ | -| `voice` | `str` | `None` | Voice identifier. _(Inherited.)_ | -| `language` | `Language \| str` | `None` | Language for synthesis. _(Inherited.)_ | -| `emphasis` | `str` | `NOT_GIVEN` | Emphasis level for SSML. | -| `force_locale` | `bool` | `NOT_GIVEN` | Wraps text in SSML `` to force the configured `language` accent instead of auto-detection. | -| `pitch` | `str` | `NOT_GIVEN` | Pitch adjustment. | -| `rate` | `str` | `NOT_GIVEN` | Speaking rate. | -| `role` | `str` | `NOT_GIVEN` | Role for SSML. | -| `style` | `str` | `NOT_GIVEN` | Speaking style. | -| `style_degree` | `str` | `NOT_GIVEN` | Degree of the speaking style. | -| `volume` | `str` | `NOT_GIVEN` | Volume level. | +| Parameter | Type | Default | Description | +| -------------- | ----------------- | ----------- | ------------------------------------------------------------------------------------------------ | +| `model` | `str` | `None` | Model identifier. _(Inherited.)_ | +| `voice` | `str` | `None` | Voice identifier. _(Inherited.)_ | +| `language` | `Language \| str` | `None` | Language for synthesis. _(Inherited.)_ | +| `emphasis` | `str` | `NOT_GIVEN` | Emphasis level for SSML. | +| `force_locale` | `bool` | `NOT_GIVEN` | Wraps text in SSML `` to force the configured `language` accent instead of auto-detection. | +| `pitch` | `str` | `NOT_GIVEN` | Pitch adjustment. | +| `rate` | `str` | `NOT_GIVEN` | Speaking rate. | +| `role` | `str` | `NOT_GIVEN` | Role for SSML. | +| `style` | `str` | `NOT_GIVEN` | Speaking style. | +| `style_degree` | `str` | `NOT_GIVEN` | Degree of the speaking style. | +| `volume` | `str` | `NOT_GIVEN` | Volume level. | ## Usage diff --git a/api-reference/server/services/tts/elevenlabs.mdx b/api-reference/server/services/tts/elevenlabs.mdx index c4145ea96..4f741345f 100644 --- a/api-reference/server/services/tts/elevenlabs.mdx +++ b/api-reference/server/services/tts/elevenlabs.mdx @@ -118,9 +118,18 @@ export ELEVENLABS_API_KEY=your_api_key instead._ - - List of pronunciation dictionary locators to use. - _Deprecated in v1.6.0. Use the `text_transforms` parameter with `replace_text` instead. Pronunciation dictionary substitutions can rewrite the spoken words in ways that no longer match the text sent to synthesis, which breaks the alignment-based word-completion tracking used to attribute spoken text back to the conversation context. Will be removed in v2.0.0._ + + List of pronunciation dictionary locators to use. _Deprecated in v1.6.0. Use + the `text_transforms` parameter with `replace_text` instead. Pronunciation + dictionary substitutions can rewrite the spoken words in ways that no longer + match the text sent to synthesis, which breaks the alignment-based + word-completion tracking used to attribute spoken text back to the + conversation context. Will be removed in v2.0.0._ @@ -145,9 +154,18 @@ The HTTP service accepts the same parameters as the WebSocket service, with thes retention mode (enterprise only). - - List of pronunciation dictionary locators to use. - _Deprecated in v1.6.0. Use the `text_transforms` parameter with `replace_text` instead. Pronunciation dictionary substitutions can rewrite the spoken words in ways that no longer match the text sent to synthesis, which breaks the alignment-based word-completion tracking used to attribute spoken text back to the conversation context. Will be removed in v2.0.0._ + + List of pronunciation dictionary locators to use. _Deprecated in v1.6.0. Use + the `text_transforms` parameter with `replace_text` instead. Pronunciation + dictionary substitutions can rewrite the spoken words in ways that no longer + match the text sent to synthesis, which breaks the alignment-based + word-completion tracking used to attribute spoken text back to the + conversation context. Will be removed in v2.0.0._ The HTTP service uses `ElevenLabsHttpTTSSettings` which also includes: diff --git a/api-reference/server/services/tts/floe.mdx b/api-reference/server/services/tts/floe.mdx index e74bd9f5f..775f6bc79 100644 --- a/api-reference/server/services/tts/floe.mdx +++ b/api-reference/server/services/tts/floe.mdx @@ -80,14 +80,18 @@ Before using the Floe TTS service, you need: Voice identifier to synthesize with. - + Floe OpenAI-compatible base URL. Optional Floe task ID sent as the `X-Floe-Task-Id` header, so a per-task - budget can bound one conversation. Attached via a custom `httpx` client; if you - pass your own `http_client`, add the header to it yourself. + budget can bound one conversation. Attached via a custom `httpx` client; if + you pass your own `http_client`, add the header to it yourself. diff --git a/api-reference/server/services/tts/google.mdx b/api-reference/server/services/tts/google.mdx index 32f0b9f20..b486234eb 100644 --- a/api-reference/server/services/tts/google.mdx +++ b/api-reference/server/services/tts/google.mdx @@ -181,7 +181,12 @@ Runtime-configurable settings passed via the `settings` constructor argument usi Streaming service using Gemini's TTS-specific models with natural voice control. Supports two backends: the Google Cloud backend (with prompts for style instructions and multi-speaker support) or the Gemini Developer API (google-genai) backend (simpler API key authentication). - + Gemini TTS model to use. Options: `"gemini-3.1-flash-tts-preview"`, `"gemini-2.5-flash-tts"`, `"gemini-2.5-pro-tts"`. _Deprecated in v0.0.105. Use `settings=GeminiTTSService.Settings(model=...)` instead._ diff --git a/api-reference/server/services/tts/gradium.mdx b/api-reference/server/services/tts/gradium.mdx index 8b55cb917..4b862b45b 100644 --- a/api-reference/server/services/tts/gradium.mdx +++ b/api-reference/server/services/tts/gradium.mdx @@ -69,11 +69,7 @@ Before using Gradium TTS services, you need: `settings=GradiumTTSService.Settings(voice=...)` instead._ - + Gradium WebSocket API endpoint. Gradium automatically routes traffic to the nearest endpoint. Override to pin to a specific region or custom deployment. diff --git a/api-reference/server/services/tts/hakim.mdx b/api-reference/server/services/tts/hakim.mdx index c8bcf8d1c..8bd25c219 100644 --- a/api-reference/server/services/tts/hakim.mdx +++ b/api-reference/server/services/tts/hakim.mdx @@ -111,15 +111,14 @@ Before using the Hakim TTS service, you need: Runtime-updatable configuration passed via the `settings` constructor argument using `HakimTTSService.Settings(...)`. -| Parameter | Type | Default | Description | -| -------------- | ------- | ------- | ------------------------------------------------------------------------------------ | -| `cfg` | `float` | `3.0` | Classifier-free-guidance weight, `0.0`-`10.0`. | -| `voice_prompt` | `str` | `None` | Free-form voice-character description. Only honoured on `model="hakim-v3"`. | +| Parameter | Type | Default | Description | +| -------------- | ------- | ------- | --------------------------------------------------------------------------- | +| `cfg` | `float` | `3.0` | Classifier-free-guidance weight, `0.0`-`10.0`. | +| `voice_prompt` | `str` | `None` | Free-form voice-character description. Only honoured on `model="hakim-v3"`. | - See the [source - repository](https://github.com/tryHakimAI/hakim-pipecat) for the - authoritative, up-to-date configuration options. + See the [source repository](https://github.com/tryHakimAI/hakim-pipecat) for + the authoritative, up-to-date configuration options. ## Usage diff --git a/api-reference/server/services/tts/inworld.mdx b/api-reference/server/services/tts/inworld.mdx index a8cab0a6e..a87c7bda6 100644 --- a/api-reference/server/services/tts/inworld.mdx +++ b/api-reference/server/services/tts/inworld.mdx @@ -11,11 +11,11 @@ Inworld provides high-quality, low-latency speech synthesis via two implementati The default model is **Realtime TTS-2** (`inworld-tts-2`). Realtime TTS-1.5-Max (`inworld-tts-1.5-max`) and Realtime TTS-1.5-Mini (`inworld-tts-1.5-mini`) remain available. -| Display name | Model ID | -| ------------------------- | ----------------------- | -| Realtime TTS-2 _(default)_ | `inworld-tts-2` | -| Realtime TTS-1.5-Max | `inworld-tts-1.5-max` | -| Realtime TTS-1.5-Mini | `inworld-tts-1.5-mini` | +| Display name | Model ID | +| -------------------------- | ---------------------- | +| Realtime TTS-2 _(default)_ | `inworld-tts-2` | +| Realtime TTS-1.5-Max | `inworld-tts-1.5-max` | +| Realtime TTS-1.5-Mini | `inworld-tts-1.5-mini` | - Custom `httpx.AsyncClient` for API requests, e.g. one with a longer request timeout. Prefer `openai.DefaultAsyncHttpxClient`, which keeps the SDK's connection limits and redirect handling; a bare `httpx.AsyncClient` uses httpx's defaults instead. Defaults to `None`, which lets the SDK build its own client. + Custom `httpx.AsyncClient` for API requests, e.g. one with a longer request + timeout. Prefer `openai.DefaultAsyncHttpxClient`, which keeps the SDK's + connection limits and redirect handling; a bare `httpx.AsyncClient` uses + httpx's defaults instead. Defaults to `None`, which lets the SDK build its own + client. diff --git a/api-reference/server/services/tts/pocket-tts.mdx b/api-reference/server/services/tts/pocket-tts.mdx index c6a1704b1..a3b6764e5 100644 --- a/api-reference/server/services/tts/pocket-tts.mdx +++ b/api-reference/server/services/tts/pocket-tts.mdx @@ -84,11 +84,11 @@ Pocket TTS runs locally and does not require an API key or external service. On Runtime-configurable settings passed via the `settings` constructor argument using `PocketTTSService.Settings(...)`. These can be updated mid-conversation with `TTSUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------- | ----------------- | ------------- | ---------------------------------------------------------------------------------------------------- | -| `model` | `str` | `None` | Model identifier. _(Inherited from base settings.)_ | +| Parameter | Type | Default | Description | +| ---------- | ----------------- | ------------- | ----------------------------------------------------------------------------------------------------- | +| `model` | `str` | `None` | Model identifier. _(Inherited from base settings.)_ | | `voice` | `str` | `"alba"` | Voice identifier (e.g. `"alba"`, `"jane"`), local `.wav` file, `.safetensors` state, or `hf://` path. | -| `language` | `Language \| str` | `Language.EN` | Language for synthesis. See supported languages below. | +| `language` | `Language \| str` | `Language.EN` | Language for synthesis. See supported languages below. | ### Supported Languages diff --git a/api-reference/server/services/tts/soniox.mdx b/api-reference/server/services/tts/soniox.mdx index de201c418..3f671335d 100644 --- a/api-reference/server/services/tts/soniox.mdx +++ b/api-reference/server/services/tts/soniox.mdx @@ -105,11 +105,11 @@ Before using Soniox TTS, you need: Runtime-configurable settings passed via the `settings` constructor argument using `SonioxTTSService.Settings(...)`. These can be updated mid-conversation with `TTSUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ---------- | ----------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `model` | `str` | `tts-rt-v1` | TTS model identifier. _(Inherited from base settings.)_ | +| Parameter | Type | Default | Description | +| ---------- | ----------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | `str` | `tts-rt-v1` | TTS model identifier. _(Inherited from base settings.)_ | | `voice` | `str` | `Adrian` | Voice name (e.g. `"Adrian"`) or the UUID of a cloned voice in the project owning the API key. _(Inherited from base settings.)_ | -| `language` | `Language \| str` | `Language.EN` | Language for synthesis. _(Inherited from base settings.)_ See [supported languages](https://soniox.com/docs/tts/concepts/languages). | +| `language` | `Language \| str` | `Language.EN` | Language for synthesis. _(Inherited from base settings.)_ See [supported languages](https://soniox.com/docs/tts/concepts/languages). | | `speed` | `float \| None` | `None` | Speech rate multiplier in the range 0.7-1.3. `None` uses the Soniox server default (1.0). Changing this flushes the current context and starts a new stream with the updated value. | ## Usage diff --git a/api-reference/server/services/tts/xai.mdx b/api-reference/server/services/tts/xai.mdx index ec045eec4..d115a59e7 100644 --- a/api-reference/server/services/tts/xai.mdx +++ b/api-reference/server/services/tts/xai.mdx @@ -95,14 +95,14 @@ export GROK_API_KEY=your_api_key Runtime-configurable settings passed via the `settings` constructor argument using `XAIHttpTTSService.Settings(...)`. These can be updated mid-conversation with `TTSUpdateSettingsFrame`. See [Service Settings](/pipecat/fundamentals/service-settings) for details. -| Parameter | Type | Default | Description | -| ------------------------------- | ----------------- | ------------- | ----------------------------------------------------------------- | -| `model` | `str` | `None` | Model identifier. _(Inherited from base settings.)_ | -| `voice` | `str` | `"eve"` | Voice identifier. _(Inherited from base settings.)_ | -| `language` | `Language \| str` | `Language.EN` | Language code. _(Inherited from base settings.)_ | -| `speed` | `float` | `None` | Speech speed multiplier from 0.7 to 1.5 (1.0 is normal). | -| `optimize_streaming_latency` | `int` | `None` | Latency optimization level (0, 1, or 2). | -| `text_normalization` | `bool` | `None` | Whether to normalize text before synthesis. | +| Parameter | Type | Default | Description | +| ---------------------------- | ----------------- | ------------- | -------------------------------------------------------- | +| `model` | `str` | `None` | Model identifier. _(Inherited from base settings.)_ | +| `voice` | `str` | `"eve"` | Voice identifier. _(Inherited from base settings.)_ | +| `language` | `Language \| str` | `Language.EN` | Language code. _(Inherited from base settings.)_ | +| `speed` | `float` | `None` | Speech speed multiplier from 0.7 to 1.5 (1.0 is normal). | +| `optimize_streaming_latency` | `int` | `None` | Latency optimization level (0, 1, or 2). | +| `text_normalization` | `bool` | `None` | Whether to normalize text before synthesis. | ### XAITTSService @@ -128,7 +128,8 @@ Runtime-configurable settings passed via the `settings` constructor argument usi Runtime-configurable settings. Includes all settings from `XAIHttpTTSService` plus `with_timestamps` for word-level timing. Changing voice, language, or - tunable parameters at runtime reconnects the WebSocket with new query parameters. + tunable parameters at runtime reconnects the WebSocket with new query + parameters. ### WebSocket Settings diff --git a/api-reference/server/services/tts/xtts.mdx b/api-reference/server/services/tts/xtts.mdx index c4cf40aa6..55ee7ec83 100644 --- a/api-reference/server/services/tts/xtts.mdx +++ b/api-reference/server/services/tts/xtts.mdx @@ -6,10 +6,12 @@ description: "Text-to-speech service implementation using Coqui's XTTS streaming `XTTSService` is deprecated as of v1.7.0 and will be removed in v2.0.0. The [Coqui XTTS streaming server](https://github.com/coqui-ai/xtts-streaming-server) it connects to has been unmaintained since February 2024 and pins a commit of the discontinued `coqui-ai/TTS` library. The XTTS-v2 model is licensed for non-commercial use only. - **Alternatives:** - - [`KokoroTTSService`](/api-reference/server/services/tts/kokoro) — Maintained local TTS service - - [`PiperTTSService`](/api-reference/server/services/tts/piper) — Maintained local TTS service - - [`pipecat-xtts-vllm`](/api-reference/server/services/tts/xtts-vllm) — Community-maintained package targeting a current XTTSv2 serving stack +**Alternatives:** + +- [`KokoroTTSService`](/api-reference/server/services/tts/kokoro) — Maintained local TTS service +- [`PiperTTSService`](/api-reference/server/services/tts/piper) — Maintained local TTS service +- [`pipecat-xtts-vllm`](/api-reference/server/services/tts/xtts-vllm) — Community-maintained package targeting a current XTTSv2 serving stack + ## Overview diff --git a/api-reference/server/services/video/simli.mdx b/api-reference/server/services/video/simli.mdx index 7b99e2648..29eece405 100644 --- a/api-reference/server/services/video/simli.mdx +++ b/api-reference/server/services/video/simli.mdx @@ -102,8 +102,8 @@ Before using Simli video services, you need: (`max_session_length`, `max_idle_time`, `enable_logging`) or `settings=SimliVideoService.Settings(...)` instead. - Additional input parameters for session configuration. See [InputParams](#inputparams) - below. + Additional input parameters for session configuration. See + [InputParams](#inputparams) below. ### InputParams diff --git a/api-reference/server/utilities/audio/audio-buffer-processor.mdx b/api-reference/server/utilities/audio/audio-buffer-processor.mdx index eedd05d11..6b89a6a66 100644 --- a/api-reference/server/utilities/audio/audio-buffer-processor.mdx +++ b/api-reference/server/utilities/audio/audio-buffer-processor.mdx @@ -52,7 +52,8 @@ Buffer size in bytes that triggers audio data events: Whether to start recording automatically when the pipeline starts, without - requiring a call to `start_recording()` or an `AudioBufferStartRecordingFrame`. + requiring a call to `start_recording()` or an + `AudioBufferStartRecordingFrame`. ## Properties diff --git a/api-reference/server/utilities/dtmf-aggregator.mdx b/api-reference/server/utilities/dtmf-aggregator.mdx index 3764220bc..27b834ee5 100644 --- a/api-reference/server/utilities/dtmf-aggregator.mdx +++ b/api-reference/server/utilities/dtmf-aggregator.mdx @@ -48,7 +48,9 @@ aggregator = DTMFAggregator( ## Output Frames - Contains the aggregated DTMF sequence as text with the configured prefix. The frame is marked as `finalized=True` to ensure proper integration with user-turn stop strategies. + Contains the aggregated DTMF sequence as text with the configured prefix. The + frame is marked as `finalized=True` to ensure proper integration with + user-turn stop strategies. All input frames are passed through downstream, including the original `InputDTMFFrame` instances. diff --git a/api-reference/server/utilities/filters/wake-check-filter.mdx b/api-reference/server/utilities/filters/wake-check-filter.mdx index 91d982bf0..6038b0efd 100644 --- a/api-reference/server/utilities/filters/wake-check-filter.mdx +++ b/api-reference/server/utilities/filters/wake-check-filter.mdx @@ -5,9 +5,9 @@ description: "Processor that passes frames only after detecting wake phrases in **Deprecated:** `WakeCheckFilter` is deprecated in favor of - [`WakePhraseUserTurnStartStrategy`](/api-reference/server/utilities/turn-management/user-turn-strategies#wakephraseuserturns tartstrategy). - The new strategy provides better integration with turn management and supports - both timeout and single-activation modes. + [`WakePhraseUserTurnStartStrategy`](/api-reference/server/utilities/turn-management/user-turn-strategies#wakephraseuserturns + tartstrategy). The new strategy provides better integration with turn + management and supports both timeout and single-activation modes. ## Overview diff --git a/api-reference/server/utilities/frame/llm-text-processor.mdx b/api-reference/server/utilities/frame/llm-text-processor.mdx index 81db8d8df..303539188 100644 --- a/api-reference/server/utilities/frame/llm-text-processor.mdx +++ b/api-reference/server/utilities/frame/llm-text-processor.mdx @@ -8,8 +8,8 @@ description: "A processor for aggregating LLMTextFrames into logical units befor `LLMTextProcessor` is a processor designed to aggregate `LLMTextFrame`s into coherent text units before passing them to downstream services, such as TTS. By utilizing text aggregators, it ensures that text is properly segmented and structured, enhancing the quality of subsequent processing. This processor expects `LLMTextFrame`s as input and outputs `AggregatedTextFrame`s containing the aggregated text. - When an `LLMTextProcessor` is in use, text aggregation is handled upstream - and the TTS service's built-in aggregator is bypassed. + When an `LLMTextProcessor` is in use, text aggregation is handled upstream and + the TTS service's built-in aggregator is bypassed. The benefit of pre-aggregating LLM text frames is that it allows for more controlled and meaningful text synthesis. Downstream services can operate on complete sentences or logical text blocks. For TTS services, this means being able to customize how certain types of text are spoken (e.g., spelling out phone numbers, stripping out url protocols, or inserting other tts-specific annotations) or even skipping over certain text segments entirely (e.g., code snippets or markup). For other services, such as RTVI, it allows for sending these logical text units as separate `bot-output` messages, supporting custom client-side handling and rendering (e.g. collapsible code blocks, clickable links, etc.). diff --git a/api-reference/server/utilities/text/markdown-text-filter.mdx b/api-reference/server/utilities/text/markdown-text-filter.mdx index 99b7f14e7..41296bcce 100644 --- a/api-reference/server/utilities/text/markdown-text-filter.mdx +++ b/api-reference/server/utilities/text/markdown-text-filter.mdx @@ -36,7 +36,8 @@ Configure the filter behavior with these options: - Whether to remove repeated sequences of 5 or more identical characters from the output + Whether to remove repeated sequences of 5 or more identical characters from + the output ## Features @@ -95,16 +96,16 @@ md_filter = MarkdownTextFilter( ## What Gets Removed -| Markdown Feature | Example | Result | -| -------------------------- | ------------------------ | ------------ | -| Bold | `**important**` | `important` | -| Italic | `*emphasized*` | `emphasized` | -| Headers | `## Section` | `Section` | -| Code (inline) | `` `code` `` | `code` | -| Code blocks (when enabled) | ` ```python\ncode\n``` ` | ` ` | -| Tables (when enabled) | `\|A\|B\|\n\|--\|--\|` | ` ` | -| HTML tags | `text` | `text` | -| Repeated characters (when enabled) | `22222` | *(removed)* | +| Markdown Feature | Example | Result | +| ---------------------------------- | ------------------------ | ------------ | +| Bold | `**important**` | `important` | +| Italic | `*emphasized*` | `emphasized` | +| Headers | `## Section` | `Section` | +| Code (inline) | `` `code` `` | `code` | +| Code blocks (when enabled) | ` ```python\ncode\n``` ` | ` ` | +| Tables (when enabled) | `\|A\|B\|\n\|--\|--\|` | ` ` | +| HTML tags | `text` | `text` | +| Repeated characters (when enabled) | `22222` | _(removed)_ | ## Notes diff --git a/api-reference/server/utilities/turn-management/external-turn-management.mdx b/api-reference/server/utilities/turn-management/external-turn-management.mdx index fe0f09fd5..6cd4bc56a 100644 --- a/api-reference/server/utilities/turn-management/external-turn-management.mdx +++ b/api-reference/server/utilities/turn-management/external-turn-management.mdx @@ -86,8 +86,8 @@ Setting `realtime_service_mode=True` adapts the pair's behavior in three ways: default="UserTurnStrategies()" > Configured strategies for starting and stopping user turns. See [User Turn - Strategies](/api-reference/server/utilities/turn-management/user-turn-strategies) for - available options. + Strategies](/api-reference/server/utilities/turn-management/user-turn-strategies) + for available options. diff --git a/api-reference/server/utilities/turn-management/transcriptions.mdx b/api-reference/server/utilities/turn-management/transcriptions.mdx index 8719242b7..fc4969eff 100644 --- a/api-reference/server/utilities/turn-management/transcriptions.mdx +++ b/api-reference/server/utilities/turn-management/transcriptions.mdx @@ -13,10 +13,9 @@ The key events for transcription collection are: - **`on_assistant_turn_stopped`** - Provides the assistant's complete transcript via `AssistantTurnStoppedMessage` - The examples below assume a cascade (STT → LLM → TTS) pipeline. With a realtime - (speech-to-speech) service and - `LLMContextAggregatorPair(context, realtime_service_mode=True)`, collect the - user transcript from + The examples below assume a cascade (STT → LLM → TTS) pipeline. With a + realtime (speech-to-speech) service and `LLMContextAggregatorPair(context, + realtime_service_mode=True)`, collect the user transcript from [`on_user_turn_message_added`](/api-reference/server/utilities/turn-management/turn-events#on_user_turn_message_added) instead — in that mode `UserTurnStoppedMessage.content` is `None`. diff --git a/api-reference/server/utilities/turn-management/turn-events.mdx b/api-reference/server/utilities/turn-management/turn-events.mdx index 4dcdd63b2..7fd4e7a0a 100644 --- a/api-reference/server/utilities/turn-management/turn-events.mdx +++ b/api-reference/server/utilities/turn-management/turn-events.mdx @@ -97,11 +97,11 @@ async def on_user_turn_stopped(aggregator, strategy, message: UserTurnStoppedMes In realtime mode (`realtime_service_mode=True` on [`LLMContextAggregatorPair`](/api-reference/server/utilities/turn-management/external-turn-management#realtime-speech-to-speech-services)), - the user message isn't finalized at turn-stop time, so - `message.content` is `None`. Subscribe to - [`on_user_turn_message_added`](#on_user_turn_message_added) instead to - get the finalized user text. Behavior in cascade (STT → LLM → TTS) - pipelines is unchanged. + the user message isn't finalized at turn-stop time, so `message.content` is + `None`. Subscribe to + [`on_user_turn_message_added`](#on_user_turn_message_added) instead to get the + finalized user text. Behavior in cascade (STT → LLM → TTS) pipelines is + unchanged. ### on_user_turn_message_added @@ -117,9 +117,9 @@ async def on_user_turn_message_added(aggregator, message: UserTurnMessageAddedMe **Parameters:** -| Parameter | Type | Description | -| ------------ | ----------------------------- | -------------------------------------------- | -| `aggregator` | `LLMUserAggregator` | The user aggregator instance | +| Parameter | Type | Description | +| ------------ | ----------------------------- | --------------------------------------------- | +| `aggregator` | `LLMUserAggregator` | The user aggregator instance | | `message` | `UserTurnMessageAddedMessage` | Contains the finalized user text and metadata | ### on_user_turn_stop_timeout @@ -314,11 +314,11 @@ from pipecat.processors.aggregators.llm_response_universal import UserTurnStoppe ``` - The complete transcribed text from the user's turn. `None` in realtime - mode (`realtime_service_mode=True`), where the user message isn't - finalized at turn-stop time — subscribe to - [`on_user_turn_message_added`](#on_user_turn_message_added) for the - finalized text instead. + The complete transcribed text from the user's turn. `None` in realtime mode + (`realtime_service_mode=True`), where the user message isn't finalized at + turn-stop time — subscribe to + [`on_user_turn_message_added`](#on_user_turn_message_added) for the finalized + text instead. diff --git a/api-reference/server/workers/base-worker.mdx b/api-reference/server/workers/base-worker.mdx index 311e3377d..d375b85b4 100644 --- a/api-reference/server/workers/base-worker.mdx +++ b/api-reference/server/workers/base-worker.mdx @@ -26,11 +26,7 @@ from pipecat.pipeline.base_worker import BaseWorker, WorkerActivationArgs `activate_worker()` call before `on_activated` fires. - + Whether to warn about tasks left running when the worker finishes. Only applies when the worker owns its task manager; a worker sharing the runner's task manager leaves the report to the runner. diff --git a/api-reference/server/workers/llm-worker.mdx b/api-reference/server/workers/llm-worker.mdx index 176c0b8cd..c93d9da07 100644 --- a/api-reference/server/workers/llm-worker.mdx +++ b/api-reference/server/workers/llm-worker.mdx @@ -229,4 +229,3 @@ async def get_weather(self, params, location: str, unit: str = "celsius"): LLM. The method signature (parameter names, types, and docstring) is automatically used to generate the tool schema. - diff --git a/api-reference/server/workers/runner.mdx b/api-reference/server/workers/runner.mdx index c1298ac85..be31ab731 100644 --- a/api-reference/server/workers/runner.mdx +++ b/api-reference/server/workers/runner.mdx @@ -52,13 +52,9 @@ Adding an agent with `add_workers()` attaches it to the runner's bus and registr Whether to force garbage collection after the main worker completes. - - Whether to warn about tasks left running on the shared task manager once - every worker has finished. + + Whether to warn about tasks left running on the shared task manager once every + worker has finished. diff --git a/client/concepts/events-and-callbacks.mdx b/client/concepts/events-and-callbacks.mdx index 2e2e191fe..bb4a159a3 100644 --- a/client/concepts/events-and-callbacks.mdx +++ b/client/concepts/events-and-callbacks.mdx @@ -4,29 +4,34 @@ description: "How to respond to bot and session events in Pipecat client applica --- - -You are currently viewing the React version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React version of this page. Use the dropdown + to the right to customize this page for your client framework. + - -You are currently viewing the JavaScript version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the JavaScript version of this page. Use the + dropdown to the right to customize this page for your client framework. + - -You are currently viewing the React Native version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React Native version of this page. Use the + dropdown to the right to customize this page for your client framework. + - -You are currently viewing the iOS version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the iOS version of this page. Use the dropdown to + the right to customize this page for your client framework. + - -You are currently viewing the Android version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the Android version of this page. Use the dropdown + to the right to customize this page for your client framework. + The Pipecat client emits events throughout the session lifecycle — when the bot connects, when the user speaks, when a transcript arrives, and more. @@ -110,7 +115,7 @@ function TranscriptDisplay() { RTVIEvent.UserTranscript, useCallback((data) => { if (data.final) setTranscript(data.text); - }, []) + }, []), ); } ``` @@ -125,7 +130,9 @@ Add handlers with `.on()` at any point — useful for dynamic subscriptions or w ```tsx client.on(RTVIEvent.BotReady, () => console.log("Bot is ready")); -client.on(RTVIEvent.UserTranscript, (data) => console.log("User said:", data.text)); +client.on(RTVIEvent.UserTranscript, (data) => + console.log("User said:", data.text), +); ``` Callbacks and event listeners are equivalent — use whichever pattern fits your architecture. @@ -221,52 +228,56 @@ Callbacks run on the main thread and can update Compose `mutableStateOf` directl These events track the connection state of the client and bot. See [Session Lifecycle](/client/concepts/session-lifecycle) for the full state progression. -| Event | Callback | When it fires | -|---|---|---| -| `Connected` | `onConnected` | Client transport connection established | -| `Disconnected` | `onDisconnected` | Client disconnected (intentional or error) | -| `TransportStateChanged` | `onTransportStateChanged` | Any transport state change; receives the new `TransportState` string | -| `BotConnected` | `onBotConnected` | Bot joined the transport; pipeline may still be initializing | -| `BotReady` | `onBotReady` | Bot pipeline is ready; safe to send messages and expect audio | -| `BotDisconnected` | `onBotDisconnected` | Bot left the session; client will also disconnect unless `disconnectOnBotDisconnect: false` | -| `ParticipantConnected` | `onParticipantJoined` | Any participant joined (bot, local, or other) | -| `ParticipantLeft` | `onParticipantLeft` | Any participant left (bot, local, or other) | +| Event | Callback | When it fires | +| ----------------------- | ------------------------- | ------------------------------------------------------------------------------------------- | +| `Connected` | `onConnected` | Client transport connection established | +| `Disconnected` | `onDisconnected` | Client disconnected (intentional or error) | +| `TransportStateChanged` | `onTransportStateChanged` | Any transport state change; receives the new `TransportState` string | +| `BotConnected` | `onBotConnected` | Bot joined the transport; pipeline may still be initializing | +| `BotReady` | `onBotReady` | Bot pipeline is ready; safe to send messages and expect audio | +| `BotDisconnected` | `onBotDisconnected` | Bot left the session; client will also disconnect unless `disconnectOnBotDisconnect: false` | +| `ParticipantConnected` | `onParticipantJoined` | Any participant joined (bot, local, or other) | +| `ParticipantLeft` | `onParticipantLeft` | Any participant left (bot, local, or other) | -`BotReady` receives a `BotReadyData` object with a `version` field — the RTVI version the bot is running. You can use this to check compatibility if your client and server may be on different versions. + `BotReady` receives a `BotReadyData` object with a `version` field — the RTVI + version the bot is running. You can use this to check compatibility if your + client and server may be on different versions. ### Voice activity These events are driven by the bot's VAD (voice activity detection) model. VAD is smarter than tracking raw audio levels — it understands turn-taking, so it can distinguish between a user who has finished speaking and one who has simply paused or is speaking slowly. -| Event | Callback | When it fires | -|---|---|---| -| `UserStartedSpeaking` | `onUserStartedSpeaking` | VAD detected the user started speaking | -| `UserStoppedSpeaking` | `onUserStoppedSpeaking` | VAD detected the user stopped speaking | -| `BotStartedSpeaking` | `onBotStartedSpeaking` | Bot started sending audio | -| `BotStoppedSpeaking` | `onBotStoppedSpeaking` | Bot stopped sending audio | -| `LocalAudioLevel` | `onLocalAudioLevel` | Local audio gain level (0–1); fires continuously | -| `RemoteAudioLevel` | `onRemoteAudioLevel` | Remote audio gain level (0–1); fires continuously | -| `UserMuteStarted` | `onUserMuteStarted` | Server started ignoring client audio (server-side mute) | -| `UserMuteStopped` | `onUserMuteStopped` | Server resumed processing client audio | +| Event | Callback | When it fires | +| --------------------- | ----------------------- | ------------------------------------------------------- | +| `UserStartedSpeaking` | `onUserStartedSpeaking` | VAD detected the user started speaking | +| `UserStoppedSpeaking` | `onUserStoppedSpeaking` | VAD detected the user stopped speaking | +| `BotStartedSpeaking` | `onBotStartedSpeaking` | Bot started sending audio | +| `BotStoppedSpeaking` | `onBotStoppedSpeaking` | Bot stopped sending audio | +| `LocalAudioLevel` | `onLocalAudioLevel` | Local audio gain level (0–1); fires continuously | +| `RemoteAudioLevel` | `onRemoteAudioLevel` | Remote audio gain level (0–1); fires continuously | +| `UserMuteStarted` | `onUserMuteStarted` | Server started ignoring client audio (server-side mute) | +| `UserMuteStopped` | `onUserMuteStopped` | Server resumed processing client audio | -`UserMuteStarted`/`UserMuteStopped` reflect server-side muting — the client continues sending audio, but the bot is ignoring it. Use these to update your UI (e.g., show a muted indicator) without actually stopping the local mic. + `UserMuteStarted`/`UserMuteStopped` reflect server-side muting — the client + continues sending audio, but the bot is ignoring it. Use these to update your + UI (e.g., show a muted indicator) without actually stopping the local mic. ### Transcription and bot output -| Event | Callback | Data | When it fires | -|---|---|---|---| +| Event | Callback | Data | When it fires | +| ---------------- | ------------------ | ---------------- | ---------------------------------------------------------------------------------- | | `UserTranscript` | `onUserTranscript` | `TranscriptData` | User speech transcribed; fires for both partial (`final: false`) and final results | -| `BotOutput` | `onBotOutput` | `BotOutputData` | Bot text output, typically aggregated by sentence or word during TTS synthesis | -| `BotLlmText` | `onBotLlmText` | `BotLLMTextData` | Raw LLM token stream | -| `BotLlmStarted` | `onBotLlmStarted` | — | LLM inference started | -| `BotLlmStopped` | `onBotLlmStopped` | — | LLM inference finished | -| `BotTtsText` | `onBotTtsText` | `BotTTSTextData` | Words from TTS as they are synthesized (streaming TTS only) | -| `BotTtsStarted` | `onBotTtsStarted` | — | TTS synthesis started | -| `BotTtsStopped` | `onBotTtsStopped` | — | TTS synthesis finished | +| `BotOutput` | `onBotOutput` | `BotOutputData` | Bot text output, typically aggregated by sentence or word during TTS synthesis | +| `BotLlmText` | `onBotLlmText` | `BotLLMTextData` | Raw LLM token stream | +| `BotLlmStarted` | `onBotLlmStarted` | — | LLM inference started | +| `BotLlmStopped` | `onBotLlmStopped` | — | LLM inference finished | +| `BotTtsText` | `onBotTtsText` | `BotTTSTextData` | Words from TTS as they are synthesized (streaming TTS only) | +| `BotTtsStarted` | `onBotTtsStarted` | — | TTS synthesis started | +| `BotTtsStopped` | `onBotTtsStopped` | — | TTS synthesis finished | `UserTranscript` fires continuously as speech is recognized. Check `data.final` to distinguish committed transcripts from work-in-progress partials: @@ -281,7 +292,7 @@ useRTVIClientEvent( } else { updatePartial(data.text); // still in progress } - }, []) + }, []), ); ``` @@ -360,7 +371,7 @@ useRTVIClientEvent( if (data.aggregated_by === "sentence") { appendSentence(data.text); } - }, []) + }, []), ); ``` @@ -422,10 +433,10 @@ override fun onBotOutput(data: BotOutputData) { ### Errors -| Event | Callback | When it fires | -|---|---|---| -| `Error` | `onError` | Bot signalled an error; `data.fatal` is `true` if the session is unrecoverable | -| `MessageError` | `onMessageError` | A client message failed or got an error response | +| Event | Callback | When it fires | +| -------------- | ---------------- | ------------------------------------------------------------------------------ | +| `Error` | `onError` | Bot signalled an error; `data.fatal` is `true` if the session is unrecoverable | +| `MessageError` | `onMessageError` | A client message failed or got an error response | Always handle `Error`. If `data.fatal` is `true`, the bot has already disconnected — update your UI accordingly: @@ -440,7 +451,7 @@ useRTVIClientEvent( } else { showToast(data.message); } - }, []) + }, []), ); ``` @@ -503,17 +514,17 @@ override fun onBackendError(message: String) { ### Devices and tracks -| Event | Callback | When it fires | -|---|---|---| -| `AvailableMicsUpdated` | `onAvailableMicsUpdated` | Mic list changed or `initDevices()` called | -| `AvailableCamsUpdated` | `onAvailableCamsUpdated` | Camera list changed or `initDevices()` called | +| Event | Callback | When it fires | +| -------------------------- | ---------------------------- | ---------------------------------------------- | +| `AvailableMicsUpdated` | `onAvailableMicsUpdated` | Mic list changed or `initDevices()` called | +| `AvailableCamsUpdated` | `onAvailableCamsUpdated` | Camera list changed or `initDevices()` called | | `AvailableSpeakersUpdated` | `onAvailableSpeakersUpdated` | Speaker list changed or `initDevices()` called | -| `MicUpdated` | `onMicUpdated` | Active microphone changed | -| `CamUpdated` | `onCamUpdated` | Active camera changed | -| `SpeakerUpdated` | `onSpeakerUpdated` | Active speaker changed | -| `DeviceError` | `onDeviceError` | Mic, camera, or permission error | -| `TrackStarted` | `onTrackStarted` | A media track (audio or video) became playable | -| `TrackStopped` | `onTrackStopped` | A media track stopped | +| `MicUpdated` | `onMicUpdated` | Active microphone changed | +| `CamUpdated` | `onCamUpdated` | Active camera changed | +| `SpeakerUpdated` | `onSpeakerUpdated` | Active speaker changed | +| `DeviceError` | `onDeviceError` | Mic, camera, or permission error | +| `TrackStarted` | `onTrackStarted` | A media track (audio or video) became playable | +| `TrackStopped` | `onTrackStopped` | A media track stopped | ### Function calling @@ -521,11 +532,11 @@ override fun onBackendError(message: String) { These events fire when the bot's LLM makes a function call. Use them to track status and display relevant UI (e.g., a loading spinner while the call is in progress). -| Event | Callback | When it fires | -|---|---|---| -| `LLMFunctionCallStarted` | `onLLMFunctionCallStarted` | LLM initiated a function call | +| Event | Callback | When it fires | +| --------------------------- | ----------------------------- | ---------------------------------------------------------------------- | +| `LLMFunctionCallStarted` | `onLLMFunctionCallStarted` | LLM initiated a function call | | `LLMFunctionCallInProgress` | `onLLMFunctionCallInProgress` | Function call is executing; triggers registered `FunctionCallHandler`s | -| `LLMFunctionCallStopped` | `onLLMFunctionCallStopped` | Function call completed or was cancelled | +| `LLMFunctionCallStopped` | `onLLMFunctionCallStopped` | Function call completed or was cancelled | @@ -533,11 +544,11 @@ These events fire when the bot's LLM makes a function call. Use them to track st These events fire when the bot's LLM makes a function call. Use them to track status and display relevant UI (e.g., a loading spinner while the call is in progress). -| Event | Callback | When it fires | -|---|---|---| -| `LLMFunctionCallStarted` | `onLLMFunctionCallStarted` | LLM initiated a function call | +| Event | Callback | When it fires | +| --------------------------- | ----------------------------- | ---------------------------------------------------------------------- | +| `LLMFunctionCallStarted` | `onLLMFunctionCallStarted` | LLM initiated a function call | | `LLMFunctionCallInProgress` | `onLLMFunctionCallInProgress` | Function call is executing; triggers registered `FunctionCallHandler`s | -| `LLMFunctionCallStopped` | `onLLMFunctionCallStopped` | Function call completed or was cancelled | +| `LLMFunctionCallStopped` | `onLLMFunctionCallStopped` | Function call completed or was cancelled | @@ -545,11 +556,11 @@ These events fire when the bot's LLM makes a function call. Use them to track st These events fire when the bot's LLM makes a function call. Use them to track status and display relevant UI (e.g., a loading spinner while the call is in progress). -| Event | Callback | When it fires | -|---|---|---| -| `LLMFunctionCallStarted` | `onLLMFunctionCallStarted` | LLM initiated a function call | +| Event | Callback | When it fires | +| --------------------------- | ----------------------------- | ---------------------------------------------------------------------- | +| `LLMFunctionCallStarted` | `onLLMFunctionCallStarted` | LLM initiated a function call | | `LLMFunctionCallInProgress` | `onLLMFunctionCallInProgress` | Function call is executing; triggers registered `FunctionCallHandler`s | -| `LLMFunctionCallStopped` | `onLLMFunctionCallStopped` | Function call completed or was cancelled | +| `LLMFunctionCallStopped` | `onLLMFunctionCallStopped` | Function call completed or was cancelled | @@ -601,10 +612,10 @@ The `onLLMFunctionCall` callback on `PipecatEventCallbacks` is also invoked for ### Other -| Event | Callback | When it fires | -|---|---|---| +| Event | Callback | When it fires | +| --------------- | ----------------- | ---------------------------------------------- | | `ServerMessage` | `onServerMessage` | Custom message sent from the bot to the client | -| `Metrics` | `onMetrics` | Pipeline performance metrics from Pipecat | +| `Metrics` | `onMetrics` | Pipeline performance metrics from Pipecat | For custom server\<-\>client messaging, see [Custom Messaging](/client/guides/custom-messaging). @@ -615,10 +626,18 @@ For custom server\<-\>client messaging, see [Custom Messaging](/client/guides/cu - + `useRTVIClientEvent` and other React-specific event utilities - + Complete callback signatures, data types, and transport compatibility @@ -628,7 +647,11 @@ For custom server\<-\>client messaging, see [Custom Messaging](/client/guides/cu - + Complete callback signatures, data types, and transport compatibility @@ -638,7 +661,11 @@ For custom server\<-\>client messaging, see [Custom Messaging](/client/guides/cu - + Complete callback signatures, data types, and transport compatibility @@ -648,7 +675,11 @@ For custom server\<-\>client messaging, see [Custom Messaging](/client/guides/cu - + Full `PipecatClientDelegate` protocol and API reference @@ -658,7 +689,11 @@ For custom server\<-\>client messaging, see [Custom Messaging](/client/guides/cu - + Full `PipecatEventCallbacks` class and API reference diff --git a/client/concepts/media-management.mdx b/client/concepts/media-management.mdx index 941985e0c..fe06cf33e 100644 --- a/client/concepts/media-management.mdx +++ b/client/concepts/media-management.mdx @@ -4,29 +4,34 @@ description: "Managing microphones, cameras, speakers, and media tracks in Pipec --- - -You are currently viewing the React version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React version of this page. Use the dropdown + to the right to customize this page for your client framework. + - -You are currently viewing the JavaScript version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the JavaScript version of this page. Use the + dropdown to the right to customize this page for your client framework. + - -You are currently viewing the React Native version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React Native version of this page. Use the + dropdown to the right to customize this page for your client framework. + - -You are currently viewing the iOS version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the iOS version of this page. Use the dropdown to + the right to customize this page for your client framework. + - -You are currently viewing the Android version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the Android version of this page. Use the dropdown + to the right to customize this page for your client framework. + The Pipecat client handles media at two levels: **local devices** (the user's mic, camera, and speakers) and **media tracks** (the live audio/video streams flowing between client and bot). This page covers how to work with both. @@ -38,13 +43,16 @@ The Pipecat client handles media at two levels: **local devices** (the user's mi Drop [`PipecatClientAudio`](/api-reference/client/react/components#pipecatclientaudio) inside your [`PipecatClientProvider`](/api-reference/client/react/components#pipecatclientprovider) and it handles everything — it mounts a hidden ` @@ -758,7 +775,7 @@ import { VoiceVisualizer } from "@pipecat-ai/client-react"; barCount={5} barWidth={4} barMaxHeight={24} -/> +/>; ``` Set `participantType="bot"` to visualize the bot's audio instead. @@ -802,7 +819,9 @@ function AudioViz() { return ( - + ); } @@ -891,14 +910,28 @@ fun AudioViz(manager: MyManager) { - + `initDevices`, `getAllMics`, `enableMic`, and more - - `usePipecatClientMediaDevices`, `usePipecatClientMicControl`, `usePipecatClientCamControl`, `usePipecatClientMediaTrack` + + `usePipecatClientMediaDevices`, `usePipecatClientMicControl`, + `usePipecatClientCamControl`, `usePipecatClientMediaTrack` - - `PipecatClientAudio`, `PipecatClientVideo`, `PipecatClientMicToggle`, `PipecatClientCamToggle`, `VoiceVisualizer` + + `PipecatClientAudio`, `PipecatClientVideo`, `PipecatClientMicToggle`, + `PipecatClientCamToggle`, `VoiceVisualizer` @@ -907,7 +940,11 @@ fun AudioViz(manager: MyManager) { - + `initDevices`, `getAllMics`, `updateMic`, `enableMic`, `tracks`, and more @@ -917,7 +954,11 @@ fun AudioViz(manager: MyManager) { - + `initDevices`, `getAllMics`, `updateMic`, `enableMic`, `tracks`, and more @@ -927,7 +968,11 @@ fun AudioViz(manager: MyManager) { - + Full API reference including `PipecatClientDelegate` and device management @@ -937,7 +982,11 @@ fun AudioViz(manager: MyManager) { - + Full API reference including `PipecatEventCallbacks` and device management diff --git a/client/concepts/session-lifecycle.mdx b/client/concepts/session-lifecycle.mdx index 0efe5a107..cc514431e 100644 --- a/client/concepts/session-lifecycle.mdx +++ b/client/concepts/session-lifecycle.mdx @@ -4,29 +4,34 @@ description: "How a Pipecat session starts, runs, and ends — and how to handle --- - -You are currently viewing the React version of this page. Use the Dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React version of this page. Use the Dropdown + to the right to customize this page for your client framework. + - -You are currently viewing the JavaScript version of this page. Use the Dropdown to the right to customize this page for your client framework. - + + You are currently viewing the JavaScript version of this page. Use the + Dropdown to the right to customize this page for your client framework. + - -You are currently viewing the React Native version of this page. Use the Dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React Native version of this page. Use the + Dropdown to the right to customize this page for your client framework. + - -You are currently viewing the iOS version of this page. Use the Dropdown to the right to customize this page for your client framework. - + + You are currently viewing the iOS version of this page. Use the Dropdown to + the right to customize this page for your client framework. + - -You are currently viewing the Android version of this page. Use the Dropdown to the right to customize this page for your client framework. - + + You are currently viewing the Android version of this page. Use the Dropdown + to the right to customize this page for your client framework. + A session is the span of time from when your client connects to a bot until it disconnects. Understanding the lifecycle helps you build correct connection flows, handle errors gracefully, and clean up reliably. @@ -39,16 +44,16 @@ The client exposes a `state` string that tracks where you are in the lifecycle. idle → authenticating → authenticated → connecting → connected → ready → disconnecting → disconnected ``` -| State | What it means | -|---|---| -| `idle` | No connection attempt has started | +| State | What it means | +| ---------------- | ------------------------------------------------------------- | +| `idle` | No connection attempt has started | | `authenticating` | `startBot()` called; waiting for your server to start the bot | -| `authenticated` | Server responded; bot is starting | -| `connecting` | Transport is establishing the WebRTC/WebSocket connection | -| `connected` | Transport is connected; bot pipeline is initializing | -| `ready` | Bot pipeline is running and ready to receive audio/messages | -| `disconnecting` | Disconnect in progress | -| `disconnected` | Session has ended | +| `authenticated` | Server responded; bot is starting | +| `connecting` | Transport is establishing the WebRTC/WebSocket connection | +| `connected` | Transport is connected; bot pipeline is initializing | +| `ready` | Bot pipeline is running and ready to receive audio/messages | +| `disconnecting` | Disconnect in progress | +| `disconnected` | Session has ended | The `ready` state is the one that matters most — it's the gate before which you should not send messages or expect audio. The `connected` state means the transport is up but the bot's pipeline may still be warming up. @@ -281,7 +286,7 @@ function MyComponent() { RTVIEvent.BotReady, useCallback((data) => { console.log("Bot ready, RTVI version:", data.version); - }, []) + }, []), ); } ``` @@ -436,7 +441,7 @@ function MyComponent() { RTVIEvent.BotDisconnected, useCallback(() => { // Update UI to show the session has ended - }, []) + }, []), ); } ``` @@ -538,7 +543,7 @@ function MyComponent() { if (fatal) { // Session is over — show reconnect UI } - }, []) + }, []), ); } ``` @@ -636,7 +641,9 @@ function StatusBar() { const transportState = usePipecatClientTransportState(); const isReady = transportState === "ready"; - const isConnecting = ["authenticating", "connecting", "connected"].includes(transportState); + const isConnecting = ["authenticating", "connecting", "connected"].includes( + transportState, + ); return ( @@ -690,7 +697,9 @@ function StatusBar() { }, []); const isReady = transportState === "ready"; - const isConnecting = ["authenticating", "connecting", "connected"].includes(transportState); + const isConnecting = ["authenticating", "connecting", "connected"].includes( + transportState, + ); return ( diff --git a/client/guides/building-a-voice-ui.mdx b/client/guides/building-a-voice-ui.mdx index 89d18000c..6e3c7dc6a 100644 --- a/client/guides/building-a-voice-ui.mdx +++ b/client/guides/building-a-voice-ui.mdx @@ -4,76 +4,95 @@ description: "Build a voice application from scratch using the Pipecat client SD --- - -You are currently viewing the React version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React version of this page. Use the dropdown + to the right to customize this page for your client framework. + - -You are currently viewing the JavaScript version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the JavaScript version of this page. Use the + dropdown to the right to customize this page for your client framework. + - -You are currently viewing the React Native version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the React Native version of this page. Use the + dropdown to the right to customize this page for your client framework. + - -You are currently viewing the iOS version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the iOS version of this page. Use the dropdown to + the right to customize this page for your client framework. + - -You are currently viewing the Android version of this page. Use the dropdown to the right to customize this page for your client framework. - + + You are currently viewing the Android version of this page. Use the dropdown + to the right to customize this page for your client framework. + -This guide walks through building a React voice app without any UI abstractions — just the Pipecat React SDK directly. You'll see exactly how the client, provider, hooks, and audio output fit together, which is useful whether you're building a fully custom UI or want to understand what the [CLI-generated app](/client/get-started/quickstart) is doing under the hood. + This guide walks through building a React voice app without any UI + abstractions — just the Pipecat React SDK directly. You'll see exactly how the + client, provider, hooks, and audio output fit together, which is useful + whether you're building a fully custom UI or want to understand what the + [CLI-generated app](/client/get-started/quickstart) is doing under the hood. -This guide walks through building a voice app in vanilla TypeScript — just the Pipecat JavaScript SDK directly, with no framework. You'll see how to set up the client, handle events, and wire up audio output manually. + This guide walks through building a voice app in vanilla TypeScript — just the + Pipecat JavaScript SDK directly, with no framework. You'll see how to set up + the client, handle events, and wire up audio output manually. -This guide walks through building a React Native voice app using the Pipecat JavaScript SDK. You'll see how to set up the client, subscribe to events with `useEffect`, and connect to your bot from a mobile app — with audio handled automatically by the platform. + This guide walks through building a React Native voice app using the Pipecat + JavaScript SDK. You'll see how to set up the client, subscribe to events with + `useEffect`, and connect to your bot from a mobile app — with audio handled + automatically by the platform. -This guide walks through building a SwiftUI voice app using the Pipecat iOS SDK. You'll see how to create the client, implement the delegate, and connect to your bot — with audio handled automatically by the SDK. + This guide walks through building a SwiftUI voice app using the Pipecat iOS + SDK. You'll see how to create the client, implement the delegate, and connect + to your bot — with audio handled automatically by the SDK. -This guide walks through building an Android voice app using the Pipecat Android SDK. You'll see how to set up a client manager, implement the event callbacks in Kotlin, and build a Jetpack Compose UI — with audio handled automatically by the transport. + This guide walks through building an Android voice app using the Pipecat + Android SDK. You'll see how to set up a client manager, implement the event + callbacks in Kotlin, and build a Jetpack Compose UI — with audio handled + automatically by the transport. ## Prerequisites -- Node.js 18+ -- A running Pipecat bot + - Node.js 18+ - A running Pipecat bot -- Node.js 18+ -- A running Pipecat bot + - Node.js 18+ - A running Pipecat bot -- Node.js 18+ -- Xcode 15+ (for iOS) or Android Studio (for Android) -- A running Pipecat bot + - Node.js 18+ - Xcode 15+ (for iOS) or Android Studio (for Android) - A + running Pipecat bot -- Xcode 15+ -- A running Pipecat bot + - Xcode 15+ - A running Pipecat bot -- Android Studio -- A running Pipecat bot + - Android Studio - A running Pipecat bot - - Follow the Pipecat server quickstart to get a bot running locally at `http://localhost:7860`. + + Follow the Pipecat server quickstart to get a bot running locally at + `http://localhost:7860`. ## Installation @@ -114,7 +133,8 @@ npx expo install @pipecat-ai/client-js @pipecat-ai/react-native-daily-media-mana These packages use native modules, so they won't run in Expo Go. You'll need a - [development build](https://docs.expo.dev/develop/development-builds/introduction/). + [development + build](https://docs.expo.dev/develop/development-builds/introduction/). Because this is a voice app, your `app.json` needs microphone permissions, the Daily config plugin, and minimum platform SDK versions: @@ -127,7 +147,7 @@ Because this is a voice app, your `app.json` needs microphone permissions, the D "NSMicrophoneUsageDescription": "This app uses the microphone to talk to your voice AI assistant.", "UIBackgroundModes": ["voip"] }, - "bundleIdentifier": "co.daily.expo.SmallWebRTCDemo", + "bundleIdentifier": "co.daily.expo.SmallWebRTCDemo" }, "android": { "package": "co.daily.expo.SmallWebRTCDemo" @@ -163,9 +183,9 @@ Because this is a voice app, your `app.json` needs microphone permissions, the D In Xcode, add the following Swift packages via **File → Add Package Dependencies**: -| Package | URL | -|---|---| -| Core SDK | `https://github.com/pipecat-ai/pipecat-client-ios.git` | +| Package | URL | +| --------------------- | ------------------------------------------------------------------- | +| Core SDK | `https://github.com/pipecat-ai/pipecat-client-ios.git` | | SmallWebRTC transport | `https://github.com/pipecat-ai/pipecat-client-ios-small-webrtc.git` | Then add `NSMicrophoneUsageDescription` to your `Info.plist`: @@ -198,9 +218,10 @@ You'll also need to request the permission at runtime before connecting. The sim - SmallWebRTC is ideal for development — no third-party account needed. - For production, swap in the [Daily transport](/api-reference/client/js/transports/daily), - which provides global infrastructure, echo cancellation, and more. + SmallWebRTC is ideal for development — no third-party account needed. For + production, swap in the [Daily + transport](/api-reference/client/js/transports/daily), which provides global + infrastructure, echo cancellation, and more. ## Step 1: Create the client @@ -216,7 +237,10 @@ import React from "react"; import ReactDOM from "react-dom/client"; import { PipecatClient } from "@pipecat-ai/client-js"; import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; -import { PipecatClientAudio, PipecatClientProvider } from "@pipecat-ai/client-react"; +import { + PipecatClientAudio, + PipecatClientProvider, +} from "@pipecat-ai/client-react"; import App from "./App"; const client = new PipecatClient({ @@ -228,7 +252,7 @@ ReactDOM.createRoot(document.getElementById("root")!).render( - + , ); ``` @@ -276,7 +300,9 @@ import { RNSmallWebRTCTransport } from "@pipecat-ai/react-native-small-webrtc-tr import { DailyMediaManager } from "@pipecat-ai/react-native-daily-media-manager"; export const client = new PipecatClient({ - transport: new RNSmallWebRTCTransport({ mediaManager: new DailyMediaManager() }), + transport: new RNSmallWebRTCTransport({ + mediaManager: new DailyMediaManager(), + }), enableMic: true, }); ``` @@ -365,19 +391,25 @@ export default function App() { const { messages } = usePipecatConversation(); const isConnected = transportState === "ready"; - const isConnecting = ["authenticating", "connecting", "connected"].includes(transportState); + const isConnecting = ["authenticating", "connecting", "connected"].includes( + transportState, + ); useRTVIClientEvent( RTVIEvent.Error, - useCallback((error) => console.error("Bot error:", error), []) + useCallback((error) => console.error("Bot error:", error), []), ); const handleConnect = async () => { - await client.connect({ webrtcRequestParams: { endpoint: "http://localhost:7860/api/offer" } }); + await client.connect({ + webrtcRequestParams: { endpoint: "http://localhost:7860/api/offer" }, + }); }; return ( -
+

Pipecat Voice App

@@ -385,11 +417,13 @@ export default function App() { onClick={isConnected ? () => client.disconnect() : handleConnect} disabled={isConnecting} > - {isConnected ? "Disconnect" : isConnecting ? "Connecting…" : "Connect"} + {isConnected + ? "Disconnect" + : isConnecting + ? "Connecting…" + : "Connect"} - - {transportState} - + {transportState}
    @@ -427,10 +461,16 @@ const messageList = document.getElementById("messages") as HTMLUListElement; client.on(RTVIEvent.TransportStateChanged, (state) => { const isConnected = state === "ready"; - const isConnecting = ["authenticating", "connecting", "connected"].includes(state); + const isConnecting = ["authenticating", "connecting", "connected"].includes( + state, + ); statusEl.textContent = state; - button.textContent = isConnected ? "Disconnect" : isConnecting ? "Connecting…" : "Connect"; + button.textContent = isConnected + ? "Disconnect" + : isConnecting + ? "Connecting…" + : "Connect"; button.disabled = isConnecting; }); @@ -450,7 +490,9 @@ button.addEventListener("click", async () => { if (client.state === "ready") { await client.disconnect(); } else { - await client.connect({ webrtcRequestParams: { endpoint: "http://localhost:7860/api/offer" } }); + await client.connect({ + webrtcRequestParams: { endpoint: "http://localhost:7860/api/offer" }, + }); } }); @@ -484,7 +526,14 @@ Replace `app/(tabs)/index.tsx` with: ```tsx import React, { useState, useEffect } from "react"; -import { View, Text, Button, ScrollView, StyleSheet, Platform } from "react-native"; +import { + View, + Text, + Button, + ScrollView, + StyleSheet, + Platform, +} from "react-native"; import { RTVIEvent } from "@pipecat-ai/client-js"; import { client } from "@/lib/client"; @@ -532,7 +581,9 @@ export default function App() { }, []); const isConnected = transportState === "ready"; - const isConnecting = ["authenticating", "connecting", "connected"].includes(transportState); + const isConnecting = ["authenticating", "connecting", "connected"].includes( + transportState, + ); const handlePress = async () => { if (isConnected) { @@ -547,7 +598,13 @@ export default function App() { Pipecat Voice App