From a5b8030f4a8aa324b3546b58d386db37d669caab Mon Sep 17 00:00:00 2001 From: Sarp Tan Doven <119648987+sarptandoven@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:50:21 -0700 Subject: [PATCH 1/3] fix: support realtime transcription intent --- src/beta/realtime/internal-base.ts | 25 ++++++++-- src/beta/realtime/websocket.ts | 23 +++++++++- src/beta/realtime/ws.ts | 26 ++++++++++- src/realtime/internal-base.ts | 56 ++++++++++++++++++---- tests/realtime.test.ts | 74 ++++++++++++++++++++++++++++-- 5 files changed, 187 insertions(+), 17 deletions(-) diff --git a/src/beta/realtime/internal-base.ts b/src/beta/realtime/internal-base.ts index a9c34017d9..649ab72a7d 100644 --- a/src/beta/realtime/internal-base.ts +++ b/src/beta/realtime/internal-base.ts @@ -84,10 +84,20 @@ export type RealtimeConnectionConfig = * Start a new Realtime session using the given model. */ model: string; + intent?: undefined; + callID?: undefined; + } + | { + /** + * Connect to the Realtime API with transcription intent. + */ + intent: 'transcription'; + model?: undefined; callID?: undefined; } | { model?: undefined; + intent?: undefined; /** * Attach to an in-progress Realtime call over a sideband control connection. */ @@ -104,12 +114,15 @@ export function buildRealtimeURL( const azure = isAzure(client); const hasModel = !!config.model; const hasCallID = !!config.callID; + const hasIntent = config.intent === 'transcription'; const path = '/realtime'; const url = new URL(baseURL + (baseURL.endsWith('/') ? path.slice(1) : path)); - if (hasModel === hasCallID) { - throw new Error('Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.'); + if ([hasModel, hasCallID, hasIntent].filter(Boolean).length !== 1) { + throw new Error( + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', + ); } url.protocol = 'wss'; @@ -119,10 +132,16 @@ export function buildRealtimeURL( throw new Error('Azure `callID` connections require the stable Realtime helpers.'); } url.searchParams.set('api-version', client.apiVersion); - url.searchParams.set('deployment', config.model!); + if (hasIntent) { + url.searchParams.set('intent', config.intent!); + } else { + url.searchParams.set('deployment', config.model!); + } } else { if (hasCallID) { url.searchParams.set('call_id', config.callID!); + } else if (hasIntent) { + url.searchParams.set('intent', config.intent!); } else { url.searchParams.set('model', config.model!); } diff --git a/src/beta/realtime/websocket.ts b/src/beta/realtime/websocket.ts index 2fa019ecec..a8d7284c69 100644 --- a/src/beta/realtime/websocket.ts +++ b/src/beta/realtime/websocket.ts @@ -117,7 +117,9 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { static async azure( client: Pick, - options: { deploymentName?: string; dangerouslyAllowBrowser?: boolean } = {}, + options: + | { deploymentName?: string; intent?: undefined; dangerouslyAllowBrowser?: boolean } + | { intent: 'transcription'; deploymentName?: undefined; dangerouslyAllowBrowser?: boolean } = {}, ): Promise { const isApiKeyProvider = await client._callApiKey(); const apiKey = client.apiKey; @@ -132,11 +134,28 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { url.searchParams.set('api-key', azureApiKey); } } + const { dangerouslyAllowBrowser } = options; + if (options.intent && options.deploymentName !== undefined) { + throw new Error( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); + } + if (options.intent) { + return new OpenAIRealtimeWebSocket( + { + intent: options.intent, + onURL, + ...(dangerouslyAllowBrowser ? { dangerouslyAllowBrowser } : {}), + __resolvedApiKey: isApiKeyProvider, + }, + client, + ); + } + const deploymentName = options.deploymentName ?? client.deploymentName; if (!deploymentName) { throw new Error('No deployment name provided'); } - const { dangerouslyAllowBrowser } = options; return new OpenAIRealtimeWebSocket( { model: deploymentName, diff --git a/src/beta/realtime/ws.ts b/src/beta/realtime/ws.ts index 713fa6c86a..d97f4adee1 100644 --- a/src/beta/realtime/ws.ts +++ b/src/beta/realtime/ws.ts @@ -76,13 +76,37 @@ export class OpenAIRealtimeWS extends OpenAIRealtimeEmitter { static async azure( client: Pick, - props: { deploymentName?: string; options?: WS.ClientOptions | undefined } = {}, + props: + | { deploymentName?: string; intent?: undefined; options?: WS.ClientOptions | undefined } + | { intent: 'transcription'; deploymentName?: undefined; options?: WS.ClientOptions | undefined } = {}, ): Promise { const isApiKeyProvider = await client._callApiKey(); const apiKey = client.apiKey; if (!apiKey) { throw new Error('Azure OpenAI Realtime requires an API key'); } + if (props.intent && props.deploymentName !== undefined) { + throw new Error( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); + } + if (props.intent) { + return new OpenAIRealtimeWS( + { + intent: props.intent, + options: { + ...props.options, + headers: { + ...props.options?.headers, + ...(isApiKeyProvider ? {} : { 'api-key': apiKey }), + }, + }, + __resolvedApiKey: isApiKeyProvider, + }, + client, + ); + } + const deploymentName = props.deploymentName ?? client.deploymentName; if (!deploymentName) { throw new Error('No deployment name provided'); diff --git a/src/realtime/internal-base.ts b/src/realtime/internal-base.ts index 7f316828f8..1af16c0ad9 100644 --- a/src/realtime/internal-base.ts +++ b/src/realtime/internal-base.ts @@ -89,10 +89,20 @@ export type RealtimeConnectionConfig = * Start a new Realtime session using the given model. */ model: string; + intent?: undefined; callID?: undefined; } | { + /** + * Connect to the Realtime API with transcription intent. + */ + intent: 'transcription'; model?: undefined; + callID?: undefined; + } + | { + model?: undefined; + intent?: undefined; /** * Attach to an in-progress Realtime call over a sideband control connection. */ @@ -105,10 +115,20 @@ export type AzureRealtimeConnectionConfig = * Override the deployment configured on the Azure client. */ deploymentName?: string; + intent?: undefined; callID?: undefined; } | { + /** + * Connect to the Azure Realtime API with transcription intent. + */ + intent: 'transcription'; deploymentName?: undefined; + callID?: undefined; + } + | { + deploymentName?: undefined; + intent?: undefined; /** * Attach to an in-progress Azure Realtime call over a sideband control connection. */ @@ -125,9 +145,12 @@ export function buildRealtimeURL( const azure = isAzure(client); const hasModel = !!config.model; const hasCallID = !!config.callID; + const hasIntent = config.intent === 'transcription'; - if (hasModel === hasCallID) { - throw new Error('Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.'); + if ([hasModel, hasCallID, hasIntent].filter(Boolean).length !== 1) { + throw new Error( + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', + ); } let url: URL; @@ -150,11 +173,17 @@ export function buildRealtimeURL( url.searchParams.set('call_id', config.callID!); } else { url.searchParams.set('api-version', client.apiVersion); - url.searchParams.set('deployment', config.model!); + if (hasIntent) { + url.searchParams.set('intent', config.intent!); + } else { + url.searchParams.set('deployment', config.model!); + } } } else { if (hasCallID) { url.searchParams.set('call_id', config.callID!); + } else if (hasIntent) { + url.searchParams.set('intent', config.intent!); } else { url.searchParams.set('model', config.model!); } @@ -166,11 +195,22 @@ export function getAzureRealtimeConnection( client: Pick, connection: AzureRealtimeConnectionConfig, ): RealtimeConnectionConfig { - if (connection.callID !== undefined) { - if (connection.deploymentName !== undefined) { - throw new Error('Pass either `deploymentName` or `callID`, but not both.'); - } - return { callID: connection.callID }; + const hasDeploymentName = connection.deploymentName !== undefined; + const hasCallID = connection.callID !== undefined; + const hasIntent = connection.intent === 'transcription'; + + if ([hasDeploymentName, hasCallID, hasIntent].filter(Boolean).length > 1) { + throw new Error( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); + } + + if (hasCallID) { + return { callID: connection.callID! }; + } + + if (hasIntent) { + return { intent: connection.intent! }; } const model = connection.deploymentName ?? client.deploymentName; diff --git a/tests/realtime.test.ts b/tests/realtime.test.ts index c3041f955d..8d8b08d287 100644 --- a/tests/realtime.test.ts +++ b/tests/realtime.test.ts @@ -1,6 +1,13 @@ import OpenAI, { AzureOpenAI } from 'openai'; import { buildRealtimeURL as buildBetaRealtimeURL } from 'openai/beta/realtime/internal-base'; -import { buildRealtimeURL as buildRealtimeURL } from 'openai/realtime/internal-base'; +import { + buildRealtimeURL as buildRealtimeURL, + getAzureRealtimeConnection, +} from 'openai/realtime/internal-base'; +import type { OpenAIRealtimeWebSocket as BetaOpenAIRealtimeWebSocket } from 'openai/beta/realtime/websocket'; +import type { OpenAIRealtimeWS as BetaOpenAIRealtimeWS } from 'openai/beta/realtime/ws'; +import type { OpenAIRealtimeWebSocket } from 'openai/realtime/websocket'; +import type { OpenAIRealtimeWS } from 'openai/realtime/ws'; const apiVersion = '2024-10-01-preview'; @@ -37,6 +44,12 @@ describe.each([ ); }); + test('uses transcription intent without a model for standard OpenAI connections', () => { + expect(buildRealtimeURL(openAIClient, { intent: 'transcription' }).toString()).toBe( + 'wss://example.com/custom/path/realtime?intent=transcription', + ); + }); + test('preserves the legacy model string form', () => { expect(buildRealtimeURL(openAIClient, 'gpt-realtime').toString()).toBe( 'wss://example.com/custom/path/realtime?model=gpt-realtime', @@ -55,19 +68,58 @@ describe.each([ ); }); + test('uses transcription intent without an Azure deployment', () => { + expect(buildRealtimeURL(azureClient, { intent: 'transcription' }).toString()).toBe( + `wss://example.com/openai/realtime?api-version=${apiVersion}&intent=transcription`, + ); + }); + test('rejects missing connection target', () => { expect(() => buildRealtimeURL(openAIClient, {} as any)).toThrow( - 'Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.', + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', ); }); test('rejects multiple connection targets', () => { expect(() => buildRealtimeURL(openAIClient, { model: 'gpt-realtime', callID: 'rtc_123' } as any)).toThrow( - 'Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.', + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', + ); + }); + + test('rejects model and transcription intent together', () => { + expect(() => + buildRealtimeURL(openAIClient, { model: 'gpt-4o-transcribe', intent: 'transcription' } as any), + ).toThrow( + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', ); }); }); +test('stable Azure helper maps transcription intent without deployment', () => { + expect(getAzureRealtimeConnection(azureClient, { intent: 'transcription' })).toEqual({ + intent: 'transcription', + }); +}); + +test('stable Azure helper rejects deployment and transcription intent together', () => { + expect(() => + getAzureRealtimeConnection(azureClient, { + deploymentName: 'my-deployment', + intent: 'transcription', + } as any), + ).toThrow( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); +}); + +test('stable Azure helper rejects call_id and transcription intent together', () => { + expect(() => + getAzureRealtimeConnection(azureClient, { callID: 'rtc_123', intent: 'transcription' } as any), + ).toThrow( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); +}); + test('stable builder uses the normalized Azure GA call_id URL', () => { for (const client of [azureClient, azureV1Client, azureEndpointClient]) { expect(buildRealtimeURL(client, { callID: 'rtc_123' }).toString()).toBe( @@ -81,3 +133,19 @@ test('beta builder directs Azure call_id users to the stable helper', () => { 'Azure `callID` connections require the stable Realtime helpers.', ); }); + +test('Azure realtime factories accept transcription intent options', () => { + const stableWebSocketOptions: Parameters[1] = { + intent: 'transcription', + }; + const stableWSOptions: Parameters[1] = { intent: 'transcription' }; + const betaWebSocketOptions: Parameters[1] = { + intent: 'transcription', + }; + const betaWSOptions: Parameters[1] = { intent: 'transcription' }; + + expect(stableWebSocketOptions.intent).toBe('transcription'); + expect(stableWSOptions.intent).toBe('transcription'); + expect(betaWebSocketOptions.intent).toBe('transcription'); + expect(betaWSOptions.intent).toBe('transcription'); +}); From a75f82116a183dfa68587b7126db68cd09c09e0d Mon Sep 17 00:00:00 2001 From: Hayden Date: Tue, 11 Aug 2026 19:26:43 -0700 Subject: [PATCH 2/3] fix: limit realtime transcription to supported GA transports --- docs/realtime.md | 16 +++- src/beta/realtime/internal-base.ts | 49 +++--------- src/beta/realtime/websocket.ts | 61 +++----------- src/beta/realtime/ws.ts | 62 ++++----------- src/realtime/internal-base.ts | 16 ++-- src/realtime/ws.ts | 2 +- tests/realtime-websocket.test.ts | 123 ++++++++++++++++------------- tests/realtime.test.ts | 98 +++++++++++------------ 8 files changed, 173 insertions(+), 254 deletions(-) diff --git a/docs/realtime.md b/docs/realtime.md index a3ad78b01b..e2bfd3183c 100644 --- a/docs/realtime.md +++ b/docs/realtime.md @@ -80,10 +80,24 @@ To start a transcription-only session, pass `intent: 'transcription'` instead of const rt = new OpenAIRealtimeWS({ intent: 'transcription' }); ``` -Azure transcription sessions also use transcription intent and do not require a model deployment: +Azure transcription sessions also use transcription intent. Do not pass a deployment in the connection options; configure the transcription deployment in a `session.update` event after the socket opens: ```ts const rt = await OpenAIRealtimeWS.azure(azureClient, { intent: 'transcription' }); + +rt.socket.on('open', () => { + rt.send({ + type: 'session.update', + session: { + type: 'transcription', + audio: { + input: { + transcription: { model: 'your-transcription-deployment' }, + }, + }, + }, + }); +}); ``` `model`, `callID`, and transcription `intent` are mutually exclusive. Azure transcription sessions must not include a `deploymentName`. The web `WebSocket` helper supports the same connection options. diff --git a/src/beta/realtime/internal-base.ts b/src/beta/realtime/internal-base.ts index 44d3061728..926ee384ac 100644 --- a/src/beta/realtime/internal-base.ts +++ b/src/beta/realtime/internal-base.ts @@ -126,7 +126,7 @@ export function isAzure(client: Pick): client is A return client instanceof AzureOpenAI; } -/** Starts a beta model or transcription session, or attaches to one existing non-Azure call. */ +/** Starts a beta Realtime model session or attaches to one existing non-Azure call. */ export type RealtimeConnectionConfig = | { /** @@ -134,29 +134,12 @@ export type RealtimeConnectionConfig = */ model: string; - /** Transcription intent; cannot be supplied when starting a model-backed session. */ - intent?: undefined; - /** Existing call identifier; cannot be supplied when starting a model-backed session. */ callID?: undefined; } - | { - /** Starts a transcription-only Realtime session without selecting a model. */ - intent: 'transcription'; - - /** Model name; cannot be supplied when starting a transcription-only session. */ - model?: undefined; - - /** Existing call identifier; cannot be supplied with transcription intent. */ - callID?: undefined; - } | { /** Model name; cannot be supplied when attaching to an existing call. */ model?: undefined; - - /** Transcription intent; cannot be supplied when attaching to an existing call. */ - intent?: undefined; - /** * Attach to an in-progress Realtime call over a sideband control connection. */ @@ -164,10 +147,10 @@ export type RealtimeConnectionConfig = }; /** - * Builds the URL for a beta model or transcription session, or a non-Azure sideband call. + * Builds the secure WebSocket URL for a beta Realtime session or non-Azure sideband call. * - * @throws {Error} If connection targets conflict, intent is invalid, or an Azure sideband - * call is requested through the beta helpers. + * @throws {Error} If both `model` and `callID`, or neither, are supplied, or an + * Azure sideband call is requested through the beta helpers. */ export function buildRealtimeURL( client: Pick, @@ -177,22 +160,14 @@ export function buildRealtimeURL( typeof connection === 'string' ? { model: connection } : connection; const baseURL = client.baseURL; const azure = isAzure(client); - const hasModel = config.model !== undefined; - const hasCallID = config.callID !== undefined; - const hasIntent = config.intent !== undefined; + const hasModel = !!config.model; + const hasCallID = !!config.callID; const path = '/realtime'; const url = new URL(baseURL + (baseURL.endsWith('/') ? path.slice(1) : path)); - if ( - Number(hasModel) + Number(hasCallID) + Number(hasIntent) !== 1 || - (hasModel && !config.model) || - (hasCallID && !config.callID) || - (hasIntent && config.intent !== 'transcription') - ) { - throw new Error( - 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', - ); + if (hasModel === hasCallID) { + throw new Error('Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.'); } url.protocol = 'wss'; @@ -202,15 +177,9 @@ export function buildRealtimeURL( throw new Error('Azure `callID` connections require the stable Realtime helpers.'); } url.searchParams.set('api-version', client.apiVersion); - if (hasIntent) { - url.searchParams.set('intent', 'transcription'); - } else { - url.searchParams.set('deployment', config.model!); - } + url.searchParams.set('deployment', config.model!); } else if (hasCallID) { url.searchParams.set('call_id', config.callID!); - } else if (hasIntent) { - url.searchParams.set('intent', 'transcription'); } else { url.searchParams.set('model', config.model!); } diff --git a/src/beta/realtime/websocket.ts b/src/beta/realtime/websocket.ts index 478d37f8e2..f3b36190ef 100644 --- a/src/beta/realtime/websocket.ts +++ b/src/beta/realtime/websocket.ts @@ -35,14 +35,14 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { socket: _WebSocket; /** - * Immediately opens a beta model or transcription session, or attaches to a non-Azure call. + * Immediately opens a beta Realtime session or attaches to an existing non-Azure call. * * Clients with function-based credentials must use * {@link OpenAIRealtimeWebSocket.create}; Azure deployment sessions should use * {@link OpenAIRealtimeWebSocket.azure}. Ephemeral credentials starting with * `ek_` are permitted in browser runtimes automatically. * - * @param props Exactly one model, transcription intent, or call ID, plus browser-safety settings. + * @param props Exactly one of `model` or `callID` and optional browser-safety settings. * @param client Existing client whose endpoint and API key should be reused. * @throws {OpenAIError} If browser access would expose an unapproved credential. */ @@ -135,7 +135,7 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { * Use this factory instead of the constructor when the client's `apiKey` is a function. * * @param client OpenAI client that owns the endpoint and refreshable or static credential. - * @param props Exactly one model, transcription intent, or call ID, plus browser-safety settings. + * @param props Exactly one of `model` or `callID` and optional browser-safety settings. */ static async create( client: Pick, @@ -148,50 +148,26 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { } /** - * Opens a native beta Azure OpenAI Realtime model or transcription session. + * Opens a native beta Azure OpenAI Realtime session for a model deployment. * * Azure credentials are redacted from the exposed `url` property immediately * after connection setup. Use the stable Realtime helper to attach to an * existing Azure call. * * @param client Azure OpenAI client that supplies the endpoint and credential. - * @param options Deployment override or transcription intent, plus browser-safety settings. + * @param options Optional deployment override and browser-safety settings. * @throws {Error} If the Azure credential or required deployment is unavailable. */ static async azure( client: Pick, - options: - | { - /** Azure model deployment; defaults to the deployment configured on the client. */ - deploymentName?: string; + options: { + /** Azure model deployment; defaults to the deployment configured on the client. */ + deploymentName?: string; - /** Transcription intent; cannot be combined with a model deployment. */ - intent?: undefined; - - /** Allows browser execution after the caller has secured the supplied Azure credential. */ - dangerouslyAllowBrowser?: boolean; - } - | { - /** Starts a transcription-only Azure Realtime session without a deployment. */ - intent: 'transcription'; - - /** Deployment override; cannot be supplied with transcription intent. */ - deploymentName?: undefined; - - /** Allows browser execution after the caller has secured the supplied Azure credential. */ - dangerouslyAllowBrowser?: boolean; - } = {}, + /** Allows browser execution after the caller has secured the supplied Azure credential. */ + dangerouslyAllowBrowser?: boolean; + } = {}, ): Promise { - if ( - (options.intent !== undefined && options.intent !== 'transcription') || - (options.intent !== undefined && options.deploymentName !== undefined) || - ('callID' in options && options.callID !== undefined) - ) { - throw new Error( - 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', - ); - } - const isApiKeyProvider = await client._callApiKey(); const apiKey = client.apiKey; if (!apiKey) { @@ -205,24 +181,11 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { url.searchParams.set('api-key', azureApiKey); } } - const { dangerouslyAllowBrowser } = options; - if (options.intent === 'transcription') { - return new OpenAIRealtimeWebSocket( - { - intent: 'transcription', - onURL, - ...(dangerouslyAllowBrowser ? { dangerouslyAllowBrowser } : {}), - __resolvedApiKey: isApiKeyProvider, - }, - client, - ); - } - const deploymentName = options.deploymentName ?? client.deploymentName; if (!deploymentName) { throw new Error('No deployment name provided'); } - + const { dangerouslyAllowBrowser } = options; return new OpenAIRealtimeWebSocket( { model: deploymentName, diff --git a/src/beta/realtime/ws.ts b/src/beta/realtime/ws.ts index c975d94929..9a578483b0 100644 --- a/src/beta/realtime/ws.ts +++ b/src/beta/realtime/ws.ts @@ -13,20 +13,20 @@ import type { RealtimeConnectionConfig } from './internal-base'; * sending client events. Use the stable Realtime helper for Azure sideband calls. */ export class OpenAIRealtimeWS extends OpenAIRealtimeEmitter { - /** Secure beta Realtime WebSocket URL with its model, transcription intent, or call ID. */ + /** Secure beta Realtime WebSocket URL, including the model or non-Azure call ID. */ url: URL; /** Underlying `ws.WebSocket` instance for connection lifecycle and transport events. */ socket: WS.WebSocket; /** - * Immediately opens a beta model or transcription session, or attaches to a non-Azure call. + * Immediately opens a beta Realtime model session or attaches to an existing non-Azure call. * * Clients with function-based credentials must use * {@link OpenAIRealtimeWS.create}; Azure deployment sessions should use * {@link OpenAIRealtimeWS.azure}. * - * @param props Exactly one model, transcription intent, or call ID, plus `ws` client settings. + * @param props Exactly one of `model` or `callID`, plus optional `ws` client settings. * @param client Existing client whose endpoint and API key should be reused. */ constructor( @@ -93,7 +93,7 @@ export class OpenAIRealtimeWS extends OpenAIRealtimeEmitter { * Use this factory instead of the constructor when the client's `apiKey` is a function. * * @param client OpenAI client that owns the endpoint and refreshable or static credential. - * @param props Exactly one model, transcription intent, or call ID, plus `ws` client settings. + * @param props Exactly one of `model` or `callID`, plus optional `ws` client settings. */ static async create( client: Pick, @@ -106,68 +106,38 @@ export class OpenAIRealtimeWS extends OpenAIRealtimeEmitter { } /** - * Opens a beta Azure OpenAI Realtime model or transcription session. + * Opens a beta Azure OpenAI Realtime session for the selected model deployment. * * Static Azure API keys are sent in the `api-key` header; function-based * credentials are resolved first and sent as bearer credentials. Use the * stable Realtime helper when attaching to an existing Azure call. * * @param client Azure OpenAI client that supplies the endpoint and credential. - * @param props Deployment override or transcription intent, plus `ws` connection settings. + * @param props Optional deployment override and `ws` connection settings. * @throws {Error} If the Azure credential or required deployment is unavailable. */ static async azure( client: Pick, - props: - | { - /** Azure model deployment; defaults to the deployment configured on the client. */ - deploymentName?: string; + props: { + /** Azure model deployment; defaults to the deployment configured on the client. */ + deploymentName?: string; - /** Transcription intent; cannot be combined with a model deployment. */ - intent?: undefined; - - /** Options passed directly to the underlying `ws.WebSocket` constructor. */ - options?: WS.ClientOptions | undefined; - } - | { - /** Starts a transcription-only Azure Realtime session without a deployment. */ - intent: 'transcription'; - - /** Deployment override; cannot be supplied with transcription intent. */ - deploymentName?: undefined; - - /** Options passed directly to the underlying `ws.WebSocket` constructor. */ - options?: WS.ClientOptions | undefined; - } = {}, + /** Options passed directly to the underlying `ws.WebSocket` constructor. */ + options?: WS.ClientOptions | undefined; + } = {}, ): Promise { - if ( - (props.intent !== undefined && props.intent !== 'transcription') || - (props.intent !== undefined && props.deploymentName !== undefined) || - ('callID' in props && props.callID !== undefined) - ) { - throw new Error( - 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', - ); - } - const isApiKeyProvider = await client._callApiKey(); const apiKey = client.apiKey; if (!apiKey) { throw new Error('Azure OpenAI Realtime requires an API key'); } - let connection: RealtimeConnectionConfig; - if (props.intent === 'transcription') { - connection = { intent: 'transcription' }; - } else { - const deploymentName = props.deploymentName ?? client.deploymentName; - if (!deploymentName) { - throw new Error('No deployment name provided'); - } - connection = { model: deploymentName }; + const deploymentName = props.deploymentName ?? client.deploymentName; + if (!deploymentName) { + throw new Error('No deployment name provided'); } return new OpenAIRealtimeWS( { - ...connection, + model: deploymentName, options: { ...props.options, headers: { diff --git a/src/realtime/internal-base.ts b/src/realtime/internal-base.ts index 211f3b44a4..59c43f17a8 100644 --- a/src/realtime/internal-base.ts +++ b/src/realtime/internal-base.ts @@ -182,7 +182,7 @@ export type AzureRealtimeConnectionConfig = callID?: undefined; } | { - /** Starts a transcription-only Azure Realtime session without a deployment. */ + /** Starts an Azure transcription session; set its deployment later in `session.update`. */ intent: 'transcription'; /** Deployment override; cannot be supplied with transcription intent. */ @@ -207,8 +207,8 @@ export type AzureRealtimeConnectionConfig = /** * Builds the secure WebSocket URL for a model or transcription session, or a sideband call. * - * Azure model sessions use deployment and API-version query parameters; Azure - * sideband calls use the versioned GA Realtime endpoint. + * Azure model sessions preserve their deployment and API-version query parameters; + * transcription sessions and sideband calls use the versioned GA Realtime endpoint. * * @throws {Error} If exactly one valid model, transcription intent, or call ID is not supplied. */ @@ -236,7 +236,7 @@ export function buildRealtimeURL( } let url: URL; - if (azure && hasCallID) { + if (azure && (hasCallID || hasIntent)) { url = new URL(baseURL); const basePath = url.pathname.replace(/\/+/g, '/').replace(/\/+$/, ''); const versionedPath = basePath.endsWith('/v1') ? basePath : `${basePath}/v1`; @@ -253,13 +253,11 @@ export function buildRealtimeURL( if (azure) { if (hasCallID) { url.searchParams.set('call_id', config.callID!); + } else if (hasIntent) { + url.searchParams.set('intent', 'transcription'); } else { url.searchParams.set('api-version', client.apiVersion); - if (hasIntent) { - url.searchParams.set('intent', 'transcription'); - } else { - url.searchParams.set('deployment', config.model!); - } + url.searchParams.set('deployment', config.model!); } } else if (hasCallID) { url.searchParams.set('call_id', config.callID!); diff --git a/src/realtime/ws.ts b/src/realtime/ws.ts index fcb1651f8f..2e82741fc7 100644 --- a/src/realtime/ws.ts +++ b/src/realtime/ws.ts @@ -18,7 +18,7 @@ import type { AzureRealtimeConnectionConfig, RealtimeConnectionConfig } from './ * and register an SDK `error` listener for API or transport failures. */ export class OpenAIRealtimeWS extends OpenAIRealtimeEmitter { - /** Secure Realtime WebSocket URL, including the selected model or existing call ID. */ + /** Secure Realtime WebSocket URL with its model, transcription intent, or call ID. */ url: URL; /** Underlying `ws.WebSocket` instance for connection lifecycle and transport events. */ diff --git a/tests/realtime-websocket.test.ts b/tests/realtime-websocket.test.ts index 89de95397b..366533d8ec 100644 --- a/tests/realtime-websocket.test.ts +++ b/tests/realtime-websocket.test.ts @@ -127,17 +127,6 @@ describe.each([ expect(sideband.url.searchParams.get('call_id')).toBe('call-123'); }); - test('opens authenticated transcription-only sessions without a model', () => { - const realtime = new Realtime({ intent: 'transcription' }, createClient()); - - expect(realtime.url.toString()).toBe('wss://example.com/v1/realtime?intent=transcription'); - expect(lastBrowserSocket().protocols).toEqual([ - 'realtime', - 'openai-insecure-api-key.test-key', - ...(beta ? ['openai-beta.realtime-v1'] : []), - ]); - }); - test('rejects function-based credentials until create resolves the token', async () => { const client = createClient(async () => 'rotating-key'); @@ -252,29 +241,55 @@ describe.each([ expect(realtime.url.searchParams.get('Authorization')).toBe(''); }); + test('rejects Azure connections without a deployment', async () => { + await expect(Realtime.azure(createAzureClient())).rejects.toThrow('No deployment name provided'); + }); + + test('rejects Azure connections without a resolved API key', async () => { + const client = createAzureClient({ deployment: 'chat' }); + client.apiKey = null; + + await expect(Realtime.azure(client)).rejects.toThrow('Azure OpenAI Realtime requires an API key'); + }); +}); + +describe('stable browser realtime transcription', () => { + test('opens authenticated transcription-only sessions without a model or beta protocol', () => { + const realtime = new StableBrowserRealtime({ intent: 'transcription' }, createClient()); + + expect(realtime.url.toString()).toBe('wss://example.com/v1/realtime?intent=transcription'); + expect(lastBrowserSocket().protocols).toEqual(['realtime', 'openai-insecure-api-key.test-key']); + }); + test.each([undefined, 'configured-deployment'])( - 'opens Azure transcription sessions without using deployment %s', + 'opens Azure transcription without using deployment %s', async (deployment) => { const client = createAzureClient(deployment === undefined ? {} : { deployment }); - const realtime = await Realtime.azure(client, { intent: 'transcription' }); + const realtime = await StableBrowserRealtime.azure(client, { intent: 'transcription' }); const connectionURL = new URL(lastBrowserSocket().url); + expect(connectionURL.pathname).toBe('/openai/v1/realtime'); expect(connectionURL.searchParams.get('intent')).toBe('transcription'); + expect(connectionURL.searchParams.has('api-version')).toBe(false); expect(connectionURL.searchParams.has('deployment')).toBe(false); expect(connectionURL.searchParams.get('api-key')).toBe('azure-key'); + expect(lastBrowserSocket().protocols).toEqual(['realtime']); expect(realtime.url.searchParams.get('api-key')).toBe(''); }, ); test('authenticates and redacts Azure token-provider transcription sessions', async () => { - const realtime = await Realtime.azure(createAzureClient({ tokenProvider: true }), { + const realtime = await StableBrowserRealtime.azure(createAzureClient({ tokenProvider: true }), { intent: 'transcription', }); const connectionURL = new URL(lastBrowserSocket().url); + expect(connectionURL.pathname).toBe('/openai/v1/realtime'); expect(connectionURL.searchParams.get('intent')).toBe('transcription'); + expect(connectionURL.searchParams.has('api-version')).toBe(false); expect(connectionURL.searchParams.has('deployment')).toBe(false); expect(connectionURL.searchParams.get('Authorization')).toBe('Bearer azure-token'); + expect(lastBrowserSocket().protocols).toEqual(['realtime']); expect(realtime.url.searchParams.get('Authorization')).toBe(''); }); @@ -286,23 +301,12 @@ describe.each([ { intent: 'unsupported' }, ])('rejects conflicting Azure transcription targets before opening a socket %#', async (options) => { await expect( - Realtime.azure(createAzureClient({ deployment: 'configured' }), options as any), + StableBrowserRealtime.azure(createAzureClient({ deployment: 'configured' }), options as any), ).rejects.toThrow( 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', ); expect(FakeBrowserSocket.instances).toHaveLength(0); }); - - test('rejects Azure connections without a deployment', async () => { - await expect(Realtime.azure(createAzureClient())).rejects.toThrow('No deployment name provided'); - }); - - test('rejects Azure connections without a resolved API key', async () => { - const client = createAzureClient({ deployment: 'chat' }); - client.apiKey = null; - - await expect(Realtime.azure(client)).rejects.toThrow('Azure OpenAI Realtime requires an API key'); - }); }); describe.each([ @@ -327,20 +331,6 @@ describe.each([ expect(sideband.url.searchParams.get('call_id')).toBe('call-123'); }); - test('opens authenticated transcription-only sessions and preserves custom headers', () => { - const realtime = new Realtime( - { intent: 'transcription', options: { headers: { 'X-Custom': 'value' } } }, - createClient(), - ); - - expect(realtime.url.toString()).toBe('wss://example.com/v1/realtime?intent=transcription'); - expect(lastNodeSocket().options.headers).toMatchObject({ - Authorization: 'Bearer test-key', - 'X-Custom': 'value', - ...(beta ? { 'OpenAI-Beta': 'realtime=v1' } : {}), - }); - }); - test('requires function-based credentials to be resolved with create', async () => { const client = createClient(async () => 'rotating-key'); @@ -425,35 +415,69 @@ describe.each([ expect(lastNodeSocket().options.headers).not.toHaveProperty('api-key'); }); + test('requires an Azure deployment', async () => { + await expect(Realtime.azure(createAzureClient())).rejects.toThrow('No deployment name provided'); + }); + + test('requires a resolved Azure API key', async () => { + const client = createAzureClient({ deployment: 'chat' }); + client.apiKey = null; + + await expect(Realtime.azure(client)).rejects.toThrow('Azure OpenAI Realtime requires an API key'); + }); +}); + +describe('stable Node realtime transcription', () => { + test('opens authenticated transcription-only sessions without a beta header', () => { + const realtime = new StableNodeRealtime( + { intent: 'transcription', options: { headers: { 'X-Custom': 'value' } } }, + createClient(), + ); + + expect(realtime.url.toString()).toBe('wss://example.com/v1/realtime?intent=transcription'); + expect(lastNodeSocket().options.headers).toMatchObject({ + Authorization: 'Bearer test-key', + 'X-Custom': 'value', + }); + expect(lastNodeSocket().options.headers).not.toHaveProperty('OpenAI-Beta'); + }); + test.each([undefined, 'configured-deployment'])( 'authenticates Azure transcription without using deployment %s', async (deployment) => { const client = createAzureClient(deployment === undefined ? {} : { deployment }); - await Realtime.azure(client, { + await StableNodeRealtime.azure(client, { intent: 'transcription', options: { headers: { 'X-Custom': 'value' } }, }); const socket = lastNodeSocket(); + expect(socket.url.pathname).toBe('/openai/v1/realtime'); expect(socket.url.searchParams.get('intent')).toBe('transcription'); + expect(socket.url.searchParams.has('api-version')).toBe(false); expect(socket.url.searchParams.has('deployment')).toBe(false); expect(socket.options.headers).toMatchObject({ 'api-key': 'azure-key', 'X-Custom': 'value', - ...(beta ? { 'OpenAI-Beta': 'realtime=v1' } : {}), }); expect(socket.options.headers).not.toHaveProperty('Authorization'); + expect(socket.options.headers).not.toHaveProperty('OpenAI-Beta'); }, ); test('authenticates Azure token-provider transcription with a bearer header', async () => { - await Realtime.azure(createAzureClient({ tokenProvider: true }), { intent: 'transcription' }); + await StableNodeRealtime.azure(createAzureClient({ tokenProvider: true }), { + intent: 'transcription', + }); const socket = lastNodeSocket(); + expect(socket.url.pathname).toBe('/openai/v1/realtime'); expect(socket.url.searchParams.get('intent')).toBe('transcription'); + expect(socket.url.searchParams.has('api-version')).toBe(false); expect(socket.url.searchParams.has('deployment')).toBe(false); expect(socket.options.headers).toMatchObject({ Authorization: 'Bearer azure-token' }); expect(socket.options.headers).not.toHaveProperty('api-key'); + expect(socket.options.headers).not.toHaveProperty('OpenAI-Beta'); }); test.each([ @@ -464,21 +488,10 @@ describe.each([ { intent: 'unsupported' }, ])('rejects conflicting Azure transcription targets before opening a socket %#', async (options) => { await expect( - Realtime.azure(createAzureClient({ deployment: 'configured' }), options as any), + StableNodeRealtime.azure(createAzureClient({ deployment: 'configured' }), options as any), ).rejects.toThrow( 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', ); expect(nodeSocketConstructor).not.toHaveBeenCalled(); }); - - test('requires an Azure deployment', async () => { - await expect(Realtime.azure(createAzureClient())).rejects.toThrow('No deployment name provided'); - }); - - test('requires a resolved Azure API key', async () => { - const client = createAzureClient({ deployment: 'chat' }); - client.apiKey = null; - - await expect(Realtime.azure(client)).rejects.toThrow('Azure OpenAI Realtime requires an API key'); - }); }); diff --git a/tests/realtime.test.ts b/tests/realtime.test.ts index 414f2910a1..45c81adb78 100644 --- a/tests/realtime.test.ts +++ b/tests/realtime.test.ts @@ -1,8 +1,6 @@ import OpenAI, { AzureOpenAI } from 'openai'; import { buildRealtimeURL as buildBetaRealtimeURL } from 'openai/beta/realtime/internal-base'; import { buildRealtimeURL, getAzureRealtimeConnection } from 'openai/realtime/internal-base'; -import type { OpenAIRealtimeWebSocket as BetaOpenAIRealtimeWebSocket } from 'openai/beta/realtime/websocket'; -import type { OpenAIRealtimeWS as BetaOpenAIRealtimeWS } from 'openai/beta/realtime/ws'; import type { OpenAIRealtimeWebSocket } from 'openai/realtime/websocket'; import type { OpenAIRealtimeWS } from 'openai/realtime/ws'; @@ -41,12 +39,6 @@ describe.each([ ); }); - test('uses transcription intent without a model for standard OpenAI connections', () => { - expect(buildRealtimeURL(openAIClient, { intent: 'transcription' }).toString()).toBe( - 'wss://example.com/custom/path/realtime?intent=transcription', - ); - }); - test('preserves the legacy model string form', () => { expect(buildRealtimeURL(openAIClient, 'gpt-realtime').toString()).toBe( 'wss://example.com/custom/path/realtime?model=gpt-realtime', @@ -65,24 +57,32 @@ describe.each([ ); }); - test('uses transcription intent without an Azure deployment', () => { - expect(buildRealtimeURL(azureClient, { intent: 'transcription' }).toString()).toBe( - `wss://example.com/openai/realtime?api-version=${apiVersion}&intent=transcription`, - ); - }); - test('rejects missing connection target', () => { - expect(() => buildRealtimeURL(openAIClient, {} as any)).toThrow( - 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', - ); + expect(() => buildRealtimeURL(openAIClient, {} as any)).toThrow('Pass exactly one of `model`'); }); test('rejects multiple connection targets', () => { expect(() => buildRealtimeURL(openAIClient, { model: 'gpt-realtime', callID: 'rtc_123' } as any)).toThrow( - 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', + 'Pass exactly one of `model`', + ); + }); +}); + +describe('stable realtime transcription', () => { + test('uses transcription intent without a model for OpenAI connections', () => { + expect(buildRealtimeURL(openAIClient, { intent: 'transcription' }).toString()).toBe( + 'wss://example.com/custom/path/realtime?intent=transcription', ); }); + test('uses the Azure GA endpoint without a deployment or preview API version', () => { + for (const client of [azureClient, azureV1Client, azureEndpointClient]) { + expect(buildRealtimeURL(client, { intent: 'transcription' }).toString()).toBe( + 'wss://example.com/openai/v1/realtime?intent=transcription', + ); + } + }); + test.each([ { model: 'gpt-realtime', intent: 'transcription' }, { callID: 'rtc_123', intent: 'transcription' }, @@ -96,28 +96,36 @@ describe.each([ 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', ); }); -}); -test('stable Azure helper maps transcription intent without using a deployment', () => { - expect( - getAzureRealtimeConnection({ deploymentName: 'configured-deployment' }, { intent: 'transcription' }), - ).toEqual({ - intent: 'transcription', + test('maps Azure transcription without using the configured deployment', () => { + expect( + getAzureRealtimeConnection({ deploymentName: 'configured-deployment' }, { intent: 'transcription' }), + ).toEqual({ intent: 'transcription' }); }); -}); -test.each([ - { deploymentName: 'my-deployment', intent: 'transcription' }, - { deploymentName: '', intent: 'transcription' }, - { callID: 'rtc_123', intent: 'transcription' }, - { callID: 'rtc_123', intent: 'unsupported' }, - { intent: 'unsupported' }, -])('stable Azure helper rejects invalid or conflicting connection targets %#', (connection) => { - expect(() => - getAzureRealtimeConnection({ deploymentName: 'configured-deployment' }, connection as any), - ).toThrow( - 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', - ); + test.each([ + { deploymentName: 'my-deployment', intent: 'transcription' }, + { deploymentName: '', intent: 'transcription' }, + { callID: 'rtc_123', intent: 'transcription' }, + { callID: 'rtc_123', intent: 'unsupported' }, + { intent: 'unsupported' }, + ])('rejects invalid or conflicting Azure connection targets %#', (connection) => { + expect(() => + getAzureRealtimeConnection({ deploymentName: 'configured-deployment' }, connection as any), + ).toThrow( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); + }); + + test('stable Azure realtime factories accept transcription intent options', () => { + const webSocketOptions: Parameters[1] = { + intent: 'transcription', + }; + const wsOptions: Parameters[1] = { intent: 'transcription' }; + + expect(webSocketOptions.intent).toBe('transcription'); + expect(wsOptions.intent).toBe('transcription'); + }); }); test('stable builder uses the normalized Azure GA call_id URL', () => { @@ -133,19 +141,3 @@ test('beta builder directs Azure call_id users to the stable helper', () => { 'Azure `callID` connections require the stable Realtime helpers.', ); }); - -test('Azure realtime factories accept transcription intent options', () => { - const stableWebSocketOptions: Parameters[1] = { - intent: 'transcription', - }; - const stableWSOptions: Parameters[1] = { intent: 'transcription' }; - const betaWebSocketOptions: Parameters[1] = { - intent: 'transcription', - }; - const betaWSOptions: Parameters[1] = { intent: 'transcription' }; - - expect(stableWebSocketOptions.intent).toBe('transcription'); - expect(stableWSOptions.intent).toBe('transcription'); - expect(betaWebSocketOptions.intent).toBe('transcription'); - expect(betaWSOptions.intent).toBe('transcription'); -}); From 66dd95f7216dec77b69926a5cdf4a74c4d0e0863 Mon Sep 17 00:00:00 2001 From: Hayden Date: Tue, 11 Aug 2026 19:46:32 -0700 Subject: [PATCH 3/3] fix(realtime): use Azure GA endpoints for stable model sessions --- docs/azure.md | 34 ++++++++++++-------- src/realtime/internal-base.ts | 18 +++-------- tests/realtime-websocket.test.ts | 55 +++++++++++++++++++++++++++++--- tests/realtime.test.ts | 30 ++++++++++++++--- 4 files changed, 101 insertions(+), 36 deletions(-) diff --git a/docs/azure.md b/docs/azure.md index 86bd8be706..2de138bb9d 100644 --- a/docs/azure.md +++ b/docs/azure.md @@ -68,21 +68,29 @@ For OpenAI workload identity on Azure-managed infrastructure, see [Authenticatio ## Realtime API -This SDK provides real-time streaming capabilities for Azure OpenAI through the `OpenAIRealtimeWS` and `OpenAIRealtimeWebSocket` clients described previously. - -To utilize the real-time features, begin by creating a fully configured `AzureOpenAI` client and passing it into either `OpenAIRealtimeWS.azure` or `OpenAIRealtimeWebSocket.azure`. For example: +Use the stable Realtime API with your Azure v1 endpoint and deployment name. The +`OpenAIRealtimeWS` and `OpenAIRealtimeWebSocket` helpers connect to the GA +`/openai/v1/realtime` endpoint without a dated `api-version` query parameter: ```ts -const cred = new DefaultAzureCredential(); -const scope = 'https://cognitiveservices.azure.com/.default'; -const deploymentName = 'gpt-4o-realtime-preview-1001'; -const azureADTokenProvider = getBearerTokenProvider(cred, scope); -const client = new AzureOpenAI({ - azureADTokenProvider, - apiVersion: '2024-10-01-preview', - deployment: deploymentName, +import OpenAI from 'openai'; +import { OpenAIRealtimeWS } from 'openai/realtime/ws'; +import { DefaultAzureCredential, getBearerTokenProvider } from '@azure/identity'; + +const endpoint = process.env['AZURE_OPENAI_ENDPOINT']; +const deploymentName = process.env['AZURE_OPENAI_DEPLOYMENT']; +if (!endpoint || !deploymentName) throw new Error('Missing Azure OpenAI configuration'); + +const tokenProvider = getBearerTokenProvider(new DefaultAzureCredential(), 'https://ai.azure.com/.default'); +const client = new OpenAI({ + baseURL: `${endpoint.replace(/\/+$/, '')}/openai/v1/`, + apiKey: tokenProvider, }); -const rt = await OpenAIRealtimeWS.azure(client); + +const rt = await OpenAIRealtimeWS.create(client, { model: deploymentName }); ``` -Once the instance has been created, you can then begin sending requests and receiving streaming responses in real time. +If you already have an `AzureOpenAI` client, `OpenAIRealtimeWS.azure(client)` and +`OpenAIRealtimeWebSocket.azure(client)` also use the GA endpoint and the client's +configured deployment. Pass `{ deploymentName: 'your-deployment' }` to override +that deployment for a specific connection. diff --git a/src/realtime/internal-base.ts b/src/realtime/internal-base.ts index 59c43f17a8..b911f6cd98 100644 --- a/src/realtime/internal-base.ts +++ b/src/realtime/internal-base.ts @@ -207,8 +207,9 @@ export type AzureRealtimeConnectionConfig = /** * Builds the secure WebSocket URL for a model or transcription session, or a sideband call. * - * Azure model sessions preserve their deployment and API-version query parameters; - * transcription sessions and sideband calls use the versioned GA Realtime endpoint. + * All Azure sessions use the versioned GA Realtime endpoint. Model-backed + * sessions select their deployment with `model`, transcription sessions use + * `intent`, and sideband connections use `call_id`. * * @throws {Error} If exactly one valid model, transcription intent, or call ID is not supplied. */ @@ -236,7 +237,7 @@ export function buildRealtimeURL( } let url: URL; - if (azure && (hasCallID || hasIntent)) { + if (azure) { url = new URL(baseURL); const basePath = url.pathname.replace(/\/+/g, '/').replace(/\/+$/, ''); const versionedPath = basePath.endsWith('/v1') ? basePath : `${basePath}/v1`; @@ -250,16 +251,7 @@ export function buildRealtimeURL( url.protocol = 'wss'; // Sideband control connections attach to an existing call via `call_id`. - if (azure) { - if (hasCallID) { - url.searchParams.set('call_id', config.callID!); - } else if (hasIntent) { - url.searchParams.set('intent', 'transcription'); - } else { - url.searchParams.set('api-version', client.apiVersion); - url.searchParams.set('deployment', config.model!); - } - } else if (hasCallID) { + if (hasCallID) { url.searchParams.set('call_id', config.callID!); } else if (hasIntent) { url.searchParams.set('intent', 'transcription'); diff --git a/tests/realtime-websocket.test.ts b/tests/realtime-websocket.test.ts index 366533d8ec..a0a78febd7 100644 --- a/tests/realtime-websocket.test.ts +++ b/tests/realtime-websocket.test.ts @@ -228,17 +228,25 @@ describe.each([ test('places Azure API keys in the URL and redacts them after opening', async () => { const realtime = await Realtime.azure(createAzureClient({ deployment: 'chat' })); + const connectionURL = new URL(lastBrowserSocket().url); expect(lastBrowserSocket().url).toContain('api-key=azure-key'); expect(lastBrowserSocket().protocols).not.toContain('openai-insecure-api-key.azure-key'); expect(realtime.url.searchParams.get('api-key')).toBe(''); + expect(connectionURL.pathname).toBe(beta ? '/openai/realtime' : '/openai/v1/realtime'); + expect(connectionURL.searchParams.get(beta ? 'deployment' : 'model')).toBe('chat'); + expect(connectionURL.searchParams.has('api-version')).toBe(beta); }); test('places rotating Azure credentials in Authorization and redacts them', async () => { const realtime = await Realtime.azure(createAzureClient({ deployment: 'chat', tokenProvider: true })); + const connectionURL = new URL(lastBrowserSocket().url); - expect(new URL(lastBrowserSocket().url).searchParams.get('Authorization')).toBe('Bearer azure-token'); + expect(connectionURL.searchParams.get('Authorization')).toBe('Bearer azure-token'); expect(realtime.url.searchParams.get('Authorization')).toBe(''); + expect(connectionURL.pathname).toBe(beta ? '/openai/realtime' : '/openai/v1/realtime'); + expect(connectionURL.searchParams.get(beta ? 'deployment' : 'model')).toBe('chat'); + expect(connectionURL.searchParams.has('api-version')).toBe(beta); }); test('rejects Azure connections without a deployment', async () => { @@ -254,6 +262,20 @@ describe.each([ }); describe('stable browser realtime transcription', () => { + test('uses an explicit Azure model deployment without preview query parameters', async () => { + const realtime = await StableBrowserRealtime.azure(createAzureClient({ deployment: 'configured' }), { + deploymentName: 'override', + }); + const connectionURL = new URL(lastBrowserSocket().url); + + expect(connectionURL.pathname).toBe('/openai/v1/realtime'); + expect(connectionURL.searchParams.get('model')).toBe('override'); + expect(connectionURL.searchParams.has('api-version')).toBe(false); + expect(connectionURL.searchParams.has('deployment')).toBe(false); + expect(lastBrowserSocket().protocols).toEqual(['realtime']); + expect(realtime.url.searchParams.get('api-key')).toBe(''); + }); + test('opens authenticated transcription-only sessions without a model or beta protocol', () => { const realtime = new StableBrowserRealtime({ intent: 'transcription' }, createClient()); @@ -403,16 +425,24 @@ describe.each([ test('authenticates Azure sessions with an API-key header', async () => { await Realtime.azure(createAzureClient({ deployment: 'chat' })); + const socket = lastNodeSocket(); - expect(lastNodeSocket().options.headers).toMatchObject({ 'api-key': 'azure-key' }); - expect(lastNodeSocket().options.headers).not.toHaveProperty('Authorization'); + expect(socket.options.headers).toMatchObject({ 'api-key': 'azure-key' }); + expect(socket.options.headers).not.toHaveProperty('Authorization'); + expect(socket.url.pathname).toBe(beta ? '/openai/realtime' : '/openai/v1/realtime'); + expect(socket.url.searchParams.get(beta ? 'deployment' : 'model')).toBe('chat'); + expect(socket.url.searchParams.has('api-version')).toBe(beta); }); test('authenticates Azure token-provider sessions with a bearer header', async () => { await Realtime.azure(createAzureClient({ deployment: 'chat', tokenProvider: true })); + const socket = lastNodeSocket(); - expect(lastNodeSocket().options.headers).toMatchObject({ Authorization: 'Bearer azure-token' }); - expect(lastNodeSocket().options.headers).not.toHaveProperty('api-key'); + expect(socket.options.headers).toMatchObject({ Authorization: 'Bearer azure-token' }); + expect(socket.options.headers).not.toHaveProperty('api-key'); + expect(socket.url.pathname).toBe(beta ? '/openai/realtime' : '/openai/v1/realtime'); + expect(socket.url.searchParams.get(beta ? 'deployment' : 'model')).toBe('chat'); + expect(socket.url.searchParams.has('api-version')).toBe(beta); }); test('requires an Azure deployment', async () => { @@ -428,6 +458,21 @@ describe.each([ }); describe('stable Node realtime transcription', () => { + test('uses an explicit Azure model deployment without preview headers or query parameters', async () => { + await StableNodeRealtime.azure(createAzureClient({ deployment: 'configured' }), { + deploymentName: 'override', + options: { headers: { 'X-Custom': 'value' } }, + }); + const socket = lastNodeSocket(); + + expect(socket.url.pathname).toBe('/openai/v1/realtime'); + expect(socket.url.searchParams.get('model')).toBe('override'); + expect(socket.url.searchParams.has('api-version')).toBe(false); + expect(socket.url.searchParams.has('deployment')).toBe(false); + expect(socket.options.headers).toMatchObject({ 'api-key': 'azure-key', 'X-Custom': 'value' }); + expect(socket.options.headers).not.toHaveProperty('OpenAI-Beta'); + }); + test('opens authenticated transcription-only sessions without a beta header', () => { const realtime = new StableNodeRealtime( { intent: 'transcription', options: { headers: { 'X-Custom': 'value' } } }, diff --git a/tests/realtime.test.ts b/tests/realtime.test.ts index 45c81adb78..2f1339e2d3 100644 --- a/tests/realtime.test.ts +++ b/tests/realtime.test.ts @@ -32,7 +32,7 @@ const azureEndpointClient = new AzureOpenAI({ describe.each([ ['stable', buildRealtimeURL], ['beta', buildBetaRealtimeURL], -] as const)('%s realtime URL builder', (_label, buildRealtimeURL) => { +] as const)('%s realtime URL builder', (label, buildRealtimeURL) => { test('uses model for standard OpenAI connections', () => { expect(buildRealtimeURL(openAIClient, { model: 'gpt-realtime' }).toString()).toBe( 'wss://example.com/custom/path/realtime?model=gpt-realtime', @@ -51,10 +51,13 @@ describe.each([ ); }); - test('uses deployment for Azure connections', () => { - expect(buildRealtimeURL(azureClient, { model: 'my-deployment' }).toString()).toBe( - `wss://example.com/openai/realtime?api-version=${apiVersion}&deployment=my-deployment`, - ); + test('routes Azure model connections using its API generation', () => { + const expectedURL = + label === 'stable' + ? 'wss://example.com/openai/v1/realtime?model=my-deployment' + : `wss://example.com/openai/realtime?api-version=${apiVersion}&deployment=my-deployment`; + + expect(buildRealtimeURL(azureClient, { model: 'my-deployment' }).toString()).toBe(expectedURL); }); test('rejects missing connection target', () => { @@ -69,6 +72,14 @@ describe.each([ }); describe('stable realtime transcription', () => { + test('uses the Azure GA model endpoint for every supported client base URL', () => { + for (const client of [azureClient, azureV1Client, azureEndpointClient]) { + expect(buildRealtimeURL(client, { model: 'my-deployment' }).toString()).toBe( + 'wss://example.com/openai/v1/realtime?model=my-deployment', + ); + } + }); + test('uses transcription intent without a model for OpenAI connections', () => { expect(buildRealtimeURL(openAIClient, { intent: 'transcription' }).toString()).toBe( 'wss://example.com/custom/path/realtime?intent=transcription', @@ -103,6 +114,15 @@ describe('stable realtime transcription', () => { ).toEqual({ intent: 'transcription' }); }); + test('uses configured Azure deployments and honors explicit overrides', () => { + const client = { deploymentName: 'configured-deployment' }; + + expect(getAzureRealtimeConnection(client, {})).toEqual({ model: 'configured-deployment' }); + expect(getAzureRealtimeConnection(client, { deploymentName: 'override-deployment' })).toEqual({ + model: 'override-deployment', + }); + }); + test.each([ { deploymentName: 'my-deployment', intent: 'transcription' }, { deploymentName: '', intent: 'transcription' },