diff --git a/src/beta/realtime/internal-base.ts b/src/beta/realtime/internal-base.ts index a9c34017d9..649ab72a7d 100644 --- a/src/beta/realtime/internal-base.ts +++ b/src/beta/realtime/internal-base.ts @@ -84,10 +84,20 @@ export type RealtimeConnectionConfig = * Start a new Realtime session using the given model. */ model: string; + intent?: undefined; + callID?: undefined; + } + | { + /** + * Connect to the Realtime API with transcription intent. + */ + intent: 'transcription'; + model?: undefined; callID?: undefined; } | { model?: undefined; + intent?: undefined; /** * Attach to an in-progress Realtime call over a sideband control connection. */ @@ -104,12 +114,15 @@ export function buildRealtimeURL( const azure = isAzure(client); const hasModel = !!config.model; const hasCallID = !!config.callID; + const hasIntent = config.intent === 'transcription'; const path = '/realtime'; const url = new URL(baseURL + (baseURL.endsWith('/') ? path.slice(1) : path)); - if (hasModel === hasCallID) { - throw new Error('Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.'); + if ([hasModel, hasCallID, hasIntent].filter(Boolean).length !== 1) { + throw new Error( + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', + ); } url.protocol = 'wss'; @@ -119,10 +132,16 @@ export function buildRealtimeURL( throw new Error('Azure `callID` connections require the stable Realtime helpers.'); } url.searchParams.set('api-version', client.apiVersion); - url.searchParams.set('deployment', config.model!); + if (hasIntent) { + url.searchParams.set('intent', config.intent!); + } else { + url.searchParams.set('deployment', config.model!); + } } else { if (hasCallID) { url.searchParams.set('call_id', config.callID!); + } else if (hasIntent) { + url.searchParams.set('intent', config.intent!); } else { url.searchParams.set('model', config.model!); } diff --git a/src/beta/realtime/websocket.ts b/src/beta/realtime/websocket.ts index 2fa019ecec..a8d7284c69 100644 --- a/src/beta/realtime/websocket.ts +++ b/src/beta/realtime/websocket.ts @@ -117,7 +117,9 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { static async azure( client: Pick, - options: { deploymentName?: string; dangerouslyAllowBrowser?: boolean } = {}, + options: + | { deploymentName?: string; intent?: undefined; dangerouslyAllowBrowser?: boolean } + | { intent: 'transcription'; deploymentName?: undefined; dangerouslyAllowBrowser?: boolean } = {}, ): Promise { const isApiKeyProvider = await client._callApiKey(); const apiKey = client.apiKey; @@ -132,11 +134,28 @@ export class OpenAIRealtimeWebSocket extends OpenAIRealtimeEmitter { url.searchParams.set('api-key', azureApiKey); } } + const { dangerouslyAllowBrowser } = options; + if (options.intent && options.deploymentName !== undefined) { + throw new Error( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); + } + if (options.intent) { + return new OpenAIRealtimeWebSocket( + { + intent: options.intent, + onURL, + ...(dangerouslyAllowBrowser ? { dangerouslyAllowBrowser } : {}), + __resolvedApiKey: isApiKeyProvider, + }, + client, + ); + } + const deploymentName = options.deploymentName ?? client.deploymentName; if (!deploymentName) { throw new Error('No deployment name provided'); } - const { dangerouslyAllowBrowser } = options; return new OpenAIRealtimeWebSocket( { model: deploymentName, diff --git a/src/beta/realtime/ws.ts b/src/beta/realtime/ws.ts index 713fa6c86a..d97f4adee1 100644 --- a/src/beta/realtime/ws.ts +++ b/src/beta/realtime/ws.ts @@ -76,13 +76,37 @@ export class OpenAIRealtimeWS extends OpenAIRealtimeEmitter { static async azure( client: Pick, - props: { deploymentName?: string; options?: WS.ClientOptions | undefined } = {}, + props: + | { deploymentName?: string; intent?: undefined; options?: WS.ClientOptions | undefined } + | { intent: 'transcription'; deploymentName?: undefined; options?: WS.ClientOptions | undefined } = {}, ): Promise { const isApiKeyProvider = await client._callApiKey(); const apiKey = client.apiKey; if (!apiKey) { throw new Error('Azure OpenAI Realtime requires an API key'); } + if (props.intent && props.deploymentName !== undefined) { + throw new Error( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); + } + if (props.intent) { + return new OpenAIRealtimeWS( + { + intent: props.intent, + options: { + ...props.options, + headers: { + ...props.options?.headers, + ...(isApiKeyProvider ? {} : { 'api-key': apiKey }), + }, + }, + __resolvedApiKey: isApiKeyProvider, + }, + client, + ); + } + const deploymentName = props.deploymentName ?? client.deploymentName; if (!deploymentName) { throw new Error('No deployment name provided'); diff --git a/src/realtime/internal-base.ts b/src/realtime/internal-base.ts index 7f316828f8..1af16c0ad9 100644 --- a/src/realtime/internal-base.ts +++ b/src/realtime/internal-base.ts @@ -89,10 +89,20 @@ export type RealtimeConnectionConfig = * Start a new Realtime session using the given model. */ model: string; + intent?: undefined; callID?: undefined; } | { + /** + * Connect to the Realtime API with transcription intent. + */ + intent: 'transcription'; model?: undefined; + callID?: undefined; + } + | { + model?: undefined; + intent?: undefined; /** * Attach to an in-progress Realtime call over a sideband control connection. */ @@ -105,10 +115,20 @@ export type AzureRealtimeConnectionConfig = * Override the deployment configured on the Azure client. */ deploymentName?: string; + intent?: undefined; callID?: undefined; } | { + /** + * Connect to the Azure Realtime API with transcription intent. + */ + intent: 'transcription'; deploymentName?: undefined; + callID?: undefined; + } + | { + deploymentName?: undefined; + intent?: undefined; /** * Attach to an in-progress Azure Realtime call over a sideband control connection. */ @@ -125,9 +145,12 @@ export function buildRealtimeURL( const azure = isAzure(client); const hasModel = !!config.model; const hasCallID = !!config.callID; + const hasIntent = config.intent === 'transcription'; - if (hasModel === hasCallID) { - throw new Error('Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.'); + if ([hasModel, hasCallID, hasIntent].filter(Boolean).length !== 1) { + throw new Error( + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', + ); } let url: URL; @@ -150,11 +173,17 @@ export function buildRealtimeURL( url.searchParams.set('call_id', config.callID!); } else { url.searchParams.set('api-version', client.apiVersion); - url.searchParams.set('deployment', config.model!); + if (hasIntent) { + url.searchParams.set('intent', config.intent!); + } else { + url.searchParams.set('deployment', config.model!); + } } } else { if (hasCallID) { url.searchParams.set('call_id', config.callID!); + } else if (hasIntent) { + url.searchParams.set('intent', config.intent!); } else { url.searchParams.set('model', config.model!); } @@ -166,11 +195,22 @@ export function getAzureRealtimeConnection( client: Pick, connection: AzureRealtimeConnectionConfig, ): RealtimeConnectionConfig { - if (connection.callID !== undefined) { - if (connection.deploymentName !== undefined) { - throw new Error('Pass either `deploymentName` or `callID`, but not both.'); - } - return { callID: connection.callID }; + const hasDeploymentName = connection.deploymentName !== undefined; + const hasCallID = connection.callID !== undefined; + const hasIntent = connection.intent === 'transcription'; + + if ([hasDeploymentName, hasCallID, hasIntent].filter(Boolean).length > 1) { + throw new Error( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); + } + + if (hasCallID) { + return { callID: connection.callID! }; + } + + if (hasIntent) { + return { intent: connection.intent! }; } const model = connection.deploymentName ?? client.deploymentName; diff --git a/tests/realtime.test.ts b/tests/realtime.test.ts index c3041f955d..8d8b08d287 100644 --- a/tests/realtime.test.ts +++ b/tests/realtime.test.ts @@ -1,6 +1,13 @@ import OpenAI, { AzureOpenAI } from 'openai'; import { buildRealtimeURL as buildBetaRealtimeURL } from 'openai/beta/realtime/internal-base'; -import { buildRealtimeURL as buildRealtimeURL } from 'openai/realtime/internal-base'; +import { + buildRealtimeURL as buildRealtimeURL, + getAzureRealtimeConnection, +} from 'openai/realtime/internal-base'; +import type { OpenAIRealtimeWebSocket as BetaOpenAIRealtimeWebSocket } from 'openai/beta/realtime/websocket'; +import type { OpenAIRealtimeWS as BetaOpenAIRealtimeWS } from 'openai/beta/realtime/ws'; +import type { OpenAIRealtimeWebSocket } from 'openai/realtime/websocket'; +import type { OpenAIRealtimeWS } from 'openai/realtime/ws'; const apiVersion = '2024-10-01-preview'; @@ -37,6 +44,12 @@ describe.each([ ); }); + test('uses transcription intent without a model for standard OpenAI connections', () => { + expect(buildRealtimeURL(openAIClient, { intent: 'transcription' }).toString()).toBe( + 'wss://example.com/custom/path/realtime?intent=transcription', + ); + }); + test('preserves the legacy model string form', () => { expect(buildRealtimeURL(openAIClient, 'gpt-realtime').toString()).toBe( 'wss://example.com/custom/path/realtime?model=gpt-realtime', @@ -55,19 +68,58 @@ describe.each([ ); }); + test('uses transcription intent without an Azure deployment', () => { + expect(buildRealtimeURL(azureClient, { intent: 'transcription' }).toString()).toBe( + `wss://example.com/openai/realtime?api-version=${apiVersion}&intent=transcription`, + ); + }); + test('rejects missing connection target', () => { expect(() => buildRealtimeURL(openAIClient, {} as any)).toThrow( - 'Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.', + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', ); }); test('rejects multiple connection targets', () => { expect(() => buildRealtimeURL(openAIClient, { model: 'gpt-realtime', callID: 'rtc_123' } as any)).toThrow( - 'Pass exactly one of `model` or `callID` when opening a Realtime WebSocket.', + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', + ); + }); + + test('rejects model and transcription intent together', () => { + expect(() => + buildRealtimeURL(openAIClient, { model: 'gpt-4o-transcribe', intent: 'transcription' } as any), + ).toThrow( + 'Pass exactly one of `model`, `callID`, or transcription `intent` when opening a Realtime WebSocket.', ); }); }); +test('stable Azure helper maps transcription intent without deployment', () => { + expect(getAzureRealtimeConnection(azureClient, { intent: 'transcription' })).toEqual({ + intent: 'transcription', + }); +}); + +test('stable Azure helper rejects deployment and transcription intent together', () => { + expect(() => + getAzureRealtimeConnection(azureClient, { + deploymentName: 'my-deployment', + intent: 'transcription', + } as any), + ).toThrow( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); +}); + +test('stable Azure helper rejects call_id and transcription intent together', () => { + expect(() => + getAzureRealtimeConnection(azureClient, { callID: 'rtc_123', intent: 'transcription' } as any), + ).toThrow( + 'Pass exactly one of `deploymentName`, `callID`, or transcription `intent` when opening an Azure Realtime WebSocket.', + ); +}); + test('stable builder uses the normalized Azure GA call_id URL', () => { for (const client of [azureClient, azureV1Client, azureEndpointClient]) { expect(buildRealtimeURL(client, { callID: 'rtc_123' }).toString()).toBe( @@ -81,3 +133,19 @@ test('beta builder directs Azure call_id users to the stable helper', () => { 'Azure `callID` connections require the stable Realtime helpers.', ); }); + +test('Azure realtime factories accept transcription intent options', () => { + const stableWebSocketOptions: Parameters[1] = { + intent: 'transcription', + }; + const stableWSOptions: Parameters[1] = { intent: 'transcription' }; + const betaWebSocketOptions: Parameters[1] = { + intent: 'transcription', + }; + const betaWSOptions: Parameters[1] = { intent: 'transcription' }; + + expect(stableWebSocketOptions.intent).toBe('transcription'); + expect(stableWSOptions.intent).toBe('transcription'); + expect(betaWebSocketOptions.intent).toBe('transcription'); + expect(betaWSOptions.intent).toBe('transcription'); +});