From a0c3f4ae74e4dde81b66248dc56e4775fb4756de Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 30 Jul 2026 10:32:44 +0000 Subject: [PATCH 1/4] fix(mllm): migrate openai realtime extension to ga api shape OpenAI retired the realtime beta shape (beta_api_shape_disabled), which broke openai_mllm_python everywhere. Drop the OpenAI-Beta header, map the renamed ga server events onto the existing message classes, and send the ga session.update shape (type realtime, output_modalities, nested audio.input/output with transcription and turn_detection). Verified against the live service: session created and updated, event stream parsed. --- .../extension/openai_mllm_python/extension.py | 61 +++++++++++-------- .../openai_mllm_python/realtime/connection.py | 10 ++- .../openai_mllm_python/realtime/struct.py | 18 ++++++ 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py b/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py index 04c9a27f1c..851004b859 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py @@ -587,36 +587,45 @@ def tool_dict(tool: LLMToolMetadata): tools = [tool_dict(t) for t in self.available_tools] prompt = self.config.prompt + # GA realtime session shape: type marker, output_modalities, and + # nested audio.input/audio.output blocks (the flat beta fields + # are rejected since the beta shape was retired). if self.config.vad_type == "server_vad": - vad_params = ServerVADUpdateParams( - threshold=self.config.vad_threshold, - prefix_padding_ms=self.config.vad_prefix_padding_ms, - silence_duration_ms=self.config.vad_silence_duration_ms, - ) + vad: dict = { + "type": "server_vad", + "threshold": self.config.vad_threshold, + "prefix_padding_ms": self.config.vad_prefix_padding_ms, + "silence_duration_ms": self.config.vad_silence_duration_ms, + } else: # semantic vad - vad_params = SemanticVADUpdateParams( - eagerness=self.config.vad_eagerness, - ) - su = SessionUpdate( - session=SessionUpdateParams( - instructions=prompt, - model=self.config.model, - tool_choice="auto" if self.available_tools else "none", - tools=tools, - turn_detection=vad_params, - ) - ) - if self.config.audio_out: - su.session.voice = self.config.voice - else: - su.session.modalities = ["text"] + vad = { + "type": "semantic_vad", + "eagerness": self.config.vad_eagerness, + } + session: dict = { + "type": "realtime", + "instructions": prompt, + "tool_choice": "auto" if self.available_tools else "none", + "tools": tools, + "output_modalities": ( + ["audio"] if self.config.audio_out else ["text"] + ), + "audio": { + "input": { + "transcription": { + "model": "gpt-4o-mini-transcribe", + "language": self.config.language, + }, + "turn_detection": vad, + }, + "output": {"voice": self.config.voice}, + }, + } + self.ten_env.log_info(f"update session {session}") - su.session.input_audio_transcription = InputAudioTranscription( - language=self.config.language, + await self.conn.send_json( + {"type": "session.update", "session": session} ) - self.ten_env.log_info(f"update session {su}") - - await self.conn.send_request(su) async def _handle_tool_call( self, tool_call_id: str, name: str, arguments: str diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/connection.py b/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/connection.py index 99479276f9..5267c2ec6d 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/connection.py +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/connection.py @@ -76,8 +76,9 @@ async def connect(self): if self.vendor == VENDOR_AZURE: headers = {"api-key": self.api_key} elif not self.vendor: + # GA realtime API: no OpenAI-Beta header (the beta shape is + # retired and rejects connections that request it). auth = aiohttp.BasicAuth("", self.api_key) if self.api_key else None - headers = {"OpenAI-Beta": "realtime=v1"} self.websocket = await self.session.ws_connect( url=self.url, @@ -91,6 +92,13 @@ async def send_audio_data(self, audio_data: bytes): message = InputAudioBufferAppend(audio=base64_audio_data) await self.send_request(message) + async def send_json(self, event: dict): + assert self.websocket is not None + message_str = json.dumps(event) + if self.verbose: + self.ten_env.log_info(f"-> {smart_str(message_str)}") + await self.websocket.send_str(message_str) + async def send_request(self, message: ClientToServerMessage): assert self.websocket is not None message_str = to_json(message) diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/struct.py b/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/struct.py index 16fc901732..d4b97c9303 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/struct.py +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/struct.py @@ -900,6 +900,24 @@ def parse_server_message(unparsed_string: str) -> ServerToClientMessage: elif data["type"] == EventType.ITEM_INPUT_AUDIO_TRANSCRIPTION_DELTA: return from_dict(ItemInputAudioTranscriptionDelta, data) + # GA realtime API event names (the beta shape was retired); map the + # renamed events onto the existing message classes. + ga_aliases = { + "response.output_audio.delta": ResponseAudioDelta, + "response.output_audio.done": ResponseAudioDone, + "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta, + "response.output_audio_transcript.done": ResponseAudioTranscriptDone, + "response.output_text.delta": ResponseTextDelta, + "response.output_text.done": ResponseTextDone, + "conversation.item.added": ItemCreated, + "conversation.item.done": ItemCreated, + } + cls = ga_aliases.get(data["type"]) + if cls is not None: + data = dict(data) + data["type"] = cls.__dataclass_fields__["type"].default + return from_dict(cls, data) + raise ValueError(f"Unknown message type: {data['type']} {data}") From 79faa61f69f75a71d6e24fe93c998890f8d2d086 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 4 Aug 2026 17:21:07 +0000 Subject: [PATCH 2/4] feat: add per-join greeting to openai mllm extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit greeting config field (default empty = off) spoken once per join after the session config is applied, as a user message item plus a response request — same semantics as the gemini mllm extension. intended to be supplied per join by the platform rather than hardcoded in graphs. --- .../extension/openai_mllm_python/extension.py | 17 +++++++++++++++++ .../extension/openai_mllm_python/manifest.json | 3 +++ 2 files changed, 20 insertions(+) diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py b/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py index 851004b859..21e4c8b387 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py @@ -78,6 +78,10 @@ class OpenAIRealtimeConfig(BaseModel): prompt: str = "" temperature: float = 0.5 max_tokens: int = 1024 + # Spoken opening line, sent once per join as a user message plus a + # response request. Empty (default) disables it; typically supplied + # per join by the platform rather than hardcoded in the graph. + greeting: str = "" voice: str = "alloy" server_vad: bool = True audio_out: bool = True @@ -106,6 +110,7 @@ def __init__(self, name: str): self.connected: bool = False self.request_transcript: str = "" + self._greeting_sent: bool = False self.response_transcript: str = "" self.available_tools: list[LLMToolMetadata] = [] self.loop: asyncio.AbstractEventLoop = None @@ -169,6 +174,7 @@ async def start_connection(self) -> None: self.connected = True self.openai_session_id = message.session.id self.openai_session = message.session + self._greeting_sent = False await self._update_session() await self._resume_context(self.message_context) case SessionUpdated(): @@ -178,6 +184,17 @@ async def start_connection(self) -> None: await self.send_server_session_ready( MLLMServerSessionReady() ) + if self.config.greeting and not self._greeting_sent: + # Session config is applied; speak the + # configured opening line once per join. + self._greeting_sent = True + await self.send_client_message_item( + MLLMClientMessageItem( + role="user", + content=self.config.greeting, + ) + ) + await self.send_client_create_response() case ItemInputAudioTranscriptionDelta(): self.ten_env.log_debug( f"On request transcript delta {message.item_id} {message.content_index}" diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json b/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json index 31bc50bc26..4b9586de54 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json @@ -90,6 +90,9 @@ }, "vad_silence_duration_ms": { "type": "int32" + }, + "greeting": { + "type": "string" } } } From b746b42a7f7942f36ab1c8f17aa7485e683c2de8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 5 Aug 2026 08:18:08 +0000 Subject: [PATCH 3/4] chore: bump openai mllm to 0.2.3 --- .../ten_packages/extension/openai_mllm_python/manifest.json | 2 +- .../ten_packages/extension/openai_mllm_python/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json b/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json index 4b9586de54..c0a51e85ee 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json @@ -1,7 +1,7 @@ { "type": "extension", "name": "openai_mllm_python", - "version": "0.2.2", + "version": "0.2.3", "dependencies": [ { "type": "system", diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/pyproject.toml b/ai_agents/agents/ten_packages/extension/openai_mllm_python/pyproject.toml index 1171d0d156..8dad1a94e1 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/pyproject.toml +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openai-mllm-python" -version = "0.2.2" +version = "0.2.3" requires-python = ">=3.10" dependencies = [ "aiohttp>=3.14.1", From 33f7e446b91c4d9968d6e4e56d34a8a6aa11c2aa Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 5 Aug 2026 13:37:28 +0000 Subject: [PATCH 4/4] chore: drop unused beta session imports --- .../ten_packages/extension/openai_mllm_python/extension.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py b/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py index 21e4c8b387..51b029b63b 100644 --- a/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py +++ b/ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py @@ -57,14 +57,9 @@ InputAudioBufferSpeechStopped, ResponseFunctionCallArgumentsDone, ErrorMessage, - SessionUpdate, - SessionUpdateParams, - InputAudioTranscription, ContentType, FunctionCallOutputItemParam, ResponseCreate, - ServerVADUpdateParams, - SemanticVADUpdateParams, )