fix: migrate openai mllm to ga realtime api and add per-join greeting

ai_agents/agents/ten_packages/extension/openai_mllm_python/extension.py

@@ -57,14 +57,9 @@
     InputAudioBufferSpeechStopped,
     ResponseFunctionCallArgumentsDone,
     ErrorMessage,
-    SessionUpdate,
-    SessionUpdateParams,
-    InputAudioTranscription,
     ContentType,
     FunctionCallOutputItemParam,
     ResponseCreate,
-    ServerVADUpdateParams,
-    SemanticVADUpdateParams,
 )
 
 
@@ -78,6 +73,10 @@ class OpenAIRealtimeConfig(BaseModel):
     prompt: str = ""
     temperature: float = 0.5
     max_tokens: int = 1024
+    # Spoken opening line, sent once per join as a user message plus a
+    # response request. Empty (default) disables it; typically supplied
+    # per join by the platform rather than hardcoded in the graph.
+    greeting: str = ""
     voice: str = "alloy"
     server_vad: bool = True
     audio_out: bool = True
@@ -106,6 +105,7 @@ def __init__(self, name: str):
         self.connected: bool = False
 
         self.request_transcript: str = ""
+        self._greeting_sent: bool = False
         self.response_transcript: str = ""
         self.available_tools: list[LLMToolMetadata] = []
         self.loop: asyncio.AbstractEventLoop = None
@@ -169,6 +169,7 @@ async def start_connection(self) -> None:
                             self.connected = True
                             self.openai_session_id = message.session.id
                             self.openai_session = message.session
+                            self._greeting_sent = False
                             await self._update_session()
                             await self._resume_context(self.message_context)
                         case SessionUpdated():
@@ -178,6 +179,17 @@ async def start_connection(self) -> None:
                             await self.send_server_session_ready(
                                 MLLMServerSessionReady()
                             )
+                            if self.config.greeting and not self._greeting_sent:
+                                # Session config is applied; speak the
+                                # configured opening line once per join.
+                                self._greeting_sent = True
+                                await self.send_client_message_item(
+                                    MLLMClientMessageItem(
+                                        role="user",
+                                        content=self.config.greeting,
+                                    )
+                                )
+                                await self.send_client_create_response()
                         case ItemInputAudioTranscriptionDelta():
                             self.ten_env.log_debug(
                                 f"On request transcript delta {message.item_id} {message.content_index}"
@@ -587,36 +599,45 @@ def tool_dict(tool: LLMToolMetadata):
             tools = [tool_dict(t) for t in self.available_tools]
         prompt = self.config.prompt
 
+        # GA realtime session shape: type marker, output_modalities, and
+        # nested audio.input/audio.output blocks (the flat beta fields
+        # are rejected since the beta shape was retired).
         if self.config.vad_type == "server_vad":
-            vad_params = ServerVADUpdateParams(
-                threshold=self.config.vad_threshold,
-                prefix_padding_ms=self.config.vad_prefix_padding_ms,
-                silence_duration_ms=self.config.vad_silence_duration_ms,
-            )
+            vad: dict = {
+                "type": "server_vad",
+                "threshold": self.config.vad_threshold,
+                "prefix_padding_ms": self.config.vad_prefix_padding_ms,
+                "silence_duration_ms": self.config.vad_silence_duration_ms,
+            }
         else:  # semantic vad
-            vad_params = SemanticVADUpdateParams(
-                eagerness=self.config.vad_eagerness,
-            )
-        su = SessionUpdate(
-            session=SessionUpdateParams(
-                instructions=prompt,
-                model=self.config.model,
-                tool_choice="auto" if self.available_tools else "none",
-                tools=tools,
-                turn_detection=vad_params,
-            )
-        )
-        if self.config.audio_out:
-            su.session.voice = self.config.voice
-        else:
-            su.session.modalities = ["text"]
+            vad = {
+                "type": "semantic_vad",
+                "eagerness": self.config.vad_eagerness,
+            }
+        session: dict = {
+            "type": "realtime",
+            "instructions": prompt,
+            "tool_choice": "auto" if self.available_tools else "none",
+            "tools": tools,
+            "output_modalities": (
+                ["audio"] if self.config.audio_out else ["text"]
+            ),
+            "audio": {
+                "input": {
+                    "transcription": {
+                        "model": "gpt-4o-mini-transcribe",
+                        "language": self.config.language,
+                    },
+                    "turn_detection": vad,
+                },
+                "output": {"voice": self.config.voice},
+            },
+        }
+        self.ten_env.log_info(f"update session {session}")
 
-        su.session.input_audio_transcription = InputAudioTranscription(
-            language=self.config.language,
+        await self.conn.send_json(
+            {"type": "session.update", "session": session}
         )
-        self.ten_env.log_info(f"update session {su}")
-
-        await self.conn.send_request(su)
 
     async def _handle_tool_call(
         self, tool_call_id: str, name: str, arguments: str

ai_agents/agents/ten_packages/extension/openai_mllm_python/manifest.json

@@ -1,7 +1,7 @@
 {
   "type": "extension",
   "name": "openai_mllm_python",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "dependencies": [
     {
       "type": "system",
@@ -90,6 +90,9 @@
         },
         "vad_silence_duration_ms": {
           "type": "int32"
+        },
+        "greeting": {
+          "type": "string"
         }
       }
     }

ai_agents/agents/ten_packages/extension/openai_mllm_python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openai-mllm-python"
-version = "0.2.2"
+version = "0.2.3"
 requires-python = ">=3.10"
 dependencies = [
     "aiohttp>=3.14.1",

ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/connection.py

@@ -76,8 +76,9 @@ async def connect(self):
         if self.vendor == VENDOR_AZURE:
             headers = {"api-key": self.api_key}
         elif not self.vendor:
+            # GA realtime API: no OpenAI-Beta header (the beta shape is
+            # retired and rejects connections that request it).
             auth = aiohttp.BasicAuth("", self.api_key) if self.api_key else None
-            headers = {"OpenAI-Beta": "realtime=v1"}
 
         self.websocket = await self.session.ws_connect(
             url=self.url,
@@ -91,6 +92,13 @@ async def send_audio_data(self, audio_data: bytes):
         message = InputAudioBufferAppend(audio=base64_audio_data)
         await self.send_request(message)
 
+    async def send_json(self, event: dict):
+        assert self.websocket is not None
+        message_str = json.dumps(event)
+        if self.verbose:
+            self.ten_env.log_info(f"-> {smart_str(message_str)}")
+        await self.websocket.send_str(message_str)
+
     async def send_request(self, message: ClientToServerMessage):
         assert self.websocket is not None
         message_str = to_json(message)

ai_agents/agents/ten_packages/extension/openai_mllm_python/realtime/struct.py

@@ -900,6 +900,24 @@ def parse_server_message(unparsed_string: str) -> ServerToClientMessage:
     elif data["type"] == EventType.ITEM_INPUT_AUDIO_TRANSCRIPTION_DELTA:
         return from_dict(ItemInputAudioTranscriptionDelta, data)
 
+    # GA realtime API event names (the beta shape was retired); map the
+    # renamed events onto the existing message classes.
+    ga_aliases = {
+        "response.output_audio.delta": ResponseAudioDelta,
+        "response.output_audio.done": ResponseAudioDone,
+        "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta,
+        "response.output_audio_transcript.done": ResponseAudioTranscriptDone,
+        "response.output_text.delta": ResponseTextDelta,
+        "response.output_text.done": ResponseTextDone,
+        "conversation.item.added": ItemCreated,
+        "conversation.item.done": ItemCreated,
+    }
+    cls = ga_aliases.get(data["type"])
+    if cls is not None:
+        data = dict(data)
+        data["type"] = cls.__dataclass_fields__["type"].default
+        return from_dict(cls, data)
+
     raise ValueError(f"Unknown message type: {data['type']} {data}")