feat: add Gradium speech-to-speech translation extension

ai_agents/.env.example

@@ -181,10 +181,17 @@ DUBVERSE_TTS_KEY=
 
 # Extension: gradium_asr_python
 # Extension: gradium_tts_python
-# Gradium ASR key
-# Gradium TTS key
+# Extension: gradium_mllm_python
+# Gradium ASR key / TTS key / speech-to-speech translation key
 # same unified key
 GRADIUM_API_KEY=
+# Extension: gradium_mllm_python (voice-assistant-realtime's gradium_translate_demo graph)
+# Must be a Gradium voice_id belonging to the graph's target_language (default:
+# en). Defaults to YTpq7expH9539ERJ (Emma, English) if unset -- confirmed by
+# Gradium. Override for other target_language values (also update
+# target_language in the graph node's property). Gradium's voice catalog is
+# also queryable via their API per Pratim (2026-08-24) -- not yet wired up here.
+GRADIUM_S2S_VOICE_ID=
 
 # Extension: inworld_tts_python
 # Inworld TTS API key (Base64-encoded)

ai_agents/agents/examples/voice-assistant-realtime/tenapp/manifest.json

@@ -30,6 +30,9 @@
     {
       "path": "../../../ten_packages/extension/glm_mllm_python"
     },
+    {
+      "path": "../../../ten_packages/extension/gradium_mllm_python"
+    },
     {
       "path": "../../../ten_packages/extension/openai_mllm_python"
     },

ai_agents/agents/examples/voice-assistant-realtime/tenapp/property.json

@@ -1,6 +1,146 @@
 {
   "ten": {
     "predefined_graphs": [
+      {
+        "name": "gradium_translate_demo",
+        "auto_start": false,
+        "graph": {
+          "nodes": [
+            {
+              "type": "extension",
+              "name": "agora_rtc",
+              "addon": "agora_rtc",
+              "extension_group": "default",
+              "property": {
+                "app_id": "${env:AGORA_APP_ID}",
+                "app_certificate": "${env:AGORA_APP_CERTIFICATE|}",
+                "channel": "ten_agent_test",
+                "stream_id": 1234,
+                "remote_stream_id": 123,
+                "subscribe_audio": true,
+                "publish_audio": true,
+                "publish_data": true,
+                "enable_agora_asr": false
+              }
+            },
+            {
+              "type": "extension",
+              "name": "main_control",
+              "addon": "main_python",
+              "extension_group": "control",
+              "property": {
+                "greeting": "Gradium translation demo connected."
+              }
+            },
+            {
+              "type": "extension",
+              "name": "message_collector",
+              "addon": "message_collector2",
+              "extension_group": "transcriber",
+              "property": {}
+            },
+            {
+              "type": "extension",
+              "name": "streamid_adapter",
+              "addon": "streamid_adapter",
+              "property": {}
+            },
+            {
+              "type": "extension",
+              "name": "v2v",
+              "addon": "gradium_mllm_python",
+              "property": {
+                "api_key": "${env:GRADIUM_API_KEY}",
+                "region": "us",
+                "path": "/api/speech/s2s",
+                "model_name": "s2s-translate",
+                "stt_model_name": "stt-translate",
+                "tts_model_name": "default",
+                "voice_id": "${env:GRADIUM_S2S_VOICE_ID|YTpq7expH9539ERJ}",
+                "target_language": "en",
+                "input_format": "pcm",
+                "output_format": "pcm",
+                "input_sample_rate": 24000
+              }
+            }
+          ],
+          "connections": [
+            {
+              "extension": "agora_rtc",
+              "audio_frame": [
+                {
+                  "name": "pcm_frame",
+                  "dest": [
+                    {
+                      "extension": "streamid_adapter"
+                    }
+                  ]
+                },
+                {
+                  "name": "pcm_frame",
+                  "source": [
+                    {
+                      "extension": "v2v"
+                    }
+                  ]
+                }
+              ],
+              "data": [
+                {
+                  "name": "data",
+                  "source": [
+                    {
+                      "extension": "message_collector"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "extension": "main_control",
+              "data": [
+                {
+                  "names": [
+                    "mllm_server_input_transcript",
+                    "mllm_server_output_transcript",
+                    "mllm_server_session_ready",
+                    "mllm_server_interrupted",
+                    "mllm_server_function_call"
+                  ],
+                  "source": [
+                    {
+                      "extension": "v2v"
+                    }
+                  ]
+                }
+              ],
+              "cmd": [
+                {
+                  "names": ["on_user_left", "on_user_joined"],
+                  "source": [
+                    {
+                      "extension": "agora_rtc"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "extension": "streamid_adapter",
+              "audio_frame": [
+                {
+                  "name": "pcm_frame",
+                  "dest": [
+                    {
+                      "extension": "v2v"
+                    }
+                  ]
+                }
+              ]
+            }
+          ]
+        }
+      },
       {
         "name": "voice_assistant_realtime",
         "auto_start": true,

ai_agents/agents/ten_packages/extension/gradium_mllm_python/README.md

@@ -0,0 +1,101 @@
+# gradium_mllm_python
+
+Real-time speech-to-speech (S2S) translation extension for TEN, using
+[Gradium](https://gradium.ai/)'s Translation API. Implements the same
+`AsyncMLLMBaseExtension` contract as `openai_mllm_python`, `azure_mllm_python`,
+`gemini_mllm_python`, `glm_mllm_python`, and `stepfun_mllm_python`, so it can
+be dropped into any graph node expecting an "mllm" addon.
+
+## Status: protocol confirmed by Gradium, not yet run against a live endpoint
+
+The full `/api/speech/s2s` protocol below was confirmed directly by Gradium
+(Pratim, 2026-08-20), building on top of `gradium_asr_python` and
+`gradium_tts_python`, which already talk to Gradium's real ASR/TTS websocket
+APIs in this repo.
+
+| | Value |
+|---|---|
+| Auth | header `x-api-key: <api_key>` |
+| Host + path | `wss://<region>.api.gradium.ai/api/speech/s2s`, region `us` or `eu` |
+| Handshake | client sends `{"type": "setup", ...}` (see below), waits for `{"type": "ready"}` before streaming audio |
+| Setup payload | `model_name: "s2s-translate"`, `stt_model_name: "stt-translate"`, `tts_model_name: "default"`, `input_format`/`output_format: "pcm"` (24kHz in, 48kHz out), `voice_id`, and `json_config: {"target_language": ...}` -- **`target_language` nests inside `json_config`, it is not a top-level field** |
+| Audio frames (both directions) | `{"type": "audio", "audio": "<base64 pcm16le>"}` |
+| Text frames | `{"type": "text", "text": ..., "final": bool, ...}` -- **translated output only**, there is no separate source-language transcript event |
+| End of turn | `{"type": "end_of_stream"}` |
+| Errors | `{"type": "error", "message": ..., "code": ...}` |
+| VAD | **not part of this protocol** -- only `ready`/`audio`/`text`/`end_of_stream`/`error` are ever sent |
+
+Supported `target_language` values (confirmed): `en`, `fr`, `de`, `es`, `pt`.
+
+`voice_id` must be a voice belonging to `target_language`, or Gradium will
+reject/mis-synthesize -- `on_init` raises if it's unset rather than guessing.
+Default is `YTpq7expH9539ERJ` ("Emma", English -- confirmed by Gradium,
+2026-08-24), matching the default `target_language: "en"`. For other
+languages, override both `voice_id` and `target_language` together (e.g. via
+`GRADIUM_S2S_VOICE_ID` for the demo graph). Per Pratim, Gradium's voice
+catalog is also queryable through their API -- not yet wired up here, so
+picking a voice for a new language is still a manual lookup for now.
+
+This has been run and passes end-to-end against a mocked Gradium client (see
+Tests below), including a real shutdown-deadlock bug caught and fixed by
+actually running it. It has **not** yet been run against Gradium's live
+endpoint -- that's still the next step, on Ben's TEN dev server.
+
+## Tests
+
+`tests/` mirrors `gradium_tts_python`'s pattern: a real TEN runtime
+(`tests/conftest.py`'s `FakeApp`) drives the actual extension lifecycle via
+`AsyncExtensionTester`, with only `GradiumS2SClient` mocked (`tests/gradium_mocks.py`)
+-- no live Gradium connection or real `voice_id` needed. Covers: session-ready
++ translated text/audio routing (`test_basic.py::test_session_ready_and_translated_output`),
+server-side error propagation, connect failures, and missing
+`api_key`/`voice_id` being reported cleanly instead of crashing the
+extension. `tests/test_config.py` separately unit-tests `GradiumMLLMConfig`
+(no TEN runtime needed) -- in particular the `json_config` nesting for
+`target_language`, which was wrong in the initial scaffold.
+
+Run from `ai_agents/` inside the dev container:
+```bash
+task test-extension EXTENSION=agents/ten_packages/extension/gradium_mllm_python
+```
+
+## Properties
+
+Refer to `api` definition in [manifest.json](manifest.json) and default
+values in [property.json](property.json).
+
+| **Property** | **Type** | **Description** |
+|---|---|---|
+| `api_key` | `string` | Gradium API key (sent as the `x-api-key` header) |
+| `region` | `string` | `us` or `eu` -- selects the websocket host |
+| `base_url` | `string` | Optional explicit host override, skips region lookup |
+| `path` | `string` | Websocket path (`/api/speech/s2s`) |
+| `model_name` | `string` | Speech-to-speech model name (`s2s-translate`) |
+| `stt_model_name` | `string` | ASR-leg model (`stt-translate`) |
+| `tts_model_name` | `string` | TTS-leg model (`default`) |
+| `voice_id` | `string` | Voice for the synthesized translated speech -- **required**, must belong to `target_language` |
+| `target_language` | `string` | Language to translate into (`en`/`fr`/`de`/`es`/`pt` confirmed); sent nested in `json_config`, not top-level, on the wire |
+| `input_format` | `string` | Input audio format (`pcm`) |
+| `output_format` | `string` | Output audio format (`pcm`, `pcm_16000`, `pcm_24000`) |
+| `input_sample_rate` | `int32` | Input PCM sample rate, Hz |
+| `dump` / `dump_path` | `bool` / `string` | Audio dump for debugging (from the shared mllm interface) |
+
+## Not implemented
+
+Gradium's S2S translation is a continuous audio pipe, not a tool-calling
+conversational LLM. `send_client_message_item`, `send_client_create_response`,
+`send_client_register_tool`, and `send_client_function_call_output` are all
+no-ops (logged at debug level) -- there's no known Gradium equivalent for
+injecting messages/tools into a translation stream. Revisit if that turns
+out to be wrong.
+
+### Audio Frame In / Out
+
+| **Name** | **Description** |
+|---|---|
+| `pcm_frame` | mic audio in / translated speech out |
+
+### Data Out
+
+`mllm_server_session_ready`, `mllm_server_output_transcript`, `error` (from
+the shared `mllm-interface.json` contract).

ai_agents/agents/ten_packages/extension/gradium_mllm_python/__init__.py

@@ -0,0 +1,7 @@
+"""
+Gradium real-time speech-to-speech translation extension for TEN framework.
+"""
+
+from .addon import GradiumMLLMExtensionAddon
+
+__all__ = ["GradiumMLLMExtensionAddon"]

ai_agents/agents/ten_packages/extension/gradium_mllm_python/addon.py

@@ -0,0 +1,23 @@
+"""
+Addon registration for the Gradium real-time speech-to-speech translation
+extension.
+"""
+
+from ten_runtime import Addon, TenEnv, register_addon_as_extension
+
+
+@register_addon_as_extension("gradium_mllm_python")
+class GradiumMLLMExtensionAddon(Addon):
+    """Addon class for registering the Gradium MLLM extension."""
+
+    def on_create_instance(
+        self, ten_env: TenEnv, addon_name: str, context
+    ) -> None:
+        from .extension import GradiumMLLMExtension
+
+        ten_env.log_info(
+            f"Creating Gradium MLLM extension instance: {addon_name}"
+        )
+        ten_env.on_create_instance_done(
+            GradiumMLLMExtension(addon_name), context
+        )

ai_agents/agents/ten_packages/extension/gradium_mllm_python/client.py

@@ -0,0 +1,87 @@
+"""
+Websocket client for Gradium's real-time speech-to-speech translation API.
+
+Protocol, confirmed directly by Gradium (2026-08-20) for /api/speech/s2s:
+  connect -> send {"type": "setup", ...} -> wait for {"type": "ready"} ->
+  stream {"type": "audio", "audio": <base64 pcm16>} both ways ->
+  {"type": "text", ...} carries translated output text (no separate
+  source-language transcript) -> {"type": "end_of_stream"} closes the turn
+  -> {"type": "error", ...} on failure. No "vad" event on this endpoint.
+"""
+
+import asyncio
+import base64
+import json
+from typing import Any, AsyncIterator
+
+import websockets
+from ten_runtime import AsyncTenEnv
+
+from .config import GradiumMLLMConfig
+from .const import WS_MSG_TYPE_ERROR, WS_MSG_TYPE_READY
+
+
+class GradiumS2SClient:
+    """Thin duplex websocket wrapper -- no reconnect/backoff logic here; that lives in extension.py."""
+
+    def __init__(self, config: GradiumMLLMConfig, ten_env: AsyncTenEnv):
+        self.config = config
+        self.ten_env = ten_env
+        self.ws: Any | None = None
+
+    async def connect(self, ready_timeout: float = 10.0) -> None:
+        headers = {"x-api-key": self.config.api_key}
+        url = self.config.websocket_url()
+        self.ten_env.log_info(f"[gradium] connecting to {url}")
+
+        self.ws = await websockets.connect(url, additional_headers=headers)
+
+        await self._send_json(self.config.setup_message())
+        await self._wait_for_ready(ready_timeout)
+
+    async def _wait_for_ready(self, timeout: float) -> None:
+        assert self.ws is not None
+        raw = await asyncio.wait_for(self.ws.recv(), timeout=timeout)
+        message = self._parse(raw)
+        if message is None:
+            raise RuntimeError("Gradium sent an unparseable message before ready")
+        if message.get("type") == WS_MSG_TYPE_ERROR:
+            raise RuntimeError(message.get("message", "Gradium setup failed"))
+        if message.get("type") != WS_MSG_TYPE_READY:
+            raise RuntimeError(
+                f"Expected 'ready' from Gradium, got {message.get('type')!r}"
+            )
+
+    async def send_audio(self, pcm_bytes: bytes) -> None:
+        assert self.ws is not None
+        audio_b64 = base64.b64encode(pcm_bytes).decode("utf-8")
+        await self._send_json({"type": "audio", "audio": audio_b64})
+
+    async def send_end_of_stream(self) -> None:
+        if self.ws is not None:
+            await self._send_json({"type": "end_of_stream"})
+
+    async def _send_json(self, payload: dict[str, Any]) -> None:
+        assert self.ws is not None
+        await self.ws.send(json.dumps(payload))
+
+    async def messages(self) -> AsyncIterator[dict[str, Any]]:
+        assert self.ws is not None
+        async for raw in self.ws:
+            message = self._parse(raw)
+            if message is not None:
+                yield message
+
+    def _parse(self, raw: str | bytes) -> dict[str, Any] | None:
+        try:
+            if isinstance(raw, bytes):
+                raw = raw.decode("utf-8")
+            return json.loads(raw)
+        except Exception as e:
+            self.ten_env.log_warn(f"[gradium] failed to parse message: {e}")
+            return None
+
+    async def close(self) -> None:
+        if self.ws is not None:
+            await self.ws.close()
+            self.ws = None