From 70e4d8da70f1b452d772778991b2c4fa3174c3ed Mon Sep 17 00:00:00 2001 From: JianYan <15005893916@163.com> Date: Sun, 27 Sep 2026 19:47:35 +0800 Subject: [PATCH 1/6] feat: add jev thinking-mode voice router --- ai_agents/.env.example | 4 + .../agents/examples/voice-assistant/README.md | 39 ++- .../voice-assistant/tenapp/manifest-lock.json | 13 + .../voice-assistant/tenapp/manifest.json | 5 +- .../voice-assistant/tenapp/property.json | 167 +++++++++- .../extension/main_python/agent/agent.py | 36 ++- .../extension/main_python/agent/events.py | 13 + .../extension/main_python/agent/llm_exec.py | 135 ++++++-- .../extension/main_python/config.py | 58 +++- .../extension/main_python/extension.py | 119 +++++++- .../extension/main_python/manifest.json | 12 + .../extension/main_python/routing.py | 93 ++++++ .../voice-assistant/tests/test_jev_graph.py | 69 +++++ .../tests/test_jev_llm_exec.py | 287 ++++++++++++++++++ .../tests/test_jev_reasoning_display.py | 218 +++++++++++++ .../voice-assistant/tests/test_jev_routing.py | 134 ++++++++ .../extension/openai_llm2_python/openai.py | 16 +- .../extension/typesafe_jev_python/README.md | 41 +++ .../extension/typesafe_jev_python/__init__.py | 9 + .../extension/typesafe_jev_python/addon.py | 9 + .../extension/typesafe_jev_python/config.py | 22 ++ .../extension/typesafe_jev_python/decision.py | 127 ++++++++ .../typesafe_jev_python/extension.py | 117 +++++++ .../typesafe_jev_python/manifest.json | 39 +++ .../typesafe_jev_python/property.json | 8 + .../typesafe_jev_python/pyproject.toml | 11 + .../typesafe_jev_python/requirements.txt | 1 + .../typesafe_jev_python/tests/__init__.py | 0 .../typesafe_jev_python/tests/bin/start | 5 + .../tests/test_decision.py | 168 ++++++++++ .../tests/test_extension.py | 187 ++++++++++++ .../src/components/Chat/MessageList.tsx | 48 ++- ai_agents/playground/src/manager/rtc/rtc.ts | 13 +- .../playground/src/store/reducers/global.ts | 31 +- ai_agents/playground/src/types/index.ts | 1 + 35 files changed, 2151 insertions(+), 104 deletions(-) create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/routing.py create mode 100644 ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py create mode 100644 ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py create mode 100644 ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py create mode 100644 ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/README.md create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/__init__.py create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/addon.py create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/config.py create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/decision.py create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/extension.py create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/manifest.json create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/property.json create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/pyproject.toml create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/requirements.txt create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/__init__.py create mode 100755 ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/bin/start create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_decision.py create mode 100644 ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_extension.py diff --git a/ai_agents/.env.example b/ai_agents/.env.example index 5288b96399..0338839d1f 100644 --- a/ai_agents/.env.example +++ b/ai_agents/.env.example @@ -78,6 +78,10 @@ QWEN_API_KEY= # Extension: deepseek DEEPSEEK_API_KEY= +# Optional voice_assistant_jev_router graph +TYPESAFE_API_KEY= +DEEPSEEK_PROXY_URL= + # Stepfun API Key STEPFUN_API_KEY= GLADIA_API_KEY= diff --git a/ai_agents/agents/examples/voice-assistant/README.md b/ai_agents/agents/examples/voice-assistant/README.md index 55a4a9673d..3ec840498b 100644 --- a/ai_agents/agents/examples/voice-assistant/README.md +++ b/ai_agents/agents/examples/voice-assistant/README.md @@ -17,14 +17,20 @@ A configurable voice assistant with real-time conversation capabilities using Ag - `DEEPGRAM_API_KEY` for the default `voice_assistant` graph - `XAI_API_KEY` for `voice_assistant_xai_asr` or `voice_assistant_xai_full` -3. **OpenAI Account**: Get credentials from [OpenAI Platform](https://platform.openai.com/) - - `OPENAI_API_KEY` - Your OpenAI API key (required) +3. **LLM Provider**: Most graphs use an OpenAI-compatible model with + `OPENAI_API_KEY`. The optional Jev graph instead uses `DEEPSEEK_API_KEY`. 4. **TTS Provider**: choose the graph you want to run - `ELEVENLABS_TTS_KEY` for the default `voice_assistant` graph or `voice_assistant_xai_asr` - `GRADIUM_API_KEY` for `voice_assistant_gradium` - `XAI_API_KEY` for `voice_assistant_xai_tts` or `voice_assistant_xai_full` +5. **Optional Jev routing graph**: `voice_assistant_jev_router` needs + `TYPESAFE_API_KEY` and `DEEPSEEK_API_KEY`. Both routes use `deepseek-flash`: + simple conversation runs without thinking, while complex requests enable + high-effort thinking. It also needs the same Deepgram and ElevenLabs keys + as the default graph. This optional graph is a chatbot and has no tools. + ### Provider-specific keys - **Deepgram Account**: Get credentials from [Deepgram Console](https://console.deepgram.com/) @@ -41,6 +47,7 @@ A configurable voice assistant with real-time conversation capabilities using Ag - `AGORA_APP_CERTIFICATE` - Agora App Certificate (optional) - `OPENAI_MODEL` - OpenAI model name (optional, defaults to configured model) - `OPENAI_PROXY_URL` - Proxy URL for OpenAI API (optional) +- `DEEPSEEK_PROXY_URL` - Proxy URL for the optional Jev graph (optional) - `WEATHERAPI_API_KEY` - Weather API key for weather tool (optional) ## Setup @@ -62,6 +69,9 @@ OPENAI_API_KEY=your_openai_api_key_here OPENAI_MODEL=gpt-4 OPENAI_PROXY_URL=your_proxy_url_here +# For voice_assistant_jev_router, use these instead of the OpenAI variables +TYPESAFE_API_KEY=your_typesafe_api_key_here +DEEPSEEK_API_KEY=your_deepseek_api_key_here # ElevenLabs (required for text-to-speech) ELEVENLABS_TTS_KEY=your_elevenlabs_api_key_here @@ -111,14 +121,39 @@ Available graph names: - `voice_assistant_xai_asr` - xAI STT + OpenAI-compatible LLM + ElevenLabs TTS - `voice_assistant_xai_tts` - Deepgram STT + OpenAI-compatible LLM + xAI TTS - `voice_assistant_xai_full` - xAI STT + OpenAI-compatible LLM + xAI TTS +- `voice_assistant_jev_router` - Deepgram STT + Jev thinking-mode routing + + DeepSeek Flash chatbot + ElevenLabs TTS Examples: ```text http://localhost:3000/?graph=voice_assistant_xai_full https://ten-demo.agora.io/?graph=voice_assistant_xai_full +http://localhost:3000/?graph=voice_assistant_jev_router ``` +The Jev graph is opt-in; the default `voice_assistant` graph is unchanged. +Its Deepgram Nova-3 ASR uses a 500 ms endpointing window. Stable `is_final` +segments are accumulated, and only `speech_final` submits the complete +utterance to Jev and DeepSeek. The default graph retains segment-final turn +handling. Barge-in behavior is unchanged. +Before each routed reply, the chat shows a separate, non-spoken Jev route +record with the choice, confidence (when available), selected LLM, latency, +and whether an error or timeout caused a fallback. +Jev evaluates the final English ASR text with a `fast`/`deep` choice. A `fast` +choice with confidence at least 0.8 uses Flash without thinking; `deep` or +an uncertain `fast` choice uses Flash with high-effort thinking. Jev errors, +invalid responses, and timeouts fall back to Flash without thinking to keep +the voice response responsive. The Jev request has a 1000 ms deadline and +the controller waits at most 1200 ms. These thresholds are demo starting +points and should be calibrated with real requests before production use. +When thinking is selected, the complete DeepSeek reasoning is shown in a +separate chat message before the final answer; it is never sent to TTS. This +graph registers no tools. +Both LLM nodes are pinned to `deepseek-flash`; the old +`DEEPSEEK_FAST_MODEL` and `DEEPSEEK_DEEP_MODEL` values do not override this +graph. + ## Configuration The voice assistant is configured in `tenapp/property.json`: diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json index 25e1b7ce14..d5332aa46b 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json @@ -440,6 +440,19 @@ ], "path": "../../../ten_packages/extension/openai_llm2_python" }, + { + "type": "extension", + "name": "typesafe_jev_python", + "version": "0.1.0", + "hash": "4533399d37b1691ec3600dc37cd9d50dcb249ff559099d0a53b298691fd8998d", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python" + } + ], + "path": "../../../ten_packages/extension/typesafe_jev_python" + }, { "type": "extension", "name": "anthropic_llm2_python", diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json index 03f83ddad0..1ad19e7860 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json @@ -87,6 +87,9 @@ { "path": "../../../ten_packages/extension/openai_llm2_python" }, + { + "path": "../../../ten_packages/extension/typesafe_jev_python" + }, { "path": "../../../ten_packages/extension/anthropic_llm2_python" }, @@ -181,4 +184,4 @@ "scripts": { "start": "scripts/start.sh" } -} \ No newline at end of file +} diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/property.json index 3b6fe0c15f..67e8f8c799 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/property.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/property.json @@ -42,8 +42,7 @@ "addon": "openai_llm2_python", "extension_group": "chatgpt", "property": { - "base_url": "https://api.openai.com/v1", - "api_key": "${env:OPENAI_API_KEY}", + "base_url": "${env:OPENAI_API_BASE|https://api.openai.com/v1}", "frequency_penalty": 0.9, "model": "${env:OPENAI_MODEL}", "max_tokens": 512, @@ -1779,6 +1778,170 @@ } ] } + }, + { + "name": "voice_assistant_jev_router", + "auto_start": false, + "graph": { + "nodes": [ + { + "type": "extension", + "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "app_certificate": "${env:AGORA_APP_CERTIFICATE|}", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false + } + }, + { + "type": "extension", + "name": "stt", + "addon": "deepgram_asr_python", + "extension_group": "stt", + "property": { + "params": { + "api_key": "${env:DEEPGRAM_API_KEY}", + "language": "en-US", + "model": "nova-3", + "endpointing": 500 + } + } + }, + { + "type": "extension", + "name": "llm_fast", + "addon": "openai_llm2_python", + "extension_group": "chatgpt_fast", + "property": { + "base_url": "https://api.deepseek.com", + "api_key": "${env:DEEPSEEK_API_KEY}", + "model": "deepseek-flash", + "max_tokens": 512, + "prompt": "You are a voice assistant. Give concise spoken answers unless the user asks for detail. Never reveal internal reasoning.", + "proxy_url": "${env:DEEPSEEK_PROXY_URL|}" + } + }, + { + "type": "extension", + "name": "llm_deep", + "addon": "openai_llm2_python", + "extension_group": "chatgpt_deep", + "property": { + "base_url": "https://api.deepseek.com", + "api_key": "${env:DEEPSEEK_API_KEY}", + "model": "deepseek-flash", + "max_tokens": 8192, + "prompt": "You are a voice assistant. Think carefully, then give a concise spoken answer unless the user asks for detail. Never reveal internal reasoning.", + "proxy_url": "${env:DEEPSEEK_PROXY_URL|}" + } + }, + { + "type": "extension", + "name": "jev", + "addon": "typesafe_jev_python", + "extension_group": "decision", + "property": { + "params": { + "api_key": "${env:TYPESAFE_API_KEY}", + "model": "jev-1.13.0", + "timeout_ms": 1000 + } + } + }, + { + "type": "extension", + "name": "tts", + "addon": "elevenlabs_tts2_python", + "extension_group": "tts", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "key": "${env:ELEVENLABS_TTS_KEY}", + "model_id": "eleven_multilingual_v2", + "voice_id": "pNInz6obpgDQGcFmaJgB", + "output_format": "pcm_16000" + } + } + }, + { + "type": "extension", + "name": "main_control", + "addon": "main_python", + "extension_group": "control", + "property": { + "greeting": "TEN Agent connected. How can I help you today?", + "turn_detection_mode": "speech_final", + "model_routing": { + "enabled": true, + "decision_dest": "jev", + "fast_dest": "llm_fast", + "deep_dest": "llm_deep", + "min_confidence": 0.8, + "timeout_ms": 1200, + "question": { + "type": "choice", + "instructions": "Does the latest user request need deliberate multi-step reasoning before a concise spoken answer? Do not judge by length alone.", + "criteria": { + "fast": "Greetings, small talk, and direct questions that can be answered without extended reasoning", + "deep": "Multi-step reasoning, planning with tradeoffs, or nuanced analysis that benefits from deliberate thought" + } + } + } + } + }, + { + "type": "extension", + "name": "message_collector", + "addon": "message_collector2", + "extension_group": "transcriber", + "property": {} + }, + { + "type": "extension", + "name": "streamid_adapter", + "addon": "streamid_adapter", + "property": {} + } + ], + "connections": [ + { + "extension": "main_control", + "cmd": [ + {"names": ["on_user_joined", "on_user_left"], "source": [{"extension": "agora_rtc"}]} + ], + "data": [ + {"name": "asr_result", "source": [{"extension": "stt"}]}, + {"name": "tts_audio_start", "source": [{"extension": "tts"}]}, + {"name": "tts_audio_end", "source": [{"extension": "tts"}]} + ] + }, + { + "extension": "agora_rtc", + "audio_frame": [ + {"name": "pcm_frame", "dest": [{"extension": "streamid_adapter"}]}, + {"name": "pcm_frame", "source": [{"extension": "tts"}]} + ], + "data": [ + {"name": "data", "source": [{"extension": "message_collector"}]} + ] + }, + { + "extension": "streamid_adapter", + "audio_frame": [ + {"name": "pcm_frame", "dest": [{"extension": "stt"}]} + ] + } + ] + } } ], "log": { diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py index 5a4b4e9b4e..29ae86337f 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py @@ -2,13 +2,25 @@ import json from typing import Awaitable, Callable, Optional from .llm_exec import LLMExec +from ..config import ModelRoutingConfig +from ..routing import RoutingDecision from ten_runtime import AsyncTenEnv, Cmd, CmdResult, Data, StatusCode from ten_ai_base.types import LLMToolMetadata -from .events import * +from .events import ( + ASRResultEvent, + AgentEvent, + LLMResponseEvent, + ModelRouteEvent, + ToolRegisterEvent, + UserJoinedEvent, + UserLeftEvent, +) class Agent: - def __init__(self, ten_env: AsyncTenEnv): + def __init__( + self, ten_env: AsyncTenEnv, routing: ModelRoutingConfig | None = None + ): self.ten_env: AsyncTenEnv = ten_env self.stopped = False @@ -28,13 +40,14 @@ def __init__(self, ten_env: AsyncTenEnv): None # currently running handler ) - self.llm_exec = LLMExec(ten_env) + self.llm_exec = LLMExec(ten_env, routing) self.llm_exec.on_response = ( self._on_llm_response ) # callback handled internally self.llm_exec.on_reasoning_response = ( self._on_llm_reasoning_response ) # callback handled internally + self.llm_exec.on_route = self._on_route # Start consumers self._asr_consumer = asyncio.create_task(self._consume_asr()) @@ -71,8 +84,6 @@ async def _dispatch(self, event: AgentEvent): for h in handlers: try: await h(event) - except asyncio.CancelledError: - raise except Exception as e: self.ten_env.log_error( f"Handler error for {etype}: {e}" @@ -156,14 +167,25 @@ async def on_data(self, data: Data): self.ten_env.log_error(f"on_data error: {e}") async def _on_llm_response( - self, ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool + self, _ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool ): await self._emit_llm( LLMResponseEvent(delta=delta, text=text, is_final=is_final) ) + async def _on_route(self, _ten_env: AsyncTenEnv, route: RoutingDecision): + await self._emit_direct( + ModelRouteEvent( + destination=route.destination, + status=route.status, + latency_ms=route.latency_ms, + choice=route.choice, + confidence=route.confidence, + ) + ) + async def _on_llm_reasoning_response( - self, ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool + self, _ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool ): """ Internal callback for streaming LLM output, wrapped as an AgentEvent. diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py index df61ec5c2f..b9a521ddb9 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py @@ -62,6 +62,18 @@ class LLMResponseEvent(AgentEventBase): is_final: bool +class ModelRouteEvent(AgentEventBase): + """The selected LLM destination for a user turn.""" + + type: Literal["data"] = "data" + name: Literal["model_route"] = "model_route" + destination: str + status: str + latency_ms: int + choice: str | None = None + confidence: float | None = None + + # ==== Unified Event Union ==== AgentEvent = Union[ @@ -70,4 +82,5 @@ class LLMResponseEvent(AgentEventBase): ToolRegisterEvent, ASRResultEvent, LLMResponseEvent, + ModelRouteEvent, ] diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py index 5e57598d1c..5640e4e686 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py @@ -25,7 +25,9 @@ ) from ten_ai_base.types import LLMToolMetadata, LLMToolResult from ..helper import _send_cmd, _send_cmd_ex -from ten_runtime import AsyncTenEnv, Loc, StatusCode +from ..config import ModelRoutingConfig +from ..routing import RoutingDecision, resolve_destination +from ten_runtime import AsyncTenEnv, StatusCode import uuid @@ -35,8 +37,11 @@ class LLMExec: This class handles the interaction with the LLM, including processing commands and data. """ - def __init__(self, ten_env: AsyncTenEnv): + def __init__( + self, ten_env: AsyncTenEnv, routing: ModelRoutingConfig | None = None + ): self.ten_env = ten_env + self.routing = routing or ModelRoutingConfig() self.input_queue = AsyncQueue() self.stopped = False self.on_response: Optional[ @@ -48,6 +53,9 @@ def __init__(self, ten_env: AsyncTenEnv): self.on_tool_call: Optional[ Callable[[AsyncTenEnv, LLMToolMetadata], Awaitable[None]] ] = None + self.on_route: Optional[ + Callable[[AsyncTenEnv, RoutingDecision], Awaitable[None]] + ] = None self.current_task: Optional[asyncio.Task] = None self.loop = asyncio.get_event_loop() self.loop.create_task(self._process_input_queue()) @@ -58,6 +66,8 @@ def __init__(self, ten_env: AsyncTenEnv): ) # Lock to ensure thread-safe access self.contexts: list[LLMMessage] = [] self.current_request_id: Optional[str] = None + self.current_decision_request_id: Optional[str] = None + self.current_llm_dest: Optional[str] = None self.current_text = None async def queue_input(self, item: str) -> None: @@ -69,14 +79,32 @@ async def flush(self) -> None: This is useful for ensuring that all pending inputs are handled before stopping. """ await self.input_queue.flush() - if self.current_request_id: - request_id = self.current_request_id - self.current_request_id = None - await _send_cmd( - self.ten_env, "abort", "llm", {"request_id": request_id} - ) + decision_id = self.current_decision_request_id + request_id = self.current_request_id + llm_dest = self.current_llm_dest + self.current_decision_request_id = None + self.current_request_id = None + self.current_llm_dest = None if self.current_task: self.current_task.cancel() + aborts = [] + if decision_id: + aborts.append( + _send_cmd( + self.ten_env, + "abort", + self.routing.decision_dest, + {"request_id": decision_id}, + ) + ) + if request_id and llm_dest: + aborts.append( + _send_cmd( + self.ten_env, "abort", llm_dest, {"request_id": request_id} + ) + ) + if aborts: + await asyncio.gather(*aborts, return_exceptions=True) async def stop(self) -> None: """ @@ -116,7 +144,7 @@ async def _process_input_queue(self): self.current_text = None if self.on_response and text: await self.on_response(self.ten_env, "", text, True) - except Exception as e: + except Exception: self.ten_env.log_error( f"Error processing input queue: {traceback.format_exc()}" ) @@ -130,7 +158,9 @@ async def _queue_context( Queue a new message to the LLM context. This method appends the new message to the existing context and sends it to the LLM. """ - ten_env.log_info(f"_queue_context: {new_message}") + ten_env.log_info( + f"_queue_context: role={getattr(new_message, 'role', 'unknown')}" + ) self.contexts.append(new_message) async def _write_context( @@ -149,37 +179,93 @@ async def _write_context( await self._queue_context(ten_env, new_message) async def _send_to_llm( - self, ten_env: AsyncTenEnv, new_message: LLMMessage + self, + ten_env: AsyncTenEnv, + new_message: LLMMessage, + llm_dest: Optional[str] = None, ) -> None: + if llm_dest is None: + llm_dest = await self._choose_llm_dest(new_message.content) messages = self.contexts.copy() messages.append(new_message) request_id = str(uuid.uuid4()) self.current_request_id = request_id + self.current_llm_dest = llm_dest + thinking = self.routing.enabled and llm_dest == self.routing.deep_dest + parameters = { + "extra_body": { + "thinking": {"type": "enabled" if thinking else "disabled"} + } + } + if thinking: + parameters["reasoning_effort"] = "high" + else: + parameters["temperature"] = 0.7 llm_input = LLMRequest( request_id=request_id, messages=messages, streaming=True, - parameters={"temperature": 0.7}, + parameters=parameters, tools=self.available_tools, ) input_json = llm_input.model_dump() - response = _send_cmd_ex(ten_env, "chat_completion", "llm", input_json) + response = _send_cmd_ex( + ten_env, "chat_completion", llm_dest, input_json + ) # Queue the new message to the context await self._queue_context(ten_env, new_message) - async for cmd_result, _ in response: - if cmd_result and cmd_result.is_final() is False: - if cmd_result.get_status_code() == StatusCode.OK: - response_json, _ = cmd_result.get_property_to_json(None) - ten_env.log_info( - f"_send_to_llm: response_json {response_json}" - ) - completion = parse_llm_response(response_json) - await self._handle_llm_response(completion) + try: + async for cmd_result, _ in response: + if cmd_result and cmd_result.is_final() is False: + if cmd_result.get_status_code() == StatusCode.OK: + response_json, _ = cmd_result.get_property_to_json(None) + completion = parse_llm_response(response_json) + await self._handle_llm_response(completion, llm_dest) + finally: + if self.current_request_id == request_id: + self.current_request_id = None + self.current_llm_dest = None + + async def _choose_llm_dest(self, text: str) -> str: + if not self.routing.enabled: + return "llm" - async def _handle_llm_response(self, llm_output: LLMResponse | None): - self.ten_env.log_info(f"_handle_llm_response: {llm_output}") + decision_id = str(uuid.uuid4()) + self.current_decision_request_id = decision_id + try: + route = await resolve_destination( + self.routing, + text, + decision_id, + lambda payload: _send_cmd( + self.ten_env, + "decision_evaluate", + self.routing.decision_dest, + payload, + ), + ) + if self.current_decision_request_id != decision_id: + raise asyncio.CancelledError + if self.on_route: + await self.on_route(self.ten_env, route) + self.ten_env.log_info( + f"Jev route: {route.status}, destination={route.destination}, " + f"latency_ms={route.latency_ms}" + ) + return route.destination + finally: + if self.current_decision_request_id == decision_id: + self.current_decision_request_id = None + + async def _handle_llm_response( + self, llm_output: LLMResponse | None, llm_dest: str + ): + if llm_output is not None: + self.ten_env.log_info( + f"_handle_llm_response: {type(llm_output).__name__}" + ) match llm_output: case LLMResponseMessageDelta(): @@ -249,6 +335,7 @@ async def _handle_llm_response(self, llm_output: LLMResponse | None): call_id=llm_output.tool_call_id, type="function_call_output", ), + llm_dest, ) else: self.ten_env.log_error( diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py index 17686708e8..a2e7b3172a 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py @@ -1,5 +1,61 @@ -from pydantic import BaseModel +from typing import Any, Literal + +from pydantic import BaseModel, Field + + +def default_route_question() -> dict[str, Any]: + return { + "type": "choice", + "instructions": ( + "Does the latest user request need deliberate multi-step " + "reasoning before a concise spoken answer? Do not judge by " + "length alone." + ), + "criteria": { + "fast": ( + "Greetings, small talk, and direct questions that can " + "be answered without extended reasoning" + ), + "deep": ( + "Multi-step reasoning, planning with tradeoffs, or " + "nuanced analysis that benefits from deliberate thought" + ), + }, + } + + +class ModelRoutingConfig(BaseModel): + enabled: bool = False + decision_dest: str = "jev" + fast_dest: str = "llm_fast" + deep_dest: str = "llm_deep" + min_confidence: float = Field(default=0.8, ge=0, le=1) + timeout_ms: int = Field(default=1200, ge=1) + question: dict[str, Any] = Field(default_factory=default_route_question) + + def select_dest(self, decision: dict[str, Any]) -> str: + """Use the fast model only for a valid, confident fast decision.""" + answers = decision.get("answers", {}) + route = answers.get("route", {}) if isinstance(answers, dict) else {} + if not isinstance(route, dict): + return self.deep_dest + confidence = route.get("confidence") + if ( + route.get("type") == "choice" + and route.get("choice") == "fast" + and isinstance(confidence, (int, float)) + and not isinstance(confidence, bool) + and self.min_confidence <= confidence <= 1 + ): + return self.fast_dest + return self.deep_dest class MainControlConfig(BaseModel): greeting: str = "Hello, I am your AI assistant." + turn_detection_mode: Literal["segment_final", "speech_final"] = ( + "segment_final" + ) + model_routing: ModelRoutingConfig = Field( + default_factory=ModelRoutingConfig + ) diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py index 923104dfb9..e3d77e3a83 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py @@ -1,4 +1,3 @@ -import asyncio import json import time from typing import Literal @@ -15,6 +14,7 @@ from .agent.events import ( ASRResultEvent, LLMResponseEvent, + ModelRouteEvent, ToolRegisterEvent, UserJoinedEvent, UserLeftEvent, @@ -42,6 +42,8 @@ def __init__(self, name: str): self.sentence_fragment: str = "" self.turn_id: int = 0 self.session_id: str = "0" + self._asr_final_segments: list[str] = [] + self._asr_interim: str = "" def _current_metadata(self) -> dict: return {"session_id": self.session_id, "turn_id": self.turn_id} @@ -53,7 +55,7 @@ async def on_init(self, ten_env: AsyncTenEnv): config_json, _ = await ten_env.get_property_to_json(None) self.config = MainControlConfig.model_validate_json(config_json) - self.agent = Agent(ten_env) + self.agent = Agent(ten_env, self.config.model_routing) # Now auto-register decorated methods for attr_name in dir(self): @@ -64,7 +66,7 @@ async def on_init(self, ten_env: AsyncTenEnv): # === Register handlers with decorators === @agent_event_handler(UserJoinedEvent) - async def _on_user_joined(self, event: UserJoinedEvent): + async def _on_user_joined(self, _event: UserJoinedEvent): self._rtc_user_count += 1 if self._rtc_user_count == 1 and self.config and self.config.greeting: await self._send_to_tts(self.config.greeting, True) @@ -73,7 +75,7 @@ async def _on_user_joined(self, event: UserJoinedEvent): ) @agent_event_handler(UserLeftEvent) - async def _on_user_left(self, event: UserLeftEvent): + async def _on_user_left(self, _event: UserLeftEvent): self._rtc_user_count -= 1 @agent_event_handler(ToolRegisterEvent) @@ -82,8 +84,18 @@ async def _on_tool_register(self, event: ToolRegisterEvent): @agent_event_handler(ASRResultEvent) async def _on_asr_result(self, event: ASRResultEvent): - self.session_id = event.metadata.get("session_id", "100") + session_id = event.metadata.get("session_id", "100") + if ( + self.config.turn_detection_mode == "speech_final" + and session_id != self.session_id + ): + self._asr_final_segments.clear() + self._asr_interim = "" + self.session_id = session_id stream_id = int(self.session_id) + if self.config.turn_detection_mode == "speech_final": + await self._on_asr_result_speech_final(event, stream_id) + return if not event.text: return if event.final or len(event.text) > 2: @@ -93,8 +105,46 @@ async def _on_asr_result(self, event: ASRResultEvent): await self.agent.queue_llm_input(event.text) await self._send_transcript("user", event.text, event.final, stream_id) + async def _on_asr_result_speech_final( + self, event: ASRResultEvent, stream_id: int + ): + asr_info = event.metadata.get("asr_info") or {} + speech_final = ( + isinstance(asr_info, dict) and asr_info.get("speech_final") is True + ) + if event.text: + # Keep the existing barge-in policy separate from turn detection. + if event.final or len(event.text) > 2: + await self._interrupt() + if event.final: + self._asr_final_segments.append(event.text) + self._asr_interim = "" + else: + self._asr_interim = event.text + + segments = self._asr_final_segments.copy() + if self._asr_interim: + segments.append(self._asr_interim) + utterance = " ".join( + segment.strip() for segment in segments if segment.strip() + ) + + if speech_final: + self._asr_final_segments.clear() + self._asr_interim = "" + if utterance: + self.turn_id += 1 + await self.agent.queue_llm_input(utterance) + await self._send_transcript("user", utterance, True, stream_id) + elif event.text and utterance: + await self._send_transcript("user", utterance, False, stream_id) + @agent_event_handler(LLMResponseEvent) async def _on_llm_response(self, event: LLMResponseEvent): + # The collector queues every snapshot as a new message. Send the + # complete reasoning once so a long chain cannot delay the answer. + if event.type == "reasoning" and not event.is_final: + return if not event.is_final and event.type == "message": sentences, self.sentence_fragment = parse_sentences( self.sentence_fragment, event.delta @@ -115,18 +165,62 @@ async def _on_llm_response(self, event: LLMResponseEvent): data_type=("reasoning" if event.type == "reasoning" else "text"), ) - async def on_start(self, ten_env: AsyncTenEnv): - ten_env.log_info("[MainControlExtension] on_start") + @agent_event_handler(ModelRouteEvent) + async def _on_model_route(self, event: ModelRouteEvent): + if event.choice: + confidence = ( + f" (confidence {event.confidence:.2f})" + if event.confidence is not None + else "" + ) + result = f"{event.choice}{confidence}" + else: + result = event.status.replace("_", " ") + fallback = ( + " (fallback)" + if event.status + not in ( + "selected_fast", + "selected_deep", + ) + else "" + ) + mode = ( + "Flash · thinking high" + if event.destination == self.config.model_routing.deep_dest + else "Flash · no thinking" + ) + label = ( + f"Jev route: {result} → {mode}{fallback} · {event.latency_ms} ms" + ) + await _send_data( + self.ten_env, + "message", + "message_collector", + { + "data_type": "raw", + "role": "assistant", + "text": json.dumps( + {"type": "routing", "data": {"text": label}} + ), + "text_ts": int(time.time() * 1000), + "is_final": True, + "stream_id": 100, + }, + ) + + async def on_start(self, _ten_env: AsyncTenEnv): + self.ten_env.log_info("[MainControlExtension] on_start") - async def on_stop(self, ten_env: AsyncTenEnv): - ten_env.log_info("[MainControlExtension] on_stop") + async def on_stop(self, _ten_env: AsyncTenEnv): + self.ten_env.log_info("[MainControlExtension] on_stop") self.stopped = True await self.agent.stop() - async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd): + async def on_cmd(self, _ten_env: AsyncTenEnv, cmd: Cmd): await self.agent.on_cmd(cmd) - async def on_data(self, ten_env: AsyncTenEnv, data: Data): + async def on_data(self, _ten_env: AsyncTenEnv, data: Data): await self.agent.on_data(data) # === helpers === @@ -177,7 +271,8 @@ async def _send_transcript( }, ) self.ten_env.log_info( - f"[MainControlExtension] Sent transcript: {role}, final={final}, text={text}" + f"[MainControlExtension] Sent transcript: role={role}, " + f"type={data_type}, final={final}, length={len(text)}" ) async def _send_to_tts(self, text: str, is_final: bool): diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json index ccd3dbb7f9..81cc5a4dbd 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json @@ -29,6 +29,18 @@ "properties": { "greeting": { "type": "string" + }, + "model_routing": { + "type": "object", + "properties": { + "enabled": {"type": "bool"}, + "decision_dest": {"type": "string"}, + "fast_dest": {"type": "string"}, + "deep_dest": {"type": "string"}, + "min_confidence": {"type": "float64"}, + "timeout_ms": {"type": "int32"}, + "question": {"type": "object", "properties": {}} + } } } } diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/routing.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/routing.py new file mode 100644 index 0000000000..b422aef72d --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/routing.py @@ -0,0 +1,93 @@ +"""Provider-independent selection of a target LLM extension.""" + +import asyncio +import json +import math +import time +from dataclasses import dataclass +from typing import Any, Awaitable, Callable + +from .config import ModelRoutingConfig + + +@dataclass(frozen=True) +class RoutingDecision: + destination: str + status: str + latency_ms: int + choice: str | None = None + confidence: float | None = None + + +async def resolve_destination( + config: ModelRoutingConfig, + text: str, + request_id: str, + send_decision: Callable[[dict[str, Any]], Awaitable[tuple[Any, Any]]], +) -> RoutingDecision: + """Select a target and retain safe metadata for route observability.""" + started = time.monotonic() + + def outcome( + destination: str, + status: str, + choice: str | None = None, + confidence: float | None = None, + ) -> RoutingDecision: + return RoutingDecision( + destination=destination, + status=status, + latency_ms=round((time.monotonic() - started) * 1000), + choice=choice, + confidence=confidence, + ) + + if not config.enabled: + return outcome("llm", "disabled") + + payload = { + "request_id": request_id, + "state": {"latest_user_text": text}, + "questions": {"route": config.question}, + } + try: + result, error = await asyncio.wait_for( + send_decision(payload), timeout=config.timeout_ms / 1000 + ) + if error or not result or result.get_status_code() != 0: + return outcome(config.fast_dest, "provider_error") + response_json, error = result.get_property_to_json(None) + if error: + return outcome(config.fast_dest, "invalid_response") + decision = json.loads(response_json) + if not isinstance(decision, dict): + return outcome(config.fast_dest, "invalid_response") + if decision.get("request_id") != request_id: + return outcome(config.fast_dest, "stale_decision") + answers = decision.get("answers") + route = answers.get("route") if isinstance(answers, dict) else None + if not isinstance(route, dict) or route.get("type") != "choice": + return outcome(config.fast_dest, "invalid_response") + choice = route.get("choice") + if choice not in ("fast", "deep"): + return outcome(config.fast_dest, "invalid_response") + raw_confidence = route.get("confidence") + confidence = ( + float(raw_confidence) + if isinstance(raw_confidence, (int, float)) + and not isinstance(raw_confidence, bool) + and math.isfinite(raw_confidence) + and 0 <= raw_confidence <= 1 + else None + ) + destination = config.select_dest(decision) + if choice == "fast" and destination == config.fast_dest: + return outcome(destination, "selected_fast", choice, confidence) + if choice == "deep": + return outcome(destination, "selected_deep", choice, confidence) + return outcome(destination, "below_threshold", choice, confidence) + except asyncio.TimeoutError: + return outcome(config.fast_dest, "timeout") + except Exception: + # Cancellation is a BaseException and still propagates to the caller. + return outcome(config.fast_dest, "provider_error") diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py new file mode 100644 index 0000000000..0a5b30d167 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py @@ -0,0 +1,69 @@ +"""Static checks for the optional Jev voice-assistant graph.""" + +import json +from pathlib import Path + + +TENAPP = Path(__file__).resolve().parents[1] / "tenapp" + + +def test_jev_graph_is_optional_and_has_required_nodes(): + app = json.loads((TENAPP / "property.json").read_text()) + graphs = {graph["name"]: graph for graph in app["ten"]["predefined_graphs"]} + assert graphs["voice_assistant"]["auto_start"] is True + assert graphs["voice_assistant_jev_router"]["auto_start"] is False + default_llm = next( + node + for node in graphs["voice_assistant"]["graph"]["nodes"] + if node["name"] == "llm" + ) + assert default_llm["property"]["model"] == "${env:OPENAI_MODEL}" + assert default_llm["property"]["base_url"] == ( + "${env:OPENAI_API_BASE|https://api.openai.com/v1}" + ) + + nodes = { + node["name"]: node + for node in graphs["voice_assistant_jev_router"]["graph"]["nodes"] + } + assert nodes["jev"]["addon"] == "typesafe_jev_python" + assert nodes["llm_fast"]["addon"] == "openai_llm2_python" + assert nodes["llm_deep"]["addon"] == "openai_llm2_python" + assert nodes["stt"]["property"]["params"]["language"] == "en-US" + assert nodes["stt"]["property"]["params"]["endpointing"] == 500 + assert nodes["jev"]["property"]["params"]["api_key"] == ( + "${env:TYPESAFE_API_KEY}" + ) + for node_name in ("llm_fast", "llm_deep"): + properties = nodes[node_name]["property"] + assert properties["base_url"] == "https://api.deepseek.com" + assert properties["api_key"] == "${env:DEEPSEEK_API_KEY}" + assert properties["proxy_url"] == "${env:DEEPSEEK_PROXY_URL|}" + assert nodes["llm_fast"]["property"]["model"] == "deepseek-flash" + assert nodes["llm_deep"]["property"]["model"] == "deepseek-flash" + assert nodes["llm_deep"]["property"]["max_tokens"] > 512 + assert "weatherapi_tool_python" not in nodes + routing = nodes["main_control"]["property"]["model_routing"] + assert ( + nodes["main_control"]["property"]["turn_detection_mode"] + == "speech_final" + ) + assert routing["enabled"] is True + assert routing["decision_dest"] == "jev" + assert routing["fast_dest"] == "llm_fast" + assert routing["deep_dest"] == "llm_deep" + main_cmds = next( + connection["cmd"] + for connection in graphs["voice_assistant_jev_router"]["graph"][ + "connections" + ] + if connection["extension"] == "main_control" + ) + assert all("tool_register" not in cmd["names"] for cmd in main_cmds) + + +def test_jev_dependency_is_declared(): + app = json.loads((TENAPP / "manifest.json").read_text()) + assert { + "path": "../../../ten_packages/extension/typesafe_jev_python" + } in app["dependencies"] diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py new file mode 100644 index 0000000000..62b389021c --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py @@ -0,0 +1,287 @@ +"""Controller tests for interruption and tool continuation targets.""" + +import asyncio +import importlib +import json +import sys +import types +from enum import IntEnum +from pathlib import Path + +import pytest + + +ROOT = ( + Path(__file__).resolve().parents[1] + / "tenapp/ten_packages/extension/main_python" +) +PACKAGE = "main_python_exec_test" +package = types.ModuleType(PACKAGE) +package.__path__ = [str(ROOT)] +sys.modules.setdefault(PACKAGE, package) +agent_package = types.ModuleType(f"{PACKAGE}.agent") +agent_package.__path__ = [str(ROOT / "agent")] +sys.modules.setdefault(f"{PACKAGE}.agent", agent_package) + + +class FakeMessage: + def __init__(self, **values): + self.__dict__.update(values) + + +class FakeToolCall(FakeMessage): + pass + + +class FakeQueue: + async def flush(self): + return None + + +base = types.ModuleType("ten_ai_base") +base.__path__ = [] +sys.modules.setdefault("ten_ai_base", base) +constants = types.ModuleType("ten_ai_base.const") +constants.CMD_PROPERTY_RESULT = "result" +sys.modules.setdefault("ten_ai_base.const", constants) +helpers = types.ModuleType("ten_ai_base.helper") +helpers.AsyncQueue = FakeQueue +sys.modules.setdefault("ten_ai_base.helper", helpers) +struct = types.ModuleType("ten_ai_base.struct") +for name in ( + "LLMMessage", + "LLMMessageContent", + "LLMMessageFunctionCall", + "LLMMessageFunctionCallOutput", + "LLMRequest", + "LLMResponse", + "LLMResponseMessageDelta", + "LLMResponseMessageDone", + "LLMResponseReasoningDelta", + "LLMResponseReasoningDone", +): + setattr(struct, name, type(name, (FakeMessage,), {})) +struct.LLMResponseToolCall = FakeToolCall +struct.parse_llm_response = lambda value: value +sys.modules.setdefault("ten_ai_base.struct", struct) +base_types = types.ModuleType("ten_ai_base.types") +base_types.LLMToolMetadata = type("LLMToolMetadata", (), {}) +base_types.LLMToolResult = dict +sys.modules.setdefault("ten_ai_base.types", base_types) + + +class StatusCode(IntEnum): + OK = 0 + ERROR = 1 + + +runtime = sys.modules.get("ten_runtime") or types.ModuleType("ten_runtime") +runtime.AsyncTenEnv = type("AsyncTenEnv", (), {}) +runtime.StatusCode = StatusCode +sys.modules.setdefault("ten_runtime", runtime) + +helper = types.ModuleType(f"{PACKAGE}.helper") +helper._send_cmd = None +helper._send_cmd_ex = None +sys.modules.setdefault(f"{PACKAGE}.helper", helper) +LLMExec = importlib.import_module(f"{PACKAGE}.agent.llm_exec").LLMExec +llm_module = sys.modules[f"{PACKAGE}.agent.llm_exec"] +ModelRoutingConfig = importlib.import_module( + f"{PACKAGE}.config" +).ModelRoutingConfig + + +class FakeEnv: + def log_info(self, _message): + pass + + def log_error(self, _message): + pass + + +@pytest.mark.asyncio +async def test_flush_aborts_jev_and_selected_llm(monkeypatch): + calls = [] + + async def send(_env, name, dest, payload): + calls.append((name, dest, payload["request_id"])) + return None, None + + monkeypatch.setattr(llm_module, "_send_cmd", send) + executor = object.__new__(LLMExec) + executor.ten_env = FakeEnv() + executor.routing = ModelRoutingConfig(enabled=True) + executor.input_queue = FakeQueue() + executor.current_decision_request_id = "decision-1" + executor.current_request_id = "llm-1" + executor.current_llm_dest = "llm_fast" + executor.current_task = asyncio.create_task(asyncio.sleep(10)) + + await executor.flush() + assert sorted(calls) == [ + ("abort", "jev", "decision-1"), + ("abort", "llm_fast", "llm-1"), + ] + assert executor.current_task.cancelled() + + +@pytest.mark.asyncio +async def test_tool_continuation_keeps_selected_llm(monkeypatch): + class ToolResult: + def get_status_code(self): + return StatusCode.OK + + def get_property_to_json(self, _path): + return json.dumps({"type": "llmresult", "content": "sunny"}), None + + async def send(_env, _name, _dest, _payload): + return ToolResult(), None + + monkeypatch.setattr(llm_module, "_send_cmd", send) + executor = object.__new__(LLMExec) + executor.ten_env = FakeEnv() + executor.tool_registry = {"weather": "weather_tool"} + executor.contexts = [] + continuation = [] + + async def send_to_llm(_env, message, dest): + continuation.append((message, dest)) + + executor._send_to_llm = send_to_llm + await executor._handle_llm_response( + FakeToolCall( + name="weather", + arguments={}, + tool_call_id="call-1", + response_id="response-1", + ), + "llm_fast", + ) + assert len(continuation) == 1 + assert continuation[0][1] == "llm_fast" + + +@pytest.mark.asyncio +async def test_route_callback_precedes_llm_selection(monkeypatch): + class DecisionResult: + def get_status_code(self): + return StatusCode.OK + + def get_property_to_json(self, _path): + return ( + json.dumps( + { + "request_id": "decision-1", + "answers": { + "route": { + "type": "choice", + "choice": "fast", + "confidence": 0.92, + } + }, + } + ), + None, + ) + + async def send(_env, name, dest, _payload): + assert (name, dest) == ("decision_evaluate", "jev") + return DecisionResult(), None + + monkeypatch.setattr(llm_module, "_send_cmd", send) + executor = object.__new__(LLMExec) + executor.ten_env = FakeEnv() + executor.routing = ModelRoutingConfig(enabled=True) + executor.current_decision_request_id = None + observed = [] + + async def on_route(_env, route): + observed.append(route) + + executor.on_route = on_route + monkeypatch.setattr(llm_module.uuid, "uuid4", lambda: "decision-1") + destination = await executor._choose_llm_dest("Hello") + assert destination == "llm_fast" + assert len(observed) == 1 + assert observed[0].destination == destination + assert observed[0].status == "selected_fast" + assert executor.current_decision_request_id is None + + +@pytest.mark.asyncio +async def test_interrupted_decision_does_not_emit_route(monkeypatch): + started = asyncio.Event() + release = asyncio.Event() + + async def send(_env, _name, _dest, _payload): + started.set() + await release.wait() + return None, RuntimeError("stale") + + monkeypatch.setattr(llm_module, "_send_cmd", send) + executor = object.__new__(LLMExec) + executor.ten_env = FakeEnv() + executor.routing = ModelRoutingConfig(enabled=True) + executor.current_decision_request_id = None + observed = [] + + async def on_route(_env, route): + observed.append(route) + + executor.on_route = on_route + pending = asyncio.create_task(executor._choose_llm_dest("Hello")) + await started.wait() + executor.current_decision_request_id = None + release.set() + with pytest.raises(asyncio.CancelledError): + await pending + assert observed == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "enabled,destination,thinking_type,effort", + [ + (True, "llm_fast", "disabled", None), + (True, "llm_deep", "enabled", "high"), + (False, "llm", "disabled", None), + ], +) +async def test_llm_request_uses_selected_thinking_mode( + monkeypatch, enabled, destination, thinking_type, effort +): + class Request(FakeMessage): + def model_dump(self): + return self.__dict__ + + async def no_results(): + if False: + yield None + + captured = [] + + def send(_env, name, dest, payload): + captured.append((name, dest, payload)) + return no_results() + + monkeypatch.setattr(llm_module, "LLMRequest", Request) + monkeypatch.setattr(llm_module, "_send_cmd_ex", send) + executor = object.__new__(LLMExec) + executor.ten_env = FakeEnv() + executor.routing = ModelRoutingConfig(enabled=enabled) + executor.contexts = [] + executor.available_tools = [] + executor.current_request_id = None + executor.current_llm_dest = None + await executor._send_to_llm( + executor.ten_env, + FakeMessage(role="user", content="Hello"), + destination, + ) + assert len(captured) == 1 + name, dest, payload = captured[0] + assert (name, dest) == ("chat_completion", destination) + params = payload["parameters"] + assert params["extra_body"]["thinking"]["type"] == thinking_type + assert params.get("reasoning_effort") == effort + assert payload["tools"] == [] diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py new file mode 100644 index 0000000000..b3d2d31e43 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py @@ -0,0 +1,218 @@ +"""The optional thinking graph shows reasoning without reading it aloud.""" + +import importlib +import json +import sys +import types +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + + +ROOT = ( + Path(__file__).resolve().parents[1] + / "tenapp/ten_packages/extension/main_python" +) +PACKAGE = "main_python_reasoning_test" +package = types.ModuleType(PACKAGE) +package.__path__ = [str(ROOT)] +sys.modules[PACKAGE] = package +agent_package = types.ModuleType(f"{PACKAGE}.agent") +agent_package.__path__ = [str(ROOT / "agent")] +sys.modules[f"{PACKAGE}.agent"] = agent_package + +decorators = types.ModuleType(f"{PACKAGE}.agent.decorators") +decorators.agent_event_handler = lambda _event_type: lambda fn: fn +sys.modules[decorators.__name__] = decorators + +agent = types.ModuleType(f"{PACKAGE}.agent.agent") +agent.Agent = type("Agent", (), {}) +sys.modules[agent.__name__] = agent + +events = types.ModuleType(f"{PACKAGE}.agent.events") +for name in ( + "ASRResultEvent", + "LLMResponseEvent", + "ModelRouteEvent", + "ToolRegisterEvent", + "UserJoinedEvent", + "UserLeftEvent", +): + setattr(events, name, type(name, (), {})) +sys.modules[events.__name__] = events + +messages = [] + + +async def fake_send_data(_env, _name, _destination, payload): + messages.append(payload) + + +helper = types.ModuleType(f"{PACKAGE}.helper") +helper._send_cmd = None +helper._send_data = fake_send_data +helper.parse_sentences = None +sys.modules[helper.__name__] = helper + +config = types.ModuleType(f"{PACKAGE}.config") +config.MainControlConfig = type("MainControlConfig", (), {}) +sys.modules[config.__name__] = config + +runtime = sys.modules.get("ten_runtime") or types.ModuleType("ten_runtime") +runtime.AsyncExtension = type( + "AsyncExtension", (), {"__init__": lambda self, _name: None} +) +runtime.AsyncTenEnv = type("AsyncTenEnv", (), {}) +runtime.Cmd = type("Cmd", (), {}) +runtime.Data = type("Data", (), {}) +sys.modules["ten_runtime"] = runtime + +MainControlExtension = importlib.import_module( + f"{PACKAGE}.extension" +).MainControlExtension + + +@pytest.mark.asyncio +async def test_complete_reasoning_is_visible_but_never_sent_to_tts(): + messages.clear() + extension = MainControlExtension("main_control") + extension.ten_env = types.SimpleNamespace(log_info=lambda _message: None) + extension._send_to_tts = AsyncMock() + + await extension._on_llm_response( + types.SimpleNamespace( + type="reasoning", delta="first", text="first", is_final=False + ) + ) + assert messages == [] + + await extension._on_llm_response( + types.SimpleNamespace( + type="reasoning", + delta="", + text="first\nsecond", + is_final=True, + ) + ) + await extension._on_llm_response( + types.SimpleNamespace( + type="message", delta="", text="Answer", is_final=True + ) + ) + + assert json.loads(messages[0]["text"]) == { + "type": "reasoning", + "data": {"text": "first\nsecond"}, + } + assert messages[0]["is_final"] is True + assert messages[1]["data_type"] == "transcribe" + assert messages[1]["text"] == "Answer" + extension._send_to_tts.assert_awaited_once_with("", True) + + +def make_asr_event(text, *, final, speech_final, session_id="100"): + return types.SimpleNamespace( + text=text, + final=final, + metadata={ + "session_id": session_id, + "asr_info": {"speech_final": speech_final}, + }, + ) + + +def make_asr_extension(mode="speech_final"): + messages.clear() + extension = MainControlExtension("main_control") + extension.ten_env = types.SimpleNamespace(log_info=lambda _message: None) + extension.config = types.SimpleNamespace(turn_detection_mode=mode) + extension.agent = types.SimpleNamespace(queue_llm_input=AsyncMock()) + extension._interrupt = AsyncMock() + return extension + + +@pytest.mark.asyncio +async def test_jev_waits_for_speech_final_and_sends_complete_utterance(): + extension = make_asr_extension() + + await extension._on_asr_result( + make_asr_event("Hello", final=True, speech_final=False) + ) + await extension._on_asr_result( + make_asr_event("world", final=False, speech_final=False) + ) + extension.agent.queue_llm_input.assert_not_awaited() + + await extension._on_asr_result( + make_asr_event("world", final=True, speech_final=True) + ) + + extension.agent.queue_llm_input.assert_awaited_once_with("Hello world") + assert extension.turn_id == 1 + assert [message["text"] for message in messages] == [ + "Hello", + "Hello world", + "Hello world", + ] + assert [message["is_final"] for message in messages] == [ + False, + False, + True, + ] + + +@pytest.mark.asyncio +async def test_empty_speech_final_commits_buffered_segments(): + extension = make_asr_extension() + await extension._on_asr_result( + make_asr_event("Ready", final=True, speech_final=False) + ) + await extension._on_asr_result( + make_asr_event("", final=False, speech_final=True) + ) + + extension.agent.queue_llm_input.assert_awaited_once_with("Ready") + assert messages[-1]["text"] == "Ready" + assert messages[-1]["is_final"] is True + + +@pytest.mark.asyncio +async def test_speech_final_can_commit_current_interim_segment(): + extension = make_asr_extension() + await extension._on_asr_result( + make_asr_event("Hello", final=False, speech_final=True) + ) + + extension.agent.queue_llm_input.assert_awaited_once_with("Hello") + assert messages[-1]["is_final"] is True + + +@pytest.mark.asyncio +async def test_speech_final_resets_buffer_for_the_next_utterance(): + extension = make_asr_extension() + await extension._on_asr_result( + make_asr_event("First", final=True, speech_final=True) + ) + await extension._on_asr_result( + make_asr_event("Second", final=True, speech_final=True) + ) + + assert [ + call.args[0] for call in extension.agent.queue_llm_input.await_args_list + ] == [ + "First", + "Second", + ] + assert extension.turn_id == 2 + + +@pytest.mark.asyncio +async def test_default_mode_keeps_segment_final_behavior(): + extension = make_asr_extension("segment_final") + await extension._on_asr_result( + make_asr_event("Hello", final=True, speech_final=False) + ) + + extension.agent.queue_llm_input.assert_awaited_once_with("Hello") + assert messages[-1]["is_final"] is True diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py new file mode 100644 index 0000000000..f8356f4e8a --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py @@ -0,0 +1,134 @@ +"""Tests for optional model routing without loading the native TEN runtime.""" + +import asyncio +import importlib +import json +import sys +import types +from pathlib import Path + +import pytest + + +PACKAGE = "main_python_routing_test" +package = types.ModuleType(PACKAGE) +package.__path__ = [ + str( + Path(__file__).resolve().parents[1] + / "tenapp/ten_packages/extension/main_python" + ) +] +sys.modules.setdefault(PACKAGE, package) +config_module = importlib.import_module(f"{PACKAGE}.config") +routing_module = importlib.import_module(f"{PACKAGE}.routing") +ModelRoutingConfig = config_module.ModelRoutingConfig +resolve_destination = routing_module.resolve_destination + + +class FakeResult: + def __init__(self, payload, status=0): + self.payload = payload + self.status = status + + def get_status_code(self): + return self.status + + def get_property_to_json(self, _path): + return json.dumps(self.payload), None + + +def decision(choice, confidence, request_id="turn-1"): + return { + "request_id": request_id, + "answers": { + "route": { + "type": "choice", + "choice": choice, + "confidence": confidence, + } + }, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "answer,expected,status", + [ + (decision("fast", 0.9), "llm_fast", "selected_fast"), + (decision("fast", 0.79), "llm_deep", "below_threshold"), + (decision("deep", 0.99), "llm_deep", "selected_deep"), + (decision("fast", 0.99, "old-turn"), "llm_fast", "stale_decision"), + (decision("fast", True), "llm_deep", "below_threshold"), + ], +) +async def test_route_selection(answer, expected, status): + config = ModelRoutingConfig(enabled=True) + + async def send(payload): + assert payload["questions"]["route"]["type"] == "choice" + assert payload["state"]["latest_user_text"] == "Hello" + return FakeResult(answer), None + + route = await resolve_destination(config, "Hello", "turn-1", send) + assert route.destination == expected + assert route.status == status + assert route.latency_ms >= 0 + if status == "stale_decision": + assert route.choice is None + else: + assert route.choice == answer["answers"]["route"]["choice"] + assert route.confidence == ( + None + if isinstance(answer["answers"]["route"]["confidence"], bool) + else answer["answers"]["route"]["confidence"] + ) + + +@pytest.mark.asyncio +async def test_provider_error_and_timeout_fall_back_to_fast(): + config = ModelRoutingConfig(enabled=True, timeout_ms=1) + + async def failed(_payload): + return None, RuntimeError("offline") + + async def slow(_payload): + await asyncio.sleep(0.1) + + provider_error = await resolve_destination( + config, "Hello", "turn-1", failed + ) + assert provider_error.destination == "llm_fast" + assert provider_error.status == "provider_error" + timeout = await resolve_destination(config, "Hello", "turn-1", slow) + assert timeout.destination == "llm_fast" + assert timeout.status == "timeout" + + +@pytest.mark.asyncio +async def test_cancellation_never_starts_fallback_request(): + config = ModelRoutingConfig(enabled=True) + started = asyncio.Event() + + async def pending(_payload): + started.set() + await asyncio.Event().wait() + + task = asyncio.create_task( + resolve_destination(config, "Hello", "turn-1", pending) + ) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +@pytest.mark.asyncio +async def test_routing_disabled_keeps_original_target(): + async def unexpected(_payload): + pytest.fail("disabled routing must not call Jev") + + route = await resolve_destination( + ModelRoutingConfig(), "Hello", "turn-1", unexpected + ) + assert route.destination == "llm" + assert route.status == "disabled" diff --git a/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py b/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py index e168075dad..596f620bac 100644 --- a/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py +++ b/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py @@ -70,9 +70,7 @@ class OpenAIChatGPT: def __init__(self, ten_env: AsyncTenEnv, config: OpenAILLM2Config): self.config = config self.ten_env = ten_env - ten_env.log_info( - f"OpenAIChatGPT initialized with config: {config.api_key}" - ) + ten_env.log_info("OpenAIChatGPT initialized") self.http_client = None if config.proxy_url: ten_env.log_info(f"Setting httpx proxy: {config.proxy_url}") @@ -236,10 +234,15 @@ async def get_chat_completions( for key, value in (request_input.parameters or {}).items(): # Check if it's a valid option and not in black list if not self.config.is_black_list_params(key): - self.ten_env.log_debug(f"set openai param: {key} = {value}") + self.ten_env.log_debug(f"set openai param: {key}") req[key] = value - self.ten_env.log_info(f"Requesting chat completions with: {req}") + thinking_config = (req.get("extra_body") or {}).get("thinking") or {} + self.ten_env.log_info( + f"Requesting chat completions: model={self.config.model}, " + f"messages={len(req['messages'])}, thinking=" + f"{thinking_config.get('type', 'default')}" + ) try: response: AsyncStream[ChatCompletionChunk] = ( @@ -263,15 +266,12 @@ async def get_chat_completions( last_chat_completion: ChatCompletionChunk | None = None async for chat_completion in response: - self.ten_env.log_debug(f"Chat completion: {chat_completion}") if chat_completion is None or len(chat_completion.choices) == 0: continue last_chat_completion = chat_completion choice = chat_completion.choices[0] delta = choice.delta - self.ten_env.log_debug(f"Processing choice: {choice}") - content = delta.content if delta and delta.content else "" raw_reasoning_content = ( delta.reasoning_content diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/README.md b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/README.md new file mode 100644 index 0000000000..5a4710b53f --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/README.md @@ -0,0 +1,41 @@ +# TypeSafe Jev decision extension + +This TEN extension exposes TypeSafe Jev as a reusable, text-only decision +provider. It does not generate chat responses. Configure `TYPESAFE_API_KEY` in +`ai_agents/.env` and install the extension with the containing tenapp. + +## Commands + +Send `decision_evaluate` with a JSON property object: + +```json +{ + "request_id": "turn-1", + "state": {"latest_user_text": "What time is it?"}, + "questions": { + "route": { + "type": "choice", + "instructions": "Which model should answer this request?", + "criteria": { + "fast": "Simple requests answerable directly", + "deep": "Multi-step or uncertain requests" + } + } + } +} +``` + +`state` may be a string, object, or array. Question types are `choice`, `score`, +and `noul` as defined by the [TypeSafe API](https://docs.typesafe.ai/api). +A successful `CmdResult` has `request_id`, actual `model`, keyed `answers`, +`usage`, and `latency_ms`. An error result has `request_id` when available and +a stable `error_code`; it never includes the user state or the API key. + +Send `abort` with `{"request_id":"turn-1"}` to cancel an in-flight call. The +abort result has a `cancelled` boolean; the cancelled evaluation returns an +error result. Evaluations are independent and may run concurrently. + +Configuration lives under `params`: `api_key`, `base_url`, `model`, and +`timeout_ms`. The default model is pinned to `jev-1.13.0`; the SDK is pinned to +`0.7.2`. Calls have no automatic retries and a hard deadline. Do not enable +SDK debug logging: it prints request and response bodies. diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/__init__.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/__init__.py new file mode 100644 index 0000000000..6179bff05a --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/__init__.py @@ -0,0 +1,9 @@ +"""TypeSafe Jev decision extension.""" + +# The TEN loader imports this package to register its addon. Pure decision +# tests may import the provider layer without the native TEN runtime present. +try: + from . import addon +except ModuleNotFoundError as error: + if error.name != "ten_runtime": + raise diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/addon.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/addon.py new file mode 100644 index 0000000000..e78c6059c9 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/addon.py @@ -0,0 +1,9 @@ +from ten_runtime import Addon, TenEnv, register_addon_as_extension + + +@register_addon_as_extension("typesafe_jev_python") +class TypeSafeJevAddon(Addon): + def on_create_instance(self, ten: TenEnv, name: str, context) -> None: + from .extension import TypeSafeJevExtension + + ten.on_create_instance_done(TypeSafeJevExtension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/config.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/config.py new file mode 100644 index 0000000000..de6128c416 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/config.py @@ -0,0 +1,22 @@ +"""Configuration for the Jev decision extension.""" + +from pydantic import BaseModel, Field + + +class JevParams(BaseModel): + api_key: str = "" + base_url: str = "https://api.typesafe.ai" + model: str = "jev-1.13.0" + timeout_ms: int = Field(default=1000, ge=1, le=30000) + + +class JevConfig(BaseModel): + params: JevParams = JevParams() + + def safe_summary(self) -> str: + """Return a loggable summary without credentials.""" + return ( + f"model={self.params.model}, " + f"timeout_ms={self.params.timeout_ms}, " + f"api_key_configured={bool(self.params.api_key)}" + ) diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/decision.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/decision.py new file mode 100644 index 0000000000..f3bddd55df --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/decision.py @@ -0,0 +1,127 @@ +"""Provider-facing evaluation logic, independent of the TEN runtime.""" + +import asyncio +import json +import time +from typing import Any + +from typesafe_sdk import ( + Choice, + Noul, + RetryPolicy, + Score, + TypeSafeAPIError, + TypeSafeAPITimeoutError, + TypeSafeAuthenticationError, + TypeSafeError, + TypeSafeRateLimitError, +) + + +class DecisionError(Exception): + """A safe, stable error to return through the TEN command result.""" + + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +def parse_request(payload: Any) -> tuple[str, Any, dict[str, Any]]: + """Validate a TEN payload and construct typed SDK questions.""" + if not isinstance(payload, dict): + raise DecisionError("invalid_request") + + request_id = payload.get("request_id") + state = payload.get("state") + raw_questions = payload.get("questions") + if not isinstance(request_id, str) or not request_id.strip(): + raise DecisionError("invalid_request") + if not isinstance(state, (str, dict, list)) or not state: + raise DecisionError("invalid_request") + if not isinstance(raw_questions, dict) or not raw_questions: + raise DecisionError("invalid_request") + try: + json.dumps(state, allow_nan=False) + json.dumps(raw_questions, allow_nan=False) + except (TypeError, ValueError) as error: + raise DecisionError("invalid_request") from error + + questions = {} + for name, question in raw_questions.items(): + if not isinstance(name, str) or not name.strip(): + raise DecisionError("invalid_request") + if not isinstance(question, dict) or not question.get("instructions"): + raise DecisionError("invalid_request") + kind = question.get("type") + criteria = question.get("criteria") + try: + if kind == "choice": + if ( + not isinstance(criteria, dict) + or not 2 <= len(criteria) <= 255 + ): + raise DecisionError("invalid_request") + questions[name] = Choice( + instructions=question["instructions"], criteria=criteria + ) + elif kind == "score": + if ( + not isinstance(criteria, list) + or not 2 <= len(criteria) <= 10 + ): + raise DecisionError("invalid_request") + questions[name] = Score( + instructions=question["instructions"], criteria=criteria + ) + elif kind == "noul": + if criteria is not None and not isinstance(criteria, dict): + raise DecisionError("invalid_request") + questions[name] = Noul( + instructions=question["instructions"], criteria=criteria + ) + else: + raise DecisionError("invalid_request") + except (TypeError, ValueError) as error: + raise DecisionError("invalid_request") from error + return request_id, state, questions + + +async def evaluate(client: Any, payload: Any, timeout_ms: int) -> dict: + """Evaluate a request once, with a hard end-to-end deadline.""" + request_id, state, questions = parse_request(payload) + if client is None: + raise DecisionError("not_configured") + + timeout_seconds = timeout_ms / 1000 + started = time.monotonic() + try: + response = await asyncio.wait_for( + client.system_one( + state, + questions, + retry=RetryPolicy(max_retries=0, timeout=timeout_seconds), + timeout=timeout_seconds, + ), + timeout=timeout_seconds, + ) + except TypeSafeAuthenticationError as error: + raise DecisionError("authentication_failed") from error + except TypeSafeRateLimitError as error: + raise DecisionError("rate_limited") from error + except (TypeSafeAPITimeoutError, asyncio.TimeoutError) as error: + raise DecisionError("timeout") from error + except TypeSafeAPIError as error: + raise DecisionError("provider_error") from error + except TypeSafeError as error: + raise DecisionError("provider_error") from error + + return { + "request_id": request_id, + "model": response.model, + "answers": { + name: answer.model_dump(mode="json") + for name, answer in response.answers.items() + }, + "usage": response.usage.model_dump(mode="json"), + "latency_ms": round((time.monotonic() - started) * 1000), + } diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/extension.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/extension.py new file mode 100644 index 0000000000..7807c1a400 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/extension.py @@ -0,0 +1,117 @@ +"""TEN adapter for Jev's typed decision API.""" + +import asyncio +import json +import logging +from typing import Optional + +from ten_runtime import AsyncExtension, AsyncTenEnv, Cmd, CmdResult, StatusCode +from typesafe_sdk import AsyncTypeSafeClient, TypeSafeError + +from .config import JevConfig +from .decision import DecisionError, evaluate + + +class TypeSafeJevExtension(AsyncExtension): + def __init__(self, name: str): + super().__init__(name) + self.config = JevConfig() + self.client: Optional[AsyncTypeSafeClient] = None + self.pending: dict[str, asyncio.Task] = {} + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + config_json, error = await ten_env.get_property_to_json("") + if error: + ten_env.log_error("config: failed to read Jev properties") + return + try: + self.config = JevConfig.model_validate_json(config_json) + except ValueError: + ten_env.log_error("config: invalid Jev properties") + return + + ten_env.log_info(f"config: {self.config.safe_summary()}") + if not self.config.params.api_key: + ten_env.log_error("config: Jev API key is missing") + return + + # The SDK's DEBUG level includes unredacted request/response bodies. + logging.getLogger("typesafe_sdk").setLevel(logging.INFO) + try: + self.client = AsyncTypeSafeClient( + api_key=self.config.params.api_key, + base_url=self.config.params.base_url, + model=self.config.params.model, + ) + except TypeSafeError: + ten_env.log_error("config: Jev client initialization failed") + + async def on_stop(self, _ten_env: AsyncTenEnv) -> None: + for task in tuple(self.pending.values()): + task.cancel() + if self.pending: + await asyncio.gather( + *tuple(self.pending.values()), return_exceptions=True + ) + if self.client: + await self.client.aclose() + self.client = None + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + request_id = "" + status = StatusCode.ERROR + result_payload: dict = {"error_code": "invalid_request"} + try: + payload_json, error = cmd.get_property_to_json(None) + if error: + raise DecisionError("invalid_request") + payload = json.loads(payload_json) + request_id = ( + payload.get("request_id", "") + if isinstance(payload, dict) + else "" + ) + if cmd.get_name() == "abort": + if not isinstance(request_id, str) or not request_id: + raise DecisionError("invalid_request") + task = self.pending.get(request_id) + if task: + task.cancel() + status = StatusCode.OK + result_payload = { + "request_id": request_id, + "cancelled": task is not None, + } + elif cmd.get_name() == "decision_evaluate": + if not isinstance(request_id, str) or not request_id: + raise DecisionError("invalid_request") + if request_id in self.pending: + raise DecisionError("duplicate_request_id") + task = asyncio.create_task( + evaluate( + self.client, payload, self.config.params.timeout_ms + ) + ) + self.pending[request_id] = task + try: + result_payload = await task + status = StatusCode.OK + finally: + self.pending.pop(request_id, None) + else: + raise DecisionError("unknown_command") + except asyncio.CancelledError: + result_payload = {"error_code": "cancelled"} + except DecisionError as error: + result_payload = {"error_code": error.code} + except (TypeError, ValueError, KeyError): + result_payload = {"error_code": "invalid_request"} + except Exception: + ten_env.log_error("Jev decision failed unexpectedly") + result_payload = {"error_code": "internal_error"} + + if isinstance(request_id, str) and request_id: + result_payload.setdefault("request_id", request_id) + cmd_result = CmdResult.create(status, cmd) + cmd_result.set_property_from_json(None, json.dumps(result_payload)) + await ten_env.return_result(cmd_result) diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/manifest.json b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/manifest.json new file mode 100644 index 0000000000..d9ff155136 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/manifest.json @@ -0,0 +1,39 @@ +{ + "type": "extension", + "name": "typesafe_jev_python", + "version": "0.1.0", + "dependencies": [ + {"type": "system", "name": "ten_runtime_python", "version": "0.11"} + ], + "api": { + "property": { + "properties": { + "params": { + "type": "object", + "properties": { + "api_key": {"type": "string"}, + "base_url": {"type": "string"}, + "model": {"type": "string"}, + "timeout_ms": {"type": "int32"} + } + } + } + }, + "cmd_in": [ + {"name": "decision_evaluate"}, + {"name": "abort"} + ] + }, + "scripts": {"test": "tests/bin/start"}, + "package": { + "include": [ + "manifest.json", + "property.json", + "**.py", + "README.md", + "pyproject.toml", + "requirements.txt", + "tests/**" + ] + } +} diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/property.json b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/property.json new file mode 100644 index 0000000000..9a31579f38 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/property.json @@ -0,0 +1,8 @@ +{ + "params": { + "api_key": "${env:TYPESAFE_API_KEY|}", + "base_url": "https://api.typesafe.ai", + "model": "jev-1.13.0", + "timeout_ms": 1000 + } +} diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/pyproject.toml b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/pyproject.toml new file mode 100644 index 0000000000..5a42f587d0 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "typesafe-jev-python" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["typesafe-sdk==0.7.2"] + +[project.optional-dependencies] +dev = ["pytest>=8,<9", "pytest-asyncio>=0.24,<2"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/requirements.txt b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/requirements.txt new file mode 100644 index 0000000000..94bb46532c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/requirements.txt @@ -0,0 +1 @@ +typesafe-sdk==0.7.2 diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/__init__.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/bin/start b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/bin/start new file mode 100755 index 0000000000..cd6cc98582 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/bin/start @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +export PYTHONPATH="$(cd ../../.. && pwd):${PYTHONPATH:-}" +python3 -m pytest -q tests/ "$@" diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_decision.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_decision.py new file mode 100644 index 0000000000..b35a8f8d20 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_decision.py @@ -0,0 +1,168 @@ +"""Unit tests for the provider-facing decision layer.""" + +import asyncio + +import httpx2 +import pytest +from typesafe_sdk import ( + Choice, + Noul, + Score, + TypeSafeAuthenticationError, + TypeSafeAPIError, + TypeSafeRateLimitError, +) + +from ten_packages.extension.typesafe_jev_python.decision import ( + DecisionError, + evaluate, + parse_request, +) + + +def request(questions=None): + return { + "request_id": "turn-1", + "state": {"latest_user_text": "Please help."}, + "questions": questions + or { + "route": { + "type": "choice", + "instructions": "Choose a route", + "criteria": {"fast": None, "deep": None}, + } + }, + } + + +class FakeModel: + def __init__(self, value): + self.value = value + + def model_dump(self, mode): + assert mode == "json" + return self.value + + +class FakeResponse: + model = "jev-1.13.0" + answers = { + "route": FakeModel( + { + "type": "choice", + "choice": "fast", + "confidence": 0.9, + "probabilities": {"fast": 0.95, "deep": 0.05}, + } + ) + } + usage = FakeModel({"input_tokens": 12}) + + +class FakeClient: + def __init__(self, response=None, error=None): + self.response = response or FakeResponse() + self.error = error + self.calls = [] + + async def system_one(self, state, questions, **kwargs): + self.calls.append((state, questions, kwargs)) + if self.error: + raise self.error + return self.response + + +def test_all_question_types_are_constructed(): + payload = request( + { + "route": { + "type": "choice", + "instructions": "Route?", + "criteria": {"fast": None, "deep": None}, + }, + "urgent": {"type": "noul", "instructions": "Urgent?"}, + "difficulty": { + "type": "score", + "instructions": "Difficulty?", + "criteria": ["easy", "medium", "hard"], + }, + } + ) + _, _, questions = parse_request(payload) + assert isinstance(questions["route"], Choice) + assert isinstance(questions["urgent"], Noul) + assert isinstance(questions["difficulty"], Score) + + +@pytest.mark.parametrize( + "payload", + [ + {}, + request({"route": {"type": "other", "instructions": "Route?"}}), + request( + { + "route": { + "type": "choice", + "instructions": "Route?", + "criteria": {"only_one": None}, + } + } + ), + request( + { + "difficulty": { + "type": "score", + "instructions": "Difficulty?", + "criteria": ["only_one"], + } + } + ), + {**request(), "state": ""}, + ], +) +def test_invalid_payload_is_rejected(payload): + with pytest.raises(DecisionError, match="invalid_request"): + parse_request(payload) + + +@pytest.mark.asyncio +async def test_success_returns_structured_answer_and_disables_retries(): + client = FakeClient() + result = await evaluate(client, request(), 1000) + assert result["request_id"] == "turn-1" + assert result["answers"]["route"]["confidence"] == 0.9 + assert result["model"] == "jev-1.13.0" + assert result["usage"] == {"input_tokens": 12} + assert result["latency_ms"] >= 0 + assert client.calls[0][2]["retry"].max_retries == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error,code", + [ + ( + TypeSafeAuthenticationError(401, {}, httpx2.Headers()), + "authentication_failed", + ), + (TypeSafeRateLimitError(429, {}, httpx2.Headers()), "rate_limited"), + (TypeSafeAPIError(529, {}, httpx2.Headers()), "provider_error"), + (asyncio.TimeoutError(), "timeout"), + ], +) +async def test_provider_errors_have_stable_codes(error, code): + with pytest.raises(DecisionError) as caught: + await evaluate(FakeClient(error=error), request(), 1000) + assert caught.value.code == code + + +@pytest.mark.asyncio +async def test_hard_timeout_and_missing_client(): + class SlowClient: + async def system_one(self, *_args, **_kwargs): + await asyncio.sleep(0.1) + + with pytest.raises(DecisionError, match="timeout"): + await evaluate(SlowClient(), request(), 1) + with pytest.raises(DecisionError, match="not_configured"): + await evaluate(None, request(), 1000) diff --git a/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_extension.py b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_extension.py new file mode 100644 index 0000000000..e51296449b --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typesafe_jev_python/tests/test_extension.py @@ -0,0 +1,187 @@ +"""TEN command tests with a small runtime double.""" + +import asyncio +import json +import sys +import types +from enum import IntEnum + +import pytest + + +class StatusCode(IntEnum): + OK = 0 + ERROR = 1 + + +class FakeResult: + def __init__(self, status): + self.status = status + self.payload = None + + @classmethod + def create(cls, status, _cmd): + return cls(status) + + def set_property_from_json(self, _path, value): + self.payload = json.loads(value) + + +runtime = types.ModuleType("ten_runtime") +runtime.AsyncExtension = type( + "AsyncExtension", (), {"__init__": lambda self, name: None} +) +runtime.Addon = type("Addon", (), {}) +runtime.AsyncTenEnv = type("AsyncTenEnv", (), {}) +runtime.TenEnv = type("TenEnv", (), {}) +runtime.Cmd = type("Cmd", (), {}) +runtime.CmdResult = FakeResult +runtime.StatusCode = StatusCode +runtime.register_addon_as_extension = lambda _name: lambda cls: cls +sys.modules.setdefault("ten_runtime", runtime) + +from ten_packages.extension.typesafe_jev_python import extension # noqa: E402 +from ten_packages.extension.typesafe_jev_python.config import ( + JevConfig, +) # noqa: E402 + + +class FakeCmd: + def __init__(self, name, payload): + self.name = name + self.payload = payload + + def get_name(self): + return self.name + + def get_property_to_json(self, _path): + return json.dumps(self.payload), None + + +class FakeEnv: + def __init__(self): + self.results = [] + self.logs = [] + + async def return_result(self, result): + self.results.append(result) + + def log_error(self, message): + self.logs.append(message) + + def log_info(self, message): + self.logs.append(message) + + async def get_property_to_json(self, _path): + return json.dumps({"params": {"api_key": ""}}), None + + +@pytest.mark.asyncio +async def test_abort_cancels_only_matching_request(monkeypatch): + started = asyncio.Event() + ongoing = asyncio.Event() + + async def fake_evaluate(_client, payload, _timeout): + started.set() + if payload["request_id"] == "one": + await ongoing.wait() + return {"request_id": payload["request_id"], "answers": {}} + + monkeypatch.setattr(extension, "evaluate", fake_evaluate) + addon = extension.TypeSafeJevExtension("jev") + env = FakeEnv() + first = asyncio.create_task( + addon.on_cmd(env, FakeCmd("decision_evaluate", {"request_id": "one"})) + ) + await started.wait() + await addon.on_cmd(env, FakeCmd("abort", {"request_id": "one"})) + await first + await addon.on_cmd(env, FakeCmd("decision_evaluate", {"request_id": "two"})) + assert len(env.results) == 3 + assert env.results[0].payload == {"request_id": "one", "cancelled": True} + assert env.results[1].payload == { + "error_code": "cancelled", + "request_id": "one", + } + assert env.results[2].status == StatusCode.OK + assert env.results[2].payload["request_id"] == "two" + + +@pytest.mark.asyncio +async def test_parallel_calls_and_duplicate_id(monkeypatch): + started = asyncio.Event() + release = asyncio.Event() + + async def fake_evaluate(_client, payload, _timeout): + started.set() + await release.wait() + return {"request_id": payload["request_id"], "answers": {}} + + monkeypatch.setattr(extension, "evaluate", fake_evaluate) + addon = extension.TypeSafeJevExtension("jev") + env = FakeEnv() + first = asyncio.create_task( + addon.on_cmd(env, FakeCmd("decision_evaluate", {"request_id": "one"})) + ) + await started.wait() + second = asyncio.create_task( + addon.on_cmd(env, FakeCmd("decision_evaluate", {"request_id": "two"})) + ) + await asyncio.sleep(0) + await addon.on_cmd(env, FakeCmd("decision_evaluate", {"request_id": "one"})) + assert env.results[0].payload["error_code"] == "duplicate_request_id" + assert len(addon.pending) == 2 + release.set() + await asyncio.gather(first, second) + assert {result.payload["request_id"] for result in env.results[1:]} == { + "one", + "two", + } + + +@pytest.mark.asyncio +async def test_invalid_command_returns_one_safe_error(): + addon = extension.TypeSafeJevExtension("jev") + env = FakeEnv() + await addon.on_cmd(env, FakeCmd("decision_evaluate", {"state": "secret"})) + assert len(env.results) == 1 + assert env.results[0].status == StatusCode.ERROR + assert env.results[0].payload == {"error_code": "invalid_request"} + assert "secret" not in str(env.logs) + + +@pytest.mark.asyncio +async def test_missing_key_is_reported_without_echoing_config(): + addon = extension.TypeSafeJevExtension("jev") + env = FakeEnv() + await addon.on_start(env) + await addon.on_cmd( + env, + FakeCmd( + "decision_evaluate", + { + "request_id": "one", + "state": "private user text", + "questions": { + "route": { + "type": "noul", + "instructions": "Simple?", + } + }, + }, + ), + ) + assert env.results[0].payload["error_code"] == "not_configured" + assert "private user text" not in str(env.logs) + + +def test_safe_config_summary_never_logs_secrets(): + config = JevConfig( + params={ + "api_key": "secret-api-key", + "base_url": "https://example.com/?token=secret-url-token", + } + ) + summary = config.safe_summary() + assert "secret-api-key" not in summary + assert "secret-url-token" not in summary diff --git a/ai_agents/playground/src/components/Chat/MessageList.tsx b/ai_agents/playground/src/components/Chat/MessageList.tsx index 0af02aec50..7f7437ca8b 100644 --- a/ai_agents/playground/src/components/Chat/MessageList.tsx +++ b/ai_agents/playground/src/components/Chat/MessageList.tsx @@ -1,4 +1,4 @@ -import { Bot, Brain } from "lucide-react"; +import { Bot, Brain, GitBranch } from "lucide-react"; import * as React from "react"; import { useAppSelector, useAutoScroll } from "@/common"; import { Avatar, AvatarFallback } from "@/components/ui/avatar"; @@ -19,8 +19,8 @@ export default function MessageList(props: { className?: string }) { ref={containerRef} className={cn("grow space-y-2 overflow-y-auto p-4", className)} > - {chatItems.map((item, _index) => { - return ; + {chatItems.map((item, index) => { + return ; })} ); @@ -42,6 +42,12 @@ export function MessageItem(props: { data: IChatItem }) { + ) : data.data_type === EMessageDataType.ROUTE ? ( + + + + + ) : ( @@ -50,19 +56,35 @@ export function MessageItem(props: { data: IChatItem }) { ) ) : null} -
+
{data.data_type === EMessageDataType.IMAGE ? ( chat ) : ( -

- {data.text} -

+ <> + {data.data_type === EMessageDataType.REASON && ( +

+ DeepSeek 思考过程 +

+ )} +

+ {data.text} +

+ )}
diff --git a/ai_agents/playground/src/manager/rtc/rtc.ts b/ai_agents/playground/src/manager/rtc/rtc.ts index e7d6e40f89..928d2a9447 100644 --- a/ai_agents/playground/src/manager/rtc/rtc.ts +++ b/ai_agents/playground/src/manager/rtc/rtc.ts @@ -174,8 +174,6 @@ export class RtcManager extends AGEventEmitter { private _parseData(data: any): void { const ascii = String.fromCharCode(...new Uint8Array(data)); - console.log("[test] textstream raw data", ascii); - // const { stream_id, is_final, text, text_ts, data_type, message_id, part_number, total_parts } = textstream; // if (total_parts > 0) { @@ -239,9 +237,6 @@ export class RtcManager extends AGEventEmitter { ); const { stream_id, is_final, text, text_ts, data_type, role } = JSON.parse(this.base64ToUtf8(completeMessage)); - console.log( - `[test] message_id: ${message_id} stream_id: ${stream_id}, text: ${text}, data_type: ${data_type}` - ); const isAgent = role === "assistant"; let textItem: IChatItem = { type: isAgent ? EMessageType.AGENT : EMessageType.USER, @@ -266,6 +261,12 @@ export class RtcManager extends AGEventEmitter { data_type: EMessageDataType.REASON, text: data.text, }; + } else if (type === "routing") { + textItem = { + ...textItem, + data_type: EMessageDataType.ROUTE, + text: data.text, + }; } else if (type === "action") { const { action, data: actionData } = data; if (action === "browse_website") { @@ -276,7 +277,7 @@ export class RtcManager extends AGEventEmitter { } } - if (text.trim().length > 0) { + if (textItem.text.trim().length > 0) { this.emit("textChanged", textItem); } diff --git a/ai_agents/playground/src/store/reducers/global.ts b/ai_agents/playground/src/store/reducers/global.ts index fc880709c3..f2f7ad32e7 100644 --- a/ai_agents/playground/src/store/reducers/global.ts +++ b/ai_agents/playground/src/store/reducers/global.ts @@ -95,12 +95,12 @@ export const globalSlice = createSlice({ state.rtmConnected = action.payload; }, addChatItem: (state, action: PayloadAction) => { - const { userId, text, isFinal, type, time } = action.payload; + const { userId, data_type, isFinal, time } = action.payload; const LastFinalIndex = state.chatItems.findLastIndex((el) => { - return el.userId === userId && el.isFinal; + return el.userId === userId && el.data_type === data_type && el.isFinal; }); const LastNonFinalIndex = state.chatItems.findLastIndex((el) => { - return el.userId === userId && !el.isFinal; + return el.userId === userId && el.data_type === data_type && !el.isFinal; }); const LastFinalItem = state.chatItems[LastFinalIndex]; const LastNonFinalItem = state.chatItems[LastNonFinalIndex]; @@ -108,44 +108,19 @@ export const globalSlice = createSlice({ // has last final Item if (time <= LastFinalItem.time) { // discard - console.log( - "[test] addChatItem, time < last final item, discard!:", - text, - isFinal, - type - ); return; } else { if (LastNonFinalItem) { - console.log( - "[test] addChatItem, update last item(none final):", - text, - isFinal, - type - ); state.chatItems[LastNonFinalIndex] = action.payload; } else { - console.log( - "[test] addChatItem, add new item:", - text, - isFinal, - type - ); state.chatItems.push(action.payload); } } } else { // no last final Item if (LastNonFinalItem) { - console.log( - "[test] addChatItem, update last item(none final):", - text, - isFinal, - type - ); state.chatItems[LastNonFinalIndex] = action.payload; } else { - console.log("[test] addChatItem, add new item:", text, isFinal, type); state.chatItems.push(action.payload); } } diff --git a/ai_agents/playground/src/types/index.ts b/ai_agents/playground/src/types/index.ts index 831b4d1305..883a459696 100644 --- a/ai_agents/playground/src/types/index.ts +++ b/ai_agents/playground/src/types/index.ts @@ -36,6 +36,7 @@ export enum EMessageType { export enum EMessageDataType { TEXT = "text", REASON = "reason", + ROUTE = "route", IMAGE = "image", } From e032d2316635d9a6c8f472f2212f680bfbc138fa Mon Sep 17 00:00:00 2001 From: JianYan <15005893916@163.com> Date: Wed, 30 Sep 2026 22:15:45 +0800 Subject: [PATCH 2/6] refactor: isolate jev voice router extensions (#2344) --- .../agents/examples/voice-assistant/README.md | 9 +- .../voice-assistant/tenapp/manifest-lock.json | 17 + .../voice-assistant/tenapp/manifest.json | 3 + .../voice-assistant/tenapp/property.json | 9 +- .../extension/main_jev_python/README.md | 19 + .../extension/main_jev_python/__init__.py | 6 + .../extension/main_jev_python/addon.py | 19 + .../main_jev_python/agent/__init__.py | 0 .../extension/main_jev_python/agent/agent.py | 249 ++++++++++ .../main_jev_python/agent/decorators.py | 16 + .../extension/main_jev_python/agent/events.py | 86 ++++ .../main_jev_python/agent/llm_exec.py | 362 ++++++++++++++ .../extension/main_jev_python/config.py | 61 +++ .../extension/main_jev_python/extension.py | 303 ++++++++++++ .../extension/main_jev_python/helper.py | 88 ++++ .../extension/main_jev_python/manifest.json | 51 ++ .../extension/main_jev_python/property.json | 1 + .../routing.py | 0 .../extension/main_python/agent/agent.py | 36 +- .../extension/main_python/agent/events.py | 13 - .../extension/main_python/agent/llm_exec.py | 135 +----- .../extension/main_python/config.py | 58 +-- .../extension/main_python/extension.py | 119 +---- .../extension/main_python/manifest.json | 12 - .../tests/test_jev_deepseek_streaming.py | 154 ++++++ .../voice-assistant/tests/test_jev_graph.py | 30 +- .../tests/test_jev_llm_exec.py | 46 +- .../tests/test_jev_reasoning_display.py | 82 +++- .../voice-assistant/tests/test_jev_routing.py | 4 +- .../extension/deepseek_llm2_python/README.md | 12 + .../deepseek_llm2_python/__init__.py | 8 + .../extension/deepseek_llm2_python/addon.py | 22 + .../extension/deepseek_llm2_python/client.py | 452 ++++++++++++++++++ .../deepseek_llm2_python/extension.py | 82 ++++ .../deepseek_llm2_python/manifest.json | 57 +++ .../deepseek_llm2_python/property.json | 9 + .../deepseek_llm2_python/pyproject.toml | 8 + .../deepseek_llm2_python/requirements.txt | 2 + .../deepseek_llm2_python/think_parser.py | 137 ++++++ .../extension/openai_llm2_python/openai.py | 4 +- .../src/components/Chat/MessageList.tsx | 48 +- ai_agents/playground/src/manager/rtc/rtc.ts | 13 +- .../playground/src/store/reducers/global.ts | 31 +- ai_agents/playground/src/types/index.ts | 1 - 44 files changed, 2477 insertions(+), 397 deletions(-) create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/README.md create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/__init__.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/addon.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/__init__.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/decorators.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/llm_exec.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/helper.py create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/manifest.json create mode 100644 ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/property.json rename ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/{main_python => main_jev_python}/routing.py (100%) create mode 100644 ai_agents/agents/examples/voice-assistant/tests/test_jev_deepseek_streaming.py create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/README.md create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/__init__.py create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/addon.py create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/client.py create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/extension.py create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/manifest.json create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/property.json create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/pyproject.toml create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/requirements.txt create mode 100644 ai_agents/agents/ten_packages/extension/deepseek_llm2_python/think_parser.py diff --git a/ai_agents/agents/examples/voice-assistant/README.md b/ai_agents/agents/examples/voice-assistant/README.md index 3ec840498b..272a352f99 100644 --- a/ai_agents/agents/examples/voice-assistant/README.md +++ b/ai_agents/agents/examples/voice-assistant/README.md @@ -28,7 +28,7 @@ A configurable voice assistant with real-time conversation capabilities using Ag 5. **Optional Jev routing graph**: `voice_assistant_jev_router` needs `TYPESAFE_API_KEY` and `DEEPSEEK_API_KEY`. Both routes use `deepseek-flash`: simple conversation runs without thinking, while complex requests enable - high-effort thinking. It also needs the same Deepgram and ElevenLabs keys + thinking. It also needs the same Deepgram and ElevenLabs keys as the default graph. This optional graph is a chatbot and has no tools. ### Provider-specific keys @@ -142,14 +142,15 @@ record with the choice, confidence (when available), selected LLM, latency, and whether an error or timeout caused a fallback. Jev evaluates the final English ASR text with a `fast`/`deep` choice. A `fast` choice with confidence at least 0.8 uses Flash without thinking; `deep` or -an uncertain `fast` choice uses Flash with high-effort thinking. Jev errors, +an uncertain `fast` choice uses Flash with thinking. Jev errors, invalid responses, and timeouts fall back to Flash without thinking to keep the voice response responsive. The Jev request has a 1000 ms deadline and the controller waits at most 1200 ms. These thresholds are demo starting points and should be calibrated with real requests before production use. When thinking is selected, the complete DeepSeek reasoning is shown in a -separate chat message before the final answer; it is never sent to TTS. This -graph registers no tools. +separate chat message before the final answer; it is never sent to TTS. The +route appears as a normal assistant message, so no frontend changes are +needed. This graph registers no tools. Both LLM nodes are pinned to `deepseek-flash`; the old `DEEPSEEK_FAST_MODEL` and `DEEPSEEK_DEEP_MODEL` values do not override this graph. diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json index d5332aa46b..3684b8fc2b 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json @@ -440,6 +440,23 @@ ], "path": "../../../ten_packages/extension/openai_llm2_python" }, + { + "type": "extension", + "name": "deepseek_llm2_python", + "version": "0.1.0", + "hash": "0f5e93ed9706f92ee48c5f9b26ba07941e5bd837c6ada0338f8eafa14fad9a1f", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python" + }, + { + "type": "system", + "name": "ten_ai_base" + } + ], + "path": "../../../ten_packages/extension/deepseek_llm2_python" + }, { "type": "extension", "name": "typesafe_jev_python", diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json index 1ad19e7860..ee631a668d 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json @@ -87,6 +87,9 @@ { "path": "../../../ten_packages/extension/openai_llm2_python" }, + { + "path": "../../../ten_packages/extension/deepseek_llm2_python" + }, { "path": "../../../ten_packages/extension/typesafe_jev_python" }, diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/property.json index 67e8f8c799..cc0fa7f148 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/property.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/property.json @@ -42,7 +42,8 @@ "addon": "openai_llm2_python", "extension_group": "chatgpt", "property": { - "base_url": "${env:OPENAI_API_BASE|https://api.openai.com/v1}", + "base_url": "https://api.openai.com/v1", + "api_key": "${env:OPENAI_API_KEY}", "frequency_penalty": 0.9, "model": "${env:OPENAI_MODEL}", "max_tokens": 512, @@ -1818,7 +1819,7 @@ { "type": "extension", "name": "llm_fast", - "addon": "openai_llm2_python", + "addon": "deepseek_llm2_python", "extension_group": "chatgpt_fast", "property": { "base_url": "https://api.deepseek.com", @@ -1832,7 +1833,7 @@ { "type": "extension", "name": "llm_deep", - "addon": "openai_llm2_python", + "addon": "deepseek_llm2_python", "extension_group": "chatgpt_deep", "property": { "base_url": "https://api.deepseek.com", @@ -1875,7 +1876,7 @@ { "type": "extension", "name": "main_control", - "addon": "main_python", + "addon": "main_jev_python", "extension_group": "control", "property": { "greeting": "TEN Agent connected. How can I help you today?", diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/README.md b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/README.md new file mode 100644 index 0000000000..23438031ac --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/README.md @@ -0,0 +1,19 @@ +# Jev voice-assistant controller + +`main_jev_python` is used only by the optional +`voice_assistant_jev_router` graph. It combines Deepgram's final English +utterance, asks the `typesafe_jev_python` extension to choose `fast` or +`deep`, and sends the chat request to `llm_fast` or `llm_deep`. + +The two LLM nodes both use DeepSeek Flash. `fast` disables thinking; `deep` +enables it. A `fast` choice needs confidence of at least 0.80. A low-confidence +choice uses thinking, while a Jev error or timeout falls back to no thinking. + +The route is sent to the message collector as a normal assistant transcript +before the answer. Reasoning is sent as the existing `reasoning` message type. +Neither the route nor the reasoning is sent to TTS. Route, reasoning, and +answer use separate stream IDs so the unchanged playground does not merge +their chat bubbles. + +The shared `main_python` controller is not imported or modified by this +extension. Other graphs continue to use it unchanged. diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/__init__.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/__init__.py new file mode 100644 index 0000000000..72593ab225 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/__init__.py @@ -0,0 +1,6 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/addon.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/addon.py new file mode 100644 index 0000000000..13c387d04e --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/addon.py @@ -0,0 +1,19 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten_runtime import ( + Addon, + register_addon_as_extension, + TenEnv, +) + + +@register_addon_as_extension("main_jev_python") +class MainJevExtensionAddon(Addon): + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + from .extension import MainControlExtension + + ten_env.log_info("on_create_instance") + ten_env.on_create_instance_done(MainControlExtension(name), context) diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/__init__.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py new file mode 100644 index 0000000000..29ae86337f --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py @@ -0,0 +1,249 @@ +import asyncio +import json +from typing import Awaitable, Callable, Optional +from .llm_exec import LLMExec +from ..config import ModelRoutingConfig +from ..routing import RoutingDecision +from ten_runtime import AsyncTenEnv, Cmd, CmdResult, Data, StatusCode +from ten_ai_base.types import LLMToolMetadata +from .events import ( + ASRResultEvent, + AgentEvent, + LLMResponseEvent, + ModelRouteEvent, + ToolRegisterEvent, + UserJoinedEvent, + UserLeftEvent, +) + + +class Agent: + def __init__( + self, ten_env: AsyncTenEnv, routing: ModelRoutingConfig | None = None + ): + self.ten_env: AsyncTenEnv = ten_env + self.stopped = False + + # Callback registry + self._callbacks: dict[ + AgentEvent, list[Callable[[AgentEvent], Awaitable]] + ] = {} + + # Queues for ordered processing + self._asr_queue: asyncio.Queue[ASRResultEvent] = asyncio.Queue() + self._llm_queue: asyncio.Queue[LLMResponseEvent] = asyncio.Queue() + + # Current consumer tasks + self._asr_consumer: Optional[asyncio.Task] = None + self._llm_consumer: Optional[asyncio.Task] = None + self._llm_active_task: Optional[asyncio.Task] = ( + None # currently running handler + ) + + self.llm_exec = LLMExec(ten_env, routing) + self.llm_exec.on_response = ( + self._on_llm_response + ) # callback handled internally + self.llm_exec.on_reasoning_response = ( + self._on_llm_reasoning_response + ) # callback handled internally + self.llm_exec.on_route = self._on_route + + # Start consumers + self._asr_consumer = asyncio.create_task(self._consume_asr()) + self._llm_consumer = asyncio.create_task(self._consume_llm()) + + # === Register handlers === + def on( + self, + event_type: AgentEvent, + handler: Callable[[AgentEvent], Awaitable] = None, + ): + """ + Register a callback for a given event type. + + Can be used in two ways: + 1) agent.on(EventType, handler) + 2) @agent.on(EventType) + async def handler(event: EventType): ... + """ + + def decorator(func: Callable[[AgentEvent], Awaitable]): + self._callbacks.setdefault(event_type, []).append(func) + return func + + if handler is None: + return decorator + else: + return decorator(handler) + + async def _dispatch(self, event: AgentEvent): + """Dispatch event to registered handlers sequentially.""" + for etype, handlers in self._callbacks.items(): + if isinstance(event, etype): + for h in handlers: + try: + await h(event) + except Exception as e: + self.ten_env.log_error( + f"Handler error for {etype}: {e}" + ) + + # === Consumers === + async def _consume_asr(self): + while not self.stopped: + event = await self._asr_queue.get() + await self._dispatch(event) + + async def _consume_llm(self): + while not self.stopped: + event = await self._llm_queue.get() + # Run handler as a task so we can cancel mid-flight + self._llm_active_task = asyncio.create_task(self._dispatch(event)) + try: + await self._llm_active_task + except asyncio.CancelledError: + self.ten_env.log_info("[Agent] Active LLM task cancelled") + finally: + self._llm_active_task = None + + # === Emit events === + async def _emit_asr(self, event: ASRResultEvent): + await self._asr_queue.put(event) + + async def _emit_llm(self, event: LLMResponseEvent): + await self._llm_queue.put(event) + + async def _emit_direct(self, event: AgentEvent): + await self._dispatch(event) + + # === Incoming from runtime === + async def on_cmd(self, cmd: Cmd): + try: + name = cmd.get_name() + if name == "on_user_joined": + await self._emit_direct(UserJoinedEvent()) + elif name == "on_user_left": + await self._emit_direct(UserLeftEvent()) + elif name == "tool_register": + tool_json, err = cmd.get_property_to_json("tool") + if err: + raise RuntimeError(f"Invalid tool metadata: {err}") + tool = LLMToolMetadata.model_validate_json(tool_json) + await self._emit_direct( + ToolRegisterEvent( + tool=tool, source=cmd.get_source().extension_name + ) + ) + else: + self.ten_env.log_warn(f"Unhandled cmd: {name}") + + await self.ten_env.return_result( + CmdResult.create(StatusCode.OK, cmd) + ) + except Exception as e: + self.ten_env.log_error(f"on_cmd error: {e}") + await self.ten_env.return_result( + CmdResult.create(StatusCode.ERROR, cmd) + ) + + async def on_data(self, data: Data): + try: + if data.get_name() == "asr_result": + asr_json, _ = data.get_property_to_json(None) + asr = json.loads(asr_json) + await self._emit_asr( + ASRResultEvent( + text=asr.get("text", ""), + final=asr.get("final", False), + metadata=asr.get("metadata", {}), + ) + ) + elif data.get_name() in ("tts_audio_start", "tts_audio_end"): + return + else: + self.ten_env.log_warn(f"Unhandled data: {data.get_name()}") + except Exception as e: + self.ten_env.log_error(f"on_data error: {e}") + + async def _on_llm_response( + self, _ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool + ): + await self._emit_llm( + LLMResponseEvent(delta=delta, text=text, is_final=is_final) + ) + + async def _on_route(self, _ten_env: AsyncTenEnv, route: RoutingDecision): + await self._emit_direct( + ModelRouteEvent( + destination=route.destination, + status=route.status, + latency_ms=route.latency_ms, + choice=route.choice, + confidence=route.confidence, + ) + ) + + async def _on_llm_reasoning_response( + self, _ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool + ): + """ + Internal callback for streaming LLM output, wrapped as an AgentEvent. + """ + await self._emit_llm( + LLMResponseEvent( + delta=delta, text=text, is_final=is_final, type="reasoning" + ) + ) + + # === LLM control === + async def register_llm_tool(self, tool: LLMToolMetadata, source: str): + """ + Register tools with the LLM. + This method sends a command to register the provided tools. + """ + await self.llm_exec.register_tool(tool, source) + + async def queue_llm_input(self, text: str): + """ + Queue a new message to the LLM context. + This method sends the text input to the LLM for processing. + """ + await self.llm_exec.queue_input(text) + + async def flush_llm(self): + """ + Flush the LLM input queue. + This will ensure that all queued inputs are processed. + """ + await self.llm_exec.flush() + + # Clear queue + while not self._llm_queue.empty(): + try: + self._llm_queue.get_nowait() + self._llm_queue.task_done() + except asyncio.QueueEmpty: + break + + # Cancel active LLM task + if self._llm_active_task and not self._llm_active_task.done(): + self._llm_active_task.cancel() + try: + await self._llm_active_task + except asyncio.CancelledError: + pass + self._llm_active_task = None + + async def stop(self): + """ + Stop the agent processing. + This will stop the event queue and any ongoing tasks. + """ + self.stopped = True + await self.llm_exec.stop() + await self.flush_llm() + if self._asr_consumer: + self._asr_consumer.cancel() + if self._llm_consumer: + self._llm_consumer.cancel() diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/decorators.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/decorators.py new file mode 100644 index 0000000000..091178135d --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/decorators.py @@ -0,0 +1,16 @@ +from .events import AgentEvent + + +def agent_event_handler(event_type: AgentEvent): + """ + Decorator to mark a method as an Agent event handler. + Usage: + @agent_event_handler(ASRResultEvent) + async def on_asr(self, event: ASRResultEvent): ... + """ + + def wrapper(func): + setattr(func, "_agent_event_type", event_type) + return func + + return wrapper diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py new file mode 100644 index 0000000000..b9a521ddb9 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py @@ -0,0 +1,86 @@ +from pydantic import BaseModel +from typing import Literal, Union, Dict, Any +from ten_ai_base.types import LLMToolMetadata + + +# ==== Base Event ==== + + +class AgentEventBase(BaseModel): + """Base class for all agent-level events.""" + + type: Literal["cmd", "data"] + name: str + + +# ==== CMD Events ==== + + +class UserJoinedEvent(AgentEventBase): + """Event triggered when a user joins the session.""" + + type: Literal["cmd"] = "cmd" + name: Literal["on_user_joined"] = "on_user_joined" + + +class UserLeftEvent(AgentEventBase): + """Event triggered when a user leaves the session.""" + + type: Literal["cmd"] = "cmd" + name: Literal["on_user_left"] = "on_user_left" + + +class ToolRegisterEvent(AgentEventBase): + """Event triggered when a tool is registered by the user.""" + + type: Literal["cmd"] = "cmd" + name: Literal["tool_register"] = "tool_register" + tool: LLMToolMetadata + source: str + + +# ==== DATA Events ==== + + +class ASRResultEvent(AgentEventBase): + """Event triggered when ASR result is received (partial or final).""" + + type: Literal["data"] = "data" + name: Literal["asr_result"] = "asr_result" + text: str + final: bool + metadata: Dict[str, Any] + + +class LLMResponseEvent(AgentEventBase): + """Event triggered when LLM returns a streaming response.""" + + type: Literal["message", "reasoning"] = "message" + name: Literal["llm_response"] = "llm_response" + delta: str + text: str + is_final: bool + + +class ModelRouteEvent(AgentEventBase): + """The selected LLM destination for a user turn.""" + + type: Literal["data"] = "data" + name: Literal["model_route"] = "model_route" + destination: str + status: str + latency_ms: int + choice: str | None = None + confidence: float | None = None + + +# ==== Unified Event Union ==== + +AgentEvent = Union[ + UserJoinedEvent, + UserLeftEvent, + ToolRegisterEvent, + ASRResultEvent, + LLMResponseEvent, + ModelRouteEvent, +] diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/llm_exec.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/llm_exec.py new file mode 100644 index 0000000000..f2e9fd749c --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/llm_exec.py @@ -0,0 +1,362 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import asyncio +import json +import traceback +from typing import Awaitable, Callable, Literal, Optional +from ten_ai_base.const import CMD_PROPERTY_RESULT +from ten_ai_base.helper import AsyncQueue +from ten_ai_base.struct import ( + LLMMessage, + LLMMessageContent, + LLMMessageFunctionCall, + LLMMessageFunctionCallOutput, + LLMRequest, + LLMResponse, + LLMResponseMessageDelta, + LLMResponseMessageDone, + LLMResponseReasoningDelta, + LLMResponseReasoningDone, + LLMResponseToolCall, + parse_llm_response, +) +from ten_ai_base.types import LLMToolMetadata, LLMToolResult +from ..helper import _send_cmd, _send_cmd_ex +from ..config import ModelRoutingConfig +from ..routing import RoutingDecision, resolve_destination +from ten_runtime import AsyncTenEnv, StatusCode +import uuid + + +class LLMExec: + """ + Context for LLM operations, including ASR and TTS. + This class handles the interaction with the LLM, including processing commands and data. + """ + + def __init__( + self, ten_env: AsyncTenEnv, routing: ModelRoutingConfig | None = None + ): + self.ten_env = ten_env + self.routing = routing or ModelRoutingConfig() + self.input_queue = AsyncQueue() + self.stopped = False + self.on_response: Optional[ + Callable[[AsyncTenEnv, str, str, bool], Awaitable[None]] + ] = None + self.on_reasoning_response: Optional[ + Callable[[AsyncTenEnv, str, str, bool], Awaitable[None]] + ] = None + self.on_tool_call: Optional[ + Callable[[AsyncTenEnv, LLMToolMetadata], Awaitable[None]] + ] = None + self.on_route: Optional[ + Callable[[AsyncTenEnv, RoutingDecision], Awaitable[None]] + ] = None + self.current_task: Optional[asyncio.Task] = None + self.loop = asyncio.get_event_loop() + self.loop.create_task(self._process_input_queue()) + self.available_tools: list[LLMToolMetadata] = [] + self.tool_registry: dict[str, str] = {} + self.available_tools_lock = ( + asyncio.Lock() + ) # Lock to ensure thread-safe access + self.contexts: list[LLMMessage] = [] + self.current_request_id: Optional[str] = None + self.current_decision_request_id: Optional[str] = None + self.current_llm_dest: Optional[str] = None + self.current_text = None + + async def queue_input(self, item: str) -> None: + await self.input_queue.put(item) + + async def flush(self) -> None: + """ + Flush the input queue to ensure all items are processed. + This is useful for ensuring that all pending inputs are handled before stopping. + """ + await self.input_queue.flush() + decision_id = self.current_decision_request_id + request_id = self.current_request_id + llm_dest = self.current_llm_dest + self.current_decision_request_id = None + self.current_request_id = None + self.current_llm_dest = None + if self.current_task: + self.current_task.cancel() + aborts = [] + if decision_id: + aborts.append( + _send_cmd( + self.ten_env, + "abort", + self.routing.decision_dest, + {"request_id": decision_id}, + ) + ) + if request_id and llm_dest: + aborts.append( + _send_cmd( + self.ten_env, "abort", llm_dest, {"request_id": request_id} + ) + ) + if aborts: + await asyncio.gather(*aborts, return_exceptions=True) + + async def stop(self) -> None: + """ + Stop the LLMExec processing. + This will stop the input queue processing and any ongoing tasks. + """ + self.stopped = True + await self.flush() + if self.current_task: + self.current_task.cancel() + + async def register_tool(self, tool: LLMToolMetadata, source: str) -> None: + """ + Register tools with the LLM. + This method sends a command to register the provided tools. + """ + async with self.available_tools_lock: + self.available_tools.append(tool) + self.tool_registry[tool.name] = source + + async def _process_input_queue(self): + """ + Process the input queue for commands and data. + This method runs in a loop, processing items from the queue. + """ + while not self.stopped: + try: + text = await self.input_queue.get() + new_message = LLMMessageContent(role="user", content=text) + self.current_task = self.loop.create_task( + self._send_to_llm(self.ten_env, new_message) + ) + await self.current_task + except asyncio.CancelledError: + self.ten_env.log_info("LLMExec processing cancelled.") + self.current_text = None + except Exception: + self.ten_env.log_error( + f"Error processing input queue: {traceback.format_exc()}" + ) + finally: + self.current_task = None + + async def _queue_context( + self, ten_env: AsyncTenEnv, new_message: LLMMessage + ) -> None: + """ + Queue a new message to the LLM context. + This method appends the new message to the existing context and sends it to the LLM. + """ + ten_env.log_info( + f"_queue_context: role={getattr(new_message, 'role', 'unknown')}" + ) + self.contexts.append(new_message) + + async def _write_context( + self, + ten_env: AsyncTenEnv, + role: Literal["user", "assistant"], + content: str, + ) -> None: + last_context = self.contexts[-1] if self.contexts else None + if last_context and last_context.role == role: + # If the last context has the same role, append to its content + last_context.content = content + else: + # Otherwise, create a new context message + new_message = LLMMessageContent(role=role, content=content) + await self._queue_context(ten_env, new_message) + + async def _send_to_llm( + self, + ten_env: AsyncTenEnv, + new_message: LLMMessage, + llm_dest: Optional[str] = None, + ) -> None: + if llm_dest is None: + llm_dest = await self._choose_llm_dest(new_message.content) + messages = self.contexts.copy() + messages.append(new_message) + request_id = str(uuid.uuid4()) + self.current_request_id = request_id + self.current_llm_dest = llm_dest + thinking = self.routing.enabled and llm_dest == self.routing.deep_dest + parameters = { + "extra_body": { + "thinking": {"type": "enabled" if thinking else "disabled"} + } + } + if thinking: + parameters["reasoning_effort"] = "high" + else: + parameters["temperature"] = 0.7 + llm_input = LLMRequest( + request_id=request_id, + messages=messages, + streaming=True, + parameters=parameters, + tools=self.available_tools, + ) + input_json = llm_input.model_dump() + response = _send_cmd_ex( + ten_env, "chat_completion", llm_dest, input_json + ) + + # Queue the new message to the context + await self._queue_context(ten_env, new_message) + + try: + async for cmd_result, _ in response: + if cmd_result and cmd_result.is_final() is False: + if cmd_result.get_status_code() == StatusCode.OK: + response_json, _ = cmd_result.get_property_to_json(None) + completion = parse_llm_response(response_json) + await self._handle_llm_response(completion, llm_dest) + finally: + if self.current_request_id == request_id: + self.current_request_id = None + self.current_llm_dest = None + + async def _choose_llm_dest(self, text: str) -> str: + if not self.routing.enabled: + return "llm" + + decision_id = str(uuid.uuid4()) + self.current_decision_request_id = decision_id + try: + route = await resolve_destination( + self.routing, + text, + decision_id, + lambda payload: _send_cmd( + self.ten_env, + "decision_evaluate", + self.routing.decision_dest, + payload, + ), + ) + if self.current_decision_request_id != decision_id: + raise asyncio.CancelledError + if self.on_route: + await self.on_route(self.ten_env, route) + self.ten_env.log_info( + f"Jev route: {route.status}, destination={route.destination}, " + f"latency_ms={route.latency_ms}" + ) + return route.destination + finally: + if self.current_decision_request_id == decision_id: + self.current_decision_request_id = None + + async def _handle_llm_response( + self, llm_output: LLMResponse | None, llm_dest: str + ): + if llm_output is not None: + self.ten_env.log_info( + f"_handle_llm_response: {type(llm_output).__name__}" + ) + + match llm_output: + case LLMResponseMessageDelta(): + delta = llm_output.delta + text = llm_output.content + self.current_text = text + if delta and self.on_response: + await self.on_response(self.ten_env, delta, text, False) + if text: + await self._write_context(self.ten_env, "assistant", text) + case LLMResponseMessageDone(): + text = llm_output.content + self.current_text = None + if self.on_response and text: + await self.on_response(self.ten_env, "", text, True) + case LLMResponseReasoningDelta(): + delta = llm_output.delta + text = llm_output.content + if delta and self.on_reasoning_response: + await self.on_reasoning_response( + self.ten_env, delta, text, False + ) + case LLMResponseReasoningDone(): + text = llm_output.content + if self.on_reasoning_response and text: + await self.on_reasoning_response( + self.ten_env, "", text, True + ) + case LLMResponseToolCall(): + self.ten_env.log_info( + f"_handle_llm_response: invoking tool call {llm_output.name}" + ) + src_extension_name = self.tool_registry.get(llm_output.name) + result, _ = await _send_cmd( + self.ten_env, + "tool_call", + src_extension_name, + { + "name": llm_output.name, + "arguments": llm_output.arguments, + }, + ) + + if result.get_status_code() == StatusCode.OK: + r, _ = result.get_property_to_json(CMD_PROPERTY_RESULT) + tool_result: LLMToolResult = json.loads(r) + + self.ten_env.log_info("Tool result received") + + context_function_call = LLMMessageFunctionCall( + name=llm_output.name, + arguments=json.dumps(llm_output.arguments), + call_id=llm_output.tool_call_id, + id=llm_output.response_id, + type="function_call", + ) + if tool_result["type"] == "llmresult": + result_content = tool_result["content"] + if isinstance(result_content, str): + await self._queue_context( + self.ten_env, context_function_call + ) + await self._send_to_llm( + self.ten_env, + LLMMessageFunctionCallOutput( + output=result_content, + call_id=llm_output.tool_call_id, + type="function_call_output", + ), + llm_dest, + ) + else: + self.ten_env.log_error( + f"Unknown tool result content: {result_content}" + ) + elif tool_result["type"] == "requery": + pass + # self.memory_cache = [] + # self.memory_cache.pop() + # result_content = tool_result["content"] + # nonlocal message + # new_message = { + # "role": "user", + # "content": self._convert_to_content_parts( + # message["content"] + # ), + # } + # new_message["content"] = new_message[ + # "content" + # ] + self._convert_to_content_parts( + # result_content + # ) + # await self.queue_input_item( + # True, messages=[new_message], no_tool=True + # ) + else: + self.ten_env.log_error("Tool call failed") diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py new file mode 100644 index 0000000000..a2e7b3172a --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py @@ -0,0 +1,61 @@ +from typing import Any, Literal + +from pydantic import BaseModel, Field + + +def default_route_question() -> dict[str, Any]: + return { + "type": "choice", + "instructions": ( + "Does the latest user request need deliberate multi-step " + "reasoning before a concise spoken answer? Do not judge by " + "length alone." + ), + "criteria": { + "fast": ( + "Greetings, small talk, and direct questions that can " + "be answered without extended reasoning" + ), + "deep": ( + "Multi-step reasoning, planning with tradeoffs, or " + "nuanced analysis that benefits from deliberate thought" + ), + }, + } + + +class ModelRoutingConfig(BaseModel): + enabled: bool = False + decision_dest: str = "jev" + fast_dest: str = "llm_fast" + deep_dest: str = "llm_deep" + min_confidence: float = Field(default=0.8, ge=0, le=1) + timeout_ms: int = Field(default=1200, ge=1) + question: dict[str, Any] = Field(default_factory=default_route_question) + + def select_dest(self, decision: dict[str, Any]) -> str: + """Use the fast model only for a valid, confident fast decision.""" + answers = decision.get("answers", {}) + route = answers.get("route", {}) if isinstance(answers, dict) else {} + if not isinstance(route, dict): + return self.deep_dest + confidence = route.get("confidence") + if ( + route.get("type") == "choice" + and route.get("choice") == "fast" + and isinstance(confidence, (int, float)) + and not isinstance(confidence, bool) + and self.min_confidence <= confidence <= 1 + ): + return self.fast_dest + return self.deep_dest + + +class MainControlConfig(BaseModel): + greeting: str = "Hello, I am your AI assistant." + turn_detection_mode: Literal["segment_final", "speech_final"] = ( + "segment_final" + ) + model_routing: ModelRoutingConfig = Field( + default_factory=ModelRoutingConfig + ) diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py new file mode 100644 index 0000000000..39dcf27c16 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py @@ -0,0 +1,303 @@ +import json +import time +from typing import Literal + +from .agent.decorators import agent_event_handler +from ten_runtime import ( + AsyncExtension, + AsyncTenEnv, + Cmd, + Data, +) + +from .agent.agent import Agent +from .agent.events import ( + ASRResultEvent, + LLMResponseEvent, + ModelRouteEvent, + ToolRegisterEvent, + UserJoinedEvent, + UserLeftEvent, +) +from .helper import _send_cmd, _send_data, parse_sentences +from .config import MainControlConfig # assume extracted from your base model + +import uuid + + +class MainControlExtension(AsyncExtension): + """ + The entry point of the agent module. + Consumes semantic AgentEvents from the Agent class and drives the runtime behavior. + """ + + def __init__(self, name: str): + super().__init__(name) + self.ten_env: AsyncTenEnv = None + self.agent: Agent = None + self.config: MainControlConfig = None + + self.stopped: bool = False + self._rtc_user_count: int = 0 + self.sentence_fragment: str = "" + self.turn_id: int = 0 + self.session_id: str = "0" + self._asr_final_segments: list[str] = [] + self._asr_interim: str = "" + self._last_text_ts: int = 0 + + def _next_text_ts(self) -> int: + self._last_text_ts = max( + int(time.time() * 1000), self._last_text_ts + 1 + ) + return self._last_text_ts + + def _current_metadata(self) -> dict: + return {"session_id": self.session_id, "turn_id": self.turn_id} + + async def on_init(self, ten_env: AsyncTenEnv): + self.ten_env = ten_env + + # Load config from runtime properties + config_json, _ = await ten_env.get_property_to_json(None) + self.config = MainControlConfig.model_validate_json(config_json) + + self.agent = Agent(ten_env, self.config.model_routing) + + # Now auto-register decorated methods + for attr_name in dir(self): + fn = getattr(self, attr_name) + event_type = getattr(fn, "_agent_event_type", None) + if event_type: + self.agent.on(event_type, fn) + + # === Register handlers with decorators === + @agent_event_handler(UserJoinedEvent) + async def _on_user_joined(self, _event: UserJoinedEvent): + self._rtc_user_count += 1 + if self._rtc_user_count == 1 and self.config and self.config.greeting: + await self._send_to_tts(self.config.greeting, True) + await self._send_transcript( + "assistant", self.config.greeting, True, 100 + ) + + @agent_event_handler(UserLeftEvent) + async def _on_user_left(self, _event: UserLeftEvent): + self._rtc_user_count -= 1 + + @agent_event_handler(ToolRegisterEvent) + async def _on_tool_register(self, event: ToolRegisterEvent): + await self.agent.register_llm_tool(event.tool, event.source) + + @agent_event_handler(ASRResultEvent) + async def _on_asr_result(self, event: ASRResultEvent): + session_id = event.metadata.get("session_id", "100") + if ( + self.config.turn_detection_mode == "speech_final" + and session_id != self.session_id + ): + self._asr_final_segments.clear() + self._asr_interim = "" + self.session_id = session_id + stream_id = int(self.session_id) + if self.config.turn_detection_mode == "speech_final": + await self._on_asr_result_speech_final(event, stream_id) + return + if not event.text: + return + if event.final or len(event.text) > 2: + await self._interrupt() + if event.final: + self.turn_id += 1 + await self.agent.queue_llm_input(event.text) + await self._send_transcript("user", event.text, event.final, stream_id) + + async def _on_asr_result_speech_final( + self, event: ASRResultEvent, stream_id: int + ): + asr_info = event.metadata.get("asr_info") or {} + speech_final = ( + isinstance(asr_info, dict) and asr_info.get("speech_final") is True + ) + if event.text: + # Keep the existing barge-in policy separate from turn detection. + if event.final or len(event.text) > 2: + await self._interrupt() + if event.final: + self._asr_final_segments.append(event.text) + self._asr_interim = "" + else: + self._asr_interim = event.text + + segments = self._asr_final_segments.copy() + if self._asr_interim: + segments.append(self._asr_interim) + utterance = " ".join( + segment.strip() for segment in segments if segment.strip() + ) + + if speech_final: + self._asr_final_segments.clear() + self._asr_interim = "" + if utterance: + self.turn_id += 1 + await self.agent.queue_llm_input(utterance) + await self._send_transcript("user", utterance, True, stream_id) + elif event.text and utterance: + await self._send_transcript("user", utterance, False, stream_id) + + @agent_event_handler(LLMResponseEvent) + async def _on_llm_response(self, event: LLMResponseEvent): + # The collector queues every snapshot as a new message. Send the + # complete reasoning once so a long chain cannot delay the answer. + if event.type == "reasoning" and not event.is_final: + return + if not event.is_final and event.type == "message": + sentences, self.sentence_fragment = parse_sentences( + self.sentence_fragment, event.delta + ) + for s in sentences: + await self._send_to_tts(s, False) + + if event.is_final and event.type == "message": + remaining_text = self.sentence_fragment or "" + self.sentence_fragment = "" + await self._send_to_tts(remaining_text, True) + + await self._send_transcript( + "assistant", + event.text, + event.is_final, + (2_000_000 if event.type == "reasoning" else 3_000_000) + + self.turn_id, + data_type=("reasoning" if event.type == "reasoning" else "text"), + ) + + @agent_event_handler(ModelRouteEvent) + async def _on_model_route(self, event: ModelRouteEvent): + if event.choice: + confidence = ( + f" (confidence {event.confidence:.2f})" + if event.confidence is not None + else "" + ) + result = f"{event.choice}{confidence}" + else: + result = "unavailable" + if event.status == "below_threshold": + reason = ( + f" (below {self.config.model_routing.min_confidence:.2f} " + "threshold)" + ) + elif event.status not in ("selected_fast", "selected_deep"): + reason = f" ({event.status.replace('_', ' ')} fallback)" + else: + reason = "" + mode = ( + "Flash · thinking on" + if event.destination == self.config.model_routing.deep_dest + else "Flash · thinking off" + ) + label = f"Jev route: {result} → {mode}{reason} · {event.latency_ms} ms" + await self._send_transcript( + "assistant", label, True, 1_000_000 + self.turn_id + ) + + async def on_start(self, _ten_env: AsyncTenEnv): + self.ten_env.log_info("[MainControlExtension] on_start") + + async def on_stop(self, _ten_env: AsyncTenEnv): + self.ten_env.log_info("[MainControlExtension] on_stop") + self.stopped = True + await self.agent.stop() + + async def on_cmd(self, _ten_env: AsyncTenEnv, cmd: Cmd): + await self.agent.on_cmd(cmd) + + async def on_data(self, _ten_env: AsyncTenEnv, data: Data): + await self.agent.on_data(data) + + # === helpers === + async def _send_transcript( + self, + role: str, + text: str, + final: bool, + stream_id: int, + data_type: Literal["text", "reasoning"] = "text", + ): + """ + Sends the transcript (ASR or LLM output) to the message collector. + """ + if data_type == "text": + await _send_data( + self.ten_env, + "message", + "message_collector", + { + "data_type": "transcribe", + "role": role, + "text": text, + "text_ts": self._next_text_ts(), + "is_final": final, + "stream_id": stream_id, + }, + ) + elif data_type == "reasoning": + await _send_data( + self.ten_env, + "message", + "message_collector", + { + "data_type": "raw", + "role": role, + "text": json.dumps( + { + "type": "reasoning", + "data": { + "text": text, + }, + } + ), + "text_ts": self._next_text_ts(), + "is_final": final, + "stream_id": stream_id, + }, + ) + self.ten_env.log_info( + f"[MainControlExtension] Sent transcript: role={role}, " + f"type={data_type}, final={final}, length={len(text)}" + ) + + async def _send_to_tts(self, text: str, is_final: bool): + """ + Sends a sentence to the TTS system. + """ + request_id = f"tts-request-{self.turn_id}" + await _send_data( + self.ten_env, + "tts_text_input", + "tts", + { + "request_id": request_id, + "text": text, + "text_input_end": is_final, + "metadata": self._current_metadata(), + }, + ) + self.ten_env.log_info( + f"[MainControlExtension] Sent to TTS: is_final={is_final}, " + f"length={len(text)}" + ) + + async def _interrupt(self): + """ + Interrupts ongoing LLM and TTS generation. Typically called when user speech is detected. + """ + self.sentence_fragment = "" + await self.agent.flush_llm() + await _send_data( + self.ten_env, "tts_flush", "tts", {"flush_id": str(uuid.uuid4())} + ) + await _send_cmd(self.ten_env, "flush", "agora_rtc") + self.ten_env.log_info("[MainControlExtension] Interrupt signal sent") diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/helper.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/helper.py new file mode 100644 index 0000000000..50a40d5aaa --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/helper.py @@ -0,0 +1,88 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# + +import json +from typing import Any, AsyncGenerator, Optional +from ten_runtime import AsyncTenEnv, Cmd, CmdResult, Data, Loc, TenError + + +def is_punctuation(char): + if char in [",", "，", ".", "。", "?", "？", "!", "！"]: + return True + return False + + +def parse_sentences(sentence_fragment, content): + sentences = [] + current_sentence = sentence_fragment + for char in content: + current_sentence += char + if is_punctuation(char): + # Check if the current sentence contains non-punctuation characters + stripped_sentence = current_sentence + if any(c.isalnum() for c in stripped_sentence): + sentences.append(stripped_sentence) + current_sentence = "" # Reset for the next sentence + + remain = current_sentence # Any remaining characters form the incomplete sentence + return sentences, remain + + +async def _send_cmd( + ten_env: AsyncTenEnv, cmd_name: str, dest: str, payload: Any = None +) -> tuple[Optional[CmdResult], Optional[TenError]]: + """ + Convenient method to send a command with a payload within app/graph w/o need to create a connection. + Note: extension using this approach will contain logics that are meaningful for this graph only, + as it will assume the target extension already exists in the graph. + For generate purpose extension, it should try to prevent using this method. + """ + cmd = Cmd.create(cmd_name) + loc = Loc("", "", dest) + cmd.set_dests([loc]) + if payload is not None: + cmd.set_property_from_json(None, json.dumps(payload)) + ten_env.log_debug(f"send_cmd: cmd_name {cmd_name}, dest {dest}") + + return await ten_env.send_cmd(cmd) + + +async def _send_cmd_ex( + ten_env: AsyncTenEnv, cmd_name: str, dest: str, payload: Any = None +) -> AsyncGenerator[tuple[Optional[CmdResult], Optional[TenError]], None]: + """Convenient method to send a command with a payload within app/graph w/o need to create a connection. + Note: extension using this approach will contain logics that are meaningful for this graph only, + as it will assume the target extension already exists in the graph. + For generate purpose extension, it should try to prevent using this method. + """ + cmd = Cmd.create(cmd_name) + loc = Loc("", "", dest) + cmd.set_dests([loc]) + if payload is not None: + cmd.set_property_from_json(None, json.dumps(payload)) + ten_env.log_debug(f"send_cmd_ex: cmd_name {cmd_name}, dest {dest}") + + async for cmd_result, ten_error in ten_env.send_cmd_ex(cmd): + if cmd_result: + ten_env.log_debug("send_cmd_ex: result received") + yield cmd_result, ten_error + + +async def _send_data( + ten_env: AsyncTenEnv, data_name: str, dest: str, payload: Any = None +) -> Optional[TenError]: + """Convenient method to send data with a payload within app/graph w/o need to create a connection. + Note: extension using this approach will contain logics that are meaningful for this graph only, + as it will assume the target extension already exists in the graph. + For generate purpose extension, it should try to prevent using this method. + """ + data = Data.create(data_name) + loc = Loc("", "", dest) + data.set_dests([loc]) + if payload is not None: + data.set_property_from_json(None, json.dumps(payload)) + ten_env.log_debug(f"send_data: data_name {data_name}, dest {dest}") + return await ten_env.send_data(data) diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/manifest.json new file mode 100644 index 0000000000..793fac20b5 --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/manifest.json @@ -0,0 +1,51 @@ +{ + "type": "extension", + "name": "main_jev_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.11" + }, + { + "type": "system", + "name": "ten_ai_base", + "version": "0.7" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "**.tent", + "**.py", + "README.md", + "tests/**" + ] + }, + "api": { + "property": { + "properties": { + "greeting": { + "type": "string" + }, + "turn_detection_mode": { + "type": "string" + }, + "model_routing": { + "type": "object", + "properties": { + "enabled": {"type": "bool"}, + "decision_dest": {"type": "string"}, + "fast_dest": {"type": "string"}, + "deep_dest": {"type": "string"}, + "min_confidence": {"type": "float64"}, + "timeout_ms": {"type": "int32"}, + "question": {"type": "object", "properties": {}} + } + } + } + } + } +} diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/property.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/property.json @@ -0,0 +1 @@ +{} diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/routing.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/routing.py similarity index 100% rename from ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/routing.py rename to ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/routing.py diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py index 29ae86337f..5a4b4e9b4e 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/agent.py @@ -2,25 +2,13 @@ import json from typing import Awaitable, Callable, Optional from .llm_exec import LLMExec -from ..config import ModelRoutingConfig -from ..routing import RoutingDecision from ten_runtime import AsyncTenEnv, Cmd, CmdResult, Data, StatusCode from ten_ai_base.types import LLMToolMetadata -from .events import ( - ASRResultEvent, - AgentEvent, - LLMResponseEvent, - ModelRouteEvent, - ToolRegisterEvent, - UserJoinedEvent, - UserLeftEvent, -) +from .events import * class Agent: - def __init__( - self, ten_env: AsyncTenEnv, routing: ModelRoutingConfig | None = None - ): + def __init__(self, ten_env: AsyncTenEnv): self.ten_env: AsyncTenEnv = ten_env self.stopped = False @@ -40,14 +28,13 @@ def __init__( None # currently running handler ) - self.llm_exec = LLMExec(ten_env, routing) + self.llm_exec = LLMExec(ten_env) self.llm_exec.on_response = ( self._on_llm_response ) # callback handled internally self.llm_exec.on_reasoning_response = ( self._on_llm_reasoning_response ) # callback handled internally - self.llm_exec.on_route = self._on_route # Start consumers self._asr_consumer = asyncio.create_task(self._consume_asr()) @@ -84,6 +71,8 @@ async def _dispatch(self, event: AgentEvent): for h in handlers: try: await h(event) + except asyncio.CancelledError: + raise except Exception as e: self.ten_env.log_error( f"Handler error for {etype}: {e}" @@ -167,25 +156,14 @@ async def on_data(self, data: Data): self.ten_env.log_error(f"on_data error: {e}") async def _on_llm_response( - self, _ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool + self, ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool ): await self._emit_llm( LLMResponseEvent(delta=delta, text=text, is_final=is_final) ) - async def _on_route(self, _ten_env: AsyncTenEnv, route: RoutingDecision): - await self._emit_direct( - ModelRouteEvent( - destination=route.destination, - status=route.status, - latency_ms=route.latency_ms, - choice=route.choice, - confidence=route.confidence, - ) - ) - async def _on_llm_reasoning_response( - self, _ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool + self, ten_env: AsyncTenEnv, delta: str, text: str, is_final: bool ): """ Internal callback for streaming LLM output, wrapped as an AgentEvent. diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py index b9a521ddb9..df61ec5c2f 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/events.py @@ -62,18 +62,6 @@ class LLMResponseEvent(AgentEventBase): is_final: bool -class ModelRouteEvent(AgentEventBase): - """The selected LLM destination for a user turn.""" - - type: Literal["data"] = "data" - name: Literal["model_route"] = "model_route" - destination: str - status: str - latency_ms: int - choice: str | None = None - confidence: float | None = None - - # ==== Unified Event Union ==== AgentEvent = Union[ @@ -82,5 +70,4 @@ class ModelRouteEvent(AgentEventBase): ToolRegisterEvent, ASRResultEvent, LLMResponseEvent, - ModelRouteEvent, ] diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py index 5640e4e686..5e57598d1c 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/agent/llm_exec.py @@ -25,9 +25,7 @@ ) from ten_ai_base.types import LLMToolMetadata, LLMToolResult from ..helper import _send_cmd, _send_cmd_ex -from ..config import ModelRoutingConfig -from ..routing import RoutingDecision, resolve_destination -from ten_runtime import AsyncTenEnv, StatusCode +from ten_runtime import AsyncTenEnv, Loc, StatusCode import uuid @@ -37,11 +35,8 @@ class LLMExec: This class handles the interaction with the LLM, including processing commands and data. """ - def __init__( - self, ten_env: AsyncTenEnv, routing: ModelRoutingConfig | None = None - ): + def __init__(self, ten_env: AsyncTenEnv): self.ten_env = ten_env - self.routing = routing or ModelRoutingConfig() self.input_queue = AsyncQueue() self.stopped = False self.on_response: Optional[ @@ -53,9 +48,6 @@ def __init__( self.on_tool_call: Optional[ Callable[[AsyncTenEnv, LLMToolMetadata], Awaitable[None]] ] = None - self.on_route: Optional[ - Callable[[AsyncTenEnv, RoutingDecision], Awaitable[None]] - ] = None self.current_task: Optional[asyncio.Task] = None self.loop = asyncio.get_event_loop() self.loop.create_task(self._process_input_queue()) @@ -66,8 +58,6 @@ def __init__( ) # Lock to ensure thread-safe access self.contexts: list[LLMMessage] = [] self.current_request_id: Optional[str] = None - self.current_decision_request_id: Optional[str] = None - self.current_llm_dest: Optional[str] = None self.current_text = None async def queue_input(self, item: str) -> None: @@ -79,32 +69,14 @@ async def flush(self) -> None: This is useful for ensuring that all pending inputs are handled before stopping. """ await self.input_queue.flush() - decision_id = self.current_decision_request_id - request_id = self.current_request_id - llm_dest = self.current_llm_dest - self.current_decision_request_id = None - self.current_request_id = None - self.current_llm_dest = None + if self.current_request_id: + request_id = self.current_request_id + self.current_request_id = None + await _send_cmd( + self.ten_env, "abort", "llm", {"request_id": request_id} + ) if self.current_task: self.current_task.cancel() - aborts = [] - if decision_id: - aborts.append( - _send_cmd( - self.ten_env, - "abort", - self.routing.decision_dest, - {"request_id": decision_id}, - ) - ) - if request_id and llm_dest: - aborts.append( - _send_cmd( - self.ten_env, "abort", llm_dest, {"request_id": request_id} - ) - ) - if aborts: - await asyncio.gather(*aborts, return_exceptions=True) async def stop(self) -> None: """ @@ -144,7 +116,7 @@ async def _process_input_queue(self): self.current_text = None if self.on_response and text: await self.on_response(self.ten_env, "", text, True) - except Exception: + except Exception as e: self.ten_env.log_error( f"Error processing input queue: {traceback.format_exc()}" ) @@ -158,9 +130,7 @@ async def _queue_context( Queue a new message to the LLM context. This method appends the new message to the existing context and sends it to the LLM. """ - ten_env.log_info( - f"_queue_context: role={getattr(new_message, 'role', 'unknown')}" - ) + ten_env.log_info(f"_queue_context: {new_message}") self.contexts.append(new_message) async def _write_context( @@ -179,93 +149,37 @@ async def _write_context( await self._queue_context(ten_env, new_message) async def _send_to_llm( - self, - ten_env: AsyncTenEnv, - new_message: LLMMessage, - llm_dest: Optional[str] = None, + self, ten_env: AsyncTenEnv, new_message: LLMMessage ) -> None: - if llm_dest is None: - llm_dest = await self._choose_llm_dest(new_message.content) messages = self.contexts.copy() messages.append(new_message) request_id = str(uuid.uuid4()) self.current_request_id = request_id - self.current_llm_dest = llm_dest - thinking = self.routing.enabled and llm_dest == self.routing.deep_dest - parameters = { - "extra_body": { - "thinking": {"type": "enabled" if thinking else "disabled"} - } - } - if thinking: - parameters["reasoning_effort"] = "high" - else: - parameters["temperature"] = 0.7 llm_input = LLMRequest( request_id=request_id, messages=messages, streaming=True, - parameters=parameters, + parameters={"temperature": 0.7}, tools=self.available_tools, ) input_json = llm_input.model_dump() - response = _send_cmd_ex( - ten_env, "chat_completion", llm_dest, input_json - ) + response = _send_cmd_ex(ten_env, "chat_completion", "llm", input_json) # Queue the new message to the context await self._queue_context(ten_env, new_message) - try: - async for cmd_result, _ in response: - if cmd_result and cmd_result.is_final() is False: - if cmd_result.get_status_code() == StatusCode.OK: - response_json, _ = cmd_result.get_property_to_json(None) - completion = parse_llm_response(response_json) - await self._handle_llm_response(completion, llm_dest) - finally: - if self.current_request_id == request_id: - self.current_request_id = None - self.current_llm_dest = None - - async def _choose_llm_dest(self, text: str) -> str: - if not self.routing.enabled: - return "llm" - - decision_id = str(uuid.uuid4()) - self.current_decision_request_id = decision_id - try: - route = await resolve_destination( - self.routing, - text, - decision_id, - lambda payload: _send_cmd( - self.ten_env, - "decision_evaluate", - self.routing.decision_dest, - payload, - ), - ) - if self.current_decision_request_id != decision_id: - raise asyncio.CancelledError - if self.on_route: - await self.on_route(self.ten_env, route) - self.ten_env.log_info( - f"Jev route: {route.status}, destination={route.destination}, " - f"latency_ms={route.latency_ms}" - ) - return route.destination - finally: - if self.current_decision_request_id == decision_id: - self.current_decision_request_id = None + async for cmd_result, _ in response: + if cmd_result and cmd_result.is_final() is False: + if cmd_result.get_status_code() == StatusCode.OK: + response_json, _ = cmd_result.get_property_to_json(None) + ten_env.log_info( + f"_send_to_llm: response_json {response_json}" + ) + completion = parse_llm_response(response_json) + await self._handle_llm_response(completion) - async def _handle_llm_response( - self, llm_output: LLMResponse | None, llm_dest: str - ): - if llm_output is not None: - self.ten_env.log_info( - f"_handle_llm_response: {type(llm_output).__name__}" - ) + async def _handle_llm_response(self, llm_output: LLMResponse | None): + self.ten_env.log_info(f"_handle_llm_response: {llm_output}") match llm_output: case LLMResponseMessageDelta(): @@ -335,7 +249,6 @@ async def _handle_llm_response( call_id=llm_output.tool_call_id, type="function_call_output", ), - llm_dest, ) else: self.ten_env.log_error( diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py index a2e7b3172a..17686708e8 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/config.py @@ -1,61 +1,5 @@ -from typing import Any, Literal - -from pydantic import BaseModel, Field - - -def default_route_question() -> dict[str, Any]: - return { - "type": "choice", - "instructions": ( - "Does the latest user request need deliberate multi-step " - "reasoning before a concise spoken answer? Do not judge by " - "length alone." - ), - "criteria": { - "fast": ( - "Greetings, small talk, and direct questions that can " - "be answered without extended reasoning" - ), - "deep": ( - "Multi-step reasoning, planning with tradeoffs, or " - "nuanced analysis that benefits from deliberate thought" - ), - }, - } - - -class ModelRoutingConfig(BaseModel): - enabled: bool = False - decision_dest: str = "jev" - fast_dest: str = "llm_fast" - deep_dest: str = "llm_deep" - min_confidence: float = Field(default=0.8, ge=0, le=1) - timeout_ms: int = Field(default=1200, ge=1) - question: dict[str, Any] = Field(default_factory=default_route_question) - - def select_dest(self, decision: dict[str, Any]) -> str: - """Use the fast model only for a valid, confident fast decision.""" - answers = decision.get("answers", {}) - route = answers.get("route", {}) if isinstance(answers, dict) else {} - if not isinstance(route, dict): - return self.deep_dest - confidence = route.get("confidence") - if ( - route.get("type") == "choice" - and route.get("choice") == "fast" - and isinstance(confidence, (int, float)) - and not isinstance(confidence, bool) - and self.min_confidence <= confidence <= 1 - ): - return self.fast_dest - return self.deep_dest +from pydantic import BaseModel class MainControlConfig(BaseModel): greeting: str = "Hello, I am your AI assistant." - turn_detection_mode: Literal["segment_final", "speech_final"] = ( - "segment_final" - ) - model_routing: ModelRoutingConfig = Field( - default_factory=ModelRoutingConfig - ) diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py index e3d77e3a83..923104dfb9 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/extension.py @@ -1,3 +1,4 @@ +import asyncio import json import time from typing import Literal @@ -14,7 +15,6 @@ from .agent.events import ( ASRResultEvent, LLMResponseEvent, - ModelRouteEvent, ToolRegisterEvent, UserJoinedEvent, UserLeftEvent, @@ -42,8 +42,6 @@ def __init__(self, name: str): self.sentence_fragment: str = "" self.turn_id: int = 0 self.session_id: str = "0" - self._asr_final_segments: list[str] = [] - self._asr_interim: str = "" def _current_metadata(self) -> dict: return {"session_id": self.session_id, "turn_id": self.turn_id} @@ -55,7 +53,7 @@ async def on_init(self, ten_env: AsyncTenEnv): config_json, _ = await ten_env.get_property_to_json(None) self.config = MainControlConfig.model_validate_json(config_json) - self.agent = Agent(ten_env, self.config.model_routing) + self.agent = Agent(ten_env) # Now auto-register decorated methods for attr_name in dir(self): @@ -66,7 +64,7 @@ async def on_init(self, ten_env: AsyncTenEnv): # === Register handlers with decorators === @agent_event_handler(UserJoinedEvent) - async def _on_user_joined(self, _event: UserJoinedEvent): + async def _on_user_joined(self, event: UserJoinedEvent): self._rtc_user_count += 1 if self._rtc_user_count == 1 and self.config and self.config.greeting: await self._send_to_tts(self.config.greeting, True) @@ -75,7 +73,7 @@ async def _on_user_joined(self, _event: UserJoinedEvent): ) @agent_event_handler(UserLeftEvent) - async def _on_user_left(self, _event: UserLeftEvent): + async def _on_user_left(self, event: UserLeftEvent): self._rtc_user_count -= 1 @agent_event_handler(ToolRegisterEvent) @@ -84,18 +82,8 @@ async def _on_tool_register(self, event: ToolRegisterEvent): @agent_event_handler(ASRResultEvent) async def _on_asr_result(self, event: ASRResultEvent): - session_id = event.metadata.get("session_id", "100") - if ( - self.config.turn_detection_mode == "speech_final" - and session_id != self.session_id - ): - self._asr_final_segments.clear() - self._asr_interim = "" - self.session_id = session_id + self.session_id = event.metadata.get("session_id", "100") stream_id = int(self.session_id) - if self.config.turn_detection_mode == "speech_final": - await self._on_asr_result_speech_final(event, stream_id) - return if not event.text: return if event.final or len(event.text) > 2: @@ -105,46 +93,8 @@ async def _on_asr_result(self, event: ASRResultEvent): await self.agent.queue_llm_input(event.text) await self._send_transcript("user", event.text, event.final, stream_id) - async def _on_asr_result_speech_final( - self, event: ASRResultEvent, stream_id: int - ): - asr_info = event.metadata.get("asr_info") or {} - speech_final = ( - isinstance(asr_info, dict) and asr_info.get("speech_final") is True - ) - if event.text: - # Keep the existing barge-in policy separate from turn detection. - if event.final or len(event.text) > 2: - await self._interrupt() - if event.final: - self._asr_final_segments.append(event.text) - self._asr_interim = "" - else: - self._asr_interim = event.text - - segments = self._asr_final_segments.copy() - if self._asr_interim: - segments.append(self._asr_interim) - utterance = " ".join( - segment.strip() for segment in segments if segment.strip() - ) - - if speech_final: - self._asr_final_segments.clear() - self._asr_interim = "" - if utterance: - self.turn_id += 1 - await self.agent.queue_llm_input(utterance) - await self._send_transcript("user", utterance, True, stream_id) - elif event.text and utterance: - await self._send_transcript("user", utterance, False, stream_id) - @agent_event_handler(LLMResponseEvent) async def _on_llm_response(self, event: LLMResponseEvent): - # The collector queues every snapshot as a new message. Send the - # complete reasoning once so a long chain cannot delay the answer. - if event.type == "reasoning" and not event.is_final: - return if not event.is_final and event.type == "message": sentences, self.sentence_fragment = parse_sentences( self.sentence_fragment, event.delta @@ -165,62 +115,18 @@ async def _on_llm_response(self, event: LLMResponseEvent): data_type=("reasoning" if event.type == "reasoning" else "text"), ) - @agent_event_handler(ModelRouteEvent) - async def _on_model_route(self, event: ModelRouteEvent): - if event.choice: - confidence = ( - f" (confidence {event.confidence:.2f})" - if event.confidence is not None - else "" - ) - result = f"{event.choice}{confidence}" - else: - result = event.status.replace("_", " ") - fallback = ( - " (fallback)" - if event.status - not in ( - "selected_fast", - "selected_deep", - ) - else "" - ) - mode = ( - "Flash · thinking high" - if event.destination == self.config.model_routing.deep_dest - else "Flash · no thinking" - ) - label = ( - f"Jev route: {result} → {mode}{fallback} · {event.latency_ms} ms" - ) - await _send_data( - self.ten_env, - "message", - "message_collector", - { - "data_type": "raw", - "role": "assistant", - "text": json.dumps( - {"type": "routing", "data": {"text": label}} - ), - "text_ts": int(time.time() * 1000), - "is_final": True, - "stream_id": 100, - }, - ) - - async def on_start(self, _ten_env: AsyncTenEnv): - self.ten_env.log_info("[MainControlExtension] on_start") + async def on_start(self, ten_env: AsyncTenEnv): + ten_env.log_info("[MainControlExtension] on_start") - async def on_stop(self, _ten_env: AsyncTenEnv): - self.ten_env.log_info("[MainControlExtension] on_stop") + async def on_stop(self, ten_env: AsyncTenEnv): + ten_env.log_info("[MainControlExtension] on_stop") self.stopped = True await self.agent.stop() - async def on_cmd(self, _ten_env: AsyncTenEnv, cmd: Cmd): + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd): await self.agent.on_cmd(cmd) - async def on_data(self, _ten_env: AsyncTenEnv, data: Data): + async def on_data(self, ten_env: AsyncTenEnv, data: Data): await self.agent.on_data(data) # === helpers === @@ -271,8 +177,7 @@ async def _send_transcript( }, ) self.ten_env.log_info( - f"[MainControlExtension] Sent transcript: role={role}, " - f"type={data_type}, final={final}, length={len(text)}" + f"[MainControlExtension] Sent transcript: {role}, final={final}, text={text}" ) async def _send_to_tts(self, text: str, is_final: bool): diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json index 81cc5a4dbd..ccd3dbb7f9 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_python/manifest.json @@ -29,18 +29,6 @@ "properties": { "greeting": { "type": "string" - }, - "model_routing": { - "type": "object", - "properties": { - "enabled": {"type": "bool"}, - "decision_dest": {"type": "string"}, - "fast_dest": {"type": "string"}, - "deep_dest": {"type": "string"}, - "min_confidence": {"type": "float64"}, - "timeout_ms": {"type": "int32"}, - "question": {"type": "object", "properties": {}} - } } } } diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_deepseek_streaming.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_deepseek_streaming.py new file mode 100644 index 0000000000..503d27bbbd --- /dev/null +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_deepseek_streaming.py @@ -0,0 +1,154 @@ +"""DeepSeek mode and reasoning tests without a network request.""" + +import importlib +import sys +import types +from pathlib import Path + +import pytest + + +ROOT = ( + Path(__file__).resolve().parents[3] + / "ten_packages/extension/deepseek_llm2_python" +) +PACKAGE = "deepseek_llm2_stream_test" +package = types.ModuleType(PACKAGE) +package.__path__ = [str(ROOT)] +sys.modules[PACKAGE] = package + + +class FakeValue: + def __init__(self, **values): + self.__dict__.update(values) + + +base = types.ModuleType("ten_ai_base") +base.__path__ = [] +sys.modules.setdefault("ten_ai_base", base) +struct = types.ModuleType("ten_ai_base.struct") +for name in ( + "ImageContent", + "LLMMessageContent", + "LLMMessageFunctionCall", + "LLMMessageFunctionCallOutput", + "LLMRequest", + "LLMResponse", + "LLMResponseMessageDelta", + "LLMResponseMessageDone", + "LLMResponseReasoningDelta", + "LLMResponseReasoningDone", + "LLMResponseToolCall", + "TextContent", +): + setattr(struct, name, type(name, (FakeValue,), {})) +sys.modules["ten_ai_base.struct"] = struct +base_types = types.ModuleType("ten_ai_base.types") +base_types.LLMToolMetadata = type("LLMToolMetadata", (), {}) +sys.modules["ten_ai_base.types"] = base_types + +runtime = sys.modules.get("ten_runtime") or types.ModuleType("ten_runtime") +runtime.__path__ = [] +sys.modules.setdefault("ten_runtime", runtime) +async_env = types.ModuleType("ten_runtime.async_ten_env") +async_env.AsyncTenEnv = type("AsyncTenEnv", (), {}) +sys.modules["ten_runtime.async_ten_env"] = async_env + +DeepSeekChatClient = importlib.import_module( + f"{PACKAGE}.client" +).DeepSeekChatClient + + +def chunk(*, content=None, reasoning_content=None): + return types.SimpleNamespace( + id="response-1", + created=1, + choices=[ + types.SimpleNamespace( + delta=types.SimpleNamespace( + content=content, + reasoning_content=reasoning_content, + tool_calls=None, + ) + ) + ], + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("thinking", [False, True]) +async def test_mode_is_forwarded_and_reasoning_is_separate(thinking): + logs = [] + captured = [] + + async def stream(): + if thinking: + yield chunk(reasoning_content="private reasoning") + yield chunk(content="spoken answer", reasoning_content="") + + async def create(**kwargs): + captured.append(kwargs) + return stream() + + config = types.SimpleNamespace( + api_key="secret-api-key", + model="deepseek-flash", + prompt="System prompt", + max_tokens=8192, + temperature=0.7, + top_p=1.0, + presence_penalty=0.0, + frequency_penalty=0.0, + seed=123, + is_black_list_params=lambda _key: False, + ) + client = object.__new__(DeepSeekChatClient) + client.config = config + client.ten_env = types.SimpleNamespace( + log_info=logs.append, + log_debug=logs.append, + log_error=logs.append, + ) + client.client = types.SimpleNamespace( + chat=types.SimpleNamespace( + completions=types.SimpleNamespace(create=create) + ) + ) + parameters = { + "extra_body": { + "thinking": {"type": "enabled" if thinking else "disabled"} + } + } + if thinking: + parameters["reasoning_effort"] = "high" + request = types.SimpleNamespace( + messages=[], + tools=[], + prompt=None, + streaming=True, + parameters=parameters, + ) + + events = [event async for event in client.get_chat_completions(request)] + + assert captured[0]["model"] == "deepseek-flash" + assert captured[0]["extra_body"] == parameters["extra_body"] + assert captured[0].get("reasoning_effort") == ("high" if thinking else None) + names = [type(event).__name__ for event in events] + if thinking: + assert names == [ + "LLMResponseReasoningDelta", + "LLMResponseReasoningDone", + "LLMResponseMessageDelta", + "LLMResponseMessageDone", + ] + assert events[1].content == "private reasoning" + else: + assert names == [ + "LLMResponseMessageDelta", + "LLMResponseMessageDone", + ] + assert events[-1].content == "spoken answer" + assert "secret-api-key" not in str(logs) + assert "private reasoning" not in str(logs) + assert "spoken answer" not in str(logs) diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py index 0a5b30d167..7f40d5a9ca 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py @@ -1,5 +1,6 @@ """Static checks for the optional Jev voice-assistant graph.""" +import hashlib import json from pathlib import Path @@ -18,17 +19,17 @@ def test_jev_graph_is_optional_and_has_required_nodes(): if node["name"] == "llm" ) assert default_llm["property"]["model"] == "${env:OPENAI_MODEL}" - assert default_llm["property"]["base_url"] == ( - "${env:OPENAI_API_BASE|https://api.openai.com/v1}" - ) + assert default_llm["property"]["base_url"] == "https://api.openai.com/v1" + assert default_llm["property"]["api_key"] == "${env:OPENAI_API_KEY}" nodes = { node["name"]: node for node in graphs["voice_assistant_jev_router"]["graph"]["nodes"] } assert nodes["jev"]["addon"] == "typesafe_jev_python" - assert nodes["llm_fast"]["addon"] == "openai_llm2_python" - assert nodes["llm_deep"]["addon"] == "openai_llm2_python" + assert nodes["llm_fast"]["addon"] == "deepseek_llm2_python" + assert nodes["llm_deep"]["addon"] == "deepseek_llm2_python" + assert nodes["main_control"]["addon"] == "main_jev_python" assert nodes["stt"]["property"]["params"]["language"] == "en-US" assert nodes["stt"]["property"]["params"]["endpointing"] == 500 assert nodes["jev"]["property"]["params"]["api_key"] == ( @@ -67,3 +68,22 @@ def test_jev_dependency_is_declared(): assert { "path": "../../../ten_packages/extension/typesafe_jev_python" } in app["dependencies"] + assert { + "path": "../../../ten_packages/extension/deepseek_llm2_python" + } in app["dependencies"] + + +def test_other_graphs_match_pre_jev_commit(): + current = json.loads((TENAPP / "property.json").read_text()) + current_graphs = [ + graph + for graph in current["ten"]["predefined_graphs"] + if graph["name"] != "voice_assistant_jev_router" + ] + # Canonical SHA-256 of predefined_graphs at 70e4d8da7^. + encoded = json.dumps( + current_graphs, sort_keys=True, separators=(",", ":") + ).encode() + assert hashlib.sha256(encoded).hexdigest() == ( + "312a4f54b9f68f4bbb94c09df9e9bca3be585a712cb106083e95dc8aec837650" + ) diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py index 62b389021c..80024a7ad3 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_llm_exec.py @@ -13,9 +13,9 @@ ROOT = ( Path(__file__).resolve().parents[1] - / "tenapp/ten_packages/extension/main_python" + / "tenapp/ten_packages/extension/main_jev_python" ) -PACKAGE = "main_python_exec_test" +PACKAGE = "main_jev_python_exec_test" package = types.ModuleType(PACKAGE) package.__path__ = [str(ROOT)] sys.modules.setdefault(PACKAGE, package) @@ -238,6 +238,48 @@ async def on_route(_env, route): assert observed == [] +@pytest.mark.asyncio +async def test_cancelled_reply_does_not_emit_stale_final_text(): + started = asyncio.Event() + + class OneInputQueue: + def __init__(self): + self.sent = False + + async def get(self): + if not self.sent: + self.sent = True + return "First request" + await asyncio.Event().wait() + + executor = object.__new__(LLMExec) + executor.ten_env = FakeEnv() + executor.input_queue = OneInputQueue() + executor.stopped = False + executor.loop = asyncio.get_running_loop() + executor.current_task = None + executor.current_text = "partial stale answer" + replies = [] + + async def on_response(_env, _delta, text, _final): + replies.append(text) + + async def pending_llm(_env, _message): + started.set() + await asyncio.Event().wait() + + executor.on_response = on_response + executor._send_to_llm = pending_llm + worker = asyncio.create_task(executor._process_input_queue()) + await started.wait() + executor.current_task.cancel() + await asyncio.sleep(0) + executor.stopped = True + worker.cancel() + await worker + assert replies == [] + + @pytest.mark.asyncio @pytest.mark.parametrize( "enabled,destination,thinking_type,effort", diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py index b3d2d31e43..c3e03a4ff5 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py @@ -12,9 +12,9 @@ ROOT = ( Path(__file__).resolve().parents[1] - / "tenapp/ten_packages/extension/main_python" + / "tenapp/ten_packages/extension/main_jev_python" ) -PACKAGE = "main_python_reasoning_test" +PACKAGE = "main_jev_python_reasoning_test" package = types.ModuleType(PACKAGE) package.__path__ = [str(ROOT)] sys.modules[PACKAGE] = package @@ -111,6 +111,84 @@ async def test_complete_reasoning_is_visible_but_never_sent_to_tts(): extension._send_to_tts.assert_awaited_once_with("", True) +@pytest.mark.asyncio +async def test_route_precedes_reasoning_and_answer_without_frontend_changes(): + messages.clear() + extension = MainControlExtension("main_control") + extension.ten_env = types.SimpleNamespace(log_info=lambda _message: None) + extension.config = types.SimpleNamespace( + model_routing=types.SimpleNamespace( + deep_dest="llm_deep", min_confidence=0.8 + ) + ) + extension.turn_id = 1 + extension._send_to_tts = AsyncMock() + + await extension._on_model_route( + types.SimpleNamespace( + choice="deep", + confidence=0.91, + status="selected_deep", + destination="llm_deep", + latency_ms=80, + ) + ) + await extension._on_llm_response( + types.SimpleNamespace( + type="reasoning", delta="", text="Think", is_final=True + ) + ) + await extension._on_llm_response( + types.SimpleNamespace( + type="message", delta="", text="Answer", is_final=True + ) + ) + + assert [item["data_type"] for item in messages] == [ + "transcribe", + "raw", + "transcribe", + ] + assert "Jev route: deep (confidence 0.91)" in messages[0]["text"] + assert "thinking on" in messages[0]["text"] + assert json.loads(messages[1]["text"])["type"] == "reasoning" + assert messages[2]["text"] == "Answer" + assert len({item["stream_id"] for item in messages}) == 3 + assert [item["text_ts"] for item in messages] == sorted( + item["text_ts"] for item in messages + ) + extension._send_to_tts.assert_awaited_once_with("", True) + + +@pytest.mark.asyncio +async def test_route_fallback_explains_error_and_uses_fast_mode(): + messages.clear() + extension = MainControlExtension("main_control") + extension.ten_env = types.SimpleNamespace(log_info=lambda _message: None) + extension.config = types.SimpleNamespace( + model_routing=types.SimpleNamespace( + deep_dest="llm_deep", min_confidence=0.8 + ) + ) + extension.turn_id = 2 + extension._send_to_tts = AsyncMock() + + await extension._on_model_route( + types.SimpleNamespace( + choice=None, + confidence=None, + status="timeout", + destination="llm_fast", + latency_ms=1200, + ) + ) + + assert messages[0]["data_type"] == "transcribe" + assert "thinking off" in messages[0]["text"] + assert "timeout fallback" in messages[0]["text"] + extension._send_to_tts.assert_not_awaited() + + def make_asr_event(text, *, final, speech_final, session_id="100"): return types.SimpleNamespace( text=text, diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py index f8356f4e8a..b3480d2284 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_routing.py @@ -10,12 +10,12 @@ import pytest -PACKAGE = "main_python_routing_test" +PACKAGE = "main_jev_python_routing_test" package = types.ModuleType(PACKAGE) package.__path__ = [ str( Path(__file__).resolve().parents[1] - / "tenapp/ten_packages/extension/main_python" + / "tenapp/ten_packages/extension/main_jev_python" ) ] sys.modules.setdefault(PACKAGE, package) diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/README.md b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/README.md new file mode 100644 index 0000000000..f697044106 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/README.md @@ -0,0 +1,12 @@ +# DeepSeek LLM2 extension + +`deepseek_llm2_python` provides the LLM2 command interface for the optional +`voice_assistant_jev_router` graph. It uses the OpenAI-compatible DeepSeek API +at `https://api.deepseek.com` with the `deepseek-flash` model. + +The controller passes `extra_body.thinking.type` as `enabled` or `disabled` +for each request. When DeepSeek streams `reasoning_content`, this extension +emits TEN reasoning events separately from answer events. It never logs the +API key, request body, generated answer, or reasoning text. + +The shared `openai_llm2_python` extension is not imported by this package. diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/__init__.py b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/__init__.py new file mode 100644 index 0000000000..8cd75ddef4 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/__init__.py @@ -0,0 +1,8 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from . import addon diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/addon.py b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/addon.py new file mode 100644 index 0000000000..623d8ffe94 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/addon.py @@ -0,0 +1,22 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from ten_runtime import ( + Addon, + register_addon_as_extension, + TenEnv, +) + + +@register_addon_as_extension("deepseek_llm2_python") +class DeepSeekLLM2ExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + from .extension import DeepSeekLLM2Extension + + ten_env.log_info("DeepSeekLLM2ExtensionAddon on_create_instance") + ten_env.on_create_instance_done(DeepSeekLLM2Extension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/client.py b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/client.py new file mode 100644 index 0000000000..76cd01cb76 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/client.py @@ -0,0 +1,452 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from collections import defaultdict +from dataclasses import dataclass, field +from enum import Enum +import json +import random +from typing import Any, AsyncGenerator, List +from pydantic import BaseModel +import httpx +from openai import AsyncOpenAI, AsyncStream +from openai.types.chat import ChatCompletionChunk + +from ten_ai_base.struct import ( + ImageContent, + LLMMessageContent, + LLMMessageFunctionCall, + LLMMessageFunctionCallOutput, + LLMRequest, + LLMResponse, + LLMResponseMessageDelta, + LLMResponseMessageDone, + LLMResponseReasoningDelta, + LLMResponseReasoningDone, + LLMResponseToolCall, + TextContent, +) +from ten_ai_base.types import LLMToolMetadata +from ten_runtime.async_ten_env import AsyncTenEnv + +from .think_parser import ThinkParser + + +@dataclass +class DeepSeekLLM2Config(BaseModel): + api_key: str = "" + base_url: str = "https://api.deepseek.com" + model: str = "deepseek-flash" + proxy_url: str = "" + temperature: float = 0.7 + top_p: float = 1.0 + presence_penalty: float = 0.0 + frequency_penalty: float = 0.0 + max_tokens: int = 4096 + seed: int = random.randint(0, 1000000) + prompt: str = "You are a helpful assistant." + custom_headers: dict[str, Any] = field(default_factory=dict) + black_list_params: List[str] = field( + default_factory=lambda: ["messages", "tools", "stream", "n", "model"] + ) + + def is_black_list_params(self, key: str) -> bool: + return key in self.black_list_params + + +class ReasoningMode(str, Enum): + ModeV1 = "v1" + + +class DeepSeekChatClient: + client = None + + def __init__(self, ten_env: AsyncTenEnv, config: DeepSeekLLM2Config): + self.config = config + self.ten_env = ten_env + ten_env.log_info("DeepSeekChatClient initialized") + self.http_client = None + if config.proxy_url: + ten_env.log_info("Setting httpx proxy") + self.http_client = httpx.AsyncClient(proxy=config.proxy_url) + + default_headers = {} + for key, value in config.custom_headers.items(): + if isinstance(value, (dict, list)): + ten_env.log_warn( + f"Skipping custom header '{key}':" + f" value must be a scalar, got {type(value).__name__}" + ) + continue + default_headers[str(key)] = str(value) + + self.client = AsyncOpenAI( + api_key=config.api_key, + base_url=config.base_url, + default_headers=default_headers, + http_client=self.http_client, + ) + + def _convert_tools_to_dict(self, tool: LLMToolMetadata): + json_dict = { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description, + "parameters": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": False, + }, + }, + "strict": True, + } + + for param in tool.parameters: + json_dict["function"]["parameters"]["properties"][param.name] = { + "type": param.type, + "description": param.description, + } + if param.required: + json_dict["function"]["parameters"]["required"].append( + param.name + ) + if param.type == "array": + json_dict["function"]["parameters"]["properties"][param.name][ + "items" + ] = param.items + + return json_dict + + async def get_chat_completions( + self, request_input: LLMRequest + ) -> AsyncGenerator[LLMResponse, None]: + messages = request_input.messages + tools = None + parsed_messages = [] + system_prompt = request_input.prompt or self.config.prompt + + self.ten_env.log_info( + f"get_chat_completions: {len(messages)} messages, streaming: {request_input.streaming}" + ) + + for message in messages: + match message: + case LLMMessageContent(): + role = message.role + content = message.content + if isinstance(content, str): + parsed_messages.append( + {"role": role, "content": content} + ) + elif isinstance(content, list): + # Assuming content is a list of objects + content_items = [] + for item in content: + match item: + case TextContent(): + content_items.append( + {"type": "text", "text": item.text} + ) + case ImageContent(): + content_items.append( + { + "type": "image_url", + "image_url": { + "url": item.image_url.url + }, + } + ) + parsed_messages.append( + {"role": role, "content": content_items} + ) + case LLMMessageFunctionCall(): + # Handle function call messages + parsed_messages.append( + { + "role": "assistant", + "tool_calls": [ + { + "id": message.call_id, + "type": "function", + "function": { + "name": message.name, + "arguments": message.arguments, + }, + } + ], + } + ) + case LLMMessageFunctionCallOutput(): + # Handle function call output messages + parsed_messages.append( + { + "role": "tool", + "tool_call_id": message.call_id, + "content": message.output, + } + ) + + for tool in request_input.tools or []: + if tools is None: + tools = [] + tools.append(self._convert_tools_to_dict(tool)) + + # Check if model is a reasoning model (gpt-5.x) that requires different parameters + is_reasoning_model = ( + self.config.model and self.config.model.lower().startswith("gpt-5") + ) + + # Build request + req = { + "model": self.config.model, + "messages": [ + {"role": "system", "content": system_prompt}, + *parsed_messages, + ], + "tools": tools, + "stream": request_input.streaming, + "n": 1, # Assuming single response for now + } + + # GPT-5.x models use max_completion_tokens and don't support sampling parameters + if is_reasoning_model: + req["max_completion_tokens"] = self.config.max_tokens + else: + req["max_tokens"] = self.config.max_tokens + req["temperature"] = self.config.temperature + req["top_p"] = self.config.top_p + req["presence_penalty"] = self.config.presence_penalty + req["frequency_penalty"] = self.config.frequency_penalty + req["seed"] = self.config.seed + + # Add additional parameters if they are not in the black list + for key, value in (request_input.parameters or {}).items(): + # Check if it's a valid option and not in black list + if not self.config.is_black_list_params(key): + self.ten_env.log_debug(f"set openai param: {key}") + req[key] = value + + thinking_config = (req.get("extra_body") or {}).get("thinking") or {} + self.ten_env.log_info( + f"Requesting chat completions: model={self.config.model}, " + f"messages={len(req['messages'])}, thinking=" + f"{thinking_config.get('type', 'default')}" + ) + + try: + response: AsyncStream[ChatCompletionChunk] = ( + await self.client.chat.completions.create(**req) + ) + + full_content = "" + # Check for tool calls + tool_calls_dict = defaultdict( + lambda: { + "id": None, + "function": {"arguments": "", "name": None}, + "type": None, + } + ) + + parser = ThinkParser() + reasoning_mode = None + reasoning_full_content = "" + + last_chat_completion: ChatCompletionChunk | None = None + + async for chat_completion in response: + if chat_completion is None or len(chat_completion.choices) == 0: + continue + last_chat_completion = chat_completion + choice = chat_completion.choices[0] + delta = choice.delta + + content = delta.content if delta and delta.content else "" + raw_reasoning_content = ( + delta.reasoning_content + if delta and hasattr(delta, "reasoning_content") + else None + ) + reasoning_content = raw_reasoning_content or "" + + if reasoning_mode is None and raw_reasoning_content is not None: + reasoning_mode = ReasoningMode.ModeV1 + + if reasoning_mode == ReasoningMode.ModeV1: + if reasoning_content: + for ( + event_type, + event_value, + ) in parser.process_reasoning_content( + reasoning_content + ): + if event_type == "reasoning_delta": + # Use a local accumulator instead of parser state + # because parser.think_content can be reset when + # reasoning_done is emitted in the same cycle. + reasoning_full_content += event_value + yield LLMResponseReasoningDelta( + response_id=chat_completion.id, + role="assistant", + content=reasoning_full_content, + delta=event_value, + created=chat_completion.created, + ) + elif parser.state == "THINK": + for ( + event_type, + event_value, + ) in parser.process_reasoning_content(""): + if event_type == "reasoning_done": + yield LLMResponseReasoningDone( + response_id=chat_completion.id, + role="assistant", + content=event_value, + created=chat_completion.created, + ) + reasoning_full_content = "" + + if content: + full_content += content + yield LLMResponseMessageDelta( + response_id=chat_completion.id, + role="assistant", + content=full_content, + delta=content, + created=chat_completion.created, + ) + elif content: + for event_type, event_value in parser.process_content( + content + ): + if event_type == "message_delta": + full_content += event_value + yield LLMResponseMessageDelta( + response_id=chat_completion.id, + role="assistant", + content=full_content, + delta=event_value, + created=chat_completion.created, + ) + elif event_type == "reasoning_delta": + # Keep reasoning delta content cumulative and stable. + reasoning_full_content += event_value + yield LLMResponseReasoningDelta( + response_id=chat_completion.id, + role="assistant", + content=reasoning_full_content, + delta=event_value, + created=chat_completion.created, + ) + elif event_type == "reasoning_done": + yield LLMResponseReasoningDone( + response_id=chat_completion.id, + role="assistant", + content=event_value, + created=chat_completion.created, + ) + reasoning_full_content = "" + + if delta.tool_calls: + try: + for tool_call in delta.tool_calls: + self.ten_env.log_info("Received a tool call") + if tool_call.index not in tool_calls_dict: + tool_calls_dict[tool_call.index] = { + "id": None, + "function": {"arguments": "", "name": None}, + "type": None, + } + + if tool_call.id: + tool_calls_dict[tool_call.index][ + "id" + ] = tool_call.id + + # If the function name is not None, set it + if tool_call.function.name: + tool_calls_dict[tool_call.index]["function"][ + "name" + ] = tool_call.function.name + + # Append the arguments if not None + if tool_call.function.arguments: + tool_calls_dict[tool_call.index]["function"][ + "arguments" + ] += tool_call.function.arguments + + # If the type is not None, set it + if tool_call.type: + tool_calls_dict[tool_call.index][ + "type" + ] = tool_call.type + except Exception as e: + self.ten_env.log_error( + f"Error processing tool call: {type(e).__name__}" + ) + + if last_chat_completion is None: + self.ten_env.log_info("No chat completion choices found.") + return + + for event_type, event_value in parser.finalize(): + if event_type == "message_delta": + full_content += event_value + yield LLMResponseMessageDelta( + response_id=last_chat_completion.id, + role="assistant", + content=full_content, + delta=event_value, + created=last_chat_completion.created, + ) + elif event_type == "reasoning_delta": + # Keep reasoning delta content cumulative and stable. + reasoning_full_content += event_value + yield LLMResponseReasoningDelta( + response_id=last_chat_completion.id, + role="assistant", + content=reasoning_full_content, + delta=event_value, + created=last_chat_completion.created, + ) + elif event_type == "reasoning_done": + yield LLMResponseReasoningDone( + response_id=last_chat_completion.id, + role="assistant", + content=event_value, + created=last_chat_completion.created, + ) + reasoning_full_content = "" + + # Convert the dictionary to a list + tool_calls_list = list(tool_calls_dict.values()) + + # Emit tool calls event (fire-and-forget) + if tool_calls_list: + for tool_call in tool_calls_list: + arguements = json.loads(tool_call["function"]["arguments"]) + self.ten_env.log_info("Emitting a tool call") + yield LLMResponseToolCall( + response_id=last_chat_completion.id, + id=last_chat_completion.id, + tool_call_id=tool_call["id"], + name=tool_call["function"]["name"], + arguments=arguements, + created=last_chat_completion.created, + ) + + # Emit content finished event after the loop completes + yield LLMResponseMessageDone( + response_id=last_chat_completion.id, + role="assistant", + content=full_content, + created=last_chat_completion.created, + ) + except Exception as e: + raise RuntimeError(f"CreateChatCompletion failed, err: {e}") from e diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/extension.py b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/extension.py new file mode 100644 index 0000000000..6a0ba75c2a --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/extension.py @@ -0,0 +1,82 @@ +# +# +# Agora Real Time Engagement +# Created by Wei Hu in 2024-08. +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +import asyncio +from typing import AsyncGenerator + +from ten_ai_base.llm2 import AsyncLLM2BaseExtension +from ten_ai_base.struct import ( + LLMRequest, + LLMRequestRetrievePrompt, + LLMResponse, + LLMResponseRetrievePrompt, +) +from ten_runtime.async_ten_env import AsyncTenEnv + +from .client import DeepSeekChatClient, DeepSeekLLM2Config + + +class DeepSeekLLM2Extension(AsyncLLM2BaseExtension): + def __init__(self, name: str): + super().__init__(name) + self.memory = [] + self.memory_cache = [] + self.config = None + self.client = None + self.sentence_fragment = "" + self.tool_task_future: asyncio.Future | None = None + self.users_count = 0 + self.last_reasoning_ts = 0 + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_info("on_init") + await super().on_init(ten_env) + + async def on_start(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_start") + await super().on_start(async_ten_env) + config_json, _ = await self.ten_env.get_property_to_json("") + self.config = DeepSeekLLM2Config.model_validate_json(config_json) + + # Mandatory properties + if not self.config.api_key: + async_ten_env.log_info("API key is missing, exiting on_start") + return + + # Create instance + try: + self.client = DeepSeekChatClient(async_ten_env, self.config) + async_ten_env.log_info( + f"initialized with max_tokens: {self.config.max_tokens}, model: {self.config.model}" + ) + except Exception as err: + async_ten_env.log_error( + f"Failed to initialize DeepSeek client: {type(err).__name__}" + ) + + async def on_stop(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_stop") + await super().on_stop(async_ten_env) + + async def on_deinit(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_deinit") + await super().on_deinit(async_ten_env) + + async def on_retrieve_prompt( + self, async_ten_env: AsyncTenEnv, request: LLMRequestRetrievePrompt + ) -> LLMResponseRetrievePrompt: + """Retrieve the current prompt from config.""" + prompt = self.config.prompt if self.config else "" + async_ten_env.log_info( + f"Retrieved prompt for request_id: {request.request_id}" + ) + return LLMResponseRetrievePrompt(prompt=prompt) + + def on_call_chat_completion( + self, async_ten_env: AsyncTenEnv, request_input: LLMRequest + ) -> AsyncGenerator[LLMResponse, None]: + return self.client.get_chat_completions(request_input) diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/manifest.json b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/manifest.json new file mode 100644 index 0000000000..f2f202492c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/manifest.json @@ -0,0 +1,57 @@ +{ + "type": "extension", + "name": "deepseek_llm2_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.11" + }, + { + "type": "system", + "name": "ten_ai_base", + "version": "0.7" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "**.py", + "README.md", + "pyproject.toml", + "requirements.txt" + ] + }, + "api": { + "interface": [ + { + "import_uri": "../../system/ten_ai_base/api/llm-interface.json" + } + ], + "property": { + "properties": { + "api_key": { + "type": "string" + }, + "model": { + "type": "string" + }, + "base_url": { + "type": "string" + }, + "proxy_url": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "custom_headers": { + "type": "object", + "properties": {} + } + } + } + } +} diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/property.json b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/property.json new file mode 100644 index 0000000000..45b0dd74e6 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/property.json @@ -0,0 +1,9 @@ +{ + "base_url": "https://api.deepseek.com", + "api_key": "${env:DEEPSEEK_API_KEY}", + "frequency_penalty": 0.9, + "model": "deepseek-flash", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:DEEPSEEK_PROXY_URL|}" +} diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/pyproject.toml b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/pyproject.toml new file mode 100644 index 0000000000..58575c5245 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "deepseek-llm2-python" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "openai>=2.44.0", + "httpx[socks]>=0.28.0", +] diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/requirements.txt b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/requirements.txt new file mode 100644 index 0000000000..e9f2ae2d19 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/requirements.txt @@ -0,0 +1,2 @@ +openai>=2.44.0 +httpx[socks]>=0.28.0 diff --git a/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/think_parser.py b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/think_parser.py new file mode 100644 index 0000000000..d8728a4e9c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/deepseek_llm2_python/think_parser.py @@ -0,0 +1,137 @@ +# +# Stream parser for extracting ... reasoning blocks from +# tokenized content deltas. +# +from __future__ import annotations + +from typing import List, Tuple + + +class ThinkParser: + OPEN_TAG = "" + CLOSE_TAG = "" + + def __init__(self): + self.state = "NORMAL" # States: 'NORMAL', 'THINK' + self.think_content = "" + self.think_delta = "" + self._pending = "" + + def _partial_suffix_len(self, text: str, tag: str) -> int: + max_len = min(len(text), len(tag) - 1) + for size in range(max_len, 0, -1): + if text.endswith(tag[:size]): + return size + return 0 + + def process_content(self, new_chars: str) -> List[Tuple[str, str]]: + events: List[Tuple[str, str]] = [] + if not new_chars: + return events + + data = self._pending + new_chars + self._pending = "" + idx = 0 + + while idx < len(data): + if self.state == "NORMAL": + open_pos = data.find(self.OPEN_TAG, idx) + if open_pos < 0: + partial = self._partial_suffix_len( + data[idx:], self.OPEN_TAG + ) + visible = ( + data[idx : len(data) - partial] + if partial + else data[idx:] + ) + if visible: + events.append(("message_delta", visible)) + if partial: + self._pending = data[len(data) - partial :] + break + + if open_pos > idx: + events.append(("message_delta", data[idx:open_pos])) + + self.state = "THINK" + self.think_delta = "" + idx = open_pos + len(self.OPEN_TAG) + continue + + close_pos = data.find(self.CLOSE_TAG, idx) + if close_pos < 0: + partial = self._partial_suffix_len(data[idx:], self.CLOSE_TAG) + think_text = ( + data[idx : len(data) - partial] if partial else data[idx:] + ) + if think_text: + self.think_content += think_text + self.think_delta = think_text + events.append(("reasoning_delta", think_text)) + if partial: + self._pending = data[len(data) - partial :] + break + + think_text = data[idx:close_pos] + if think_text: + self.think_content += think_text + self.think_delta = think_text + events.append(("reasoning_delta", think_text)) + + events.append(("reasoning_done", self.think_content)) + self.think_content = "" + self.think_delta = "" + self.state = "NORMAL" + idx = close_pos + len(self.CLOSE_TAG) + + return events + + def process_reasoning_content( + self, reasoning_content: str + ) -> List[Tuple[str, str]]: + events: List[Tuple[str, str]] = [] + if reasoning_content: + if self.state == "NORMAL": + self.state = "THINK" + self.think_content += reasoning_content + self.think_delta = reasoning_content + events.append(("reasoning_delta", reasoning_content)) + elif self.state == "THINK": + events.append(("reasoning_done", self.think_content)) + self.state = "NORMAL" + self.think_delta = "" + self.think_content = "" + return events + + def process(self, new_chars): + prev_state = self.state + events = self.process_content(new_chars) + return prev_state != self.state or any( + event_type == "reasoning_done" for event_type, _ in events + ) + + def process_by_reasoning_content(self, reasoning_content): + prev_state = self.state + events = self.process_reasoning_content(reasoning_content) + return prev_state != self.state or any( + event_type == "reasoning_done" for event_type, _ in events + ) + + def finalize(self) -> List[Tuple[str, str]]: + events: List[Tuple[str, str]] = [] + if self._pending: + if self.state == "NORMAL": + events.append(("message_delta", self._pending)) + else: + self.think_content += self._pending + self.think_delta = self._pending + events.append(("reasoning_delta", self._pending)) + self._pending = "" + + if self.state == "THINK": + events.append(("reasoning_done", self.think_content)) + self.state = "NORMAL" + self.think_delta = "" + self.think_content = "" + return events diff --git a/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py b/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py index 596f620bac..f773873763 100644 --- a/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py +++ b/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py @@ -237,11 +237,9 @@ async def get_chat_completions( self.ten_env.log_debug(f"set openai param: {key}") req[key] = value - thinking_config = (req.get("extra_body") or {}).get("thinking") or {} self.ten_env.log_info( f"Requesting chat completions: model={self.config.model}, " - f"messages={len(req['messages'])}, thinking=" - f"{thinking_config.get('type', 'default')}" + f"messages={len(req['messages'])}" ) try: diff --git a/ai_agents/playground/src/components/Chat/MessageList.tsx b/ai_agents/playground/src/components/Chat/MessageList.tsx index 7f7437ca8b..0af02aec50 100644 --- a/ai_agents/playground/src/components/Chat/MessageList.tsx +++ b/ai_agents/playground/src/components/Chat/MessageList.tsx @@ -1,4 +1,4 @@ -import { Bot, Brain, GitBranch } from "lucide-react"; +import { Bot, Brain } from "lucide-react"; import * as React from "react"; import { useAppSelector, useAutoScroll } from "@/common"; import { Avatar, AvatarFallback } from "@/components/ui/avatar"; @@ -19,8 +19,8 @@ export default function MessageList(props: { className?: string }) { ref={containerRef} className={cn("grow space-y-2 overflow-y-auto p-4", className)} > - {chatItems.map((item, index) => { - return ; + {chatItems.map((item, _index) => { + return ; })} ); @@ -42,12 +42,6 @@ export function MessageItem(props: { data: IChatItem }) { - ) : data.data_type === EMessageDataType.ROUTE ? ( - - - - - ) : ( @@ -56,35 +50,19 @@ export function MessageItem(props: { data: IChatItem }) { ) ) : null} -
+
{data.data_type === EMessageDataType.IMAGE ? ( chat ) : ( - <> - {data.data_type === EMessageDataType.REASON && ( -

- DeepSeek 思考过程 -

- )} -

- {data.text} -

- +

+ {data.text} +

)}
diff --git a/ai_agents/playground/src/manager/rtc/rtc.ts b/ai_agents/playground/src/manager/rtc/rtc.ts index 928d2a9447..e7d6e40f89 100644 --- a/ai_agents/playground/src/manager/rtc/rtc.ts +++ b/ai_agents/playground/src/manager/rtc/rtc.ts @@ -174,6 +174,8 @@ export class RtcManager extends AGEventEmitter { private _parseData(data: any): void { const ascii = String.fromCharCode(...new Uint8Array(data)); + console.log("[test] textstream raw data", ascii); + // const { stream_id, is_final, text, text_ts, data_type, message_id, part_number, total_parts } = textstream; // if (total_parts > 0) { @@ -237,6 +239,9 @@ export class RtcManager extends AGEventEmitter { ); const { stream_id, is_final, text, text_ts, data_type, role } = JSON.parse(this.base64ToUtf8(completeMessage)); + console.log( + `[test] message_id: ${message_id} stream_id: ${stream_id}, text: ${text}, data_type: ${data_type}` + ); const isAgent = role === "assistant"; let textItem: IChatItem = { type: isAgent ? EMessageType.AGENT : EMessageType.USER, @@ -261,12 +266,6 @@ export class RtcManager extends AGEventEmitter { data_type: EMessageDataType.REASON, text: data.text, }; - } else if (type === "routing") { - textItem = { - ...textItem, - data_type: EMessageDataType.ROUTE, - text: data.text, - }; } else if (type === "action") { const { action, data: actionData } = data; if (action === "browse_website") { @@ -277,7 +276,7 @@ export class RtcManager extends AGEventEmitter { } } - if (textItem.text.trim().length > 0) { + if (text.trim().length > 0) { this.emit("textChanged", textItem); } diff --git a/ai_agents/playground/src/store/reducers/global.ts b/ai_agents/playground/src/store/reducers/global.ts index f2f7ad32e7..fc880709c3 100644 --- a/ai_agents/playground/src/store/reducers/global.ts +++ b/ai_agents/playground/src/store/reducers/global.ts @@ -95,12 +95,12 @@ export const globalSlice = createSlice({ state.rtmConnected = action.payload; }, addChatItem: (state, action: PayloadAction) => { - const { userId, data_type, isFinal, time } = action.payload; + const { userId, text, isFinal, type, time } = action.payload; const LastFinalIndex = state.chatItems.findLastIndex((el) => { - return el.userId === userId && el.data_type === data_type && el.isFinal; + return el.userId === userId && el.isFinal; }); const LastNonFinalIndex = state.chatItems.findLastIndex((el) => { - return el.userId === userId && el.data_type === data_type && !el.isFinal; + return el.userId === userId && !el.isFinal; }); const LastFinalItem = state.chatItems[LastFinalIndex]; const LastNonFinalItem = state.chatItems[LastNonFinalIndex]; @@ -108,19 +108,44 @@ export const globalSlice = createSlice({ // has last final Item if (time <= LastFinalItem.time) { // discard + console.log( + "[test] addChatItem, time < last final item, discard!:", + text, + isFinal, + type + ); return; } else { if (LastNonFinalItem) { + console.log( + "[test] addChatItem, update last item(none final):", + text, + isFinal, + type + ); state.chatItems[LastNonFinalIndex] = action.payload; } else { + console.log( + "[test] addChatItem, add new item:", + text, + isFinal, + type + ); state.chatItems.push(action.payload); } } } else { // no last final Item if (LastNonFinalItem) { + console.log( + "[test] addChatItem, update last item(none final):", + text, + isFinal, + type + ); state.chatItems[LastNonFinalIndex] = action.payload; } else { + console.log("[test] addChatItem, add new item:", text, isFinal, type); state.chatItems.push(action.payload); } } diff --git a/ai_agents/playground/src/types/index.ts b/ai_agents/playground/src/types/index.ts index 883a459696..831b4d1305 100644 --- a/ai_agents/playground/src/types/index.ts +++ b/ai_agents/playground/src/types/index.ts @@ -36,7 +36,6 @@ export enum EMessageType { export enum EMessageDataType { TEXT = "text", REASON = "reason", - ROUTE = "route", IMAGE = "image", } From f5c7431fd01f1e9d0d2328b2dfd03b15c0dd033e Mon Sep 17 00:00:00 2001 From: JianYan <15005893916@163.com> Date: Wed, 30 Sep 2026 22:17:59 +0800 Subject: [PATCH 3/6] fix: keep deepseek reasoning separate from spoken answer (#2344) --- ai_agents/agents/examples/voice-assistant/tenapp/property.json | 2 +- .../agents/examples/voice-assistant/tests/test_jev_graph.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/property.json index cc0fa7f148..e9fe596502 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/property.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/property.json @@ -1840,7 +1840,7 @@ "api_key": "${env:DEEPSEEK_API_KEY}", "model": "deepseek-flash", "max_tokens": 8192, - "prompt": "You are a voice assistant. Think carefully, then give a concise spoken answer unless the user asks for detail. Never reveal internal reasoning.", + "prompt": "You are a voice assistant. Think carefully, then give a concise final answer unless the user asks for detail. Keep reasoning out of the spoken answer; it is delivered separately.", "proxy_url": "${env:DEEPSEEK_PROXY_URL|}" } }, diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py index 7f40d5a9ca..d511941763 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py @@ -43,6 +43,9 @@ def test_jev_graph_is_optional_and_has_required_nodes(): assert nodes["llm_fast"]["property"]["model"] == "deepseek-flash" assert nodes["llm_deep"]["property"]["model"] == "deepseek-flash" assert nodes["llm_deep"]["property"]["max_tokens"] > 512 + deep_prompt = nodes["llm_deep"]["property"]["prompt"] + assert "it is delivered separately" in deep_prompt + assert "Never reveal internal reasoning" not in deep_prompt assert "weatherapi_tool_python" not in nodes routing = nodes["main_control"]["property"]["model_routing"] assert ( From 7f613f370842d90ffbc9babdbee2927d5d74d9e5 Mon Sep 17 00:00:00 2001 From: JianYan <15005893916@163.com> Date: Fri, 2 Oct 2026 20:58:34 +0800 Subject: [PATCH 4/6] fix: restore shared openai adapter (#2344) --- .../extension/openai_llm2_python/openai.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py b/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py index f773873763..e168075dad 100644 --- a/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py +++ b/ai_agents/agents/ten_packages/extension/openai_llm2_python/openai.py @@ -70,7 +70,9 @@ class OpenAIChatGPT: def __init__(self, ten_env: AsyncTenEnv, config: OpenAILLM2Config): self.config = config self.ten_env = ten_env - ten_env.log_info("OpenAIChatGPT initialized") + ten_env.log_info( + f"OpenAIChatGPT initialized with config: {config.api_key}" + ) self.http_client = None if config.proxy_url: ten_env.log_info(f"Setting httpx proxy: {config.proxy_url}") @@ -234,13 +236,10 @@ async def get_chat_completions( for key, value in (request_input.parameters or {}).items(): # Check if it's a valid option and not in black list if not self.config.is_black_list_params(key): - self.ten_env.log_debug(f"set openai param: {key}") + self.ten_env.log_debug(f"set openai param: {key} = {value}") req[key] = value - self.ten_env.log_info( - f"Requesting chat completions: model={self.config.model}, " - f"messages={len(req['messages'])}" - ) + self.ten_env.log_info(f"Requesting chat completions with: {req}") try: response: AsyncStream[ChatCompletionChunk] = ( @@ -264,12 +263,15 @@ async def get_chat_completions( last_chat_completion: ChatCompletionChunk | None = None async for chat_completion in response: + self.ten_env.log_debug(f"Chat completion: {chat_completion}") if chat_completion is None or len(chat_completion.choices) == 0: continue last_chat_completion = chat_completion choice = chat_completion.choices[0] delta = choice.delta + self.ten_env.log_debug(f"Processing choice: {choice}") + content = delta.content if delta and delta.content else "" raw_reasoning_content = ( delta.reasoning_content From 92b9bfae292afc247d2c0953ec949497e82e4d1c Mon Sep 17 00:00:00 2001 From: JianYan <15005893916@163.com> Date: Fri, 2 Oct 2026 22:23:24 +0800 Subject: [PATCH 5/6] fix: allow more time for Jev routing (#2344) --- ai_agents/agents/examples/voice-assistant/README.md | 4 ++-- .../agents/examples/voice-assistant/tenapp/property.json | 4 ++-- .../agents/examples/voice-assistant/tests/test_jev_graph.py | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ai_agents/agents/examples/voice-assistant/README.md b/ai_agents/agents/examples/voice-assistant/README.md index 272a352f99..90ae191c67 100644 --- a/ai_agents/agents/examples/voice-assistant/README.md +++ b/ai_agents/agents/examples/voice-assistant/README.md @@ -144,8 +144,8 @@ Jev evaluates the final English ASR text with a `fast`/`deep` choice. A `fast` choice with confidence at least 0.8 uses Flash without thinking; `deep` or an uncertain `fast` choice uses Flash with thinking. Jev errors, invalid responses, and timeouts fall back to Flash without thinking to keep -the voice response responsive. The Jev request has a 1000 ms deadline and -the controller waits at most 1200 ms. These thresholds are demo starting +the voice response responsive. The Jev request has a 2000 ms deadline and +the controller waits at most 2300 ms. These thresholds are demo starting points and should be calibrated with real requests before production use. When thinking is selected, the complete DeepSeek reasoning is shown in a separate chat message before the final answer; it is never sent to TTS. The diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/property.json index e9fe596502..c13717b5e6 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/property.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/property.json @@ -1853,7 +1853,7 @@ "params": { "api_key": "${env:TYPESAFE_API_KEY}", "model": "jev-1.13.0", - "timeout_ms": 1000 + "timeout_ms": 2000 } } }, @@ -1887,7 +1887,7 @@ "fast_dest": "llm_fast", "deep_dest": "llm_deep", "min_confidence": 0.8, - "timeout_ms": 1200, + "timeout_ms": 2300, "question": { "type": "choice", "instructions": "Does the latest user request need deliberate multi-step reasoning before a concise spoken answer? Do not judge by length alone.", diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py index d511941763..b9bb4b57f5 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py @@ -35,6 +35,7 @@ def test_jev_graph_is_optional_and_has_required_nodes(): assert nodes["jev"]["property"]["params"]["api_key"] == ( "${env:TYPESAFE_API_KEY}" ) + assert nodes["jev"]["property"]["params"]["timeout_ms"] == 2000 for node_name in ("llm_fast", "llm_deep"): properties = nodes[node_name]["property"] assert properties["base_url"] == "https://api.deepseek.com" @@ -56,6 +57,7 @@ def test_jev_graph_is_optional_and_has_required_nodes(): assert routing["decision_dest"] == "jev" assert routing["fast_dest"] == "llm_fast" assert routing["deep_dest"] == "llm_deep" + assert routing["timeout_ms"] == 2300 main_cmds = next( connection["cmd"] for connection in graphs["voice_assistant_jev_router"]["graph"][ From c5f8135c3b97cb92fdc8910649ec9e6a99f75ab1 Mon Sep 17 00:00:00 2001 From: JianYan <15005893916@163.com> Date: Sat, 3 Oct 2026 10:52:48 +0800 Subject: [PATCH 6/6] fix: submit Jev turns when speech final is delayed (#2344) --- .../agents/examples/voice-assistant/README.md | 9 +- .../voice-assistant/tenapp/property.json | 1 + .../extension/main_jev_python/agent/agent.py | 17 +- .../extension/main_jev_python/agent/events.py | 12 ++ .../extension/main_jev_python/config.py | 1 + .../extension/main_jev_python/extension.py | 129 ++++++++++++++-- .../voice-assistant/tests/test_jev_graph.py | 1 + .../tests/test_jev_reasoning_display.py | 146 +++++++++++++++++- 8 files changed, 291 insertions(+), 25 deletions(-) diff --git a/ai_agents/agents/examples/voice-assistant/README.md b/ai_agents/agents/examples/voice-assistant/README.md index 90ae191c67..8c81d5ccf5 100644 --- a/ai_agents/agents/examples/voice-assistant/README.md +++ b/ai_agents/agents/examples/voice-assistant/README.md @@ -134,9 +134,12 @@ http://localhost:3000/?graph=voice_assistant_jev_router The Jev graph is opt-in; the default `voice_assistant` graph is unchanged. Its Deepgram Nova-3 ASR uses a 500 ms endpointing window. Stable `is_final` -segments are accumulated, and only `speech_final` submits the complete -utterance to Jev and DeepSeek. The default graph retains segment-final turn -handling. Barge-in behavior is unchanged. +segments are accumulated. `speech_final` submits the complete utterance to Jev +and DeepSeek immediately; if it does not arrive, a 1000 ms deadline after the +last nonempty ASR result submits the buffered stable text once. Empty ASR +results do not extend the deadline, and late results from an already submitted +segment are ignored. The default graph retains segment-final turn handling. +Barge-in behavior is unchanged. Before each routed reply, the chat shows a separate, non-spoken Jev route record with the choice, confidence (when available), selected LLM, latency, and whether an error or timeout caused a fallback. diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/property.json index c13717b5e6..8c688e058e 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/property.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/property.json @@ -1881,6 +1881,7 @@ "property": { "greeting": "TEN Agent connected. How can I help you today?", "turn_detection_mode": "speech_final", + "asr_final_fallback_ms": 1000, "model_routing": { "enabled": true, "decision_dest": "jev", diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py index 29ae86337f..cff2dce350 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/agent.py @@ -7,6 +7,7 @@ from ten_runtime import AsyncTenEnv, Cmd, CmdResult, Data, StatusCode from ten_ai_base.types import LLMToolMetadata from .events import ( + ASRCommitTimeoutEvent, ASRResultEvent, AgentEvent, LLMResponseEvent, @@ -30,7 +31,9 @@ def __init__( ] = {} # Queues for ordered processing - self._asr_queue: asyncio.Queue[ASRResultEvent] = asyncio.Queue() + self._asr_queue: asyncio.Queue[ + ASRResultEvent | ASRCommitTimeoutEvent + ] = asyncio.Queue() self._llm_queue: asyncio.Queue[LLMResponseEvent] = asyncio.Queue() # Current consumer tasks @@ -111,6 +114,17 @@ async def _consume_llm(self): async def _emit_asr(self, event: ASRResultEvent): await self._asr_queue.put(event) + async def queue_asr_timeout( + self, session_id: str, stream_id: int, generation: int + ): + await self._asr_queue.put( + ASRCommitTimeoutEvent( + session_id=session_id, + stream_id=stream_id, + generation=generation, + ) + ) + async def _emit_llm(self, event: LLMResponseEvent): await self._llm_queue.put(event) @@ -157,6 +171,7 @@ async def on_data(self, data: Data): text=asr.get("text", ""), final=asr.get("final", False), metadata=asr.get("metadata", {}), + start_ms=asr.get("start_ms"), ) ) elif data.get_name() in ("tts_audio_start", "tts_audio_end"): diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py index b9a521ddb9..e8924fc9f8 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/agent/events.py @@ -50,6 +50,17 @@ class ASRResultEvent(AgentEventBase): text: str final: bool metadata: Dict[str, Any] + start_ms: int | None = None + + +class ASRCommitTimeoutEvent(AgentEventBase): + """The speech-final fallback deadline for one ASR session.""" + + type: Literal["data"] = "data" + name: Literal["asr_commit_timeout"] = "asr_commit_timeout" + session_id: str + stream_id: int + generation: int class LLMResponseEvent(AgentEventBase): @@ -81,6 +92,7 @@ class ModelRouteEvent(AgentEventBase): UserLeftEvent, ToolRegisterEvent, ASRResultEvent, + ASRCommitTimeoutEvent, LLMResponseEvent, ModelRouteEvent, ] diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py index a2e7b3172a..9843e0dfd0 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/config.py @@ -56,6 +56,7 @@ class MainControlConfig(BaseModel): turn_detection_mode: Literal["segment_final", "speech_final"] = ( "segment_final" ) + asr_final_fallback_ms: int = Field(default=1000, ge=1) model_routing: ModelRoutingConfig = Field( default_factory=ModelRoutingConfig ) diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py index 39dcf27c16..f2c56ab523 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py +++ b/ai_agents/agents/examples/voice-assistant/tenapp/ten_packages/extension/main_jev_python/extension.py @@ -1,3 +1,4 @@ +import asyncio import json import time from typing import Literal @@ -12,6 +13,7 @@ from .agent.agent import Agent from .agent.events import ( + ASRCommitTimeoutEvent, ASRResultEvent, LLMResponseEvent, ModelRouteEvent, @@ -44,8 +46,62 @@ def __init__(self, name: str): self.session_id: str = "0" self._asr_final_segments: list[str] = [] self._asr_interim: str = "" + self._asr_buffer_start_ms: int | None = None + self._asr_final_start_ms: set[int] = set() + self._asr_last_committed_start_ms: int | None = None + self._asr_fallback_task: asyncio.Task | None = None + self._asr_fallback_generation: int = 0 self._last_text_ts: int = 0 + def _cancel_asr_fallback(self): + self._asr_fallback_generation += 1 + if self._asr_fallback_task is not None: + self._asr_fallback_task.cancel() + self._asr_fallback_task = None + + def _schedule_asr_fallback(self, stream_id: int): + self._cancel_asr_fallback() + generation = self._asr_fallback_generation + session_id = self.session_id + + async def commit_after_silence(): + try: + await asyncio.sleep(self.config.asr_final_fallback_ms / 1000) + # Use the ASR queue so the deadline and arriving ASR results + # cannot race while modifying the utterance buffer. + await self.agent.queue_asr_timeout( + session_id, stream_id, generation + ) + except asyncio.CancelledError: + pass + + self._asr_fallback_task = asyncio.create_task(commit_after_silence()) + + def _clear_asr_buffer(self): + self._asr_final_segments.clear() + self._asr_final_start_ms.clear() + self._asr_interim = "" + self._asr_buffer_start_ms = None + + async def _commit_asr_utterance(self, stream_id: int): + segments = self._asr_final_segments.copy() + if self._asr_interim: + segments.append(self._asr_interim) + utterance = " ".join( + segment.strip() for segment in segments if segment.strip() + ) + if self._asr_buffer_start_ms is not None: + previous = self._asr_last_committed_start_ms + self._asr_last_committed_start_ms = max( + previous if previous is not None else -1, + self._asr_buffer_start_ms, + ) + self._clear_asr_buffer() + if utterance: + self.turn_id += 1 + await self.agent.queue_llm_input(utterance) + await self._send_transcript("user", utterance, True, stream_id) + def _next_text_ts(self) -> int: self._last_text_ts = max( int(time.time() * 1000), self._last_text_ts + 1 @@ -84,6 +140,8 @@ async def _on_user_joined(self, _event: UserJoinedEvent): @agent_event_handler(UserLeftEvent) async def _on_user_left(self, _event: UserLeftEvent): self._rtc_user_count -= 1 + self._cancel_asr_fallback() + self._clear_asr_buffer() @agent_event_handler(ToolRegisterEvent) async def _on_tool_register(self, event: ToolRegisterEvent): @@ -96,8 +154,9 @@ async def _on_asr_result(self, event: ASRResultEvent): self.config.turn_detection_mode == "speech_final" and session_id != self.session_id ): - self._asr_final_segments.clear() - self._asr_interim = "" + self._cancel_asr_fallback() + self._clear_asr_buffer() + self._asr_last_committed_start_ms = None self.session_id = session_id stream_id = int(self.session_id) if self.config.turn_detection_mode == "speech_final": @@ -119,32 +178,64 @@ async def _on_asr_result_speech_final( speech_final = ( isinstance(asr_info, dict) and asr_info.get("speech_final") is True ) - if event.text: + start_ms = getattr(event, "start_ms", None) + already_committed = ( + start_ms is not None + and self._asr_last_committed_start_ms is not None + and start_ms <= self._asr_last_committed_start_ms + ) + already_buffered = ( + event.final + and start_ms is not None + and start_ms in self._asr_final_start_ms + ) + accepted_text = bool( + event.text and not already_committed and not already_buffered + ) + if accepted_text: # Keep the existing barge-in policy separate from turn detection. if event.final or len(event.text) > 2: await self._interrupt() if event.final: self._asr_final_segments.append(event.text) self._asr_interim = "" + if start_ms is not None: + self._asr_final_start_ms.add(start_ms) else: self._asr_interim = event.text - - segments = self._asr_final_segments.copy() - if self._asr_interim: - segments.append(self._asr_interim) - utterance = " ".join( - segment.strip() for segment in segments if segment.strip() - ) + if start_ms is not None: + previous = self._asr_buffer_start_ms + self._asr_buffer_start_ms = max( + previous if previous is not None else -1, start_ms + ) if speech_final: - self._asr_final_segments.clear() - self._asr_interim = "" + self._cancel_asr_fallback() + await self._commit_asr_utterance(stream_id) + elif accepted_text: + if self._asr_final_segments: + self._schedule_asr_fallback(stream_id) + segments = self._asr_final_segments.copy() + if self._asr_interim: + segments.append(self._asr_interim) + utterance = " ".join( + segment.strip() for segment in segments if segment.strip() + ) if utterance: - self.turn_id += 1 - await self.agent.queue_llm_input(utterance) - await self._send_transcript("user", utterance, True, stream_id) - elif event.text and utterance: - await self._send_transcript("user", utterance, False, stream_id) + await self._send_transcript("user", utterance, False, stream_id) + + @agent_event_handler(ASRCommitTimeoutEvent) + async def _on_asr_commit_timeout(self, event: ASRCommitTimeoutEvent): + if ( + self.stopped + or self.config.turn_detection_mode != "speech_final" + or event.session_id != self.session_id + or event.generation != self._asr_fallback_generation + or not self._asr_final_segments + ): + return + self._cancel_asr_fallback() + await self._commit_asr_utterance(event.stream_id) @agent_event_handler(LLMResponseEvent) async def _on_llm_response(self, event: LLMResponseEvent): @@ -209,6 +300,10 @@ async def on_start(self, _ten_env: AsyncTenEnv): async def on_stop(self, _ten_env: AsyncTenEnv): self.ten_env.log_info("[MainControlExtension] on_stop") self.stopped = True + fallback_task = self._asr_fallback_task + self._cancel_asr_fallback() + if fallback_task is not None: + await asyncio.gather(fallback_task, return_exceptions=True) await self.agent.stop() async def on_cmd(self, _ten_env: AsyncTenEnv, cmd: Cmd): diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py index b9bb4b57f5..67e798f977 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_graph.py @@ -53,6 +53,7 @@ def test_jev_graph_is_optional_and_has_required_nodes(): nodes["main_control"]["property"]["turn_detection_mode"] == "speech_final" ) + assert nodes["main_control"]["property"]["asr_final_fallback_ms"] == 1000 assert routing["enabled"] is True assert routing["decision_dest"] == "jev" assert routing["fast_dest"] == "llm_fast" diff --git a/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py b/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py index c3e03a4ff5..534dcc3776 100644 --- a/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py +++ b/ai_agents/agents/examples/voice-assistant/tests/test_jev_reasoning_display.py @@ -1,5 +1,6 @@ """The optional thinking graph shows reasoning without reading it aloud.""" +import asyncio import importlib import json import sys @@ -32,6 +33,7 @@ events = types.ModuleType(f"{PACKAGE}.agent.events") for name in ( + "ASRCommitTimeoutEvent", "ASRResultEvent", "LLMResponseEvent", "ModelRouteEvent", @@ -189,10 +191,13 @@ async def test_route_fallback_explains_error_and_uses_fast_mode(): extension._send_to_tts.assert_not_awaited() -def make_asr_event(text, *, final, speech_final, session_id="100"): +def make_asr_event( + text, *, final, speech_final, session_id="100", start_ms=None +): return types.SimpleNamespace( text=text, final=final, + start_ms=start_ms, metadata={ "session_id": session_id, "asr_info": {"speech_final": speech_final}, @@ -200,12 +205,27 @@ def make_asr_event(text, *, final, speech_final, session_id="100"): ) -def make_asr_extension(mode="speech_final"): +def make_asr_extension(mode="speech_final", fallback_ms=1000): messages.clear() extension = MainControlExtension("main_control") extension.ten_env = types.SimpleNamespace(log_info=lambda _message: None) - extension.config = types.SimpleNamespace(turn_detection_mode=mode) - extension.agent = types.SimpleNamespace(queue_llm_input=AsyncMock()) + extension.config = types.SimpleNamespace( + turn_detection_mode=mode, asr_final_fallback_ms=fallback_ms + ) + extension.agent = types.SimpleNamespace( + queue_llm_input=AsyncMock(), queue_asr_timeout=AsyncMock() + ) + + async def deliver_timeout(session_id, stream_id, generation): + await extension._on_asr_commit_timeout( + types.SimpleNamespace( + session_id=session_id, + stream_id=stream_id, + generation=generation, + ) + ) + + extension.agent.queue_asr_timeout.side_effect = deliver_timeout extension._interrupt = AsyncMock() return extension @@ -294,3 +314,121 @@ async def test_default_mode_keeps_segment_final_behavior(): extension.agent.queue_llm_input.assert_awaited_once_with("Hello") assert messages[-1]["is_final"] is True + + +@pytest.mark.asyncio +async def test_missing_speech_final_commits_once_after_last_nonempty_result(): + extension = make_asr_extension(fallback_ms=20) + committed = asyncio.Event() + + async def record_commit(_text): + committed.set() + + extension.agent.queue_llm_input.side_effect = record_commit + await extension._on_asr_result( + make_asr_event( + "Okay. Just introduce yourself.", + final=True, + speech_final=False, + start_ms=4430, + ) + ) + for start_ms in (5000, 5500, 6000): + await extension._on_asr_result( + make_asr_event( + "", final=False, speech_final=False, start_ms=start_ms + ) + ) + + await asyncio.wait_for(committed.wait(), timeout=0.5) + extension.agent.queue_llm_input.assert_awaited_once_with( + "Okay. Just introduce yourself." + ) + assert messages[-1]["is_final"] is True + + # The delayed endpoint and its final duplicate must not make a second turn. + await extension._on_asr_result( + make_asr_event("", final=False, speech_final=True, start_ms=7000) + ) + await extension._on_asr_result( + make_asr_event( + "Okay. Just introduce yourself.", + final=True, + speech_final=True, + start_ms=4430, + ) + ) + assert extension.agent.queue_llm_input.await_count == 1 + assert extension.turn_id == 1 + + # Repeating the same words later is a real new turn, not a text duplicate. + await extension._on_asr_result( + make_asr_event( + "Okay. Just introduce yourself.", + final=True, + speech_final=True, + start_ms=20000, + ) + ) + assert extension.agent.queue_llm_input.await_count == 2 + + +@pytest.mark.asyncio +async def test_stale_deadline_cannot_commit_newer_buffer(): + extension = make_asr_extension() + await extension._on_asr_result( + make_asr_event("Hello", final=True, speech_final=False, start_ms=1000) + ) + old_generation = extension._asr_fallback_generation + await extension._on_asr_result( + make_asr_event("world", final=False, speech_final=False, start_ms=1600) + ) + new_generation = extension._asr_fallback_generation + assert new_generation > old_generation + + await extension._on_asr_commit_timeout( + types.SimpleNamespace( + session_id="100", stream_id=100, generation=old_generation + ) + ) + extension.agent.queue_llm_input.assert_not_awaited() + await extension._on_asr_commit_timeout( + types.SimpleNamespace( + session_id="100", stream_id=100, generation=new_generation + ) + ) + extension.agent.queue_llm_input.assert_awaited_once_with("Hello world") + + +@pytest.mark.asyncio +async def test_session_change_discards_buffer_and_old_timeout(): + extension = make_asr_extension() + await extension._on_asr_result( + make_asr_event("Old", final=True, speech_final=False, start_ms=1000) + ) + old_generation = extension._asr_fallback_generation + await extension._on_asr_result( + make_asr_event( + "New", + final=True, + speech_final=False, + session_id="101", + start_ms=100, + ) + ) + await extension._on_asr_commit_timeout( + types.SimpleNamespace( + session_id="100", stream_id=100, generation=old_generation + ) + ) + extension.agent.queue_llm_input.assert_not_awaited() + await extension._on_asr_result( + make_asr_event( + "", + final=False, + speech_final=True, + session_id="101", + start_ms=200, + ) + ) + extension.agent.queue_llm_input.assert_awaited_once_with("New")