feat(tts): add kotoba_tts_python extension (Kotoba TTS on fal)

ai_agents/agents/integration_tests/tts_guarder/tests/bin/start

@@ -16,7 +16,7 @@ else
 fi
 
 # Only websocket TTS extensions report connection status.
-if [[ "${EXTENSION_NAME}" == "fish_audio_tts_python" || "${EXTENSION_NAME}" == "bytedance_tts" || "${EXTENSION_NAME}" == "bytedance_tts_duplex" || "${EXTENSION_NAME}" == "minimax_tts_websocket" || "${EXTENSION_NAME}" == "minimax_tts_websocket_python" || "${EXTENSION_NAME}" == "rime_tts" ]]; then
+if [[ "${EXTENSION_NAME}" == "fish_audio_tts_python" || "${EXTENSION_NAME}" == "bytedance_tts" || "${EXTENSION_NAME}" == "bytedance_tts_duplex" || "${EXTENSION_NAME}" == "minimax_tts_websocket" || "${EXTENSION_NAME}" == "minimax_tts_websocket_python" || "${EXTENSION_NAME}" == "rime_tts" || "${EXTENSION_NAME}" == "kotoba_tts_python" ]]; then
     export ENABLE_CONNECTION_STATUS="True"
 else
     export ENABLE_CONNECTION_STATUS="False"

ai_agents/agents/integration_tests/tts_guarder/tests/test_connection_status.py

@@ -25,6 +25,7 @@
     "fish_audio_tts_python",
     "bytedance_tts",
     "bytedance_tts_duplex",
+    "kotoba_tts_python",
     "minimax_tts_websocket",
     "minimax_tts_websocket_python",
     "rime_tts",

ai_agents/agents/ten_packages/extension/kotoba_tts_python/README.md

@@ -0,0 +1,96 @@
+# kotoba_tts_python
+
+TEN framework TTS extension for [Kotoba](https://kotoba.tech) text-to-speech,
+served on [fal](https://fal.ai) as the public app `Kotoba-Technologies/kotoba-tts`.
+
+The extension implements `AsyncTTS2BaseExtension` (WebSocket path) on top of
+the official [`kotoba-sdk`](https://pypi.org/project/kotoba-sdk/) client.
+Audio is delivered as 16-bit mono PCM at the configured sample rate.
+
+## Configuration
+
+Set the fal API key in the environment the TEN app runs in:
+
+```bash
+export KOTOBA_TTS_KEY=<your fal key>
+```
+
+`property.json` / graph node `params`:
+
+| key | default | description |
+|---|---|---|
+| `key` | `${env:KOTOBA_TTS_KEY}` | fal API key (`api_key` is accepted as an alias) |
+| `base_url` | `wss://fal.run/Kotoba-Technologies/kotoba-tts/v2/tts/ws` | WebSocket endpoint; a self-hosted URL uses `Bearer` auth automatically |
+| `language` | `ja` | language served by the endpoint (the public app serves Japanese) |
+| `speaker_id` | `male-01` | `male-01` or `female-01` |
+| `sample_rate` | `16000` | `8000`, `16000` or `24000` |
+| `request_timeout_seconds` | `30` | longest wait for the first audio chunk of a sentence, and between chunks |
+| `segment_soft_limit_chars` | `120` | buffered text longer than this is cut at a comma; twice this is cut unconditionally |
+| `max_pending_chars` | `5000` | text accepted for a request but not yet sent to the service; past this the request ends with a non-fatal `pending_text_limit` error and `tts_audio_end(ERROR)` (about 15 minutes of Japanese speech) |
+
+Top-level `dump` / `dump_path` write one `kotoba_tts_dump_<request_id>.pcm`
+per request for debugging. If `dump_path` cannot be created, recording is
+disabled for that request and the request still completes.
+
+Metrics per request: `ttfb` (text sent to first audio chunk), `connect_delay`
+(WebSocket handshake, only when a new connection was opened), request bytes
+per sentence, and the usage counters on every terminal path including
+interruption.
+
+## How it works
+
+The service synthesizes one complete text per response, while TEN streams
+LLM output as deltas. The extension buffers deltas per `request_id` and sends
+a request as soon as a sentence is complete (`。！？!?.` or a newline), then
+the remainder on `text_input_end`.
+
+One WebSocket is reused across requests; it reconnects after the service's
+idle timeout or any connection drop, and retries a sentence once when the
+connection fails before any audio for it arrived. Connection state is
+reported through `connection_status_changed` (`connecting`, `connected`,
+`disconnected`).
+
+Flush cancels the in-flight response (`response.cancel`), drops buffered text
+and reports `tts_audio_end` with reason `INTERRUPTED`. Empty or
+punctuation-only input ends immediately without contacting the service.
+
+Errors: a rejected key or an unsupported language / sample rate is reported
+as `FATAL_ERROR`; worker and transport failures are `NON_FATAL_ERROR`, and
+the next request starts on a fresh connection. A response the service stops
+at its generation length limit keeps the audio already sent and ends the
+request with a non-fatal `response_truncated` error and
+`tts_audio_end(ERROR)`; it is not retried.
+
+## Tests
+
+Extension-local tests run against an in-process fake of the Kotoba service
+and need the TEN runtime. Inside the `ten_agent_build` container:
+
+```bash
+cd ten_packages/extension/kotoba_tts_python
+tman -y install --standalone
+pip install -r requirements.txt -r .ten/app/ten_packages/system/ten_ai_base/requirements.txt pytest
+tests/bin/start
+```
+
+The TEN TTS guarder suite runs from a `ten-framework` checkout that
+contains this extension (`scripts/sync-to-ten-fork.sh`). Against the live
+service:
+
+```bash
+export KOTOBA_TTS_KEY=<your fal key>
+task tts-guarder-test EXTENSION=kotoba_tts_python
+```
+
+Against the fake service, which needs no key (`tests/configs_fake/` points
+at port 8765):
+
+```bash
+python3 ten_packages/extension/kotoba_tts_python/tests/run_fake_server.py 8765 &
+export KOTOBA_TTS_KEY=fake-key
+task tts-guarder-test EXTENSION=kotoba_tts_python CONFIG_DIR=tests/configs_fake
+```
+
+Guarder skips: `test_subtitle_alignment` is skipped because the service
+exposes no word timestamps. `test_connection_status` runs for this
+extension (listed in the guarder's WebSocket allowlist).

ai_agents/agents/ten_packages/extension/kotoba_tts_python/__init__.py

@@ -0,0 +1,5 @@
+#
+# Kotoba TTS extension for the TEN framework.
+# Licensed under the Apache License, Version 2.0.
+#
+from . import addon  # noqa: F401

ai_agents/agents/ten_packages/extension/kotoba_tts_python/addon.py

@@ -0,0 +1,19 @@
+#
+# Kotoba TTS extension for the TEN framework.
+# Licensed under the Apache License, Version 2.0.
+#
+from ten_runtime import (
+    Addon,
+    register_addon_as_extension,
+    TenEnv,
+)
+
+
+@register_addon_as_extension("kotoba_tts_python")
+class KotobaTTSExtensionAddon(Addon):
+
+    def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None:
+        from .extension import KotobaTTSExtension
+
+        ten_env.log_info("KotobaTTSExtensionAddon on_create_instance")
+        ten_env.on_create_instance_done(KotobaTTSExtension(name), context)

ai_agents/agents/ten_packages/extension/kotoba_tts_python/aio.py

@@ -0,0 +1,37 @@
+#
+# Kotoba TTS extension for the TEN framework.
+# Licensed under the Apache License, Version 2.0.
+#
+"""Waiting on another task without mistaking its cancellation for ours.
+
+``await task`` and ``await asyncio.shield(task)`` raise ``CancelledError``
+both when the caller is cancelled and when somebody else cancelled
+``task``. Python 3.10 cannot tell the two apart afterwards, so code that
+waits on a task it does not own uses these helpers instead: the task's own
+outcome is never raised, and only the caller's cancellation propagates.
+"""
+import asyncio
+
+
+async def wait_done(task: asyncio.Future, timeout: float | None = None) -> bool:
+    """Wait until ``task`` is done, at most ``timeout`` seconds.
+
+    Returns whether it is done. Never cancels ``task``; never raises its
+    result, its error or its cancellation. Cancellation of the caller
+    propagates as usual.
+    """
+    done, _ = await asyncio.wait({task}, timeout=timeout)
+    if task in done and not task.cancelled():
+        # Mark the outcome as retrieved; the task's owner reports failures.
+        task.exception()
+    return task in done
+
+
+async def cancel_and_wait(task: asyncio.Future) -> None:
+    """Cancel ``task`` and wait until it has finished.
+
+    Only the task's own cancellation is absorbed; a cancellation of the
+    caller during the wait propagates.
+    """
+    task.cancel()
+    await wait_done(task)

ai_agents/agents/ten_packages/extension/kotoba_tts_python/config.py

@@ -0,0 +1,188 @@
+#
+# Kotoba TTS extension for the TEN framework.
+# Licensed under the Apache License, Version 2.0.
+#
+import copy
+import re
+from typing import Any
+
+from pydantic import BaseModel, Field
+from ten_ai_base import utils
+
+DEFAULT_BASE_URL = "wss://fal.run/Kotoba-Technologies/kotoba-tts/v2/tts/ws"
+SUPPORTED_SAMPLE_RATES = (8000, 16000, 24000)
+
+# Keys the extension understands under ``params``. Everything else is
+# reported as ignored and never forwarded to the service.
+_KNOWN_PARAMS = (
+    "key",
+    "api_key",
+    "base_url",
+    "language",
+    "speaker_id",
+    "sample_rate",
+    "request_timeout_seconds",
+    "segment_soft_limit_chars",
+    "max_pending_chars",
+)
+
+
+class KotobaTTSConfig(BaseModel):
+    dump: bool = False
+    dump_path: str = "./"
+    params: dict[str, Any] = Field(default_factory=dict)
+
+    # Resolved from ``params`` by ``update_params()``.
+    key: str = ""
+    base_url: str = DEFAULT_BASE_URL
+    language: str = "ja"
+    speaker_id: str = "male-01"
+    sample_rate: int = 16000
+    # Longest gap tolerated between sending a segment and its first audio
+    # chunk, and between two audio chunks.
+    request_timeout_seconds: float = 30.0
+    # Buffered text longer than this is cut at a comma; twice this length
+    # is cut unconditionally so buffering stays bounded.
+    segment_soft_limit_chars: int = 120
+    # Text accepted for a request but not yet sent to the service. Past
+    # this, the request ends with a pending_text_limit error instead of
+    # growing without bound while synthesis falls behind. 5,000 characters
+    # is about 15 minutes of Japanese speech, far above one LLM turn.
+    max_pending_chars: int = 5000
+
+    def update_params(self) -> None:
+        """Move known keys from ``params`` into typed fields.
+
+        ``params`` keeps only the keys the extension does not understand,
+        so the caller can log them as ignored.
+        """
+        params = dict(self.params)
+
+        key = params.pop("key", None)
+        api_key = params.pop("api_key", None)
+        resolved_key = key if key else api_key
+        if isinstance(resolved_key, str):
+            self.key = resolved_key
+
+        for name in ("base_url", "language", "speaker_id"):
+            value = params.pop(name, None)
+            if isinstance(value, str) and value.strip():
+                setattr(self, name, value.strip())
+
+        sample_rate = params.pop("sample_rate", None)
+        if sample_rate is not None:
+            self.sample_rate = _as_int(sample_rate, "sample_rate")
+
+        timeout = params.pop("request_timeout_seconds", None)
+        if timeout is not None:
+            self.request_timeout_seconds = _as_float(
+                timeout, "request_timeout_seconds"
+            )
+
+        soft_limit = params.pop("segment_soft_limit_chars", None)
+        if soft_limit is not None:
+            self.segment_soft_limit_chars = _as_int(
+                soft_limit, "segment_soft_limit_chars"
+            )
+
+        max_pending = params.pop("max_pending_chars", None)
+        if max_pending is not None:
+            self.max_pending_chars = _as_int(max_pending, "max_pending_chars")
+
+        self.params = params
+
+    def validate_config(self) -> None:
+        """Raise ``ValueError`` when a request could not succeed."""
+        if not self.key:
+            raise ValueError("params.key (fal API key) is required")
+        if not self.base_url.startswith(("ws://", "wss://")):
+            raise ValueError("params.base_url must be a ws:// or wss:// URL")
+        if not self.language:
+            raise ValueError("params.language must not be empty")
+        if not self.speaker_id:
+            raise ValueError("params.speaker_id must not be empty")
+        if self.sample_rate not in SUPPORTED_SAMPLE_RATES:
+            raise ValueError(
+                "params.sample_rate must be one of "
+                f"{SUPPORTED_SAMPLE_RATES}, got {self.sample_rate}"
+            )
+        if not self.request_timeout_seconds > 0:
+            raise ValueError("params.request_timeout_seconds must be > 0")
+        if self.segment_soft_limit_chars < 1:
+            raise ValueError("params.segment_soft_limit_chars must be >= 1")
+        if self.max_pending_chars < 1:
+            raise ValueError("params.max_pending_chars must be >= 1")
+
+    def ignored_params(self) -> list[str]:
+        return sorted(self.params.keys())
+
+    def redacted_base_url(self) -> str:
+        """The endpoint with credentials and signed query values masked."""
+        return _redact_url(self.base_url)
+
+    def to_str(self, sensitive_handling: bool = True) -> str:
+        if not sensitive_handling:
+            return f"{self}"
+        config = copy.deepcopy(self)
+        if config.key:
+            config.key = utils.encrypt(config.key)
+        # Unknown params may hold credentials under any name: mask every
+        # value, and strip credentials from the URL as well.
+        config.params = {
+            name: (
+                utils.encrypt(str(value)) if value not in (None, "") else value
+            )
+            for name, value in config.params.items()
+        }
+        config.base_url = _redact_url(config.base_url)
+        return f"{config}"
+
+
+_USERINFO_RE = re.compile(r"^([a-z][a-z0-9+.-]*://)[^/@]+@")
+_QUERY_VALUE_RE = re.compile(r"=[^&#]*")
+
+
+def _redact_url(url: str) -> str:
+    """Mask every credential-bearing part of a URL for diagnostics.
+
+    No name list: a custom endpoint may carry a token under any query name
+    (``access_token``, ``X-Amz-Signature``, ...), so every query value is
+    masked and only the names stay. ``scheme://user:secret@host`` becomes
+    ``scheme://***@host``.
+    """
+    if not url:
+        return url
+    url = _USERINFO_RE.sub(r"\1***@", url)
+    query_start = url.find("?")
+    if query_start < 0:
+        return url
+    head, query = url[: query_start + 1], url[query_start + 1 :]
+    return head + _QUERY_VALUE_RE.sub("=***", query)
+
+
+def _as_int(value: Any, name: str) -> int:
+    if isinstance(value, bool):
+        raise ValueError(f"params.{name} must be an integer, got a boolean")
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float) and value.is_integer():
+        return int(value)
+    if isinstance(value, str) and value.strip().isdigit():
+        return int(value.strip())
+    raise ValueError(
+        f"params.{name} must be an integer, got {type(value).__name__}"
+    )
+
+
+def _as_float(value: Any, name: str) -> float:
+    if isinstance(value, bool):
+        raise ValueError(f"params.{name} must be a number, got a boolean")
+    try:
+        result = float(value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(
+            f"params.{name} must be a number, got {type(value).__name__}"
+        ) from exc
+    if result != result or result in (float("inf"), float("-inf")):
+        raise ValueError(f"params.{name} must be a finite number")
+    return result