diff --git a/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start b/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start index 84bb6baf11..97d8d74bd0 100755 --- a/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start +++ b/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start @@ -16,7 +16,7 @@ else fi # Only websocket TTS extensions report connection status. -if [[ "${EXTENSION_NAME}" == "fish_audio_tts_python" || "${EXTENSION_NAME}" == "bytedance_tts" || "${EXTENSION_NAME}" == "bytedance_tts_duplex" || "${EXTENSION_NAME}" == "minimax_tts_websocket" || "${EXTENSION_NAME}" == "minimax_tts_websocket_python" || "${EXTENSION_NAME}" == "rime_tts" ]]; then +if [[ "${EXTENSION_NAME}" == "fish_audio_tts_python" || "${EXTENSION_NAME}" == "bytedance_tts" || "${EXTENSION_NAME}" == "bytedance_tts_duplex" || "${EXTENSION_NAME}" == "minimax_tts_websocket" || "${EXTENSION_NAME}" == "minimax_tts_websocket_python" || "${EXTENSION_NAME}" == "rime_tts" || "${EXTENSION_NAME}" == "kotoba_tts_python" ]]; then export ENABLE_CONNECTION_STATUS="True" else export ENABLE_CONNECTION_STATUS="False" diff --git a/ai_agents/agents/integration_tests/tts_guarder/tests/test_connection_status.py b/ai_agents/agents/integration_tests/tts_guarder/tests/test_connection_status.py index b47491546c..c2cb98e5ae 100644 --- a/ai_agents/agents/integration_tests/tts_guarder/tests/test_connection_status.py +++ b/ai_agents/agents/integration_tests/tts_guarder/tests/test_connection_status.py @@ -25,6 +25,7 @@ "fish_audio_tts_python", "bytedance_tts", "bytedance_tts_duplex", + "kotoba_tts_python", "minimax_tts_websocket", "minimax_tts_websocket_python", "rime_tts", diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/README.md b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/README.md new file mode 100644 index 0000000000..2f0bed1235 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/README.md @@ -0,0 +1,96 @@ +# kotoba_tts_python + +TEN framework TTS extension for [Kotoba](https://kotoba.tech) text-to-speech, +served on [fal](https://fal.ai) as the public app `Kotoba-Technologies/kotoba-tts`. + +The extension implements `AsyncTTS2BaseExtension` (WebSocket path) on top of +the official [`kotoba-sdk`](https://pypi.org/project/kotoba-sdk/) client. +Audio is delivered as 16-bit mono PCM at the configured sample rate. + +## Configuration + +Set the fal API key in the environment the TEN app runs in: + +```bash +export KOTOBA_TTS_KEY= +``` + +`property.json` / graph node `params`: + +| key | default | description | +|---|---|---| +| `key` | `${env:KOTOBA_TTS_KEY}` | fal API key (`api_key` is accepted as an alias) | +| `base_url` | `wss://fal.run/Kotoba-Technologies/kotoba-tts/v2/tts/ws` | WebSocket endpoint; a self-hosted URL uses `Bearer` auth automatically | +| `language` | `ja` | language served by the endpoint (the public app serves Japanese) | +| `speaker_id` | `male-01` | `male-01` or `female-01` | +| `sample_rate` | `16000` | `8000`, `16000` or `24000` | +| `request_timeout_seconds` | `30` | longest wait for the first audio chunk of a sentence, and between chunks | +| `segment_soft_limit_chars` | `120` | buffered text longer than this is cut at a comma; twice this is cut unconditionally | +| `max_pending_chars` | `5000` | text accepted for a request but not yet sent to the service; past this the request ends with a non-fatal `pending_text_limit` error and `tts_audio_end(ERROR)` (about 15 minutes of Japanese speech) | + +Top-level `dump` / `dump_path` write one `kotoba_tts_dump_.pcm` +per request for debugging. If `dump_path` cannot be created, recording is +disabled for that request and the request still completes. + +Metrics per request: `ttfb` (text sent to first audio chunk), `connect_delay` +(WebSocket handshake, only when a new connection was opened), request bytes +per sentence, and the usage counters on every terminal path including +interruption. + +## How it works + +The service synthesizes one complete text per response, while TEN streams +LLM output as deltas. The extension buffers deltas per `request_id` and sends +a request as soon as a sentence is complete (`。！？!?.` or a newline), then +the remainder on `text_input_end`. + +One WebSocket is reused across requests; it reconnects after the service's +idle timeout or any connection drop, and retries a sentence once when the +connection fails before any audio for it arrived. Connection state is +reported through `connection_status_changed` (`connecting`, `connected`, +`disconnected`). + +Flush cancels the in-flight response (`response.cancel`), drops buffered text +and reports `tts_audio_end` with reason `INTERRUPTED`. Empty or +punctuation-only input ends immediately without contacting the service. + +Errors: a rejected key or an unsupported language / sample rate is reported +as `FATAL_ERROR`; worker and transport failures are `NON_FATAL_ERROR`, and +the next request starts on a fresh connection. A response the service stops +at its generation length limit keeps the audio already sent and ends the +request with a non-fatal `response_truncated` error and +`tts_audio_end(ERROR)`; it is not retried. + +## Tests + +Extension-local tests run against an in-process fake of the Kotoba service +and need the TEN runtime. Inside the `ten_agent_build` container: + +```bash +cd ten_packages/extension/kotoba_tts_python +tman -y install --standalone +pip install -r requirements.txt -r .ten/app/ten_packages/system/ten_ai_base/requirements.txt pytest +tests/bin/start +``` + +The TEN TTS guarder suite runs from a `ten-framework` checkout that +contains this extension (`scripts/sync-to-ten-fork.sh`). Against the live +service: + +```bash +export KOTOBA_TTS_KEY= +task tts-guarder-test EXTENSION=kotoba_tts_python +``` + +Against the fake service, which needs no key (`tests/configs_fake/` points +at port 8765): + +```bash +python3 ten_packages/extension/kotoba_tts_python/tests/run_fake_server.py 8765 & +export KOTOBA_TTS_KEY=fake-key +task tts-guarder-test EXTENSION=kotoba_tts_python CONFIG_DIR=tests/configs_fake +``` + +Guarder skips: `test_subtitle_alignment` is skipped because the service +exposes no word timestamps. `test_connection_status` runs for this +extension (listed in the guarder's WebSocket allowlist). diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/__init__.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/__init__.py new file mode 100644 index 0000000000..d1f701bc0c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/__init__.py @@ -0,0 +1,5 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +from . import addon # noqa: F401 diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/addon.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/addon.py new file mode 100644 index 0000000000..9d56ed31e2 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/addon.py @@ -0,0 +1,19 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +from ten_runtime import ( + Addon, + register_addon_as_extension, + TenEnv, +) + + +@register_addon_as_extension("kotoba_tts_python") +class KotobaTTSExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + from .extension import KotobaTTSExtension + + ten_env.log_info("KotobaTTSExtensionAddon on_create_instance") + ten_env.on_create_instance_done(KotobaTTSExtension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/aio.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/aio.py new file mode 100644 index 0000000000..14512c5d1a --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/aio.py @@ -0,0 +1,37 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Waiting on another task without mistaking its cancellation for ours. + +``await task`` and ``await asyncio.shield(task)`` raise ``CancelledError`` +both when the caller is cancelled and when somebody else cancelled +``task``. Python 3.10 cannot tell the two apart afterwards, so code that +waits on a task it does not own uses these helpers instead: the task's own +outcome is never raised, and only the caller's cancellation propagates. +""" +import asyncio + + +async def wait_done(task: asyncio.Future, timeout: float | None = None) -> bool: + """Wait until ``task`` is done, at most ``timeout`` seconds. + + Returns whether it is done. Never cancels ``task``; never raises its + result, its error or its cancellation. Cancellation of the caller + propagates as usual. + """ + done, _ = await asyncio.wait({task}, timeout=timeout) + if task in done and not task.cancelled(): + # Mark the outcome as retrieved; the task's owner reports failures. + task.exception() + return task in done + + +async def cancel_and_wait(task: asyncio.Future) -> None: + """Cancel ``task`` and wait until it has finished. + + Only the task's own cancellation is absorbed; a cancellation of the + caller during the wait propagates. + """ + task.cancel() + await wait_done(task) diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/config.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/config.py new file mode 100644 index 0000000000..1eccbeef11 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/config.py @@ -0,0 +1,188 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +import copy +import re +from typing import Any + +from pydantic import BaseModel, Field +from ten_ai_base import utils + +DEFAULT_BASE_URL = "wss://fal.run/Kotoba-Technologies/kotoba-tts/v2/tts/ws" +SUPPORTED_SAMPLE_RATES = (8000, 16000, 24000) + +# Keys the extension understands under ``params``. Everything else is +# reported as ignored and never forwarded to the service. +_KNOWN_PARAMS = ( + "key", + "api_key", + "base_url", + "language", + "speaker_id", + "sample_rate", + "request_timeout_seconds", + "segment_soft_limit_chars", + "max_pending_chars", +) + + +class KotobaTTSConfig(BaseModel): + dump: bool = False + dump_path: str = "./" + params: dict[str, Any] = Field(default_factory=dict) + + # Resolved from ``params`` by ``update_params()``. + key: str = "" + base_url: str = DEFAULT_BASE_URL + language: str = "ja" + speaker_id: str = "male-01" + sample_rate: int = 16000 + # Longest gap tolerated between sending a segment and its first audio + # chunk, and between two audio chunks. + request_timeout_seconds: float = 30.0 + # Buffered text longer than this is cut at a comma; twice this length + # is cut unconditionally so buffering stays bounded. + segment_soft_limit_chars: int = 120 + # Text accepted for a request but not yet sent to the service. Past + # this, the request ends with a pending_text_limit error instead of + # growing without bound while synthesis falls behind. 5,000 characters + # is about 15 minutes of Japanese speech, far above one LLM turn. + max_pending_chars: int = 5000 + + def update_params(self) -> None: + """Move known keys from ``params`` into typed fields. + + ``params`` keeps only the keys the extension does not understand, + so the caller can log them as ignored. + """ + params = dict(self.params) + + key = params.pop("key", None) + api_key = params.pop("api_key", None) + resolved_key = key if key else api_key + if isinstance(resolved_key, str): + self.key = resolved_key + + for name in ("base_url", "language", "speaker_id"): + value = params.pop(name, None) + if isinstance(value, str) and value.strip(): + setattr(self, name, value.strip()) + + sample_rate = params.pop("sample_rate", None) + if sample_rate is not None: + self.sample_rate = _as_int(sample_rate, "sample_rate") + + timeout = params.pop("request_timeout_seconds", None) + if timeout is not None: + self.request_timeout_seconds = _as_float( + timeout, "request_timeout_seconds" + ) + + soft_limit = params.pop("segment_soft_limit_chars", None) + if soft_limit is not None: + self.segment_soft_limit_chars = _as_int( + soft_limit, "segment_soft_limit_chars" + ) + + max_pending = params.pop("max_pending_chars", None) + if max_pending is not None: + self.max_pending_chars = _as_int(max_pending, "max_pending_chars") + + self.params = params + + def validate_config(self) -> None: + """Raise ``ValueError`` when a request could not succeed.""" + if not self.key: + raise ValueError("params.key (fal API key) is required") + if not self.base_url.startswith(("ws://", "wss://")): + raise ValueError("params.base_url must be a ws:// or wss:// URL") + if not self.language: + raise ValueError("params.language must not be empty") + if not self.speaker_id: + raise ValueError("params.speaker_id must not be empty") + if self.sample_rate not in SUPPORTED_SAMPLE_RATES: + raise ValueError( + "params.sample_rate must be one of " + f"{SUPPORTED_SAMPLE_RATES}, got {self.sample_rate}" + ) + if not self.request_timeout_seconds > 0: + raise ValueError("params.request_timeout_seconds must be > 0") + if self.segment_soft_limit_chars < 1: + raise ValueError("params.segment_soft_limit_chars must be >= 1") + if self.max_pending_chars < 1: + raise ValueError("params.max_pending_chars must be >= 1") + + def ignored_params(self) -> list[str]: + return sorted(self.params.keys()) + + def redacted_base_url(self) -> str: + """The endpoint with credentials and signed query values masked.""" + return _redact_url(self.base_url) + + def to_str(self, sensitive_handling: bool = True) -> str: + if not sensitive_handling: + return f"{self}" + config = copy.deepcopy(self) + if config.key: + config.key = utils.encrypt(config.key) + # Unknown params may hold credentials under any name: mask every + # value, and strip credentials from the URL as well. + config.params = { + name: ( + utils.encrypt(str(value)) if value not in (None, "") else value + ) + for name, value in config.params.items() + } + config.base_url = _redact_url(config.base_url) + return f"{config}" + + +_USERINFO_RE = re.compile(r"^([a-z][a-z0-9+.-]*://)[^/@]+@") +_QUERY_VALUE_RE = re.compile(r"=[^&#]*") + + +def _redact_url(url: str) -> str: + """Mask every credential-bearing part of a URL for diagnostics. + + No name list: a custom endpoint may carry a token under any query name + (``access_token``, ``X-Amz-Signature``, ...), so every query value is + masked and only the names stay. ``scheme://user:secret@host`` becomes + ``scheme://***@host``. + """ + if not url: + return url + url = _USERINFO_RE.sub(r"\1***@", url) + query_start = url.find("?") + if query_start < 0: + return url + head, query = url[: query_start + 1], url[query_start + 1 :] + return head + _QUERY_VALUE_RE.sub("=***", query) + + +def _as_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise ValueError(f"params.{name} must be an integer, got a boolean") + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, str) and value.strip().isdigit(): + return int(value.strip()) + raise ValueError( + f"params.{name} must be an integer, got {type(value).__name__}" + ) + + +def _as_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"params.{name} must be a number, got a boolean") + try: + result = float(value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"params.{name} must be a number, got {type(value).__name__}" + ) from exc + if result != result or result in (float("inf"), float("-inf")): + raise ValueError(f"params.{name} must be a finite number") + return result diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/extension.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/extension.py new file mode 100644 index 0000000000..9f433a51d9 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/extension.py @@ -0,0 +1,801 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +import asyncio +import hashlib +import os +import re +import time +import traceback +from collections import deque + +from pydantic import ValidationError +from ten_ai_base.const import LOG_CATEGORY_KEY_POINT +from ten_ai_base.helper import PCMWriter +from ten_ai_base.message import ( + ModuleError, + ModuleErrorCode, + ModuleErrorVendorInfo, + ModuleType, + TTSAudioEndReason, +) +from ten_ai_base.struct import TTSTextInput +from ten_ai_base.tts2 import AsyncTTS2BaseExtension +from ten_runtime import AsyncTenEnv + +from .aio import cancel_and_wait, wait_done +from .config import KotobaTTSConfig +from .kotoba_tts import KotobaTTSClient, KotobaTTSError, classify_error +from .segmenter import SentenceBuffer + +# Marker put on a request's segment queue after its last segment. +_END_OF_REQUEST = None +# How many finished request ids to remember for late-input detection. +_COMPLETED_HISTORY = 512 +# How long flush and shutdown wait for a worker that is already in its +# terminal path before taking it over. A latency bound only: whatever the +# worker did not complete, the caller completes (see ``_finalize()``). +_TERMINAL_JOIN_SECONDS = 2.0 + + +def dump_file_name(request_id: str) -> str: + """Dump file name for a request id: a sanitized form of the id for + readability plus a hash of the raw id, so ids that sanitize to the same + text (``a/b`` and ``a_b``) never share a file.""" + safe_id = re.sub(r"[^A-Za-z0-9_.-]", "_", request_id) + digest = hashlib.sha256(request_id.encode("utf-8")).hexdigest()[:8] + return f"kotoba_tts_dump_{safe_id}_{digest}.pcm" + + +def _describe_init_error(exc: BaseException) -> str: + """A configuration error message without any rejected input values.""" + if isinstance(exc, ValidationError): + parts = [] + for err in exc.errors(include_input=False): + loc = ".".join(str(x) for x in err.get("loc", ())) or "config" + parts.append(f"{loc}: {err.get('type', 'invalid')}") + return "invalid property: " + "; ".join(parts) + if isinstance(exc, ValueError): + # Our own validate_config() messages name fields, never values. + return str(exc) + return f"{type(exc).__name__}" + + +class _RequestState: + """Everything owned by one TEN request id.""" + + def __init__( + self, request_id: str, generation: int, soft_limit_chars: int + ) -> None: + self.request_id = request_id + self.generation = generation + self.buffer = SentenceBuffer(soft_limit_chars) + self.segments: asyncio.Queue[str | None] = asyncio.Queue() + self.worker: asyncio.Task | None = None + self.audio_started_at: float | None = None + self.emitted_bytes = 0 + self.ttfb_sent = False + self.recorder: PCMWriter | None = None + # A trailing half sample held back until the next chunk of the same + # response completes it, so only whole samples are emitted and + # counted. Cleared at every response boundary. + self.pending_bytes = b"" + # A handshake time taken from the client but not yet reported. + # Kept here so a send interrupted by flush is repeated, not lost. + self.pending_connect_delay_ms: int | None = None + # Terminal progress. ``ended``: a terminal path has started. Each + # ``*_sent`` flag is set right after its event has gone out, so a + # second caller of ``_finalize()`` repeats only what was cut off. + self.ended = False + # text_input_end has arrived: later input for this id is late. + self.input_ended = False + # Characters of the sentences queued for the worker and not yet + # taken; with the buffered fragment, the request's pending text. + self.queued_chars = 0 + self.error_sent = False + self.audio_end_sent = False + self.usage_sent = False + self.finished = False + # Set once every event is out and the request no longer owns the + # extension; only the dump flush may still be running. + self.settled = asyncio.Event() + # Flush and shutdown may finalize concurrently; one at a time. + self.finalize_lock = asyncio.Lock() + # The dump close, run as its own task once the events are out so + # that flush never waits on disk I/O. Joined by on_stop. + self.cleanup: asyncio.Task | None = None + + +class KotobaTTSExtension(AsyncTTS2BaseExtension): + # Injectable monotonic clock; tests replace it to check accounting. + _clock = staticmethod(time.monotonic) + + def __init__(self, name: str) -> None: + super().__init__(name) + self.config: KotobaTTSConfig | None = None + self.client: KotobaTTSClient | None = None + self._init_error: str | None = None + self._request: _RequestState | None = None + # Requests whose terminal path has started and whose dump is not + # closed yet. ``_request`` may already point at the next request. + self._finishing: set[_RequestState] = set() + # Set first thing in on_stop(): no request may start after that. + self._stopping = False + self._generation = 0 + self._completed_ids: set[str] = set() + self._completed_order: deque[str] = deque(maxlen=_COMPLETED_HISTORY) + + # -- lifecycle --------------------------------------------------------- + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + await super().on_init(ten_env) + try: + config_json, _ = await ten_env.get_property_to_json("") + config = KotobaTTSConfig.model_validate_json(config_json or "{}") + config.update_params() + config.validate_config() + self.config = config + ignored = config.ignored_params() + if ignored: + # Names only: an unknown key may hold a credential. + ten_env.log_warn(f"ignoring unknown params: {ignored}") + ten_env.log_info( + f"config: {config.to_str(sensitive_handling=True)}", + category=LOG_CATEGORY_KEY_POINT, + ) + self.client = KotobaTTSClient( + config, + ten_env, + self.vendor(), + on_connecting=self.on_connecting, + on_connected=self.on_connected, + on_disconnected=self.on_disconnected, + clock=self._clock, + ) + except Exception as exc: + # Never echo rejected input: a malformed property may carry + # the API key inside the exception text. + self._init_error = _describe_init_error(exc) + ten_env.log_error(f"on_init failed: {self._init_error}") + await self.send_tts_error( + request_id="", + error=ModuleError( + message=f"Initialization failed: {self._init_error}", + module=ModuleType.TTS, + code=ModuleErrorCode.FATAL_ERROR, + vendor_info=ModuleErrorVendorInfo(vendor=self.vendor()), + ), + ) + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + # The base class's input loop keeps running until its own on_stop; + # refuse new requests first, so a request released by a finishing + # one during the waits below cannot start behind our back. + self._stopping = True + while True: + requests = list(self._finishing) + current = self._request + if current is not None and current not in self._finishing: + requests.append(current) + if not requests: + break + for request in requests: + if request.ended: + # Terminal events or the dump close may be mid-flight: + # let them land, bounded, then complete what is left. + await self._join_bounded(request, request.worker, "stop") + await self._finalize( + request, TTSAudioEndReason.INTERRUPTED, finish=False + ) + await self._join_bounded( + request, request.cleanup, "dump close" + ) + else: + # A request that never reached its end gets no terminal + # events on shutdown (the base class emits none + # either); only its worker is stopped and its dump + # closed. + await self._stop_worker(request) + request.ended = True + # Shutdown is the one caller that must wait for the file: + # whatever the dump task did not finish is closed here. + await self._close_recorder(request) + self._finishing.discard(request) + if self._request is request: + self._request = None + if self.client is not None: + await self.client.close() + await super().on_stop(ten_env) + + # -- vendor contract --------------------------------------------------- + + def vendor(self) -> str: + return "kotoba" + + def vendor_metadata(self) -> dict: + if self.config is None: + return {} + return { + # Metadata travels on every event: never the raw endpoint. + "url": self.config.redacted_base_url(), + "language": self.config.language, + "speaker_id": self.config.speaker_id, + "sample_rate": self.config.sample_rate, + } + + def synthesize_audio_sample_rate(self) -> int: + if self.client is not None: + return self.client.negotiated_sample_rate + if self.config is not None: + return self.config.sample_rate + return 16000 + + def synthesize_audio_channels(self) -> int: + return 1 + + def synthesize_audio_sample_width(self) -> int: + return 2 + + # -- requests ---------------------------------------------------------- + + async def request_tts(self, t: TTSTextInput) -> None: + """Buffer text; a worker task synthesizes complete sentences. + + Returns quickly: the base class awaits this call inline. + """ + current = self._request + if ( + current is not None + and current.request_id == t.request_id + and (current.input_ended or current.ended) + ): + # Late input for the request still in progress: its input is + # closed, or its terminal sequence is running. It is neither + # spoken nor counted, and the queue is not released here: the + # request's own _finalize() calls finish_request() once its + # usage is out, so the next request cannot start early and + # share the usage counters with it. + self.ten_env.log_warn( + f"ignoring late text for {t.request_id}: its input has ended" + ) + return + + if t.request_id in self._completed_ids: + # Late input for a request that has fully finished (for example + # a repeated empty request, or more deltas after an init-failed + # request was reported). Not an error: release the queue. + self.ten_env.log_warn( + f"ignoring text for finished request {t.request_id}" + ) + await self.finish_request(t.request_id) + return + + if self._stopping: + # Shutdown has begun: nothing may start now. The base class is + # about to flush its queue and clear request state itself. + self.ten_env.log_warn( + f"ignoring text for {t.request_id}: extension is stopping" + ) + return + + if self.client is None or self.config is None: + await self.send_tts_error( + t.request_id, + ModuleError( + message=f"TTS not initialized: {self._init_error}", + module=ModuleType.TTS, + code=ModuleErrorCode.FATAL_ERROR, + vendor_info=ModuleErrorVendorInfo(vendor=self.vendor()), + ), + ) + # Consumers close a request on its terminal event: send one, + # and only one; further deltas of this id take the branch above. + await self.send_tts_audio_end( + request_id=t.request_id, + request_event_interval_ms=0, + request_total_audio_duration_ms=0, + reason=TTSAudioEndReason.ERROR, + ) + self._remember_completed(t.request_id) + await self.finish_request( + t.request_id, reason=TTSAudioEndReason.ERROR + ) + return + + request = self._request + if request is None or request.request_id != t.request_id: + if request is not None and not request.ended: + # The base class finishes a request before it hands us the + # next one, so this is unexpected. Interrupt the old one + # so it cannot stay open forever. + self.ten_env.log_warn( + f"request {request.request_id} superseded by " + f"{t.request_id} without text_input_end" + ) + await self._interrupt(request) + request = self._start_request(t.request_id) + + if t.text: + self.metrics_add_input_characters(len(t.text)) + for segment in request.buffer.push(t.text): + request.queued_chars += len(segment) + await request.segments.put(segment) + pending = request.queued_chars + len(request.buffer.pending) + if pending > self.config.max_pending_chars: + # Synthesis has fallen this far behind the input: end the + # request with an explicit error rather than keep growing + # or drop text silently. Later deltas for it are ignored. + self.ten_env.log_warn( + f"request {t.request_id}: {pending} characters pending " + f"exceed max_pending_chars " + f"{self.config.max_pending_chars}; ending it" + ) + self._abort( + request, + TTSAudioEndReason.ERROR, + error=self._module_error( + KotobaTTSError( + f"pending text exceeded max_pending_chars " + f"({self.config.max_pending_chars})", + code="pending_text_limit", + fatal=False, + retryable=False, + ) + ), + finish=True, + ) + return + if t.text_input_end: + request.input_ended = True + await self._finalize_input(request) + + def _start_request(self, request_id: str) -> _RequestState: + assert self.config is not None + self._generation += 1 + request = _RequestState( + request_id, self._generation, self.config.segment_soft_limit_chars + ) + if self.config.dump: + request.recorder = self._open_recorder(request_id) + request.worker = asyncio.create_task(self._run_request(request)) + self._request = request + return request + + def _open_recorder(self, request_id: str) -> PCMWriter | None: + """Create the dump writer, or ``None`` when the dump location is + unusable. A bad ``dump_path`` must never strand the request queue.""" + assert self.config is not None + try: + os.makedirs(self.config.dump_path, exist_ok=True) + path = os.path.join( + self.config.dump_path, dump_file_name(request_id) + ) + # PCMWriter appends: start this request's file empty so a + # stale file from an earlier run never receives new audio. + with open(path, "wb"): + pass + return PCMWriter(path) + except Exception as exc: + self.ten_env.log_error( + f"dump disabled for {request_id}: cannot use " + f"dump_path {self.config.dump_path!r}: {exc}" + ) + return None + + async def _finalize_input(self, request: _RequestState) -> None: + rest = request.buffer.flush() + if rest: + await request.segments.put(rest) + await request.segments.put(_END_OF_REQUEST) + + async def _run_request(self, request: _RequestState) -> None: + try: + while True: + segment = await request.segments.get() + if segment is _END_OF_REQUEST: + break + request.queued_chars -= len(segment) + await self._synthesize_segment(request, segment) + await self._end_request(request, TTSAudioEndReason.REQUEST_END) + # CancelledError is a BaseException: it passes through the handlers + # below untouched, which is what flush relies on. + except KotobaTTSError as err: + await self._end_request( + request, TTSAudioEndReason.ERROR, error=self._module_error(err) + ) + except Exception as exc: + self.ten_env.log_error( + f"request {request.request_id} failed: " + f"{traceback.format_exc()}" + ) + await self._end_request( + request, + TTSAudioEndReason.ERROR, + error=self._module_error(classify_error(exc)), + ) + + async def _synthesize_segment( + self, request: _RequestState, segment: str + ) -> None: + """Send one sentence; retry once on a fresh socket when the failure + happened before any audio for this sentence arrived.""" + assert self.client is not None + for attempt in range(2): + audio_in_segment = False + await self.send_tts_request_metrics( + request.request_id, + segment, + request_time_ms=int(time.time() * 1000), + ) + stream = self.client.synthesize(segment) + try: + async for chunk in stream: + if not self._is_current(request): + return + audio_in_segment = True + await self._emit_audio(request, chunk) + return + except KotobaTTSError as err: + if not self._is_current(request): + return + if err.retryable and not audio_in_segment and attempt == 0: + self.ten_env.log_warn( + f"retrying segment after {err.code}: {err}", + category=LOG_CATEGORY_KEY_POINT, + ) + continue + raise + finally: + await stream.aclose() + self._end_response(request) + + def _end_response(self, request: _RequestState) -> None: + """A service response is over (done, failed, retried or abandoned). + + Each response is encoded independently, so a half sample left at + its end can never be completed by the next one: drop it instead of + joining it to the next response's first byte. + """ + if request.pending_bytes: + self.ten_env.log_warn( + f"request {request.request_id}: response ended on a partial " + f"sample; dropped {len(request.pending_bytes)} byte(s)" + ) + request.pending_bytes = b"" + + async def _emit_audio(self, request: _RequestState, chunk: bytes) -> None: + assert self.client is not None and self.config is not None + frame_bytes = ( + self.synthesize_audio_sample_width() + * self.synthesize_audio_channels() + ) + data = request.pending_bytes + chunk + cut = len(data) - len(data) % frame_bytes + request.pending_bytes = data[cut:] + chunk = data[:cut] + if not chunk: + return + ttfb_ms: int | None = None + if not request.ttfb_sent: + request.ttfb_sent = True + # TTFB runs from the moment the text was handed to the socket, + # not from connection setup (reported as connect_delay below). + # Read before any await: the metric sends take time that is + # not part of the time to first byte. + origin = self.client.last_send_at + ttfb_ms = ( + int((self._clock() - origin) * 1000) + if origin is not None + else 0 + ) + await self._report_connect_delay(request) + if ttfb_ms is not None: + await self.send_tts_ttfb_metrics( + request_id=request.request_id, + ttfb_ms=ttfb_ms, + extra_metadata={ + "speaker_id": self.config.speaker_id, + "language": self.config.language, + }, + ) + if request.audio_started_at is None: + # The event interval starts when tts_audio_start is emitted. + request.audio_started_at = self._clock() + await self.send_tts_audio_start(request_id=request.request_id) + await self.send_tts_audio_data(chunk) + # Count only audio that was handed to the runtime. + request.emitted_bytes += len(chunk) + self.metrics_add_recv_audio_chunks(chunk) + if request.recorder is not None: + try: + await request.recorder.write(chunk) + except Exception as exc: + # A dump failure must not fail the request: log once and + # stop recording this request. + self.ten_env.log_error( + f"dump write failed for {request.request_id}: {exc}" + ) + request.recorder = None + + async def _end_request( + self, + request: _RequestState, + reason: TTSAudioEndReason, + error: ModuleError | None = None, + ) -> None: + """Worker entry to the terminal path, for completion and failure.""" + if not self._is_current(request) or request.ended: + return + self._generation += 1 + await self._finalize(request, reason, error) + + async def cancel_tts(self) -> None: + """Flush: stop the worker, cancel the in-flight response, and + report INTERRUPTED. The base class clears request state itself. + + If the worker is already emitting its terminal events, wait for + them so that no event of the old request lands after + tts_flush_end; if they do not land in time, take over. + """ + request = self._request + if request is None: + return + if not request.ended: + await self._interrupt(request) + return + await self._await_terminal(request) + + async def _interrupt(self, request: _RequestState) -> None: + """Stop a request that is still synthesizing, end it as + INTERRUPTED, and wait (bounded) until its events are out. + ``finish_request()`` is not called: the base class owns the state + transition on flush.""" + self._abort(request, TTSAudioEndReason.INTERRUPTED, finish=False) + await self._await_terminal(request) + + async def _await_terminal(self, request: _RequestState) -> None: + """Wait up to _TERMINAL_JOIN_SECONDS for the request's terminal + events. If they do not land, the task running them is cancelled + and the rest is completed here, as INTERRUPTED.""" + if await self._wait_settled(request): + return + await self._finalize( + request, TTSAudioEndReason.INTERRUPTED, finish=False + ) + + def _abort( + self, + request: _RequestState, + reason: TTSAudioEndReason, + error: ModuleError | None = None, + *, + finish: bool, + ) -> None: + """End a request that is still synthesizing: cancel its worker and + start the abort as the request's own task, which waits for the + worker, cancels the in-flight response and runs the terminal + sequence. + + Nothing here awaits, so ``request.worker`` holds the abort from the + moment the request is marked ended. A flush or a shutdown can then + cancel and take it over however long a step stalls, including the + ``response.cancel`` write; a step awaited on the caller's task (for + example, the base class's input loop) could neither be taken over + nor be stopped from emitting after the takeover. Callers that must + not return before the events are out wait with + ``_await_terminal()``. + """ + request.ended = True + self._generation += 1 + worker = request.worker + if worker is not None: + worker.cancel() + request.worker = asyncio.create_task( + self._run_terminal(request, worker, reason, error, finish) + ) + + async def _run_terminal( + self, + request: _RequestState, + worker: asyncio.Task | None, + reason: TTSAudioEndReason, + error: ModuleError | None, + finish: bool, + ) -> None: + """Task body for ``_abort()``: wait for the cancelled worker, cancel + the in-flight response, then the terminal sequence. A failure is + logged rather than left unobserved on the task; a waiter then + takes over after its bound, as for a stalled send.""" + try: + if worker is not None: + await wait_done(worker) + if self.client is not None: + await self.client.cancel() + await self._finalize(request, reason, error, finish=finish) + except Exception: + self.ten_env.log_error( + f"request {request.request_id} terminal sequence failed: " + f"{traceback.format_exc()}" + ) + + async def _finalize( + self, + request: _RequestState, + reason: TTSAudioEndReason, + error: ModuleError | None = None, + *, + finish: bool = True, + ) -> None: + """The single terminal sequence: error, audio_end, usage, state + transition, release of the extension, dump flush. + + Idempotent: every step records its completion on the request right + after its await returns, so a caller that takes over after the + worker was cancelled mid-way repeats only the steps that did not + complete. ``finish`` is False when the base class performs the + state transition itself (flush, shutdown). + """ + request.ended = True + self._finishing.add(request) + async with request.finalize_lock: + if error is not None and not request.error_sent: + # Error first, then the terminal audio_end, so consumers + # learn why the request ends before they see it end. + await self.send_tts_error(request.request_id, error) + request.error_sent = True + if not request.audio_end_sent: + interval_ms = 0 + if request.audio_started_at is not None: + interval_ms = int( + (self._clock() - request.audio_started_at) * 1000 + ) + bytes_per_second = ( + self.synthesize_audio_sample_rate() + * self.synthesize_audio_sample_width() + * self.synthesize_audio_channels() + ) + duration_ms = request.emitted_bytes * 1000 // bytes_per_second + await self.send_tts_audio_end( + request_id=request.request_id, + request_event_interval_ms=interval_ms, + request_total_audio_duration_ms=duration_ms, + reason=reason, + ) + request.audio_end_sent = True + self._remember_completed(request.request_id) + if not request.usage_sent: + # A handshake paid for by this request but never followed + # by audio (flushed early) is still this request's metric. + await self._report_connect_delay(request) + # Usage for this request only; this also resets the base + # counters so the next request does not inherit them. + await self.send_usage_metrics(request.request_id) + request.usage_sent = True + if finish and not request.finished: + await self.finish_request(request.request_id, reason=reason) + request.finished = True + # Every event is out: the extension is free for the next + # request, and the caller (flush included) is not made to wait + # for disk I/O. The dump closes in its own task; the request + # stays in _finishing until that task is through, so on_stop + # can still join it. + if self._request is request: + self._request = None + request.settled.set() + if request.recorder is not None and request.cleanup is None: + request.cleanup = asyncio.create_task(self._close_dump(request)) + if request.recorder is None: + self._finishing.discard(request) + + async def _close_dump(self, request: _RequestState) -> None: + await self._close_recorder(request) + self._finishing.discard(request) + + async def _report_connect_delay(self, request: _RequestState) -> None: + """Report the handshake time of a new connection under this + request, once. The value is parked on the request before the send + so an interrupted send is repeated by the next caller.""" + if request.pending_connect_delay_ms is None and self.client is not None: + request.pending_connect_delay_ms = ( + self.client.take_connect_delay_ms() + ) + if request.pending_connect_delay_ms is None: + return + await self.metrics_connect_delay( + request.pending_connect_delay_ms, request_id=request.request_id + ) + request.pending_connect_delay_ms = None + + async def _wait_settled(self, request: _RequestState) -> bool: + """Wait for the request's terminal events to land. Returns False + when they did not land in time, in which case the task running them + has been cancelled and the caller takes over.""" + try: + await asyncio.wait_for( + request.settled.wait(), _TERMINAL_JOIN_SECONDS + ) + return True + except asyncio.TimeoutError: + self.ten_env.log_warn( + f"request {request.request_id} did not finish within " + f"{_TERMINAL_JOIN_SECONDS:g} s; cancelling it" + ) + await self._stop_worker(request) + return False + + async def _join_bounded( + self, request: _RequestState, task: asyncio.Task | None, what: str + ) -> None: + """Shutdown found a request's worker or dump close in flight: wait + for it, bounded. The task may end in any way, including being + cancelled by a concurrent flush takeover: that is a completed join. + On timeout cancel it; the caller completes the rest.""" + if task is None or task.done() or task is asyncio.current_task(): + return + if await wait_done(task, _TERMINAL_JOIN_SECONDS): + return + self.ten_env.log_warn( + f"request {request.request_id}: {what} did not finish " + f"within {_TERMINAL_JOIN_SECONDS:g} s of stop; cancelling it" + ) + await cancel_and_wait(task) + if task is request.worker: + request.worker = None + + # -- helpers ----------------------------------------------------------- + + def _is_current(self, request: _RequestState) -> bool: + return ( + self._request is request and request.generation == self._generation + ) + + def _module_error(self, err: KotobaTTSError) -> ModuleError: + return ModuleError( + message=str(err), + module=ModuleType.TTS, + code=( + ModuleErrorCode.FATAL_ERROR + if err.fatal + else ModuleErrorCode.NON_FATAL_ERROR + ), + vendor_info=err.vendor_info(self.vendor()), + ) + + def _remember_completed(self, request_id: str) -> None: + if len(self._completed_order) == self._completed_order.maxlen: + oldest = self._completed_order[0] + self._completed_ids.discard(oldest) + self._completed_order.append(request_id) + self._completed_ids.add(request_id) + + async def _stop_worker(self, request: _RequestState) -> None: + worker = request.worker + request.worker = None + if worker is None or worker.done(): + return + if worker is asyncio.current_task(): + return + await cancel_and_wait(worker) + + async def _close_recorder(self, request: _RequestState) -> None: + recorder = request.recorder + if recorder is None: + return + try: + # PCMWriter.flush() returns early while a background write is + # running and leaves bytes buffered since then; repeat until + # the buffer is empty so the dump holds every emitted frame. + # Shielded: cancelling the caller must not cancel the writer's + # own task, which has already taken the bytes out of the buffer. + for _ in range(8): + await asyncio.shield(recorder.flush()) + if not getattr(recorder, "buffer", None): + break + except Exception as exc: + self.ten_env.log_error( + f"failed to flush dump for {request.request_id}: {exc}" + ) + # Detached only once the flush is through: a close interrupted by + # cancellation leaves the recorder for the caller that takes over. + request.recorder = None diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/kotoba_tts.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/kotoba_tts.py new file mode 100644 index 0000000000..5efadc30a4 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/kotoba_tts.py @@ -0,0 +1,540 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Thin client over the Kotoba Python SDK (``kotoba-sdk``). + +One ``AsyncTTSSession`` is kept open across requests. The service closes +idle sockets and connections can drop at any time, so every request goes +through ``connect()``, which reconnects when the socket is gone. +""" +import asyncio +import re +import time +from collections.abc import AsyncIterator, Awaitable, Callable + +import websockets +from kotoba._ws_tts import AsyncTTSSession +from kotoba.errors import APIError, AuthError, KotobaError, ProtocolError +from kotoba.errors import TimeoutError as KotobaTimeoutError +from ten_ai_base.const import LOG_CATEGORY_VENDOR +from ten_ai_base.message import ModuleErrorVendorInfo +from ten_runtime import AsyncTenEnv + +from .aio import cancel_and_wait, wait_done +from .config import KotobaTTSConfig + +# Server error codes that mean the configuration can never work. +_FATAL_CODES = frozenset( + { + "unsupported_language", + "unsupported_format", + "invalid_sample_rate", + "invalid_open_message", + "unauthorized", + } +) + +# Server error codes that reject the text of one request: not fatal for the +# session, and not worth a retry with the same text. +_REQUEST_CODES = frozenset({"invalid_text", "text_too_long"}) + +# ``HTTP 429`` inside a handshake failure message. +_HTTP_STATUS_RE = re.compile(r"\bHTTP (\d{3})\b") + +# How long ``cancel()`` waits for the server's ``response.done`` before the +# socket is dropped instead. +CANCEL_GRACE_SECONDS = 2.0 + +# ``scheme://user:secret@host`` -> ``scheme://***@host`` inside free text. +_USERINFO_RE = re.compile(r"([a-z][a-z0-9+.-]*://)[^/@\s]+@") + + +class KotobaTTSError(Exception): + """A failed service call, classified for the extension. + + ``fatal`` maps to ``ModuleErrorCode.FATAL_ERROR``. ``retryable`` says a + fresh connection may succeed when no audio was produced yet. + """ + + def __init__( + self, message: str, *, code: str, fatal: bool, retryable: bool + ) -> None: + super().__init__(message) + self.code = code + self.fatal = fatal + self.retryable = retryable + + def vendor_info(self, vendor: str) -> ModuleErrorVendorInfo: + return ModuleErrorVendorInfo( + vendor=vendor, code=self.code, message=str(self) + ) + + +def classify_error(exc: BaseException) -> KotobaTTSError: + """Map an SDK or transport exception onto ``KotobaTTSError``.""" + if isinstance(exc, KotobaTTSError): + return exc + if isinstance(exc, AuthError): + return KotobaTTSError( + "fal rejected the API key", + code="unauthorized", + fatal=True, + retryable=False, + ) + if isinstance(exc, ProtocolError): + # The service names the condition in ``message`` on ``error`` + # frames and in ``error.code`` on failed responses; check both. + text = f"{exc.code or ''} {exc}" + fatal_code = next((c for c in _FATAL_CODES if c in text), None) + if fatal_code is None and "sample_rate" in text: + fatal_code = "invalid_sample_rate" + code = fatal_code or exc.code or "error" + return KotobaTTSError( + str(exc), + code=code, + fatal=fatal_code is not None, + retryable=fatal_code is None and code not in _REQUEST_CODES, + ) + if _is_policy_close(exc): + # fal accepts the upgrade and then closes with 1008 "Unauthorized" + # for an unknown key, instead of refusing the HTTP handshake. + return KotobaTTSError( + "fal rejected the API key", + code="unauthorized", + fatal=True, + retryable=False, + ) + if isinstance(exc, KotobaTimeoutError): + return KotobaTTSError( + str(exc), code="handshake_timeout", fatal=False, retryable=True + ) + if isinstance(exc, (APIError, KotobaError)): + # A refused handshake names its HTTP status; 429 is the gateway's + # quota or rate limit, transient and worth one retry. + status = _HTTP_STATUS_RE.search(str(exc)) + if status is not None and status.group(1) == "429": + return KotobaTTSError( + str(exc), code="rate_limited", fatal=False, retryable=True + ) + return KotobaTTSError( + str(exc), code="transport_error", fatal=False, retryable=True + ) + if isinstance(exc, websockets.exceptions.ConnectionClosed): + close_code = exc.rcvd.code if exc.rcvd is not None else None + return KotobaTTSError( + f"connection closed: code={close_code}", + code="connection_closed", + fatal=False, + retryable=True, + ) + # Unknown exception: the message may quote arbitrary data, so only the + # type is emitted; the full traceback goes to the log. + return KotobaTTSError( + f"unexpected {type(exc).__name__}", + code="unexpected_error", + fatal=False, + retryable=False, + ) + + +def _is_policy_close(exc: BaseException) -> bool: + """True for a WebSocket close with code 1008 (policy violation), which + the fal gateway uses for authentication failures.""" + if isinstance(exc, websockets.exceptions.ConnectionClosed): + return exc.rcvd is not None and exc.rcvd.code == 1008 + if isinstance(exc, APIError) and not isinstance(exc, ProtocolError): + text = str(exc) + return "code=1008" in text or "unauthorized" in text.lower() + return False + + +class KotobaTTSClient: + def __init__( + self, + config: KotobaTTSConfig, + ten_env: AsyncTenEnv, + vendor: str, + *, + on_connecting: Callable[[], Awaitable[None]] | None = None, + on_connected: Callable[[], Awaitable[None]] | None = None, + on_disconnected: Callable[..., Awaitable[None]] | None = None, + session_factory: Callable[..., AsyncTTSSession] = AsyncTTSSession, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.config = config + self.ten_env = ten_env + self.vendor = vendor + self._on_connecting = on_connecting + self._on_connected = on_connected + self._on_disconnected = on_disconnected + self._session_factory = session_factory + self._clock = clock + + self._session: AsyncTTSSession | None = None + self._connect_lock = asyncio.Lock() + # True from the moment text is handed to the socket until the + # server's response.done has been consumed. Stays True when the + # consumer stops early, so cancel() knows there is work to undo. + self._response_active = False + self._drain_task: asyncio.Task | None = None + # Watches the SDK receiver so a remote close is reported while idle. + self._watch_task: asyncio.Task | None = None + # Set by the watcher when the transport ended during a response, so + # synthesize() retires the session once that response is consumed. + self._remote_closed: AsyncTTSSession | None = None + self._closed = False + + # Rate echoed by ``session.created``; equals the configured rate + # unless the server negotiated differently. + self.negotiated_sample_rate: int = config.sample_rate + # Monotonic time of the last text send, the TTFB origin. + self.last_send_at: float | None = None + # Handshake duration of the most recent new connection, consumed + # once by ``take_connect_delay_ms()``. + self._pending_connect_delay_ms: int | None = None + + # -- connection -------------------------------------------------------- + + def take_connect_delay_ms(self) -> int | None: + """Return and clear the handshake time of the last new connection.""" + delay, self._pending_connect_delay_ms = ( + self._pending_connect_delay_ms, + None, + ) + return delay + + def _scrub(self, text: str) -> str: + """Remove endpoint credentials from diagnostic text. + + SDK and transport errors quote the URL they tried (``Could not + reach ``); a custom endpoint may carry a password or a signed + query, so every string we log or emit goes through here. + """ + if not text: + return text + raw = self.config.base_url + if raw and raw in text: + text = text.replace(raw, self.config.redacted_base_url()) + return _USERINFO_RE.sub(r"\1***@", text) + + def _classify(self, exc: BaseException) -> KotobaTTSError: + err = classify_error(exc) + clean = self._scrub(str(err)) + if clean == str(err): + return err + return KotobaTTSError( + clean, code=err.code, fatal=err.fatal, retryable=err.retryable + ) + + async def connect(self) -> AsyncTTSSession: + """Return the open session, connecting when needed.""" + if self._response_active: + # A previous caller abandoned its response; cancel it so the + # socket carries no stale audio into the next request. + await self.cancel() + await self._await_drain() + async with self._connect_lock: + if self._closed: + raise KotobaTTSError( + "client is closed", + code="client_closed", + fatal=False, + retryable=False, + ) + if self._session is not None: + return self._session + await self._report_connecting() + started = self._clock() + session = self._session_factory( + self.config.base_url, + language=self.config.language, + speaker_id=self.config.speaker_id, + audio_format="pcm16", + sample_rate=self.config.sample_rate, + api_key=self.config.key, + ) + try: + await session.start() + except asyncio.CancelledError: + await session.close() + await self._report_disconnected(message="connect cancelled") + raise + except Exception as exc: + err = self._classify(exc) + self.ten_env.log_warn( + f"vendor_status: connect failed code={err.code}", + category=LOG_CATEGORY_VENDOR, + ) + await self._report_disconnected( + message="connect failed", + vendor_info=err.vendor_info(self.vendor), + ) + raise err from exc + self._session = session + self._response_active = False + self.negotiated_sample_rate = int(session.sample_rate) + if session.audio_format not in ("pcm16", "pcm_16"): + await self._drop_locked(message="unexpected audio format") + raise KotobaTTSError( + f"server negotiated {session.audio_format}, " + "expected pcm16", + code="unsupported_format", + fatal=True, + retryable=False, + ) + delay_ms = int((self._clock() - started) * 1000) + self._pending_connect_delay_ms = delay_ms + self.ten_env.log_info( + f"vendor_status: connected in {delay_ms} ms, " + f"sample_rate={self.negotiated_sample_rate}", + category=LOG_CATEGORY_VENDOR, + ) + await self._report_connected() + self._watch_task = asyncio.create_task( + self._watch_remote_close(session) + ) + return session + + async def _watch_remote_close(self, session: AsyncTTSSession) -> None: + """Report a remote close as soon as the SDK receiver ends. + + Only the watched session is retired; a response in progress is left + to ``synthesize()``, which sees the same termination. + """ + receiver = getattr(session, "_receiver_task", None) + if receiver is None: + return + # Only the fact that the receiver ended matters here; its errors + # are reported through synthesize(). A receiver cancelled by + # session.close() also counts as ended. + await wait_done(receiver) + if self._session is not session: + return + if self._response_active: + # synthesize() is still consuming this response's queued + # events; it retires the session when the response completes. + self._remote_closed = session + return + await self.drop(message="remote closed") + + async def drop(self, *, message: str) -> None: + """Close the socket; the next ``connect()`` opens a new one.""" + async with self._connect_lock: + await self._drop_locked(message=message) + + async def _drop_locked(self, *, message: str) -> None: + session, self._session = self._session, None + self._response_active = False + self._remote_closed = None + watch, self._watch_task = self._watch_task, None + if ( + watch is not None + and not watch.done() + and watch is not asyncio.current_task() + ): + watch.cancel() + if session is None: + return + await session.close() + self.ten_env.log_info( + f"vendor_status: disconnected ({message})", + category=LOG_CATEGORY_VENDOR, + ) + await self._report_disconnected(message=message) + + async def close(self) -> None: + self._closed = True + if self._drain_task is not None: + await cancel_and_wait(self._drain_task) + self._drain_task = None + await self.drop(message="closed") + + # -- status callbacks (a reporting failure never breaks the client) --- + + async def _report_connecting(self) -> None: + if self._on_connecting is None: + return + try: + await self._on_connecting() + except Exception as exc: + self.ten_env.log_error(f"on_connecting callback failed: {exc}") + + async def _report_connected(self) -> None: + if self._on_connected is None: + return + try: + await self._on_connected() + except Exception as exc: + self.ten_env.log_error(f"on_connected callback failed: {exc}") + + async def _report_disconnected( + self, + *, + message: str, + vendor_info: ModuleErrorVendorInfo | None = None, + ) -> None: + if self._on_disconnected is None: + return + try: + await self._on_disconnected( + code=0, message=message, vendor_info=vendor_info + ) + except Exception as exc: + self.ten_env.log_error(f"on_disconnected callback failed: {exc}") + + # -- synthesis --------------------------------------------------------- + + async def synthesize(self, text: str) -> AsyncIterator[bytes]: + """Yield PCM16 chunks for ``text`` until the server reports done. + + Raises ``KotobaTTSError``. After an error the socket is dropped, so + a retry reconnects. ``last_send_at`` is set when the text has been + handed to the socket. + """ + session = await self.connect() + timeout = self.config.request_timeout_seconds + + # Mark the response active before the send: if the send is + # interrupted the server may still have received the text. + self._response_active = True + try: + await session.synthesize(text) + except asyncio.CancelledError: + await self.drop(message="send interrupted") + raise + except Exception as exc: + # A socket the server closed while idle fails here first. + await self.drop(message="send failed") + raise self._classify(exc) from exc + self.last_send_at = self._clock() + + done = False + finished = False + finish_reason = None + try: + while True: + try: + event = await asyncio.wait_for( + session.__anext__(), timeout=timeout + ) + except StopAsyncIteration: + break + if event.type == "audio_chunk": + if event.audio: + yield event.audio + elif event.type == "done": + done = True + status = event.metadata.get("status") + # The SDK lifts only ``status`` out of the response. + response = event.metadata.get("response") or {} + finish_reason = response.get("finish_reason") + self.ten_env.log_debug( + f"vendor_status: response done status={status} " + f"finish_reason={finish_reason}", + category=LOG_CATEGORY_VENDOR, + ) + finished = True + except asyncio.TimeoutError as exc: + await self.drop(message="response timeout") + raise KotobaTTSError( + f"no audio from the service for {timeout:g} s", + code="response_timeout", + fatal=False, + retryable=False, + ) from exc + except KotobaError as exc: + await self.drop(message="service error") + raise self._classify(exc) from exc + finally: + # Left True when the consumer stopped early (cancellation or an + # abandoned generator): ``cancel()`` then clears the server side. + if finished: + self._response_active = False + + if done and self._remote_closed is session: + # The server closed right after response.done while we were + # still draining its queue: retire the socket now. + await self.drop(message="remote closed after response") + + if not done: + await self.drop(message="closed before response.done") + raise KotobaTTSError( + "connection closed before the response completed", + code="connection_closed", + fatal=False, + retryable=True, + ) + + if finish_reason == "length": + # The service stopped at its generation length limit and still + # reports the response completed. The audio already yielded + # stands; the rest of the text was never synthesized, and the + # same text would stop at the same place again. + raise KotobaTTSError( + "the service stopped at its generation length limit; " + "the rest of the text was not synthesized", + code="response_truncated", + fatal=False, + retryable=False, + ) + + async def cancel(self) -> None: + """Cancel the in-flight response, if any. + + The server answers with ``response.done(cancelled)``; a drain task + consumes it so the socket can be reused. If the server does not + answer in time, the socket is dropped. + """ + session = self._session + if session is None or not self._response_active: + return + self._response_active = False + try: + await session.cancel() + except asyncio.CancelledError: + # The cancel may not have reached the service, so the old + # response can still be streaming on this socket. + await self.drop(message="cancel interrupted") + raise + except Exception as exc: + await self.drop(message=f"cancel failed: {self._scrub(str(exc))}") + return + self._drain_task = asyncio.create_task(self._drain(session)) + + async def _drain(self, session: AsyncTTSSession) -> None: + try: + await asyncio.wait_for( + self._consume_until_done(session), CANCEL_GRACE_SECONDS + ) + except (asyncio.TimeoutError, KotobaError) as exc: + if self._session is session: + await self.drop( + message=f"cancel drain failed: {self._scrub(str(exc))}" + ) + + @staticmethod + async def _consume_until_done(session: AsyncTTSSession) -> None: + async for _event in session: + pass + + async def _await_drain(self) -> None: + """Wait for a pending cancel drain. + + Caller cancellation propagates: an interrupted drain leaves the + socket in an unknown state, so it is dropped first. + """ + task = self._drain_task + if task is None: + return + try: + # The drain's own outcome, including being cancelled by + # close(), is not this caller's failure. + await wait_done(task) + except asyncio.CancelledError: + task.cancel() + await self.drop(message="drain interrupted") + raise + finally: + if self._drain_task is task: + self._drain_task = None diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/manifest.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/manifest.json new file mode 100644 index 0000000000..03c401599d --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/manifest.json @@ -0,0 +1,72 @@ +{ + "type": "extension", + "name": "kotoba_tts_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.11" + }, + { + "type": "system", + "name": "ten_ai_base", + "version": "0.7" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "BUILD.gn", + "**.tent", + "**.py", + "README.md", + "pyproject.toml", + "requirements.txt" + ] + }, + "api": { + "interface": [ + { + "import_uri": "../../system/ten_ai_base/api/tts-interface.json" + } + ], + "property": { + "properties": { + "params": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "base_url": { + "type": "string" + }, + "language": { + "type": "string" + }, + "speaker_id": { + "type": "string" + }, + "sample_rate": { + "type": "int64" + }, + "request_timeout_seconds": { + "type": "float64" + }, + "segment_soft_limit_chars": { + "type": "int64" + } + } + }, + "dump": { + "type": "bool" + }, + "dump_path": { + "type": "string" + } + } + } + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/property.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/property.json new file mode 100644 index 0000000000..d017c6b7bc --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/property.json @@ -0,0 +1,11 @@ +{ + "dump": false, + "dump_path": "./", + "params": { + "key": "${env:KOTOBA_TTS_KEY|}", + "base_url": "wss://fal.run/Kotoba-Technologies/kotoba-tts/v2/tts/ws", + "language": "ja", + "speaker_id": "male-01", + "sample_rate": 16000 + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/pyproject.toml b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/pyproject.toml new file mode 100644 index 0000000000..8d6eebe07e --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "kotoba-tts-python" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "kotoba-sdk>=0.3.0", + "pydantic>=2.0.0", + "websockets>=15.0", +] diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/requirements.txt b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/requirements.txt new file mode 100644 index 0000000000..37fa889a16 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/requirements.txt @@ -0,0 +1,3 @@ +kotoba-sdk>=0.3.0 +pydantic>=2.0.0 +websockets>=15.0 diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/segmenter.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/segmenter.py new file mode 100644 index 0000000000..38228c7c60 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/segmenter.py @@ -0,0 +1,104 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Sentence buffering for streamed LLM text. + +The Kotoba TTS service synthesizes one complete text per response, while +TEN delivers LLM output as many small deltas. ``SentenceBuffer`` collects +deltas and releases complete sentences, so each sentence becomes one +service request as soon as it is complete. +""" + +# Terminators that always end a sentence, wherever they appear. +_HARD_TERMINATORS = "。！？\n" +# Terminators that end a sentence only when followed by a space, a closing +# quote or bracket, or the end of the delta (so ``3.14`` stays whole). +_ASCII_TERMINATORS = ".!?" +# Soft break points used only when the buffer grows past the soft limit. +_SOFT_TERMINATORS = "、，,;；" +# Characters that belong to the sentence they close. +_CLOSERS = '」』”’")）】》' + + +class SentenceBuffer: + def __init__(self, soft_limit_chars: int = 120) -> None: + if soft_limit_chars < 1: + raise ValueError("soft_limit_chars must be >= 1") + self._soft_limit = soft_limit_chars + self._buffer = "" + + @property + def pending(self) -> str: + """Text received but not yet released.""" + return self._buffer + + def push(self, text: str) -> list[str]: + """Append ``text`` and return the sentences that are now complete.""" + if not text: + return [] + self._buffer += text + segments: list[str] = [] + while True: + cut = self._find_boundary(self._buffer) + if cut is None: + break + segments.append(self._buffer[:cut]) + self._buffer = self._buffer[cut:].lstrip() + segments.extend(self._cut_oversized()) + return [s.strip() for s in segments if _has_speech(s)] + + def flush(self) -> str | None: + """Release whatever is buffered, or ``None`` when nothing in it + can be spoken (blank or punctuation only).""" + rest = self._buffer.strip() + self._buffer = "" + return rest if _has_speech(rest) else None + + def _find_boundary(self, text: str) -> int | None: + """Index just past the first sentence end, or ``None``.""" + length = len(text) + for i, ch in enumerate(text): + if ch in _HARD_TERMINATORS: + return self._extend_over_closers(text, i + 1) + if ch in _ASCII_TERMINATORS: + prev_is_digit = i > 0 and text[i - 1].isdigit() + if i + 1 >= length: + if ch == "." and prev_is_digit: + return None + return length + nxt = text[i + 1] + if nxt.isspace() or nxt in _CLOSERS: + return self._extend_over_closers(text, i + 1) + return None + + @staticmethod + def _extend_over_closers(text: str, index: int) -> int: + while index < len(text) and text[index] in _CLOSERS: + index += 1 + return index + + def _cut_oversized(self) -> list[str]: + """Bound the buffer: cut at a soft break past the soft limit, and + unconditionally at twice the soft limit.""" + segments: list[str] = [] + while len(self._buffer) > self._soft_limit: + window = self._buffer[: self._soft_limit * 2] + cut = -1 + for j in range(len(window) - 1, -1, -1): + if window[j] in _SOFT_TERMINATORS: + cut = j + 1 + break + if cut == -1: + if len(self._buffer) <= self._soft_limit * 2: + break + space = window.rfind(" ") + cut = space + 1 if space > 0 else len(window) + segments.append(self._buffer[:cut]) + self._buffer = self._buffer[cut:].lstrip() + return segments + + +def _has_speech(text: str) -> bool: + """True when ``text`` holds at least one letter or digit.""" + return any(ch.isalnum() for ch in text) diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/__init__.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/bin/start b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/bin/start new file mode 100755 index 0000000000..f6a1cf283d --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/bin/start @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." + +export PYTHONPATH=.ten/app:.ten/app/ten_packages/system/ten_runtime_python/lib:.ten/app/ten_packages/system/ten_runtime_python/interface:.ten/app/ten_packages/system/ten_ai_base/interface:$PYTHONPATH + +# If the Python app imports some modules that are compiled with a different +# version of libstdc++ (ex: PyTorch), the Python app may encounter confusing +# errors. To solve this problem, we can preload the correct version of +# libstdc++. +# +# export LD_PRELOAD=/lib/x86_64-linux-gnu/libstdc++.so.6 +# +# Another solution is to make sure the module 'ten_runtime_python' is imported +# _after_ the module that requires another version of libstdc++ is imported. +# +# Refer to https://github.com/pytorch/pytorch/issues/102360?from_wecom=1#issuecomment-1708989096 + +pytest -s tests/ "$@" \ No newline at end of file diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_basic_audio_setting1.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_basic_audio_setting1.json new file mode 100644 index 0000000000..b80336787e --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_basic_audio_setting1.json @@ -0,0 +1,8 @@ +{ + "dump": true, + "dump_path": "./tests/keep_dump_output/", + "params": { + "key": "${env:KOTOBA_TTS_KEY}", + "sample_rate": 16000 + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_basic_audio_setting2.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_basic_audio_setting2.json new file mode 100644 index 0000000000..e8e513da42 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_basic_audio_setting2.json @@ -0,0 +1,8 @@ +{ + "dump": true, + "dump_path": "./tests/keep_dump_output/", + "params": { + "key": "${env:KOTOBA_TTS_KEY}", + "sample_rate": 24000 + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_dump.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_dump.json new file mode 100644 index 0000000000..1324f5e6a6 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_dump.json @@ -0,0 +1,7 @@ +{ + "dump": true, + "dump_path": "./tests/dump_output/", + "params": { + "key": "${env:KOTOBA_TTS_KEY}" + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_invalid.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_invalid.json new file mode 100644 index 0000000000..eca2fc1216 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_invalid.json @@ -0,0 +1,5 @@ +{ + "params": { + "key": "invalid" + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_miss_required.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_miss_required.json new file mode 100644 index 0000000000..7292ac4c63 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs/property_miss_required.json @@ -0,0 +1,5 @@ +{ + "params": { + "key": "" + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/README.md b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/README.md new file mode 100644 index 0000000000..b32a36e61d --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/README.md @@ -0,0 +1,3 @@ +Guarder configs that point at `tests/run_fake_server.py` instead of fal. +Use: `task tts-guarder-test EXTENSION=kotoba_tts_python CONFIG_DIR=tests/configs_fake` +with `KOTOBA_TTS_KEY=fake-key` exported and the fake server running. diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_basic_audio_setting1.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_basic_audio_setting1.json new file mode 100644 index 0000000000..9c94b2652e --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_basic_audio_setting1.json @@ -0,0 +1,9 @@ +{ + "dump": true, + "dump_path": "./tests/keep_dump_output/", + "params": { + "key": "${env:KOTOBA_TTS_KEY}", + "base_url": "ws://127.0.0.1:8765/v2/tts/ws", + "sample_rate": 16000 + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_basic_audio_setting2.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_basic_audio_setting2.json new file mode 100644 index 0000000000..b4aeb83268 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_basic_audio_setting2.json @@ -0,0 +1,9 @@ +{ + "dump": true, + "dump_path": "./tests/keep_dump_output/", + "params": { + "key": "${env:KOTOBA_TTS_KEY}", + "base_url": "ws://127.0.0.1:8765/v2/tts/ws", + "sample_rate": 24000 + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_dump.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_dump.json new file mode 100644 index 0000000000..27f4acd38b --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_dump.json @@ -0,0 +1,8 @@ +{ + "dump": true, + "dump_path": "./tests/dump_output/", + "params": { + "key": "${env:KOTOBA_TTS_KEY}", + "base_url": "ws://127.0.0.1:8765/v2/tts/ws" + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_invalid.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_invalid.json new file mode 100644 index 0000000000..21b09f8d49 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_invalid.json @@ -0,0 +1,6 @@ +{ + "params": { + "key": "invalid", + "base_url": "ws://127.0.0.1:8765/v2/tts/ws" + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_miss_required.json b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_miss_required.json new file mode 100644 index 0000000000..596cabae1e --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/configs_fake/property_miss_required.json @@ -0,0 +1,6 @@ +{ + "params": { + "key": "", + "base_url": "ws://127.0.0.1:8765/v2/tts/ws" + } +} diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/conftest.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/conftest.py new file mode 100644 index 0000000000..001977148c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/conftest.py @@ -0,0 +1,99 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import json +import threading +from typing_extensions import override +import pytest +from ten_runtime import ( + App, + TenEnv, +) + + +class FakeApp(App): + def __init__(self): + super().__init__() + self.event: threading.Event | None = None + + # In the case of a fake app, we use `on_init` to allow the blocked testing + # fixture to continue execution, rather than using `on_configure`. The + # reason is that in the TEN runtime C core, the relationship between the + # addon manager and the (fake) app is bound after `on_configure_done` is + # called. So we only need to let the testing fixture continue execution + # after this action in the TEN runtime C core, and at the upper layer + # timing, the earliest point is within the `on_init()` function of the upper + # TEN app. Therefore, we release the testing fixture lock within the user + # layer's `on_init()` of the TEN app. + @override + def on_init(self, ten_env: TenEnv) -> None: + assert self.event + self.event.set() + + ten_env.on_init_done() + + @override + def on_configure(self, ten_env: TenEnv) -> None: + ten_env.init_property_from_json( + json.dumps( + { + "ten": { + "log": { + "handlers": [ + { + "matchers": [{"level": "debug"}], + "formatter": { + "type": "plain", + "colored": True, + }, + "emitter": { + "type": "console", + "config": {"stream": "stdout"}, + }, + } + ] + } + } + } + ), + ) + + ten_env.on_configure_done() + + +class FakeAppCtx: + def __init__(self, event: threading.Event): + self.fake_app: FakeApp | None = None + self.event = event + + +def run_fake_app(fake_app_ctx: FakeAppCtx): + app = FakeApp() + app.event = fake_app_ctx.event + fake_app_ctx.fake_app = app + app.run(False) + + +@pytest.fixture(scope="session", autouse=True) +def global_setup_and_teardown(): + event = threading.Event() + fake_app_ctx = FakeAppCtx(event) + + fake_app_thread = threading.Thread( + target=run_fake_app, args=(fake_app_ctx,) + ) + fake_app_thread.start() + + event.wait() + + assert fake_app_ctx.fake_app is not None + + # Yield control to the test; after the test execution is complete, continue + # with the teardown process. + yield + + # Teardown part. + fake_app_ctx.fake_app.close() + fake_app_thread.join() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/fake_kotoba_server.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/fake_kotoba_server.py new file mode 100644 index 0000000000..8d99ddcf3c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/fake_kotoba_server.py @@ -0,0 +1,398 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""In-process stand-in for the Kotoba TTS WebSocket service. + +Speaks the ``/v2/tts/ws`` protocol the real app uses (``open`` -> +``session.created``, ``response.create`` -> ``audio.chunk``* -> +``response.done``, ``response.cancel`` -> ``response.done(cancelled)``), +so the extension and the real ``kotoba-sdk`` client run unchanged in tests. +Runs on its own thread with its own event loop because the extension under +test runs inside the TEN runtime. +""" +import asyncio +import base64 +import json +import threading +import time +from dataclasses import dataclass, field +from http import HTTPStatus + +import websockets +from websockets.asyncio.server import serve + + +def pcm16_ramp(num_samples: int, start: int = 0) -> bytes: + """Deterministic little-endian int16 samples.""" + out = bytearray() + for i in range(num_samples): + value = ((start + i) % 2000) - 1000 + out += int(value).to_bytes(2, "little", signed=True) + return bytes(out) + + +@dataclass +class Scenario: + """Knobs that shape the server's behaviour for one test.""" + + # Audio returned for every response.create. + chunks_per_response: int = 3 + samples_per_chunk: int = 800 + chunk_delay_s: float = 0.02 + # Reject the WebSocket handshake with this HTTP status (e.g. 401). + # ``handshake_status_count`` limits it to the first N connections + # (None = every connection), imitating a transient 429. + handshake_status: int | None = None + handshake_status_count: int | None = None + # Keys rejected for an unknown key. ``reject_mode`` "http" answers the + # handshake with 401; "close" accepts the upgrade and then closes with + # code 1008 "Unauthorized", which is what the fal gateway does. + reject_keys: frozenset[str] = frozenset({"", "invalid"}) + reject_mode: str = "close" + # Answer ``open`` with this error message instead of session.created. + open_error: str | None = None + # Finish the N-th response (1-based) with response.done(status=failed) + # carrying this error code, after ``fail_after_chunks`` chunks. + fail_response_index: int | None = None + fail_code: str = "worker_unavailable" + fail_after_chunks: int = 0 + # Close the socket without response.done on the N-th response, after + # ``close_after_chunks`` chunks. + close_response_index: int | None = None + close_after_chunks: int = 0 + # Stop the N-th response at the generation length limit after + # ``length_after_chunks`` chunks: response.done(status=completed, + # finish_reason=length), as the service does for a truncated response. + length_response_index: int | None = None + length_after_chunks: int = 0 + # Close the socket this long after the last response completed, + # imitating the service's idle timeout. + idle_close_s: float | None = None + # Hold the first chunk of every response this long (TTFB / stall). + first_chunk_delay_s: float = 0.0 + # Delay before answering ``open`` (slow handshake). + handshake_delay_s: float = 0.0 + # Delay before answering ``response.cancel`` with done(cancelled). + cancel_done_delay_s: float = 0.0 + # Send this many stray audio.chunk frames after a cancel, before + # done(cancelled), imitating audio already in flight. + stray_chunks_after_cancel: int = 0 + # Never answer the N-th response.create (1-based): no chunks, no done. + hold_response_index: int | None = None + # Append this many extra bytes to every chunk (1 = split samples). + extra_chunk_bytes: int = 0 + # Echo this format in session.created instead of the requested one. + negotiate_format: str | None = None + # Close the socket right after a completed response.done. + close_after_done: bool = False + + +@dataclass +class Recorded: + connections: int = 0 + # ("open" | "close", connection index, monotonic time) + events: list = field(default_factory=list) + opens: list[dict] = field(default_factory=list) + texts: list[str] = field(default_factory=list) + cancels: int = 0 + responses_completed: int = 0 + responses_cancelled: int = 0 + + +class FakeKotobaServer: + def __init__(self, scenario: Scenario | None = None) -> None: + self.scenario = scenario or Scenario() + self.recorded = Recorded() + self.port: int = 0 + self.fixed_port: int = 0 + self._loop: asyncio.AbstractEventLoop | None = None + self._thread: threading.Thread | None = None + self._ready = threading.Event() + self._stop: asyncio.Event | None = None + self._response_counter = 0 + self._lock = threading.Lock() + + # -- lifecycle --------------------------------------------------------- + + @property + def url(self) -> str: + return f"ws://127.0.0.1:{self.port}/v2/tts/ws" + + def start(self) -> "FakeKotobaServer": + self._thread = threading.Thread( + target=self._run, name="fake-kotoba-server", daemon=True + ) + self._thread.start() + assert self._ready.wait(5), "fake server did not start" + return self + + def stop(self) -> None: + if self._loop is not None and self._stop is not None: + self._loop.call_soon_threadsafe(self._stop.set) + if self._thread is not None: + self._thread.join(timeout=5) + + def expected_pcm(self, responses: int = 1) -> bytes: + """Audio bytes the server emits for ``responses`` full responses.""" + chunk = pcm16_ramp(self.scenario.samples_per_chunk) + return chunk * self.scenario.chunks_per_response * responses + + def _run(self) -> None: + asyncio.run(self._serve()) + + async def _serve(self) -> None: + self._loop = asyncio.get_running_loop() + self._stop = asyncio.Event() + async with serve( + self._handle, + "127.0.0.1", + self.fixed_port, + process_request=self._process_request, + ) as server: + self.port = server.sockets[0].getsockname()[1] + self._ready.set() + await self._stop.wait() + + # -- protocol ---------------------------------------------------------- + + def _process_request(self, connection, request): + with self._lock: + self.recorded.connections += 1 + connection_index = self.recorded.connections + status = self.scenario.handshake_status + count = self.scenario.handshake_status_count + if count is not None and connection_index > count: + status = None + if status is None and self.scenario.reject_mode == "http": + if self._key_of(request) in self.scenario.reject_keys: + status = HTTPStatus.UNAUTHORIZED + if status is None: + return None + return connection.respond(HTTPStatus(status), "rejected\n") + + @staticmethod + def _key_of(request) -> str: + auth = request.headers.get("Authorization", "") + return auth.split(" ", 1)[1] if " " in auth else "" + + async def _handle(self, ws) -> None: + if ( + self.scenario.reject_mode == "close" + and self._key_of(ws.request) in self.scenario.reject_keys + ): + await ws.close(code=1008, reason="Unauthorized") + return + with self._lock: + conn_index = self.recorded.connections + self.recorded.events.append(("open", conn_index, time.monotonic())) + try: + await self._serve_connection(ws) + finally: + with self._lock: + self.recorded.events.append( + ("close", conn_index, time.monotonic()) + ) + + async def _serve_connection(self, ws) -> None: + try: + raw = await ws.recv() + body = json.loads(raw) + if body.get("type") != "open": + await self._send( + ws, {"type": "error", "message": "invalid open message"} + ) + await ws.close(code=1003) + return + self.recorded.opens.append(body) + if self.scenario.handshake_delay_s: + await asyncio.sleep(self.scenario.handshake_delay_s) + if self.scenario.open_error: + await self._send( + ws, {"type": "error", "message": self.scenario.open_error} + ) + await ws.close(code=1003) + return + sample_rate = int(body.get("sample_rate") or 24000) + await self._send( + ws, + { + "type": "session.created", + "format": ( + self.scenario.negotiate_format + or str(body.get("format") or "pcm_f32").lower() + ), + "sample_rate": sample_rate, + "channels": 1, + "language": body.get("language"), + "speaker_id": body.get("speaker_id"), + "client_id": "fake-client", + }, + ) + await self._session_loop(ws) + except websockets.exceptions.ConnectionClosed: + return + + async def _session_loop(self, ws) -> None: + active: asyncio.Task | None = None + cancel_event = asyncio.Event() + idle = self.scenario.idle_close_s + loop = asyncio.get_running_loop() + idle_since: float | None = loop.time() + while True: + try: + raw = await asyncio.wait_for(ws.recv(), timeout=0.05) + except asyncio.TimeoutError: + busy = active is not None and not active.done() + if busy: + idle_since = None + elif idle_since is None: + idle_since = loop.time() + elif idle is not None and loop.time() - idle_since > idle: + await ws.close(code=1000, reason="idle") + return + continue + body = json.loads(raw) + msg_type = body.get("type") + if msg_type == "response.create": + idle_since = None + if active is not None and not active.done(): + await self._send( + ws, + {"type": "error", "message": "response already active"}, + ) + continue + text = body.get("text") + if not isinstance(text, str) or not text: + await self._send( + ws, + { + "type": "error", + "message": "text must be a non-empty string", + }, + ) + continue + self.recorded.texts.append(text) + cancel_event = asyncio.Event() + active = asyncio.create_task( + self._respond(ws, text, cancel_event) + ) + elif msg_type == "response.cancel": + self.recorded.cancels += 1 + if active is None or active.done(): + await self._send( + ws, {"type": "error", "message": "no active response"} + ) + continue + cancel_event.set() + else: + await self._send( + ws, {"type": "error", "message": "invalid message type"} + ) + + async def _respond( + self, ws, text: str, cancel_event: asyncio.Event + ) -> None: + try: + await self._respond_inner(ws, text, cancel_event) + except websockets.exceptions.ConnectionClosed: + # The client went away mid-response; nothing left to send. + return + + async def _respond_inner( + self, ws, text: str, cancel_event: asyncio.Event + ) -> None: + with self._lock: + self._response_counter += 1 + index = self._response_counter + response_id = f"resp-{index}" + sc = self.scenario + await self._send( + ws, {"type": "response.created", "response": {"id": response_id}} + ) + fail_here = sc.fail_response_index == index + close_here = sc.close_response_index == index + length_here = sc.length_response_index == index + chunks = ( + sc.length_after_chunks if length_here else sc.chunks_per_response + ) + if sc.hold_response_index == index: + # Imitate a stalled service: wait for cancel or forever. + await cancel_event.wait() + await self._finish_cancelled(ws, response_id) + return + for i in range(chunks): + if fail_here and i >= sc.fail_after_chunks: + await self._send( + ws, + { + "type": "response.done", + "response": { + "id": response_id, + "status": "failed", + "error": { + "code": sc.fail_code, + "message": sc.fail_code, + }, + }, + }, + ) + return + if close_here and i >= sc.close_after_chunks: + await ws.close(code=1011, reason="simulated drop") + return + delay = sc.chunk_delay_s + (sc.first_chunk_delay_s if i == 0 else 0) + try: + await asyncio.wait_for(cancel_event.wait(), timeout=delay) + except asyncio.TimeoutError: + pass + if cancel_event.is_set(): + await self._finish_cancelled(ws, response_id) + return + chunk = pcm16_ramp(sc.samples_per_chunk) + if sc.extra_chunk_bytes: + chunk += bytes([0x7F]) * sc.extra_chunk_bytes + await self._send( + ws, + { + "type": "audio.chunk", + "audio": base64.b64encode(chunk).decode("ascii"), + "isFinal": i == chunks - 1, + "response_id": response_id, + }, + ) + self.recorded.responses_completed += 1 + done = {"id": response_id, "status": "completed"} + if length_here: + done["finish_reason"] = "length" + await self._send(ws, {"type": "response.done", "response": done}) + if sc.close_after_done: + await ws.close(code=1000, reason="done") + + async def _finish_cancelled(self, ws, response_id: str) -> None: + sc = self.scenario + self.recorded.responses_cancelled += 1 + for _ in range(sc.stray_chunks_after_cancel): + chunk = pcm16_ramp(sc.samples_per_chunk, start=999) + await self._send( + ws, + { + "type": "audio.chunk", + "audio": base64.b64encode(chunk).decode("ascii"), + "isFinal": False, + "response_id": response_id, + }, + ) + if sc.cancel_done_delay_s: + await asyncio.sleep(sc.cancel_done_delay_s) + await self._send( + ws, + { + "type": "response.done", + "response": {"id": response_id, "status": "cancelled"}, + }, + ) + + @staticmethod + async def _send(ws, payload: dict) -> None: + await ws.send(json.dumps(payload)) diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/helpers.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/helpers.py new file mode 100644 index 0000000000..c2e93df882 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/helpers.py @@ -0,0 +1,296 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Shared tester for the extension-local tests. + +``RecordingTester`` drives ``kotoba_tts_python`` inside the TEN runtime, +records every event it emits, and lets a test script text inputs and +flushes. Tests subclass it or pass a ``script`` coroutine. +""" +import asyncio +import json +import threading +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field + +from ten_runtime import ( + AsyncExtensionTester, + AsyncTenEnvTester, + AudioFrame, + Data, + TenError, + TenErrorCode, +) + +EXTENSION_NAME = "kotoba_tts_python" + + +@dataclass +class Event: + name: str + payload: dict + at: float + + +@dataclass +class Recording: + events: list[Event] = field(default_factory=list) + frames: list[tuple[str, bytes, int]] = field(default_factory=list) + + def named(self, name: str) -> list[Event]: + return [e for e in self.events if e.name == name] + + def audio_bytes(self, request_id: str | None = None) -> bytes: + return b"".join( + data + for rid, data, _rate in self.frames + if request_id is None or rid == request_id + ) + + REQUEST_EVENTS = ("tts_audio_start", "pcm_frame", "tts_audio_end", "error") + + def sequence(self, request_id: str | None = None) -> list[str]: + """Request-scoped event names in arrival order; consecutive frames + collapse to one ``pcm_frame`` entry. Base-class bookkeeping events + (provide_features, metrics, connection status) are left out.""" + out: list[str] = [] + for e in self.events: + if e.name not in self.REQUEST_EVENTS: + continue + if request_id is not None and e.payload.get("request_id") not in ( + request_id, + "", + None, + ): + continue + if e.name == "pcm_frame" and out and out[-1] == "pcm_frame": + continue + out.append(e.name) + return out + + +def make_config(base_url: str, **overrides) -> dict: + params = { + "key": "test-key", + "base_url": base_url, + "sample_rate": 16000, + "request_timeout_seconds": 5, + } + params.update(overrides.pop("params", {})) + config = { + "dump": False, + "dump_path": "./tests/dump_output/", + "params": params, + } + config.update(overrides) + return config + + +class RecordingTester(AsyncExtensionTester): + def __init__( + self, + script: Callable[ + ["RecordingTester", AsyncTenEnvTester], Awaitable[None] + ], + *, + timeout_s: float = 20.0, + ) -> None: + super().__init__() + self._script = script + self._timeout_s = timeout_s + self.recording = Recording() + self.failure: str | None = None + self._waiters: list[ + tuple[Callable[[Recording], bool], asyncio.Future] + ] = [] + + # -- driving ----------------------------------------------------------- + + async def on_start(self, ten_env: AsyncTenEnvTester) -> None: + asyncio.create_task(self._run_script(ten_env)) + + async def _run_script(self, ten_env: AsyncTenEnvTester) -> None: + try: + await asyncio.wait_for(self._script(self, ten_env), self._timeout_s) + except asyncio.TimeoutError: + self.failure = f"script timed out; sequence so far: {self.recording.sequence()}" + except AssertionError as exc: + self.failure = f"assertion failed: {exc}" + except Exception as exc: + self.failure = f"{type(exc).__name__}: {exc}" + if self.failure: + ten_env.stop_test( + TenError.create(TenErrorCode.ErrorCodeGeneric, self.failure) + ) + else: + ten_env.stop_test() + + async def send_text( + self, + ten_env: AsyncTenEnvTester, + request_id: str, + text: str, + *, + end: bool, + metadata: dict | None = None, + ) -> None: + data = Data.create("tts_text_input") + data.set_property_string("request_id", request_id) + data.set_property_string("text", text) + data.set_property_bool("text_input_end", end) + if metadata is None: + # The base class echoes input metadata on every frame and + # event, which is how frames are attributed to a request. + metadata = {"session_id": "test-session", "request_id": request_id} + data.set_property_from_json("metadata", json.dumps(metadata)) + await ten_env.send_data(data) + + async def send_flush( + self, ten_env: AsyncTenEnvTester, flush_id: str = "flush-1" + ) -> None: + data = Data.create("tts_flush") + data.set_property_string("flush_id", flush_id) + data.set_property_from_json( + "metadata", json.dumps({"flush_id": flush_id}) + ) + await ten_env.send_data(data) + + async def wait_for( + self, predicate: Callable[[Recording], bool], timeout_s: float = 10.0 + ) -> None: + if predicate(self.recording): + return + fut: asyncio.Future = asyncio.get_running_loop().create_future() + self._waiters.append((predicate, fut)) + await asyncio.wait_for(fut, timeout_s) + + async def wait_for_event( + self, name: str, count: int = 1, timeout_s: float = 10.0 + ) -> list[Event]: + await self.wait_for(lambda r: len(r.named(name)) >= count, timeout_s) + return self.recording.named(name) + + async def wait_for_audio_end( + self, request_id: str, timeout_s: float = 10.0 + ) -> Event: + def done(r: Recording) -> bool: + return any( + e.payload.get("request_id") == request_id + for e in r.named("tts_audio_end") + ) + + await self.wait_for(done, timeout_s) + return [ + e + for e in self.recording.named("tts_audio_end") + if e.payload.get("request_id") == request_id + ][0] + + # -- recording --------------------------------------------------------- + + async def on_data(self, ten_env: AsyncTenEnvTester, data: Data) -> None: + name = data.get_name() + raw, _ = data.get_property_to_json(None) + payload = json.loads(raw) if raw else {} + self.recording.events.append(Event(name, payload, time.monotonic())) + self._notify() + + async def on_audio_frame( + self, ten_env: AsyncTenEnvTester, audio_frame: AudioFrame + ) -> None: + buf = audio_frame.lock_buf() + try: + data = bytes(buf) + finally: + audio_frame.unlock_buf(buf) + raw, _ = audio_frame.get_property_to_json("metadata") + request_id = "" + if raw: + try: + request_id = json.loads(raw).get("request_id", "") + except json.JSONDecodeError: + request_id = "" + rate = audio_frame.get_sample_rate() + self.recording.frames.append((request_id, data, rate)) + self.recording.events.append( + Event( + "pcm_frame", + {"request_id": request_id, "bytes": len(data)}, + time.monotonic(), + ) + ) + self._notify() + + def _notify(self) -> None: + remaining = [] + for predicate, fut in self._waiters: + if fut.done(): + continue + if predicate(self.recording): + fut.set_result(None) + else: + remaining.append((predicate, fut)) + self._waiters = remaining + + +def metrics_with(tester, key: str): + """Metrics events whose payload carries ``key``.""" + return [ + e + for e in tester.recording.named("metrics") + if key in e.payload.get("metrics", {}) + ] + + +def events_after(tester, name: str, after_event) -> list: + """Events named ``name`` recorded after ``after_event``.""" + return [e for e in tester.recording.named(name) if e.at > after_event.at] + + +def run_tester( + script, config: dict, *, timeout_s: float = 20.0 +) -> RecordingTester: + tester = RecordingTester(script, timeout_s=timeout_s) + tester.set_test_mode_single(EXTENSION_NAME, json.dumps(config)) + error = tester.run() + assert ( + error is None + ), f"tester failed: {error.error_message() if error else 'unknown'}" + assert tester.failure is None, tester.failure + return tester + + +def dump_path(dump_dir: str, request_id: str) -> str: + """Path of the extension's dump file for ``request_id``.""" + import os + + from kotoba_tts_python.extension import dump_file_name + + return os.path.join(dump_dir, dump_file_name(request_id)) + + +class ThreadGate: + """A one-shot flag shared between the tester and the extension. + + The tester and the extension under test run separate asyncio loops on + separate threads, and ``asyncio.Event`` is not thread-safe: setting it + from the other loop can leave the waiter asleep. Same interface as + ``asyncio.Event`` (``set``, ``is_set``, ``wait``), backed by a + ``threading.Event`` that each side polls on its own loop. + """ + + def __init__(self) -> None: + self._flag = threading.Event() + + def set(self) -> None: + self._flag.set() + + def is_set(self) -> bool: + return self._flag.is_set() + + async def wait(self) -> bool: + while not self._flag.is_set(): + await asyncio.sleep(0.005) + return True diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/run_fake_server.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/run_fake_server.py new file mode 100644 index 0000000000..167055fa5b --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/run_fake_server.py @@ -0,0 +1,34 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Run the fake Kotoba service on a fixed port, for the TEN guarder suite. + +python3 tests/run_fake_server.py 8765 +""" +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 + + +def main() -> None: + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8765 + server = FakeKotobaServer( + Scenario(chunks_per_response=25, samples_per_chunk=1600) + ) + server.fixed_port = port + server.start() + print(f"fake kotoba tts listening on {server.url}", flush=True) + try: + while True: + time.sleep(3600) + except KeyboardInterrupt: + server.stop() + + +if __name__ == "__main__": + main() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_accounting.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_accounting.py new file mode 100644 index 0000000000..2b3f3df5e6 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_accounting.py @@ -0,0 +1,241 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Exact audio-end accounting and metric origins under a controlled clock.""" +import asyncio +import sys +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import kotoba_tts_python.extension as ext_mod # noqa: E402 +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import make_config, metrics_with, run_tester # noqa: E402 + + +class FakeClock: + """Monotonic clock that advances a fixed step on every read.""" + + def __init__(self, step_s: float) -> None: + self.t = 1000.0 + self.step = step_s + self.reads = 0 + + def __call__(self) -> float: + self.reads += 1 + self.t += self.step + return self.t + + +def test_interval_is_measured_from_audio_start_emission(): + """With a clock that jumps 100 ms per read, the interval equals the + number of reads between the audio_start read and the audio_end read, + independent of anything that happened before audio_start.""" + clock = FakeClock(step_s=0.1) + server = FakeKotobaServer( + Scenario(chunks_per_response=3, handshake_delay_s=0.3) + ).start() + try: + with patch.object( + ext_mod.KotobaTTSExtension, "_clock", staticmethod(clock) + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "clk", "時計。", end=True) + end = await tester.wait_for_audio_end("clk") + interval = end.payload["request_event_interval_ms"] + # Reads between audio_start and audio_end in _emit_audio / + # _send_audio_end: exactly one (the end read), so 100 ms. + assert interval == 100, interval + ttfb = metrics_with(tester, "ttfb")[0].payload["metrics"][ + "ttfb" + ] + # TTFB: one read at send, one read at first chunk. + assert ttfb == 100, ttfb + connect = metrics_with(tester, "connect_delay") + assert len(connect) == 1 + # Connect delay: one read before start(), one after. + assert connect[0].payload["metrics"]["connect_delay"] == 100 + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_duration_counts_only_emitted_bytes_on_every_terminal_path(): + """Success, partial error, and interrupted paths report exactly the + bytes the tester received.""" + server = FakeKotobaServer( + Scenario( + chunks_per_response=6, + chunk_delay_s=0.05, + fail_response_index=2, + fail_after_chunks=2, + ) + ).start() + try: + + async def script(tester, ten_env): + rec = tester.recording + # success + await tester.send_text(ten_env, "ok", "成功。", end=True) + end = await tester.wait_for_audio_end("ok") + assert end.payload["request_total_audio_duration_ms"] == ( + len(rec.audio_bytes("ok")) * 1000 // 32000 + ) + # partial error (response 2 fails after 2 chunks) + await tester.send_text(ten_env, "err", "失敗。", end=True) + end = await tester.wait_for_audio_end("err") + assert end.payload["reason"] == 3 + assert len(rec.audio_bytes("err")) == 2 * 800 * 2 + assert end.payload["request_total_audio_duration_ms"] == ( + len(rec.audio_bytes("err")) * 1000 // 32000 + ) + # interrupted + await tester.send_text(ten_env, "int", "中断。", end=True) + await tester.wait_for( + lambda r: len(r.audio_bytes("int")) >= 2 * 800 * 2 + ) + await tester.send_flush(ten_env) + await tester.wait_for_event("tts_flush_end") + end = await tester.wait_for_audio_end("int", timeout_s=0.5) + assert end.payload["reason"] == 2 + assert end.payload["request_total_audio_duration_ms"] == ( + len(rec.audio_bytes("int")) * 1000 // 32000 + ) + await asyncio.sleep(0.3) + # no-audio termination + await tester.send_text(ten_env, "none", "", end=True) + end = await tester.wait_for_audio_end("none", timeout_s=0.5) + assert end.payload["request_total_audio_duration_ms"] == 0 + assert end.payload["request_event_interval_ms"] == 0 + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_ttfb_excludes_handshake_time(): + server = FakeKotobaServer( + Scenario( + chunks_per_response=2, + handshake_delay_s=0.6, + first_chunk_delay_s=0.2, + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "t", "計測。", end=True) + await tester.wait_for_audio_end("t") + ttfb = metrics_with(tester, "ttfb")[0].payload["metrics"]["ttfb"] + assert 150 <= ttfb < 500, ttfb + connect = metrics_with(tester, "connect_delay")[0] + assert connect.payload["metrics"]["connect_delay"] >= 550 + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_interval_under_controlled_clock_on_error_and_interrupt(): + """Error after partial audio and flush after audio: the interval is + exactly the reads between audio_start and audio_end (one read).""" + clock = FakeClock(step_s=0.1) + server = FakeKotobaServer( + Scenario( + chunks_per_response=4, + chunk_delay_s=0.05, + fail_response_index=1, + fail_after_chunks=2, + ) + ).start() + try: + with patch.object( + ext_mod.KotobaTTSExtension, "_clock", staticmethod(clock) + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "err", "失敗。", end=True) + end = await tester.wait_for_audio_end("err") + assert end.payload["reason"] == 3 + assert end.payload["request_event_interval_ms"] == 100 + await tester.send_text(ten_env, "int", "中断。", end=True) + await tester.wait_for( + lambda r: len(r.audio_bytes("int")) >= 2 * 1600 + ) + await tester.send_flush(ten_env) + await tester.wait_for_event("tts_flush_end") + end = await tester.wait_for_audio_end("int", timeout_s=0.5) + assert end.payload["reason"] == 2 + assert end.payload["request_event_interval_ms"] == 100 + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_split_samples_are_aligned_and_never_cross_requests(): + """Each service chunk carries one extra byte. Only whole samples may be + emitted and counted; the half sample held back at the end of a request + must not leak into the next one.""" + server = FakeKotobaServer( + Scenario(chunks_per_response=3, extra_chunk_bytes=1) + ).start() + try: + + async def script(tester, ten_env): + rec = tester.recording + await tester.send_text(ten_env, "odd1", "奇数。", end=True) + end1 = await tester.wait_for_audio_end("odd1") + emitted1 = rec.audio_bytes("odd1") + # 3 chunks * (1600 + 1) bytes = 4803 -> 4802 emitted, 1 held + assert len(emitted1) == 4802 + assert all(len(d) % 2 == 0 for _, d, _ in rec.frames) + assert end1.payload["request_total_audio_duration_ms"] == ( + 4802 * 1000 // 32000 + ) + await tester.send_text(ten_env, "odd2", "次。", end=True) + end2 = await tester.wait_for_audio_end("odd2") + emitted2 = rec.audio_bytes("odd2") + assert len(emitted2) == 4802 + assert end2.payload["request_total_audio_duration_ms"] == ( + 4802 * 1000 // 32000 + ) + # The first request's held byte did not prefix the second. + assert emitted2[:2] == server.expected_pcm(1)[:2] + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_half_sample_never_crosses_response_boundary(): + """Two sentences in one request, every chunk one byte too long: the + half sample left at the end of the first sentence's response must be + dropped, not glued onto the second response's first byte.""" + server = FakeKotobaServer( + Scenario(chunks_per_response=3, extra_chunk_bytes=1) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "two", "一文目。二文目。", end=True) + end = await tester.wait_for_audio_end("two") + audio = tester.recording.audio_bytes("two") + # Per response: 3 * 1601 = 4803 bytes, 4802 emitted, 1 dropped. + assert len(audio) == 2 * 4802, len(audio) + # The second response starts with the ramp's first sample, not + # with the first response's stray byte. + ramp = server.expected_pcm(1) + assert audio[4802:4804] == ramp[:2] + assert end.payload["request_total_audio_duration_ms"] == ( + 2 * 4802 * 1000 // 32000 + ) + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["一文目。", "二文目。"] + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_basic.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_basic.py new file mode 100644 index 0000000000..7bae0f1a46 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_basic.py @@ -0,0 +1,130 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Core request lifecycle against the fake Kotoba service.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import make_config, run_tester # noqa: E402 + +SAMPLE_WIDTH = 2 +CHANNELS = 1 + + +def test_single_request_emits_ordered_events_and_exact_pcm(): + server = FakeKotobaServer(Scenario(chunks_per_response=4)).start() + try: + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "req-1", "こんにちは、世界。", end=True + ) + end = await tester.wait_for_audio_end("req-1") + rec = tester.recording + assert rec.sequence() == [ + "tts_audio_start", + "pcm_frame", + "tts_audio_end", + ], rec.sequence() + starts = rec.named("tts_audio_start") + assert len(starts) == 1 + assert starts[0].payload["request_id"] == "req-1" + assert rec.audio_bytes("req-1") == server.expected_pcm(1) + assert {rate for _, _, rate in rec.frames} == {16000} + assert end.payload["reason"] == 1 + expected_ms = ( + len(server.expected_pcm(1)) + * 1000 + // (16000 * SAMPLE_WIDTH * CHANNELS) + ) + assert end.payload["request_total_audio_duration_ms"] == expected_ms + assert end.payload["request_event_interval_ms"] >= 0 + assert rec.named("error") == [] + assert len(rec.named("tts_audio_end")) == 1 + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["こんにちは、世界。"] + assert server.recorded.opens[0]["format"] == "pcm16" + assert server.recorded.opens[0]["sample_rate"] == 16000 + assert server.recorded.opens[0]["language"] == "ja" + assert server.recorded.opens[0]["speaker_id"] == "male-01" + assert "key" not in server.recorded.opens[0] + finally: + server.stop() + + +def test_configured_sample_rate_and_voice_reach_the_service(): + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "req-24k", "テスト。", end=True) + end = await tester.wait_for_audio_end("req-24k") + rec = tester.recording + assert {rate for _, _, rate in rec.frames} == {24000} + expected_ms = ( + len(server.expected_pcm(1)) * 1000 // (24000 * SAMPLE_WIDTH) + ) + assert end.payload["request_total_audio_duration_ms"] == expected_ms + + run_tester( + script, + make_config( + server.url, + params={"sample_rate": 24000, "speaker_id": "female-01"}, + ), + ) + assert server.recorded.opens[0]["sample_rate"] == 24000 + assert server.recorded.opens[0]["speaker_id"] == "female-01" + finally: + server.stop() + + +def test_empty_text_ends_fast_without_contacting_the_service(): + server = FakeKotobaServer().start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "empty-1", " ", end=True) + end = await tester.wait_for_audio_end("empty-1", timeout_s=0.5) + rec = tester.recording + assert rec.frames == [] + assert rec.named("tts_audio_start") == [] + assert rec.named("error") == [] + assert end.payload["request_total_audio_duration_ms"] == 0 + assert end.payload["request_event_interval_ms"] == 0 + assert end.payload["reason"] == 1 + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 0 + finally: + server.stop() + + +def test_repeated_empty_requests_with_same_id_end_once_without_errors(): + server = FakeKotobaServer().start() + try: + + async def script(tester, ten_env): + for text in ("", " ", ""): + await tester.send_text(ten_env, "999", text, end=True) + await tester.wait_for_audio_end("999", timeout_s=1.0) + # Give late inputs time to arrive; they must stay silent. + import asyncio + + await asyncio.sleep(0.3) + rec = tester.recording + assert len(rec.named("tts_audio_end")) == 1 + assert rec.named("error") == [] + # The queue is released: a new request still works. + await tester.send_text(ten_env, "after-999", "次。", end=True) + end = await tester.wait_for_audio_end("after-999") + assert end.payload["request_total_audio_duration_ms"] > 0 + + run_tester(script, make_config(server.url)) + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_cancellation.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_cancellation.py new file mode 100644 index 0000000000..b8ef601771 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_cancellation.py @@ -0,0 +1,1303 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Cancellation under blocked transport, terminal races, double flush, +late events from an old session, and shutdown while blocked. + +Every test asserts the same invariants: nothing for the old request after +``tts_flush_end``, exactly one ``tts_audio_end`` per request, flush +completes quickly, and the next request works. +""" +import asyncio +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import ( # noqa: E402 + ThreadGate, + dump_path, + events_after, + make_config, + metrics_with, + run_tester, +) + +FLUSH_BUDGET_S = 1.0 + + +async def _wait_until(predicate, timeout_s: float = 5.0) -> None: + deadline = asyncio.get_running_loop().time() + timeout_s + while not predicate(): + if asyncio.get_running_loop().time() > deadline: + raise AssertionError("condition not met in time") + await asyncio.sleep(0.02) + + +def _assert_clean_after_flush(tester, request_id: str) -> None: + rec = tester.recording + flush_end = rec.named("tts_flush_end")[-1] + ends = [ + e + for e in rec.named("tts_audio_end") + if e.payload["request_id"] == request_id + ] + assert len(ends) == 1, f"expected one audio_end, got {len(ends)}" + assert ends[0].at <= flush_end.at, "audio_end landed after flush_end" + late_frames = [ + f + for f in events_after(tester, "pcm_frame", flush_end) + if f.payload.get("request_id") == request_id + ] + assert late_frames == [], "frames for the old request after flush_end" + assert rec.named("error") == [] + + +def test_flush_while_send_is_blocked_discards_the_socket(): + """The service never answers response.create (held send/response). + Flush must not leave that response on a socket the next request reuses. + """ + server = FakeKotobaServer( + Scenario(chunks_per_response=3, hold_response_index=1) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "held", "止まった応答。", end=True) + await _wait_until( + lambda: server.recorded.texts == ["止まった応答。"] + ) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-held") + await tester.wait_for_event("tts_flush_end") + assert time.monotonic() - t0 < FLUSH_BUDGET_S + end = await tester.wait_for_audio_end("held", timeout_s=0.5) + assert end.payload["reason"] == 2 + assert end.payload["request_total_audio_duration_ms"] == 0 + _assert_clean_after_flush(tester, "held") + # Next request gets clean audio, never the held response's. + await tester.send_text(ten_env, "next", "次。", end=True) + end2 = await tester.wait_for_audio_end("next") + assert end2.payload["reason"] == 1 + assert tester.recording.audio_bytes("next") == server.expected_pcm( + 1 + ) + + run_tester(script, make_config(server.url)) + assert server.recorded.cancels == 1 + finally: + server.stop() + + +def test_flush_during_slow_handshake_is_bounded(): + server = FakeKotobaServer( + Scenario(chunks_per_response=2, handshake_delay_s=1.5) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "hs", "握手中。", end=True) + await _wait_until(lambda: server.recorded.connections >= 1) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-hs") + await tester.wait_for_event("tts_flush_end") + assert time.monotonic() - t0 < FLUSH_BUDGET_S + end = await tester.wait_for_audio_end("hs", timeout_s=0.5) + assert end.payload["reason"] == 2 + assert tester.recording.named("tts_audio_start") == [] + _assert_clean_after_flush(tester, "hs") + await tester.send_text(ten_env, "after", "次。", end=True) + end2 = await tester.wait_for_audio_end("after", timeout_s=5.0) + assert tester.recording.audio_bytes("after") == server.expected_pcm( + 1 + ) + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_double_flush_with_pending_drain_stays_bounded(): + """Flush A, start B while A's cancel drain is pending, flush B.""" + server = FakeKotobaServer( + Scenario( + chunks_per_response=40, + chunk_delay_s=0.05, + cancel_done_delay_s=1.0, + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "A", "最初の長い文。", end=True) + await tester.wait_for_event("pcm_frame", count=2) + await tester.send_flush(ten_env, "flush-A") + await tester.wait_for_event("tts_flush_end") + await tester.send_text(ten_env, "B", "二番目。", end=True) + await asyncio.sleep(0.1) # B is waiting on A's drain + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-B") + await tester.wait_for_event("tts_flush_end", count=2) + assert time.monotonic() - t0 < FLUSH_BUDGET_S + await tester.wait_for_audio_end("B", timeout_s=0.5) + _assert_clean_after_flush(tester, "A") + _assert_clean_after_flush(tester, "B") + await tester.send_text(ten_env, "C", "三番目。", end=True) + end_c = await tester.wait_for_audio_end("C", timeout_s=8.0) + assert end_c.payload["reason"] == 1 + assert len(tester.recording.audio_bytes("C")) == len( + server.expected_pcm(1) + ) + + run_tester(script, make_config(server.url)) + assert server.recorded.texts[:1] == ["最初の長い文。"] + assert "二番目。" not in server.recorded.texts + finally: + server.stop() + + +def test_stray_audio_after_cancel_never_reaches_the_next_request(): + server = FakeKotobaServer( + Scenario( + chunks_per_response=40, + chunk_delay_s=0.05, + stray_chunks_after_cancel=3, + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "old", "古い要求。", end=True) + await tester.wait_for_event("pcm_frame", count=2) + await tester.send_flush(ten_env, "flush-old") + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("old", timeout_s=0.5) + await tester.send_text(ten_env, "new", "新しい要求。", end=True) + end = await tester.wait_for_audio_end("new", timeout_s=8.0) + assert end.payload["reason"] == 1 + # The stray chunks carry a distinct ramp; none may appear. + assert tester.recording.audio_bytes("new") == server.expected_pcm(1) + _assert_clean_after_flush(tester, "old") + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_flush_during_terminal_path_keeps_audio_end_before_flush_end(): + """Block the dump flush that _end_request performs after audio_end and + flush during it: the old request's events must all precede flush_end + and there must be exactly one audio_end.""" + import tempfile + import shutil + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_cancel_") + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + gate = {"event": None} + + original_flush = ext_mod.PCMWriter.flush + + async def slow_flush(self): + if gate["event"] is not None: + await gate["event"].wait() + await original_flush(self) + + try: + with patch.object(ext_mod.PCMWriter, "flush", slow_flush): + + async def script(tester, ten_env): + gate["event"] = ThreadGate() + await tester.send_text(ten_env, "term", "終了中。", end=True) + end = await tester.wait_for_audio_end("term") + # Worker is now inside _end_request, blocked on the dump. + await tester.send_flush(ten_env, "flush-term") + flush_end = (await tester.wait_for_event("tts_flush_end"))[0] + assert end.at <= flush_end.at + gate["event"].set() + await asyncio.sleep(0.3) + ends = [ + e + for e in tester.recording.named("tts_audio_end") + if e.payload["request_id"] == "term" + ] + assert len(ends) == 1 + assert ends[0].payload["reason"] == 1 + await tester.send_text(ten_env, "after", "次。", end=True) + await tester.wait_for_audio_end("after") + + run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_flush_takes_over_when_terminal_worker_hangs(): + """If the worker's terminal event never goes out, flush waits its bound + and then emits exactly one INTERRUPTED end itself.""" + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + state = {"hang": True, "calls": 0} + original = ext_mod.KotobaTTSExtension.send_tts_audio_end + + async def hanging_first_audio_end(self, *args, **kwargs): + state["calls"] += 1 + if state["hang"] and state["calls"] == 1: + await asyncio.sleep(60) # never returns before cancellation + return await original(self, *args, **kwargs) + + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.5), patch.object( + ext_mod.KotobaTTSExtension, + "send_tts_audio_end", + hanging_first_audio_end, + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "hang", "固まる。", end=True) + await tester.wait_for( + lambda r: len(r.frames) >= 2 and state["calls"] >= 1, + timeout_s=5.0, + ) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-hang") + await tester.wait_for_event("tts_flush_end") + elapsed = time.monotonic() - t0 + assert 0.4 < elapsed < 2.0, elapsed + end = await tester.wait_for_audio_end("hang", timeout_s=0.5) + assert end.payload["reason"] == 2 + await asyncio.sleep(0.3) + assert len(tester.recording.named("tts_audio_end")) == 1 + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_usage_after_flush_belongs_to_the_next_request_only(): + server = FakeKotobaServer( + Scenario(chunks_per_response=40, chunk_delay_s=0.05) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "A", "あいうえお、かきくけこ。", end=True + ) + await tester.wait_for_event("pcm_frame", count=2) + await tester.send_flush(ten_env, "flush-A") + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("A", timeout_s=0.5) + await tester.send_text(ten_env, "B", "さしすせそ。", end=True) + await tester.wait_for_audio_end("B", timeout_s=8.0) + # Usage is emitted right after audio_end; wait for B's. + await tester.wait_for( + lambda r: any( + e.payload["metadata"].get("request_id") == "B" + for e in r.named("metrics") + if "recv_audio_duration" in e.payload.get("metrics", {}) + ), + timeout_s=2.0, + ) + usage = [ + e + for e in metrics_with(tester, "recv_audio_duration") + if e.payload["metadata"].get("request_id") + ] + by_id = {e.payload["metadata"]["request_id"]: e for e in usage} + assert set(by_id) == {"A", "B"} + assert by_id["A"].payload["metrics"]["input_characters"] == len( + "あいうえお、かきくけこ。" + ) + assert by_id["B"].payload["metrics"]["input_characters"] == len( + "さしすせそ。" + ) + b_bytes = len(tester.recording.audio_bytes("B")) + assert by_id["B"].payload["metrics"]["recv_audio_duration"] == ( + b_bytes * 1000 / (16000 * 2) + ) + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_stop_during_blocked_handshake_is_bounded(): + server = FakeKotobaServer( + Scenario(chunks_per_response=2, handshake_delay_s=5.0) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "stop", "停止。", end=True) + await _wait_until(lambda: server.recorded.connections >= 1) + # Returning ends the test: the runtime calls on_stop while the + # handshake is still pending. run_tester enforces the bound. + + t0 = time.monotonic() + run_tester(script, make_config(server.url), timeout_s=10.0) + assert time.monotonic() - t0 < 4.0 + finally: + server.stop() + + +def test_disconnect_callback_failure_does_not_break_the_client(): + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer( + Scenario(chunks_per_response=2, idle_close_s=0.3) + ).start() + + original = ext_mod.KotobaTTSExtension.on_disconnected + + async def failing_on_disconnected(self, **kwargs): + # The base class itself reports "stopped" at shutdown; only the + # client's own disconnect reports are made to fail here. + if kwargs.get("message") == "stopped": + return await original(self, **kwargs) + raise RuntimeError("status sink down") + + try: + with patch.object( + ext_mod.KotobaTTSExtension, + "on_disconnected", + failing_on_disconnected, + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "one", "一。", end=True) + await tester.wait_for_audio_end("one") + await asyncio.sleep(0.9) # idle close -> drop -> callback + await tester.send_text(ten_env, "two", "二。", end=True) + end = await tester.wait_for_audio_end("two") + assert end.payload["reason"] == 1 + assert tester.recording.audio_bytes( + "two" + ) == server.expected_pcm(1) + assert tester.recording.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 2 + finally: + server.stop() + + +def test_flush_while_the_write_itself_is_suspended(): + """Gate AsyncTTSSession.synthesize (the actual write) and flush while it + is suspended: the send-cleanup branch must drop the socket, report + INTERRUPTED once, and the next request must get clean audio.""" + from unittest.mock import patch + import kotoba._ws_tts as sdk_tts + + server = FakeKotobaServer(Scenario(chunks_per_response=3)).start() + gate = {"event": None, "entered": None} + original = sdk_tts.AsyncTTSSession.synthesize + + async def gated_synthesize(self, text, **kwargs): + event, gate["event"] = gate["event"], None # gate the first send only + if event is not None: + gate["entered"].set() + await event.wait() + return await original(self, text, **kwargs) + + try: + with patch.object( + sdk_tts.AsyncTTSSession, "synthesize", gated_synthesize + ): + + async def script(tester, ten_env): + gate["event"] = ThreadGate() + gate["entered"] = ThreadGate() + await tester.send_text(ten_env, "w", "書き込み中。", end=True) + await asyncio.wait_for(gate["entered"].wait(), 5.0) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-write") + await tester.wait_for_event("tts_flush_end") + assert time.monotonic() - t0 < FLUSH_BUDGET_S + end = await tester.wait_for_audio_end("w", timeout_s=0.5) + assert end.payload["reason"] == 2 + _assert_clean_after_flush(tester, "w") + await tester.send_text(ten_env, "n", "次。", end=True) + end2 = await tester.wait_for_audio_end("n") + assert end2.payload["reason"] == 1 + assert tester.recording.audio_bytes("n") == server.expected_pcm( + 1 + ) + + run_tester(script, make_config(server.url)) + # The interrupted write dropped its socket; the next request opened a new one. + assert server.recorded.connections == 2 + assert server.recorded.texts == ["次。"] + finally: + server.stop() + + +def test_stop_while_the_write_is_suspended_is_bounded(): + from unittest.mock import patch + import kotoba._ws_tts as sdk_tts + + server = FakeKotobaServer(Scenario(chunks_per_response=3)).start() + gate = {"event": None, "entered": None} + original = sdk_tts.AsyncTTSSession.synthesize + + async def gated_synthesize(self, text, **kwargs): + if gate["event"] is not None: + gate["entered"].set() + await gate["event"].wait() + return await original(self, text, **kwargs) + + try: + with patch.object( + sdk_tts.AsyncTTSSession, "synthesize", gated_synthesize + ): + + async def script(tester, ten_env): + gate["event"] = ThreadGate() + gate["entered"] = ThreadGate() + await tester.send_text(ten_env, "s", "停止。", end=True) + await asyncio.wait_for(gate["entered"].wait(), 5.0) + # Return: the runtime stops the extension mid-write. + + t0 = time.monotonic() + run_tester(script, make_config(server.url), timeout_s=10.0) + assert time.monotonic() - t0 < 4.0 + finally: + server.stop() + + +def test_late_old_session_cleanup_preserves_the_replacement(): + """A's cancel drain outlives the grace period (server answers late). B + must get its own connection, and the old session's late cleanup must + not drop B's socket or leak A's audio. + + By construction the replacement cannot exist while the old session is + still being drained: ``connect()`` awaits the drain (or its timeout and + drop) before opening a new socket, and ``_drain`` / the watcher only + act on a session that is still ``self._session``. The server-side + open/close order asserted below is the proof of that ordering.""" + server = FakeKotobaServer( + Scenario( + chunks_per_response=40, + chunk_delay_s=0.05, + cancel_done_delay_s=3.0, + stray_chunks_after_cancel=2, + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "A", "古い。", end=True) + await tester.wait_for_event("pcm_frame", count=2) + await tester.send_flush(ten_env, "flush-A") + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("A", timeout_s=0.5) + await tester.send_text(ten_env, "B", "新しい。", end=True) + end_b = await tester.wait_for_audio_end("B", timeout_s=12.0) + assert end_b.payload["reason"] == 1 + assert tester.recording.audio_bytes("B") == server.expected_pcm(1) + await asyncio.sleep(1.5) # old session's late done(cancelled) lands + await tester.send_text(ten_env, "C", "三。", end=True) + end_c = await tester.wait_for_audio_end("C", timeout_s=8.0) + assert end_c.payload["reason"] == 1 + assert tester.recording.audio_bytes("C") == server.expected_pcm(1) + statuses = [ + e.payload.get("current") + for e in tester.recording.named("connection_status_changed") + ] + assert statuses.count("connected") == 2, statuses + + run_tester(script, make_config(server.url)) + # A's connection was dropped after the grace period; B and C share the replacement. + assert server.recorded.connections == 2 + kinds = [(k, i) for k, i, _ in server.recorded.events] + opens = [i for k, i in kinds if k == "open"] + assert len(opens) == 2, kinds + first, second = opens + assert kinds.index(("close", first)) < kinds.index( + ("open", second) + ), kinds + finally: + server.stop() + + +def test_flush_takeover_finalizes_the_dump(): + """Same as the hung-terminal test, with dumping enabled: when flush + takes over, the dump must still hold every emitted frame.""" + import os + import shutil + import tempfile + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_takeover_") + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + state = {"calls": 0} + original = ext_mod.KotobaTTSExtension.send_tts_audio_end + + async def hanging_first_audio_end(self, *args, **kwargs): + state["calls"] += 1 + if state["calls"] == 1: + await asyncio.sleep(60) + return await original(self, *args, **kwargs) + + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.5), patch.object( + ext_mod.KotobaTTSExtension, + "send_tts_audio_end", + hanging_first_audio_end, + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "hang", "固まる。", end=True) + await tester.wait_for( + lambda r: len(r.frames) >= 2 and state["calls"] >= 1, + timeout_s=5.0, + ) + await tester.send_flush(ten_env, "flush-hang") + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("hang", timeout_s=0.5) + + tester = run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + with open(dump_path(dump_dir, "hang"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("hang") + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_stop_during_recorder_finalization_still_finalizes_the_dump(): + """The app stops while the terminal path is inside the (slow) dump + flush: on_stop must join the finishing worker instead of cancelling + it, and the file must be complete.""" + import os + import shutil + import tempfile + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_stopflush_") + server = FakeKotobaServer(Scenario(chunks_per_response=3)).start() + original_flush = ext_mod.PCMWriter.flush + + async def slow_flush(self): + await asyncio.sleep(0.5) + await original_flush(self) + + try: + with patch.object(ext_mod.PCMWriter, "flush", slow_flush): + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "stopdump", "停止中。", end=True + ) + await tester.wait_for_audio_end("stopdump") + # Return now: the worker is inside the 0.5 s flush and the + # runtime stops the extension while it is pending. + + tester = run_tester( + script, + make_config(server.url, dump=True, dump_path=dump_dir), + timeout_s=10.0, + ) + with open(dump_path(dump_dir, "stopdump"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("stopdump") + assert len(tester.recording.audio_bytes("stopdump")) == len( + server.expected_pcm(1) + ) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def _usage_for(tester, request_id: str) -> list: + return [ + e + for e in metrics_with(tester, "recv_audio_duration") + if e.payload["metadata"].get("request_id") == request_id + ] + + +def test_flush_takeover_after_audio_end_still_reports_usage(): + """The worker hangs after audio_end went out, inside the usage send. + Flush takes over after the bound: no second audio_end, but usage for + the old request is emitted, and the next request's usage holds only + its own counters.""" + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + state = {"calls": 0} + original = ext_mod.KotobaTTSExtension.send_usage_metrics + + async def hanging_first_usage(self, *args, **kwargs): + state["calls"] += 1 + if state["calls"] == 1: + await asyncio.sleep(60) + return await original(self, *args, **kwargs) + + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.5), patch.object( + ext_mod.KotobaTTSExtension, + "send_usage_metrics", + hanging_first_usage, + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "U1", "使用量。", end=True) + await tester.wait_for_audio_end("U1") + await _wait_until(lambda: state["calls"] >= 1) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-usage") + await tester.wait_for_event("tts_flush_end") + assert 0.4 < time.monotonic() - t0 < 2.0 + await tester.send_text(ten_env, "U2", "次。", end=True) + await tester.wait_for_audio_end("U2") + await tester.wait_for( + lambda r: len(_usage_for(tester, "U2")) >= 1, + timeout_s=2.0, + ) + ends = [ + e.payload["request_id"] + for e in tester.recording.named("tts_audio_end") + ] + assert ends == ["U1", "U2"], ends + u1 = _usage_for(tester, "U1") + u2 = _usage_for(tester, "U2") + assert len(u1) == 1 and len(u2) == 1 + assert u1[0].payload["metrics"]["input_characters"] == len( + "使用量。" + ) + assert u2[0].payload["metrics"]["input_characters"] == len( + "次。" + ) + assert u2[0].payload["metrics"]["recv_audio_duration"] == ( + len(tester.recording.audio_bytes("U2")) * 1000 / (16000 * 2) + ) + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_stop_during_gated_dump_flush_completes_the_file(): + """Shutdown arrives while the dump flush is suspended for longer than + the join bound: on_stop cancels the worker and finishes the flush + itself; the file holds every emitted frame.""" + import os + import shutil + import tempfile + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_stopgate_") + server = FakeKotobaServer(Scenario(chunks_per_response=3)).start() + original_flush = ext_mod.PCMWriter.flush + state = {"calls": 0} + + async def gated_flush(self): + state["calls"] += 1 + if state["calls"] == 1: + # Longer than the (patched) join bound, shorter than the test. + await asyncio.sleep(1.0) + await original_flush(self) + + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.3), patch.object( + ext_mod.PCMWriter, "flush", gated_flush + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "gate", "停止。", end=True) + await tester.wait_for_audio_end("gate") + await _wait_until(lambda: state["calls"] >= 1) + # Return: the runtime stops the extension inside the flush. + + t0 = time.monotonic() + tester = run_tester( + script, + make_config(server.url, dump=True, dump_path=dump_dir), + timeout_s=10.0, + ) + assert time.monotonic() - t0 < 6.0 + with open(dump_path(dump_dir, "gate"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("gate") + assert len(tester.recording.audio_bytes("gate")) == len( + server.expected_pcm(1) + ) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_next_request_during_previous_dump_flush_then_stop(): + """The previous request's dump is still being written when the next + request starts and the app stops: on_stop must finish the previous + request's flush (it is no longer the current request) and close the + current request's dump. Events emitted during on_stop are not + observable here, so the second file is checked against the frames + recorded before the stop.""" + import os + import shutil + import tempfile + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_overlap_") + server = FakeKotobaServer( + Scenario(chunks_per_response=3, chunk_delay_s=0.1) + ).start() + original_flush = ext_mod.PCMWriter.flush + gate = {"event": None, "entered": None} + + async def gated_flush(self): + event, gate["event"] = gate["event"], None # gate the first flush + if event is not None: + gate["entered"].set() + await event.wait() + await original_flush(self) + + try: + with patch.object(ext_mod.PCMWriter, "flush", gated_flush): + + async def script(tester, ten_env): + release = ThreadGate() + gate["event"] = release + gate["entered"] = ThreadGate() + await tester.send_text(ten_env, "P", "前。", end=True) + await tester.wait_for_audio_end("P") + await asyncio.wait_for(gate["entered"].wait(), 5.0) + # P is now stuck in its dump flush; start the next request. + await tester.send_text(ten_env, "Q", "後ろ。", end=True) + await tester.wait_for(lambda r: len(r.audio_bytes("Q")) > 0) + release.set() + # Return: the runtime stops the extension while Q streams + # and P's flush has just been released. + + tester = run_tester( + script, + make_config(server.url, dump=True, dump_path=dump_dir), + timeout_s=10.0, + ) + rec = tester.recording + ends = {e.payload["request_id"]: e for e in rec.named("tts_audio_end")} + assert ends["P"].payload["reason"] == 1 + with open(dump_path(dump_dir, "P"), "rb") as f: + assert f.read() == rec.audio_bytes("P") + assert len(rec.audio_bytes("P")) == len(server.expected_pcm(1)) + with open(dump_path(dump_dir, "Q"), "rb") as f: + dump_q = f.read() + recorded_q = rec.audio_bytes("Q") + assert recorded_q and dump_q.startswith(recorded_q) + # Frames emitted between the script's return and on_stop are in + # the file but not in the recording; the file ends on a frame. + assert len(dump_q) % 1600 == 0, len(dump_q) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_flush_waits_for_terminal_events_that_land_in_time(): + """Flush arrives while the worker is between its terminal events and + finish_request (gated here). The events land within the bound, so + flush waits for them instead of taking over: audio_end keeps its + REQUEST_END reason, there is exactly one of it, it precedes + flush_end, and the next request works.""" + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + gate = {"release": None, "entered": None} + original = ext_mod.KotobaTTSExtension.finish_request + + async def gated_finish(self, *args, **kwargs): + release, gate["release"] = gate["release"], None # first call only + if release is not None: + gate["entered"].set() + await release.wait() + return await original(self, *args, **kwargs) + + try: + with patch.object( + ext_mod.KotobaTTSExtension, "finish_request", gated_finish + ): + + async def script(tester, ten_env): + release = ThreadGate() + gate["release"] = release + gate["entered"] = ThreadGate() + await tester.send_text(ten_env, "wait", "待つ。", end=True) + end = await tester.wait_for_audio_end("wait") + await asyncio.wait_for(gate["entered"].wait(), 5.0) + flush_task = asyncio.ensure_future( + tester.send_flush(ten_env, "flush-wait") + ) + await asyncio.sleep(0.3) + assert tester.recording.named("tts_flush_end") == [] + t_release = time.monotonic() + release.set() + await flush_task + flush_end = (await tester.wait_for_event("tts_flush_end"))[0] + assert flush_end.at >= t_release + assert end.at <= flush_end.at + ends = [ + e + for e in tester.recording.named("tts_audio_end") + if e.payload["request_id"] == "wait" + ] + assert len(ends) == 1 and ends[0].payload["reason"] == 1 + assert len(_usage_for(tester, "wait")) == 1 + await tester.send_text(ten_env, "after", "次。", end=True) + end2 = await tester.wait_for_audio_end("after") + assert end2.payload["reason"] == 1 + assert tester.recording.audio_bytes( + "after" + ) == server.expected_pcm(1) + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_request_released_during_shutdown_never_starts(): + """A's dump flush is gated, B is streaming, C is queued behind B. The + app stops. While on_stop waits for A, B completes and the base class + releases C: C must not start (no worker, no dump, no text sent).""" + import os + import shutil + import tempfile + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_stopq_") + server = FakeKotobaServer( + Scenario(chunks_per_response=4, chunk_delay_s=0.1) + ).start() + original_flush = ext_mod.PCMWriter.flush + gate = {"entered": None} + + async def gated_flush(self): + entered, gate["entered"] = gate["entered"], None # first flush only + if entered is not None: + entered.set() + await asyncio.sleep(1.0) # longer than B needs to finish + await original_flush(self) + + try: + with patch.object(ext_mod.PCMWriter, "flush", gated_flush): + + async def script(tester, ten_env): + entered = ThreadGate() + gate["entered"] = entered + await tester.send_text(ten_env, "A", "前。", end=True) + await tester.wait_for_audio_end("A") + await asyncio.wait_for(entered.wait(), 5.0) + # A is stuck in its dump flush. B streams, C waits behind B. + await tester.send_text(ten_env, "B", "中。", end=True) + await tester.send_text(ten_env, "C", "後。", end=True) + await tester.wait_for(lambda r: len(r.audio_bytes("B")) > 0) + # Return: the runtime stops the extension while A flushes + # and B still has chunks to go. + + t0 = time.monotonic() + tester = run_tester( + script, + make_config(server.url, dump=True, dump_path=dump_dir), + timeout_s=10.0, + ) + assert time.monotonic() - t0 < 8.0 + files = sorted(os.listdir(dump_dir)) + assert os.path.basename(dump_path(dump_dir, "C")) not in files, files + assert "後。" not in server.recorded.texts, server.recorded.texts + with open(dump_path(dump_dir, "A"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("A") + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_flush_during_streaming_does_not_wait_for_dump_io(): + """Dump enabled, the dump write is held: flush during streaming must + still produce tts_flush_end and accept the next request while the + write is pending. Shutdown then finishes the file exactly.""" + import os + import shutil + import tempfile + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_flushio_") + server = FakeKotobaServer( + Scenario(chunks_per_response=40, chunk_delay_s=0.05) + ).start() + original_flush = ext_mod.PCMWriter.flush + gate = {"release": None, "entered": None} + + async def gated_flush(self): + release, gate["release"] = gate["release"], None # first flush only + if release is not None: + gate["entered"].set() + await release.wait() + await original_flush(self) + + try: + with patch.object(ext_mod.PCMWriter, "flush", gated_flush): + + async def script(tester, ten_env): + release = ThreadGate() + gate["release"] = release + gate["entered"] = ThreadGate() + await tester.send_text( + ten_env, "io", "書き込み待ち。", end=True + ) + await tester.wait_for_event("pcm_frame", count=2) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-io") + await tester.wait_for_event("tts_flush_end", timeout_s=2.0) + assert time.monotonic() - t0 < FLUSH_BUDGET_S + end = await tester.wait_for_audio_end("io", timeout_s=0.5) + assert end.payload["reason"] == 2 + _assert_clean_after_flush(tester, "io") + # The dump close is parked on the gate, not yet released. + await asyncio.wait_for(gate["entered"].wait(), 2.0) + assert not release.is_set() + # The input gate is open: the next request runs meanwhile. + await tester.send_text(ten_env, "next", "次。", end=True) + end2 = await tester.wait_for_audio_end("next", timeout_s=5.0) + assert end2.payload["reason"] == 1 + release.set() + + tester = run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + with open(dump_path(dump_dir, "io"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("io") + with open(dump_path(dump_dir, "next"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("next") + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_flush_takeover_during_shutdown_does_not_abort_stop(): + """A terminal event hangs. Flush starts waiting for it, then the app + stops while that wait is still running. Flush times out first and + cancels the worker that on_stop is joining. on_stop must treat that as + a completed join, finish its cleanup (dump file exact, socket closed) + and return normally instead of letting the CancelledError escape.""" + import os + import shutil + import tempfile + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_stopjoin_") + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + state = {"calls": 0} + original = ext_mod.KotobaTTSExtension.send_tts_audio_end + + async def hanging_first_audio_end(self, *args, **kwargs): + state["calls"] += 1 + if state["calls"] == 1: + await asyncio.sleep(60) # the worker's own terminal event + return await original(self, *args, **kwargs) + + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.6), patch.object( + ext_mod.KotobaTTSExtension, + "send_tts_audio_end", + hanging_first_audio_end, + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "sj", "停止と接続。", end=True) + await tester.wait_for( + lambda r: len(r.frames) >= 2 and state["calls"] >= 1, + timeout_s=5.0, + ) + await tester.send_flush(ten_env, "flush-sj") + # Return while flush is inside its 0.6 s wait: on_stop + # starts its own 0.6 s join on the same worker, and flush's + # timeout cancels that worker first. + await asyncio.sleep(0.2) + + t0 = time.monotonic() + tester = run_tester( + script, + make_config(server.url, dump=True, dump_path=dump_dir), + timeout_s=10.0, + ) + assert time.monotonic() - t0 < 6.0 + with open(dump_path(dump_dir, "sj"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("sj") + assert len(tester.recording.audio_bytes("sj")) == len( + server.expected_pcm(1) + ) + # The transport was closed by on_stop (client.close() ran). + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and not any( + kind == "close" for kind, _conn, _t in server.recorded.events + ): + time.sleep(0.02) + assert any( + kind == "close" for kind, _conn, _t in server.recorded.events + ), server.recorded.events + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def _hold_first_audio_end(ext_mod, request_id: str, hold_s: float = 8.0): + """Patch send_tts_audio_end so its first call for ``request_id`` stalls + for ``hold_s`` (cancellable); later calls go through.""" + original = ext_mod.KotobaTTSExtension.send_tts_audio_end + state = {"held": False} + + async def held(self, *args, **kwargs): + if kwargs.get("request_id") == request_id and not state["held"]: + state["held"] = True + await asyncio.sleep(hold_s) + return await original(self, *args, **kwargs) + + return held, state + + +def test_flush_takes_over_a_stalled_pending_limit_end(): + """The pending-text limit ends a request and its audio_end send stalls. + A flush must still be acknowledged within the takeover bound, with one + audio_end for that request, and the next request must succeed.""" + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + sentences = [ + f"これは{i}番目の長めの文で、上限を超えるために続けて送ります。" + for i in range(6) + ] + server = FakeKotobaServer( + Scenario(chunks_per_response=20, chunk_delay_s=0.05) + ).start() + held, state = _hold_first_audio_end(ext_mod, "flood") + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.5), patch.object( + ext_mod.KotobaTTSExtension, "send_tts_audio_end", held + ): + + async def script(tester, ten_env): + rec = tester.recording + await tester.send_text( + ten_env, "flood", sentences[0], end=False + ) + await tester.wait_for(lambda r: len(r.audio_bytes("flood")) > 0) + for s in sentences[1:]: + await tester.send_text(ten_env, "flood", s, end=False) + await tester.wait_for( + lambda r: any( + e.payload.get("id") == "flood" for e in r.named("error") + ), + timeout_s=3.0, + ) + await _wait_until(lambda: state["held"]) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-limit") + await tester.wait_for_event("tts_flush_end", timeout_s=3.0) + assert time.monotonic() - t0 < 2.0 + ends = [ + e + for e in rec.named("tts_audio_end") + if e.payload["request_id"] == "flood" + ] + assert len(ends) == 1 and ends[0].payload["reason"] == 2 + flush_end = rec.named("tts_flush_end")[0] + assert ends[0].at <= flush_end.at + await tester.send_text(ten_env, "after", "復旧。", end=True) + end = await tester.wait_for_audio_end("after", timeout_s=5.0) + assert end.payload["reason"] == 1 + assert rec.audio_bytes("after") == server.expected_pcm(1) + + run_tester( + script, + make_config(server.url, params={"max_pending_chars": 60}), + timeout_s=20.0, + ) + finally: + server.stop() + + +def test_flush_is_bounded_when_its_own_audio_end_stalls(): + """Flush interrupts a speaking request and the audio_end it sends for + that request stalls: tts_flush_end must still arrive within the + takeover bound, with exactly one audio_end.""" + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer( + Scenario(chunks_per_response=40, chunk_delay_s=0.05) + ).start() + held, state = _hold_first_audio_end(ext_mod, "x") + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.5), patch.object( + ext_mod.KotobaTTSExtension, "send_tts_audio_end", held + ): + + async def script(tester, ten_env): + rec = tester.recording + await tester.send_text(ten_env, "x", "長い文です。", end=True) + await tester.wait_for_event("pcm_frame", count=2) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-own") + await tester.wait_for_event("tts_flush_end", timeout_s=3.0) + assert time.monotonic() - t0 < 2.0 + assert state["held"] + ends = [ + e + for e in rec.named("tts_audio_end") + if e.payload["request_id"] == "x" + ] + assert len(ends) == 1 and ends[0].payload["reason"] == 2 + _assert_clean_after_flush(tester, "x") + await tester.send_text(ten_env, "y", "次。", end=True) + end = await tester.wait_for_audio_end("y", timeout_s=5.0) + assert end.payload["reason"] == 1 + + run_tester(script, make_config(server.url), timeout_s=20.0) + finally: + server.stop() + + +def _hold_first_cancel(hold_s: float): + """Patch AsyncTTSSession.cancel so its first call (the response.cancel + write) stalls for ``hold_s`` (cancellable); later calls go through.""" + import kotoba._ws_tts as sdk_tts + + original = sdk_tts.AsyncTTSSession.cancel + state = {"held": False} + + async def held(self): + if not state["held"]: + state["held"] = True + await asyncio.sleep(hold_s) + return await original(self) + + return held, state + + +def test_flush_takes_over_while_the_limit_cancel_send_stalls(): + """The pending-text limit ends a request and its response.cancel write + stalls. Flush takes over within the bound; when the write would have + resumed, nothing more for the old request may follow tts_flush_end + (no late error, no second audio_end). The socket whose cancel never + went out is dropped, and the next request gets clean audio.""" + from unittest.mock import patch + import kotoba._ws_tts as sdk_tts + import kotoba_tts_python.extension as ext_mod + + hold_s = 1.5 + sentences = [ + f"これは{i}番目の長めの文で、上限を超えるために続けて送ります。" + for i in range(6) + ] + server = FakeKotobaServer( + Scenario(chunks_per_response=20, chunk_delay_s=0.05) + ).start() + held, state = _hold_first_cancel(hold_s) + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.5), patch.object( + sdk_tts.AsyncTTSSession, "cancel", held + ): + + async def script(tester, ten_env): + rec = tester.recording + await tester.send_text( + ten_env, "flood", sentences[0], end=False + ) + await tester.wait_for(lambda r: len(r.audio_bytes("flood")) > 0) + for s in sentences[1:]: + await tester.send_text(ten_env, "flood", s, end=False) + await _wait_until(lambda: state["held"]) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-limit-cancel") + await tester.wait_for_event("tts_flush_end", timeout_s=3.0) + assert time.monotonic() - t0 < 2.0 + # Outlast the stalled write, then look for late events. + await asyncio.sleep(hold_s) + flush_end = rec.named("tts_flush_end")[0] + for name in ("error", "tts_audio_end", "pcm_frame"): + late = [ + e + for e in events_after(tester, name, flush_end) + if "flood" + in (e.payload.get("id"), e.payload.get("request_id")) + ] + assert ( + late == [] + ), f"{name} for the old request after flush_end" + ends = [ + e + for e in rec.named("tts_audio_end") + if e.payload["request_id"] == "flood" + ] + assert len(ends) == 1 and ends[0].payload["reason"] == 2 + await tester.send_text(ten_env, "after", "復旧。", end=True) + end = await tester.wait_for_audio_end("after", timeout_s=5.0) + assert end.payload["reason"] == 1 + assert rec.audio_bytes("after") == server.expected_pcm(1) + + run_tester( + script, + make_config(server.url, params={"max_pending_chars": 60}), + timeout_s=20.0, + ) + assert server.recorded.connections == 2 + finally: + server.stop() + + +def test_flush_is_bounded_when_its_own_cancel_send_stalls(): + """Flush interrupts a speaking request and the response.cancel write + stalls: tts_flush_end must still arrive within the takeover bound, + with exactly one audio_end, and the next request gets clean audio on + a new socket.""" + from unittest.mock import patch + import kotoba._ws_tts as sdk_tts + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer( + Scenario(chunks_per_response=40, chunk_delay_s=0.05) + ).start() + held, state = _hold_first_cancel(4.0) + try: + with patch.object(ext_mod, "_TERMINAL_JOIN_SECONDS", 0.5), patch.object( + sdk_tts.AsyncTTSSession, "cancel", held + ): + + async def script(tester, ten_env): + rec = tester.recording + await tester.send_text(ten_env, "x", "長い文です。", end=True) + await tester.wait_for_event("pcm_frame", count=2) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-own-cancel") + await tester.wait_for_event("tts_flush_end", timeout_s=6.0) + assert time.monotonic() - t0 < 2.0 + assert state["held"] + end = await tester.wait_for_audio_end("x", timeout_s=0.5) + assert end.payload["reason"] == 2 + _assert_clean_after_flush(tester, "x") + await tester.send_text(ten_env, "y", "次。", end=True) + end = await tester.wait_for_audio_end("y", timeout_s=5.0) + assert end.payload["reason"] == 1 + assert rec.audio_bytes("y") == server.expected_pcm(1) + + run_tester(script, make_config(server.url), timeout_s=20.0) + assert server.recorded.connections == 2 + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_dump.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_dump.py new file mode 100644 index 0000000000..450084f531 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_dump.py @@ -0,0 +1,278 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Dump files hold exactly the emitted PCM, one file per request.""" +import os +import shutil +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import dump_path as _dump_file # noqa: E402 +from helpers import make_config, run_tester # noqa: E402 + + +def test_dump_matches_emitted_audio_per_request(): + dump_dir = tempfile.mkdtemp(prefix="kotoba_dump_") + server = FakeKotobaServer(Scenario(chunks_per_response=3)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "d1", "一文目。二文目。", end=True) + await tester.wait_for_audio_end("d1") + await tester.send_text(ten_env, "d2", "三文目。", end=True) + await tester.wait_for_audio_end("d2") + + tester = run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + files = sorted(os.listdir(dump_dir)) + assert files == sorted( + os.path.basename(_dump_file(dump_dir, rid)) for rid in ("d1", "d2") + ) + with open(_dump_file(dump_dir, "d1"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("d1") + with open(_dump_file(dump_dir, "d2"), "rb") as f: + assert f.read() == tester.recording.audio_bytes("d2") + assert tester.recording.audio_bytes("d1") == server.expected_pcm(2) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_dump_is_finalized_on_flush_and_sanitizes_request_id(): + dump_dir = tempfile.mkdtemp(prefix="kotoba_dump_") + server = FakeKotobaServer( + Scenario(chunks_per_response=40, chunk_delay_s=0.05) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "../evil/id", "長い文。", end=True) + await tester.wait_for_event("pcm_frame", count=2) + await tester.send_flush(ten_env) + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("../evil/id", timeout_s=1.0) + + tester = run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + files = os.listdir(dump_dir) + assert files == [os.path.basename(_dump_file(dump_dir, "../evil/id"))] + assert "/" not in files[0] and files[0].startswith("kotoba_tts_dump_") + with open(os.path.join(dump_dir, files[0]), "rb") as f: + data = f.read() + assert data == tester.recording.audio_bytes("../evil/id") + assert 0 < len(data) < len(server.expected_pcm(1)) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_dump_write_failure_does_not_fail_the_request(): + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_dump_") + server = FakeKotobaServer(Scenario(chunks_per_response=3)).start() + + async def failing_write(self, data): + raise OSError("disk full") + + try: + with patch.object(ext_mod.PCMWriter, "write", failing_write): + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "nodisk", "書けない。", end=True + ) + end = await tester.wait_for_audio_end("nodisk") + assert end.payload["reason"] == 1 + assert tester.recording.audio_bytes( + "nodisk" + ) == server.expected_pcm(1) + assert end.payload["request_total_audio_duration_ms"] == ( + len(server.expected_pcm(1)) * 1000 // 32000 + ) + assert tester.recording.named("error") == [] + + run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_unusable_dump_path_does_not_strand_the_queue(): + """dump_path names an existing file, so the directory cannot be created. + The request must still complete, and so must the next one.""" + blocker = tempfile.NamedTemporaryFile( + prefix="kotoba_not_a_dir_", delete=False + ) + blocker.close() + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "d1", "一。", end=True) + end1 = await tester.wait_for_audio_end("d1") + assert end1.payload["reason"] == 1 + await tester.send_text(ten_env, "d2", "二。", end=True) + end2 = await tester.wait_for_audio_end("d2") + assert end2.payload["reason"] == 1 + assert tester.recording.named("error") == [] + assert tester.recording.audio_bytes("d2") == server.expected_pcm(1) + + run_tester( + script, make_config(server.url, dump=True, dump_path=blocker.name) + ) + finally: + server.stop() + os.unlink(blocker.name) + + +def test_dump_is_finalized_on_error_with_partial_audio(): + dump_dir = tempfile.mkdtemp(prefix="kotoba_dump_") + server = FakeKotobaServer( + Scenario( + chunks_per_response=4, fail_response_index=1, fail_after_chunks=2 + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "err", "途中失敗。", end=True) + end = await tester.wait_for_audio_end("err") + assert end.payload["reason"] == 3 + + tester = run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + with open(_dump_file(dump_dir, "err"), "rb") as f: + data = f.read() + assert data == tester.recording.audio_bytes("err") + assert len(data) == 2 * 800 * 2 + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_large_dump_with_slow_writer_is_exact_and_isolated(): + """96 KB of audio crosses PCMWriter's 64 KiB auto-flush while the file + write itself is slowed down; the finalized dump must equal the emitted + PCM, and a flush under load must not lose the buffered tail.""" + import time as _time + from unittest.mock import patch + import ten_ai_base.helper as helper_mod + + dump_dir = tempfile.mkdtemp(prefix="kotoba_dump_big_") + server = FakeKotobaServer( + Scenario( + chunks_per_response=60, samples_per_chunk=800, chunk_delay_s=0.01 + ) + ).start() + original_write = helper_mod.write_pcm_to_file + + def slow_write(data, path): + _time.sleep(0.15) + return original_write(data, path) + + try: + with patch.object(helper_mod, "write_pcm_to_file", slow_write): + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "big", "長い長い文章。", end=True + ) + await tester.wait_for_audio_end("big", timeout_s=15.0) + # flush under load: second request interrupted mid-stream + await tester.send_text( + ten_env, "cut", "途中で止める。", end=True + ) + await tester.wait_for( + lambda r: len(r.audio_bytes("cut")) >= 70_000, + timeout_s=15.0, + ) + await tester.send_flush(ten_env) + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("cut", timeout_s=2.0) + await tester.send_text(ten_env, "after", "次。", end=True) + await tester.wait_for_audio_end("after", timeout_s=15.0) + + tester = run_tester( + script, + make_config(server.url, dump=True, dump_path=dump_dir), + timeout_s=60.0, + ) + for rid in ("big", "cut", "after"): + with open(_dump_file(dump_dir, rid), "rb") as f: + data = f.read() + assert data == tester.recording.audio_bytes(rid), rid + assert len(tester.recording.audio_bytes("big")) == 60 * 1600 + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_colliding_request_ids_get_separate_dump_files(): + """``a/b`` and ``a_b`` sanitize to the same text; each request must + still get its own file holding exactly its own audio.""" + dump_dir = tempfile.mkdtemp(prefix="kotoba_dump_") + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "a/b", "一つ目。", end=True) + await tester.wait_for_audio_end("a/b") + await tester.send_text(ten_env, "a_b", "二つ目。", end=True) + await tester.wait_for_audio_end("a_b") + + tester = run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + first, second = _dump_file(dump_dir, "a/b"), _dump_file(dump_dir, "a_b") + assert first != second + assert sorted(os.listdir(dump_dir)) == sorted( + os.path.basename(f) for f in (first, second) + ) + with open(first, "rb") as f: + assert f.read() == tester.recording.audio_bytes("a/b") + with open(second, "rb") as f: + assert f.read() == tester.recording.audio_bytes("a_b") + assert len(tester.recording.audio_bytes("a/b")) == len( + server.expected_pcm(1) + ) + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) + + +def test_existing_dump_file_is_replaced_not_appended(): + """A stale file with this request's name (an earlier run, the same id + reused) must not be appended to: the file holds only this run's audio.""" + dump_dir = tempfile.mkdtemp(prefix="kotoba_dump_") + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + stale = _dump_file(dump_dir, "again") + with open(stale, "wb") as f: + f.write(b"\x7f" * 4096) + + async def script(tester, ten_env): + await tester.send_text(ten_env, "again", "もう一度。", end=True) + await tester.wait_for_audio_end("again") + + tester = run_tester( + script, make_config(server.url, dump=True, dump_path=dump_dir) + ) + with open(stale, "rb") as f: + data = f.read() + assert data == tester.recording.audio_bytes("again") + assert b"\x7f" * 16 not in data + finally: + server.stop() + shutil.rmtree(dump_dir, ignore_errors=True) diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_error_msg.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_error_msg.py new file mode 100644 index 0000000000..0a77f4dc16 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_error_msg.py @@ -0,0 +1,505 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Vendor error classification and secret handling.""" +import asyncio +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import make_config, run_tester # noqa: E402 + +FATAL = -1000 +NON_FATAL = 1000 + + +def _assert_no_secret(tester, secret: str) -> None: + for e in tester.recording.events: + assert secret not in json.dumps(e.payload, ensure_ascii=False) + + +def test_rejected_key_is_fatal_and_never_logged(): + server = FakeKotobaServer( + Scenario(handshake_status=401, reject_mode="http") + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "auth", "認証失敗。", end=True) + end = await tester.wait_for_audio_end("auth") + errors = tester.recording.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == FATAL + assert errors[0].payload["vendor_info"]["code"] == "unauthorized" + assert errors[0].payload["id"] == "auth" + assert end.payload["reason"] == 3 + assert tester.recording.frames == [] + _assert_no_secret(tester, "secret-key-123") + _assert_no_secret(tester, "認証失敗") + + run_tester( + script, make_config(server.url, params={"key": "secret-key-123"}) + ) + # A fatal error is not retried. + assert server.recorded.connections == 1 + finally: + server.stop() + + +def test_policy_close_1008_is_fatal_like_fal(): + """fal accepts the upgrade and closes with 1008 for a bad key.""" + server = FakeKotobaServer(Scenario(reject_mode="close")).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "auth2", "鍵が違う。", end=True) + end = await tester.wait_for_audio_end("auth2") + errors = tester.recording.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == FATAL + assert errors[0].payload["vendor_info"]["code"] == "unauthorized" + assert end.payload["reason"] == 3 + _assert_no_secret(tester, "invalid") + + run_tester(script, make_config(server.url, params={"key": "invalid"})) + assert server.recorded.connections == 1 + finally: + server.stop() + + +def test_unsupported_language_is_fatal(): + server = FakeKotobaServer( + Scenario(open_error="unsupported_language") + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "lang", "言語。", end=True) + await tester.wait_for_audio_end("lang") + errors = tester.recording.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == FATAL + assert ( + errors[0].payload["vendor_info"]["code"] + == "unsupported_language" + ) + + run_tester(script, make_config(server.url, params={"language": "xx"})) + assert server.recorded.connections == 1 + finally: + server.stop() + + +def test_worker_failure_before_audio_is_retried_silently(): + server = FakeKotobaServer( + Scenario( + chunks_per_response=2, + fail_response_index=1, + fail_code="worker_unavailable", + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "retry", "再試行。", end=True) + end = await tester.wait_for_audio_end("retry") + assert end.payload["reason"] == 1 + assert tester.recording.named("error") == [] + assert tester.recording.audio_bytes("retry") == server.expected_pcm( + 1 + ) + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["再試行。", "再試行。"] + assert server.recorded.connections == 2 + finally: + server.stop() + + +def test_worker_failure_after_audio_is_non_fatal_and_recoverable(): + server = FakeKotobaServer( + Scenario( + chunks_per_response=4, + fail_response_index=1, + fail_after_chunks=1, + fail_code="worker_unavailable", + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "fail", "途中失敗。", end=True) + end = await tester.wait_for_audio_end("fail") + rec = tester.recording + assert end.payload["reason"] == 3 + assert end.payload["request_total_audio_duration_ms"] == ( + 800 * 2 * 1000 // (16000 * 2) + ) + errors = rec.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == NON_FATAL + assert ( + errors[0].payload["vendor_info"]["code"] == "worker_unavailable" + ) + await tester.send_text(ten_env, "ok", "復旧。", end=True) + end2 = await tester.wait_for_audio_end("ok") + assert end2.payload["reason"] == 1 + assert rec.audio_bytes("ok") == server.expected_pcm(1) + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_missing_key_fails_at_init_and_on_request(): + server = FakeKotobaServer().start() + try: + + async def script(tester, ten_env): + await tester.wait_for_event("error") + init_errors = tester.recording.named("error") + assert init_errors[0].payload["code"] == FATAL + assert "key" in init_errors[0].payload["message"] + await tester.send_text(ten_env, "nokey", "鍵なし。", end=True) + await tester.wait_for( + lambda r: any( + e.payload.get("id") == "nokey" for e in r.named("error") + ) + ) + # The request still gets its terminal event. + end = await tester.wait_for_audio_end("nokey", timeout_s=2.0) + assert end.payload["reason"] == 3 + assert end.payload["request_total_audio_duration_ms"] == 0 + assert tester.recording.frames == [] + + run_tester(script, make_config(server.url, params={"key": ""})) + assert server.recorded.connections == 0 + finally: + server.stop() + + +def test_event_metadata_never_carries_endpoint_secrets(): + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + # Query names outside any known list: every value must be masked. + secret_url = ( + server.url + + "?sign=sig-secret-123&access_token=tok-secret-456" + + "&X-Amz-Signature=amz-secret-789" + ) + + async def script(tester, ten_env): + await tester.send_text(ten_env, "meta", "メタデータ。", end=True) + await tester.wait_for_audio_end("meta") + for e in tester.recording.events: + payload = json.dumps(e.payload) + for secret in ( + "sig-secret-123", + "tok-secret-456", + "amz-secret-789", + ): + assert secret not in payload, (e.name, secret) + start = tester.recording.named("tts_audio_start")[0] + assert ( + "127.0.0.1" + in start.payload["metadata"]["vendor_metadata"]["url"] + ) + + run_tester(script, make_config(secret_url)) + finally: + server.stop() + + +def test_unexpected_negotiated_format_is_fatal(): + server = FakeKotobaServer( + Scenario(chunks_per_response=2, negotiate_format="pcm_f32") + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "fmt", "形式。", end=True) + end = await tester.wait_for_audio_end("fmt") + errors = tester.recording.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == FATAL + assert ( + errors[0].payload["vendor_info"]["code"] == "unsupported_format" + ) + assert end.payload["reason"] == 3 + assert tester.recording.frames == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 1 + assert server.recorded.texts == [] + finally: + server.stop() + + +def test_unexpected_worker_exception_ends_request_and_recovers(): + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + original = ext_mod.KotobaTTSClient.synthesize + state = {"raised": False} + + def exploding_synthesize(self, text): + if not state["raised"]: + state["raised"] = True + raise RuntimeError("boom") + return original(self, text) + + try: + with patch.object( + ext_mod.KotobaTTSClient, "synthesize", exploding_synthesize + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "boom", "爆発。", end=True) + end = await tester.wait_for_audio_end("boom") + assert end.payload["reason"] == 3 + errors = tester.recording.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == NON_FATAL + assert ( + errors[0].payload["vendor_info"]["code"] + == "unexpected_error" + ) + assert "爆発" not in errors[0].payload["message"] + await tester.send_text(ten_env, "ok", "回復。", end=True) + end2 = await tester.wait_for_audio_end("ok") + assert end2.payload["reason"] == 1 + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_transport_errors_never_quote_endpoint_secrets(): + """Connection refused on a credential-bearing endpoint: the SDK quotes + the URL it tried; nothing we emit or log may carry the secrets.""" + secret_url = "ws://user:pw-secret-77@127.0.0.1:9/ws?sign=sig-secret-77" + + async def script(tester, ten_env): + await tester.send_text(ten_env, "refused", "接続拒否。", end=True) + end = await tester.wait_for_audio_end("refused", timeout_s=10.0) + assert end.payload["reason"] == 3 + for e in tester.recording.events: + text = json.dumps(e.payload, ensure_ascii=False) + assert "pw-secret-77" not in text, e.name + assert "sig-secret-77" not in text, e.name + errors = tester.recording.named("error") + assert len(errors) == 1 and errors[0].payload["code"] == NON_FATAL + + run_tester( + script, make_config(secret_url, params={"request_timeout_seconds": 2}) + ) + + +def test_malformed_property_error_never_echoes_rejected_input(): + """``params`` given as a bare string (someone pasted the key) fails + pydantic validation; the rejected value must not appear anywhere.""" + server = FakeKotobaServer().start() + try: + + async def script(tester, ten_env): + await tester.wait_for_event("error") + await tester.send_text(ten_env, "bad", "設定不正。", end=True) + await tester.wait_for_audio_end("bad", timeout_s=2.0) + for e in tester.recording.events: + assert "KEY-MARKER-4242" not in json.dumps(e.payload) + errors = tester.recording.named("error") + assert all(e.payload["code"] == FATAL for e in errors) + assert "params" in errors[0].payload["message"] + + run_tester(script, {"params": "KEY-MARKER-4242"}) + assert server.recorded.connections == 0 + finally: + server.stop() + + +def test_transient_rate_limit_on_handshake_is_retried_silently(): + """The gateway answers the first handshake with 429: the retry on a + fresh socket succeeds and the consumer sees nothing of it.""" + server = FakeKotobaServer( + Scenario( + chunks_per_response=2, + handshake_status=429, + handshake_status_count=1, + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "rl1", "混雑。", end=True) + end = await tester.wait_for_audio_end("rl1") + assert end.payload["reason"] == 1 + assert tester.recording.named("error") == [] + assert tester.recording.audio_bytes("rl1") == server.expected_pcm(1) + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 2 + finally: + server.stop() + + +def test_persistent_rate_limit_is_non_fatal_and_recoverable(): + """429 on both attempts: one NON_FATAL error naming the limit, one + audio_end(ERROR), and the next request connects normally.""" + server = FakeKotobaServer( + Scenario( + chunks_per_response=2, + handshake_status=429, + handshake_status_count=2, + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "rl2", "制限中。", end=True) + end = await tester.wait_for_audio_end("rl2") + rec = tester.recording + assert end.payload["reason"] == 3 + assert end.payload["request_total_audio_duration_ms"] == 0 + errors = rec.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == NON_FATAL + assert errors[0].payload["vendor_info"]["code"] == "rate_limited" + assert errors[0].payload["id"] == "rl2" + await tester.send_text(ten_env, "rl3", "復旧。", end=True) + end2 = await tester.wait_for_audio_end("rl3") + assert end2.payload["reason"] == 1 + assert rec.audio_bytes("rl3") == server.expected_pcm(1) + assert len(rec.named("error")) == 1 + + run_tester(script, make_config(server.url)) + # Two refused handshakes for rl2, one accepted for rl3. + assert server.recorded.connections == 3 + assert server.recorded.texts == ["復旧。"] + finally: + server.stop() + + +def test_text_rejected_by_the_service_is_non_fatal_and_not_retried(): + """The service fails the response with a request-level code: one + NON_FATAL error carrying that code, no retry with the same text, and + the next request works on the same connection.""" + server = FakeKotobaServer( + Scenario( + chunks_per_response=2, + fail_response_index=1, + fail_code="invalid_text", + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "bad", "却下される文。", end=True) + end = await tester.wait_for_audio_end("bad") + rec = tester.recording + assert end.payload["reason"] == 3 + assert end.payload["request_total_audio_duration_ms"] == 0 + errors = rec.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == NON_FATAL + assert errors[0].payload["vendor_info"]["code"] == "invalid_text" + assert errors[0].payload["id"] == "bad" + assert rec.audio_bytes("bad") == b"" + await tester.send_text(ten_env, "good", "次の文。", end=True) + end2 = await tester.wait_for_audio_end("good") + assert end2.payload["reason"] == 1 + assert rec.audio_bytes("good") == server.expected_pcm(1) + assert len(rec.named("error")) == 1 + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["却下される文。", "次の文。"] + finally: + server.stop() + + +def test_length_truncated_response_is_a_non_fatal_error(): + """The service stops a response at its generation length limit + (completed with finish_reason=length): the audio already sent is kept, + followed by one NON_FATAL response_truncated error and one ERROR end + whose duration is that audio. The truncated text is not retried, the + rest of the request is not spoken, and the next request works on the + same connection.""" + server = FakeKotobaServer( + Scenario( + chunks_per_response=4, + length_response_index=1, + length_after_chunks=2, + ) + ).start() + try: + + async def script(tester, ten_env): + rec = tester.recording + await tester.send_text( + ten_env, "long", "途中で切れる長い文。続きの文。", end=True + ) + end = await tester.wait_for_audio_end("long") + assert end.payload["reason"] == 3 + emitted = rec.audio_bytes("long") + assert len(emitted) == 2 * server.scenario.samples_per_chunk * 2 + assert end.payload["request_total_audio_duration_ms"] == ( + len(emitted) * 1000 // (16000 * 2) + ) + errors = rec.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == NON_FATAL + assert ( + errors[0].payload["vendor_info"]["code"] == "response_truncated" + ) + assert errors[0].payload["id"] == "long" + assert errors[0].at <= end.at + await tester.send_text(ten_env, "next", "次の文。", end=True) + end2 = await tester.wait_for_audio_end("next") + assert end2.payload["reason"] == 1 + assert rec.audio_bytes("next") == server.expected_pcm(1) + assert len(rec.named("error")) == 1 + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["途中で切れる長い文。", "次の文。"] + assert server.recorded.connections == 1 + finally: + server.stop() + + +def test_init_failure_reports_each_request_once_across_deltas(): + """With the extension uninitialized, a request arriving as several + deltas gets exactly one error and one audio_end, not one per delta.""" + server = FakeKotobaServer().start() + try: + + async def script(tester, ten_env): + await tester.wait_for_event("error") # the init error + await tester.send_text(ten_env, "multi", "一つ、", end=False) + await tester.send_text(ten_env, "multi", "二つ、", end=False) + await tester.send_text(ten_env, "multi", "三つ。", end=True) + await tester.wait_for_audio_end("multi", timeout_s=2.0) + await asyncio.sleep(0.3) + errors = [ + e + for e in tester.recording.named("error") + if e.payload.get("id") == "multi" + ] + ends = [ + e + for e in tester.recording.named("tts_audio_end") + if e.payload["request_id"] == "multi" + ] + assert len(errors) == 1, len(errors) + assert len(ends) == 1 and ends[0].payload["reason"] == 3 + # The queue is released: a following request is reported too. + await tester.send_text(ten_env, "next", "次。", end=True) + await tester.wait_for_audio_end("next", timeout_s=2.0) + + run_tester(script, make_config(server.url, params={"key": ""})) + assert server.recorded.connections == 0 + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_flush.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_flush.py new file mode 100644 index 0000000000..95ee8ac5c2 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_flush.py @@ -0,0 +1,131 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Flush / cancellation boundaries.""" +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import make_config, run_tester # noqa: E402 + + +async def _wait_until(predicate, timeout_s: float = 5.0) -> None: + deadline = asyncio.get_running_loop().time() + timeout_s + while not predicate(): + if asyncio.get_running_loop().time() > deadline: + raise AssertionError("condition not met in time") + await asyncio.sleep(0.02) + + +def test_flush_after_audio_started_interrupts_once_and_stays_silent(): + server = FakeKotobaServer( + Scenario(chunks_per_response=40, chunk_delay_s=0.05) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "long", "長い文章です。", end=True) + await tester.wait_for_event("pcm_frame", count=2) + await tester.send_flush(ten_env, "flush-A") + flush_end = (await tester.wait_for_event("tts_flush_end"))[0] + assert flush_end.payload["flush_id"] == "flush-A" + end = await tester.wait_for_audio_end("long", timeout_s=1.0) + assert end.payload["reason"] == 2 + rec = tester.recording + emitted = len(rec.audio_bytes("long")) + assert 0 < emitted < len(server.expected_pcm(1)) + assert end.payload["request_total_audio_duration_ms"] == ( + emitted * 1000 // (16000 * 2) + ) + assert end.payload["request_event_interval_ms"] > 0 + frames_at_flush = len(rec.frames) + await asyncio.sleep(0.6) + assert len(rec.frames) == frames_at_flush + assert len(rec.named("tts_audio_end")) == 1 + assert rec.named("error") == [] + # The next request works on the same socket. + await tester.send_text(ten_env, "after", "次。", end=True) + end2 = await tester.wait_for_audio_end("after") + assert end2.payload["reason"] == 1 + assert rec.audio_bytes("after") == server.expected_pcm(1) + + run_tester(script, make_config(server.url)) + assert server.recorded.cancels == 1 + assert server.recorded.responses_cancelled == 1 + assert server.recorded.connections == 1 + finally: + server.stop() + + +def test_flush_before_first_audio_reports_zero_and_no_start(): + server = FakeKotobaServer( + Scenario(chunks_per_response=3, first_chunk_delay_s=1.5) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "slow", "遅い応答。", end=True) + await _wait_until(lambda: server.recorded.texts == ["遅い応答。"]) + await tester.send_flush(ten_env, "flush-B") + await tester.wait_for_event("tts_flush_end") + end = await tester.wait_for_audio_end("slow", timeout_s=1.0) + rec = tester.recording + assert end.payload["reason"] == 2 + assert end.payload["request_total_audio_duration_ms"] == 0 + assert end.payload["request_event_interval_ms"] == 0 + assert rec.named("tts_audio_start") == [] + await asyncio.sleep(0.5) + assert rec.frames == [] + await tester.send_text(ten_env, "after", "次。", end=True) + await tester.wait_for_audio_end("after", timeout_s=5.0) + assert rec.audio_bytes("after") == server.expected_pcm(1) + + run_tester(script, make_config(server.url)) + assert server.recorded.cancels == 1 + finally: + server.stop() + + +def test_flush_with_nothing_active_only_acknowledges(): + server = FakeKotobaServer().start() + try: + + async def script(tester, ten_env): + await tester.send_flush(ten_env, "flush-C") + await tester.wait_for_event("tts_flush_end") + await asyncio.sleep(0.2) + rec = tester.recording + assert rec.named("tts_audio_end") == [] + assert rec.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 0 + finally: + server.stop() + + +def test_buffered_text_is_dropped_by_flush(): + server = FakeKotobaServer( + Scenario(chunks_per_response=30, chunk_delay_s=0.05) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "multi", "一文目。二文目。三文目。", end=True + ) + await tester.wait_for_event("pcm_frame") + await tester.send_flush(ten_env, "flush-D") + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("multi", timeout_s=1.0) + await asyncio.sleep(0.5) + + run_tester(script, make_config(server.url)) + # Only the first sentence ever reached the service. + assert server.recorded.texts == ["一文目。"] + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_long_input.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_long_input.py new file mode 100644 index 0000000000..e108957de0 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_long_input.py @@ -0,0 +1,202 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Long input: one long text, many streamed deltas, and flush under a deep +sentence backlog. The segmenter and the per-request segment queue are this +extension's own logic, so they are exercised at length here.""" +import asyncio +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import events_after, make_config, run_tester # noqa: E402 + + +def _sentences(n: int) -> list[str]: + return [ + f"これは長い入力の{i}番目の文で、文字数を増やすために同じ言い回しを" + f"少し長めに続けていますが、百二十文字の上限には届きません。" + for i in range(1, n + 1) + ] + + +def test_long_single_input_is_segmented_and_spoken_in_order(): + sentences = _sentences(40) + text = "".join(sentences) + assert len(text) > 2000 + server = FakeKotobaServer( + Scenario(chunks_per_response=1, chunk_delay_s=0.005) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "long", text, end=True) + end = await tester.wait_for_audio_end("long", timeout_s=15.0) + assert end.payload["reason"] == 1 + assert tester.recording.audio_bytes("long") == server.expected_pcm( + len(sentences) + ) + assert len(tester.recording.named("tts_audio_start")) == 1 + assert tester.recording.named("error") == [] + + run_tester(script, make_config(server.url), timeout_s=30.0) + # One service request per sentence, in order, nothing lost. + assert server.recorded.texts == sentences + assert server.recorded.connections == 1 + finally: + server.stop() + + +def test_streamed_deltas_under_backlog_keep_sentence_order(): + """Sixty deltas, two per sentence, sent without waiting: the worker + must speak the thirty sentences in order while the queue is deep.""" + sentences = _sentences(30) + server = FakeKotobaServer( + Scenario(chunks_per_response=1, chunk_delay_s=0.02) + ).start() + try: + + async def script(tester, ten_env): + for i, sentence in enumerate(sentences): + cut = len(sentence) // 2 + last = i == len(sentences) - 1 + await tester.send_text( + ten_env, "stream", sentence[:cut], end=False + ) + await tester.send_text( + ten_env, "stream", sentence[cut:], end=last + ) + end = await tester.wait_for_audio_end("stream", timeout_s=15.0) + assert end.payload["reason"] == 1 + assert tester.recording.audio_bytes( + "stream" + ) == server.expected_pcm(len(sentences)) + + run_tester(script, make_config(server.url), timeout_s=30.0) + assert server.recorded.texts == sentences + finally: + server.stop() + + +def test_flush_under_backlog_cancels_once_and_recovers(): + """Thirty sentences queued, flush while the first is streaming: one + INTERRUPTED end, the backlog is dropped (at most the sentence in flight + and the one already sent reach the service), and the next request + works on the same connection.""" + sentences = _sentences(30) + server = FakeKotobaServer( + Scenario(chunks_per_response=4, chunk_delay_s=0.05) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "backlog", "".join(sentences), end=True + ) + await tester.wait_for_event("pcm_frame", count=2) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-backlog") + flush_end = (await tester.wait_for_event("tts_flush_end"))[0] + assert time.monotonic() - t0 < 1.0 + end = await tester.wait_for_audio_end("backlog", timeout_s=0.5) + assert end.payload["reason"] == 2 + assert end.at <= flush_end.at + late = [ + f + for f in events_after(tester, "pcm_frame", flush_end) + if f.payload.get("request_id") == "backlog" + ] + assert late == [] + sent_before_flush = len(server.recorded.texts) + assert sent_before_flush <= 2, sent_before_flush + await tester.send_text(ten_env, "after", "復旧。", end=True) + end2 = await tester.wait_for_audio_end("after") + assert end2.payload["reason"] == 1 + assert tester.recording.audio_bytes("after") == server.expected_pcm( + 1 + ) + # Nothing from the dropped backlog was sent after the flush. + assert server.recorded.texts[sent_before_flush:] == ["復旧。"] + + run_tester(script, make_config(server.url)) + assert server.recorded.cancels == 1 + finally: + server.stop() + + +def test_pending_text_limit_ends_the_request_cleanly(): + """The service speaks one sentence per second while ten sentences + arrive at once. Past max_pending_chars the request ends with one + non-fatal pending_text_limit error and one audio_end(ERROR); the rest + of its text is never sent. Flush then stays responsive on the next + request, and the request after that succeeds.""" + sentences = _sentences(10) + limit = 100 + assert len(sentences[0]) * 2 > limit # crossed while the first speaks + server = FakeKotobaServer( + Scenario(chunks_per_response=20, chunk_delay_s=0.05) + ).start() + try: + + async def script(tester, ten_env): + rec = tester.recording + await tester.send_text(ten_env, "flood", sentences[0], end=False) + # The first sentence is being spoken; the rest arrives at once. + await tester.wait_for(lambda r: len(r.audio_bytes("flood")) > 0) + for i, sentence in enumerate(sentences[1:], start=1): + last = i == len(sentences) - 1 + await tester.send_text(ten_env, "flood", sentence, end=last) + end = await tester.wait_for_audio_end("flood", timeout_s=5.0) + assert end.payload["reason"] == 3 + errors = [ + e for e in rec.named("error") if e.payload.get("id") == "flood" + ] + assert len(errors) == 1, errors + assert errors[0].payload["code"] == 1000 # NON_FATAL_ERROR + assert ( + errors[0].payload["vendor_info"]["code"] == "pending_text_limit" + ) + assert errors[0].at <= end.at + await asyncio.sleep(0.5) + ends = [ + e + for e in rec.named("tts_audio_end") + if e.payload["request_id"] == "flood" + ] + assert len(ends) == 1 + late = [ + f + for f in events_after(tester, "pcm_frame", end) + if f.payload.get("request_id") == "flood" + ] + assert late == [] + # Only the sentence in flight reached the service. + assert server.recorded.texts == sentences[:1], server.recorded.texts + + # Flush stays responsive on the next request. + await tester.send_text(ten_env, "next", sentences[1], end=True) + await tester.wait_for(lambda r: len(r.audio_bytes("next")) > 0) + t0 = time.monotonic() + await tester.send_flush(ten_env, "flush-after-limit") + await tester.wait_for_event("tts_flush_end", timeout_s=2.0) + assert time.monotonic() - t0 < 1.0 + end2 = await tester.wait_for_audio_end("next", timeout_s=1.0) + assert end2.payload["reason"] == 2 + + # And the request after that succeeds. + await tester.send_text(ten_env, "after", "復旧。", end=True) + end3 = await tester.wait_for_audio_end("after", timeout_s=5.0) + assert end3.payload["reason"] == 1 + assert rec.audio_bytes("after") == server.expected_pcm(1) + + run_tester( + script, + make_config(server.url, params={"max_pending_chars": limit}), + timeout_s=30.0, + ) + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_metrics.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_metrics.py new file mode 100644 index 0000000000..2ad00f9686 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_metrics.py @@ -0,0 +1,149 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""TTFB, request and usage metrics are request-scoped and emitted once.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import make_config, run_tester # noqa: E402 + + +def _metrics(tester, key: str): + return [ + e + for e in tester.recording.named("metrics") + if key in e.payload.get("metrics", {}) + ] + + +def test_ttfb_once_per_request_and_request_bytes_match_text(): + server = FakeKotobaServer( + Scenario(chunks_per_response=3, first_chunk_delay_s=0.2) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "m1", "一文目。二文目。", end=True) + await tester.wait_for_audio_end("m1") + await tester.send_text(ten_env, "m2", "三文目。", end=True) + await tester.wait_for_audio_end("m2") + # Usage follows audio_end; wait for m2's before reading. + await tester.wait_for( + lambda r: any( + e.payload["metadata"].get("request_id") == "m2" + for e in r.named("metrics") + if "recv_audio_duration" in e.payload.get("metrics", {}) + ), + timeout_s=2.0, + ) + + ttfb = _metrics(tester, "ttfb") + assert [e.payload["metadata"]["request_id"] for e in ttfb] == [ + "m1", + "m2", + ] + for e in ttfb: + assert e.payload["vendor"] == "kotoba" + # First chunk is held 200 ms by the fake service. + assert 150 <= e.payload["metrics"]["ttfb"] < 3000 + assert e.payload["metadata"]["speaker_id"] == "male-01" + + requests = _metrics(tester, "request_bytes") + sizes = [e.payload["metrics"]["request_bytes"] for e in requests] + assert sizes == [ + len("一文目。".encode()), + len("二文目。".encode()), + len("三文目。".encode()), + ] + + usage = _metrics(tester, "recv_audio_duration") + per_request = [ + e for e in usage if e.payload["metadata"].get("request_id") + ] + assert [ + e.payload["metadata"]["request_id"] for e in per_request + ] == [ + "m1", + "m2", + ] + chunk_ms = 800 * 1000 / 16000 + assert per_request[0].payload["metrics"]["recv_audio_duration"] == ( + 2 * 3 * chunk_ms + ) + assert per_request[1].payload["metrics"]["recv_audio_duration"] == ( + 3 * chunk_ms + ) + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_no_ttfb_when_no_audio(): + server = FakeKotobaServer(Scenario(handshake_status=401)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "none", "失敗。", end=True) + await tester.wait_for_audio_end("none") + assert _metrics(tester, "ttfb") == [] + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_very_fast_response_still_reports_events_once(): + server = FakeKotobaServer( + Scenario(chunks_per_response=1, chunk_delay_s=0.0) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "fast", "速い。", end=True) + end = await tester.wait_for_audio_end("fast") + rec = tester.recording + assert len(rec.named("tts_audio_start")) == 1 + assert len(rec.named("tts_audio_end")) == 1 + ttfb = _metrics(tester, "ttfb") + assert len(ttfb) == 1 + assert 0 <= ttfb[0].payload["metrics"]["ttfb"] < 500 + assert end.payload["request_total_audio_duration_ms"] == ( + len(server.expected_pcm(1)) * 1000 // 32000 + ) + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_connect_delay_belongs_to_the_request_that_paid_for_it(): + """First request on a new socket is flushed before any audio: its + handshake must be reported under its own id, not the next request's.""" + server = FakeKotobaServer( + Scenario(chunks_per_response=3, first_chunk_delay_s=1.5) + ).start() + try: + + async def script(tester, ten_env): + import asyncio + + await tester.send_text(ten_env, "first", "最初。", end=True) + await asyncio.sleep(0.3) # connected, waiting for first chunk + await tester.send_flush(ten_env) + await tester.wait_for_event("tts_flush_end") + await tester.wait_for_audio_end("first", timeout_s=1.0) + await tester.send_text(ten_env, "second", "次。", end=True) + await tester.wait_for_audio_end("second", timeout_s=8.0) + delays = _metrics(tester, "connect_delay") + assert [d.payload["metadata"]["request_id"] for d in delays] == [ + "first" + ], [d.payload for d in delays] + + run_tester(script, make_config(server.url)) + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_params.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_params.py new file mode 100644 index 0000000000..2efac68112 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_params.py @@ -0,0 +1,213 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Configuration mapping tests; no TEN runtime needed.""" +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from config import DEFAULT_BASE_URL, KotobaTTSConfig # noqa: E402 + + +def _config(params: dict, **top) -> KotobaTTSConfig: + config = KotobaTTSConfig(params=params, **top) + config.update_params() + return config + + +def test_defaults_apply_when_only_key_given(): + config = _config({"key": "k"}) + config.validate_config() + assert config.base_url == DEFAULT_BASE_URL + assert config.language == "ja" + assert config.speaker_id == "male-01" + assert config.sample_rate == 16000 + assert config.request_timeout_seconds == 30.0 + assert config.max_pending_chars == 5000 + assert config.ignored_params() == [] + + +def test_api_key_alias_and_overrides(): + config = _config( + { + "api_key": "alias", + "base_url": "wss://example.test/v2/tts/ws", + "language": "ja", + "speaker_id": "female-01", + "sample_rate": "24000", + "request_timeout_seconds": 5, + "segment_soft_limit_chars": 40, + "voice_settings": {"x": 1}, + } + ) + config.validate_config() + assert config.key == "alias" + assert config.base_url == "wss://example.test/v2/tts/ws" + assert config.speaker_id == "female-01" + assert config.sample_rate == 24000 + assert config.request_timeout_seconds == 5.0 + assert config.segment_soft_limit_chars == 40 + assert config.ignored_params() == ["voice_settings"] + + +def test_key_wins_over_api_key(): + assert _config({"key": "a", "api_key": "b"}).key == "a" + + +@pytest.mark.parametrize( + ("params", "message"), + [ + pytest.param({"key": ""}, "params.key", id="missing-key"), + pytest.param( + {"key": "k", "sample_rate": 44100}, + "params.sample_rate", + id="unsupported-sample-rate", + ), + pytest.param( + {"key": "k", "base_url": "https://fal.run/x"}, + "params.base_url", + id="http-url", + ), + pytest.param( + {"key": "k", "request_timeout_seconds": 0}, + "request_timeout_seconds", + id="zero-timeout", + ), + pytest.param( + {"key": "k", "language": " "}, + "params.language", + id="blank-language-keeps-default", + marks=pytest.mark.xfail( + reason="blank values fall back to the default", strict=True + ), + ), + ], +) +def test_validate_rejects(params, message): + config = _config(params) + with pytest.raises(ValueError, match=message): + config.validate_config() + + +@pytest.mark.parametrize( + "params", + [ + pytest.param({"key": "k", "sample_rate": True}, id="bool-as-int"), + pytest.param({"key": "k", "sample_rate": "16k"}, id="non-numeric"), + pytest.param( + {"key": "k", "request_timeout_seconds": float("nan")}, + id="nan-timeout", + ), + pytest.param( + {"key": "k", "request_timeout_seconds": "fast"}, + id="text-timeout", + ), + pytest.param( + {"key": "k", "max_pending_chars": "lots"}, id="text-pending" + ), + ], +) +def test_update_params_rejects_bad_types(params): + with pytest.raises(ValueError): + _config(params) + + +def test_to_str_masks_key(): + config = _config({"key": "secret-key-value"}) + text = config.to_str(sensitive_handling=True) + assert "secret-key-value" not in text + assert "secret-key-value" in config.to_str(sensitive_handling=False) + + +def test_to_str_redacts_unknown_param_values_and_url_credentials(): + config = _config( + { + "key": "k-secret", + "token": "tok-secret", + "custom_note": "plain-value", + "base_url": "wss://user:pw-secret@example.test/ws?sign=sig-secret", + } + ) + text = config.to_str(sensitive_handling=True) + for secret in ( + "k-secret", + "tok-secret", + "plain-value", + "pw-secret", + "sig-secret", + ): + assert secret not in text, secret + assert "example.test" in text + assert config.ignored_params() == ["custom_note", "token"] + + +def test_redacted_base_url_masks_userinfo_and_signed_query(): + config = _config( + { + "key": "k", + "base_url": "wss://user:pw-secret@example.test/ws?sign=sig-secret&x=1", + } + ) + url = config.redacted_base_url() + assert "pw-secret" not in url and "sig-secret" not in url + assert url.startswith("wss://***@example.test/ws?") + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + pytest.param("language", "", "params.language", id="blank-language"), + pytest.param("speaker_id", "", "params.speaker_id", id="blank-speaker"), + pytest.param( + "segment_soft_limit_chars", + 0, + "segment_soft_limit_chars", + id="zero-soft-limit", + ), + pytest.param( + "max_pending_chars", 0, "max_pending_chars", id="zero-pending" + ), + ], +) +def test_validate_rejects_blank_or_zero_fields(field, value, message): + config = KotobaTTSConfig(key="k", **{field: value}) + with pytest.raises(ValueError, match=message): + config.validate_config() + + +def test_boolean_for_float_is_rejected(): + with pytest.raises(ValueError, match="request_timeout_seconds"): + _config({"key": "k", "request_timeout_seconds": True}) + + +@pytest.mark.parametrize( + "url, expected", + [ + pytest.param( + "wss://host/v2/tts/ws", "wss://host/v2/tts/ws", id="no-query" + ), + pytest.param( + "wss://host/ws?access_token=tok123&X-Amz-Signature=amz456", + "wss://host/ws?access_token=***&X-Amz-Signature=***", + id="any-query-name", + ), + pytest.param( + "wss://user:pw@host/ws?k=v#frag", + "wss://***@host/ws?k=***#frag", + id="userinfo-and-fragment", + ), + pytest.param( + "wss://host/ws?flag", "wss://host/ws?flag", id="bare-flag" + ), + ], +) +def test_redacted_base_url_masks_every_query_value(url, expected): + config = _config({"key": "k", "base_url": url}) + assert config.redacted_base_url() == expected + rendered = config.to_str(sensitive_handling=True) + for secret in ("tok123", "amz456", "user:pw"): + assert secret not in rendered diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_reconnect.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_reconnect.py new file mode 100644 index 0000000000..4be1fae614 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_reconnect.py @@ -0,0 +1,222 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Connection loss before audio, after partial audio, and while idle.""" +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import make_config, run_tester # noqa: E402 + + +def test_drop_before_audio_retries_once_on_a_fresh_socket(): + server = FakeKotobaServer( + Scenario(chunks_per_response=3, close_response_index=1) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "retry", "再接続。", end=True) + end = await tester.wait_for_audio_end("retry") + rec = tester.recording + assert end.payload["reason"] == 1 + assert rec.audio_bytes("retry") == server.expected_pcm(1) + assert len(rec.named("tts_audio_start")) == 1 + assert rec.named("error") == [] + statuses = [ + e.payload.get("current") + for e in rec.named("connection_status_changed") + ] + assert statuses.count("connected") == 2 + assert "disconnected" in statuses + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 2 + assert server.recorded.texts == ["再接続。", "再接続。"] + finally: + server.stop() + + +def test_drop_after_partial_audio_ends_with_error_and_recovers(): + server = FakeKotobaServer( + Scenario( + chunks_per_response=5, + close_response_index=1, + close_after_chunks=2, + ) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text( + ten_env, "partial", "途中で切れる。", end=True + ) + end = await tester.wait_for_audio_end("partial") + rec = tester.recording + assert end.payload["reason"] == 3 + emitted = len(rec.audio_bytes("partial")) + assert emitted == 2 * 800 * 2 + assert end.payload["request_total_audio_duration_ms"] == ( + emitted * 1000 // (16000 * 2) + ) + errors = rec.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == 1000 + assert errors[0].payload["vendor_info"]["vendor"] == "kotoba" + assert "途中で切れる" not in errors[0].payload["message"] + await tester.send_text(ten_env, "again", "回復。", end=True) + end2 = await tester.wait_for_audio_end("again") + assert end2.payload["reason"] == 1 + assert rec.audio_bytes("again") == server.expected_pcm(1) + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 2 + assert server.recorded.texts == ["途中で切れる。", "回復。"] + finally: + server.stop() + + +def test_idle_close_between_requests_reconnects_transparently(): + server = FakeKotobaServer( + Scenario(chunks_per_response=2, idle_close_s=0.3) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "one", "一。", end=True) + await tester.wait_for_audio_end("one") + await asyncio.sleep(0.9) + await tester.send_text(ten_env, "two", "二。", end=True) + end = await tester.wait_for_audio_end("two") + rec = tester.recording + assert end.payload["reason"] == 1 + assert rec.audio_bytes("two") == server.expected_pcm(1) + assert rec.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 2 + assert server.recorded.texts == ["一。", "二。"] + finally: + server.stop() + + +def test_response_stall_times_out_without_hanging(): + server = FakeKotobaServer( + Scenario(chunks_per_response=3, first_chunk_delay_s=3.0) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "stall", "止まる。", end=True) + end = await tester.wait_for_audio_end("stall", timeout_s=4.0) + rec = tester.recording + assert end.payload["reason"] == 3 + assert end.payload["request_total_audio_duration_ms"] == 0 + errors = rec.named("error") + assert len(errors) == 1 + assert errors[0].payload["code"] == 1000 + assert ( + errors[0].payload["vendor_info"]["code"] == "response_timeout" + ) + + run_tester( + script, + make_config(server.url, params={"request_timeout_seconds": 0.5}), + ) + finally: + server.stop() + + +def test_idle_close_is_reported_before_the_next_text(): + server = FakeKotobaServer( + Scenario(chunks_per_response=2, idle_close_s=0.3) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "one", "一。", end=True) + await tester.wait_for_audio_end("one") + # The service closes the idle socket; the status must flip + # without any further request from us. + await tester.wait_for( + lambda r: any( + e.payload.get("current") == "disconnected" + and e.payload.get("last") == "connected" + for e in r.named("connection_status_changed") + ), + timeout_s=2.0, + ) + await tester.send_text(ten_env, "two", "二。", end=True) + end = await tester.wait_for_audio_end("two") + assert end.payload["reason"] == 1 + assert tester.recording.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 2 + finally: + server.stop() + + +def test_connecting_and_connected_callback_failures_are_logged_only(): + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + + async def failing(self, *args, **kwargs): + raise RuntimeError("status sink down") + + try: + with patch.object( + ext_mod.KotobaTTSExtension, "on_connecting", failing + ), patch.object(ext_mod.KotobaTTSExtension, "on_connected", failing): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "cb", "回調。", end=True) + end = await tester.wait_for_audio_end("cb") + assert end.payload["reason"] == 1 + assert tester.recording.audio_bytes( + "cb" + ) == server.expected_pcm(1) + assert tester.recording.named("error") == [] + + run_tester(script, make_config(server.url)) + finally: + server.stop() + + +def test_close_right_after_done_retires_the_session(): + """Server closes immediately after response.done: the status must flip + to disconnected without another request, and the next request opens a + new connection.""" + server = FakeKotobaServer( + Scenario(chunks_per_response=3, close_after_done=True) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "one", "一。", end=True) + end = await tester.wait_for_audio_end("one") + assert end.payload["reason"] == 1 + await tester.wait_for( + lambda r: any( + e.payload.get("current") == "disconnected" + and e.payload.get("last") == "connected" + for e in r.named("connection_status_changed") + ), + timeout_s=2.0, + ) + await tester.send_text(ten_env, "two", "二。", end=True) + end2 = await tester.wait_for_audio_end("two") + assert end2.payload["reason"] == 1 + assert tester.recording.audio_bytes("two") == server.expected_pcm(1) + assert tester.recording.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 2 + finally: + server.stop() diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_segmenter.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_segmenter.py new file mode 100644 index 0000000000..3a3060af13 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_segmenter.py @@ -0,0 +1,110 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Pure unit tests for ``SentenceBuffer``; no TEN runtime needed.""" +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from segmenter import SentenceBuffer # noqa: E402 + + +@pytest.mark.parametrize( + ("deltas", "released", "pending"), + [ + pytest.param( + ["こんにちは。", "元気ですか？"], + ["こんにちは。", "元気ですか？"], + "", + id="japanese-one-sentence-per-delta", + ), + pytest.param( + ["今日は", "いい天気", "ですね。明日も"], + ["今日はいい天気ですね。"], + "明日も", + id="japanese-split-mid-delta", + ), + pytest.param( + ["Hello there.", " How are you?"], + ["Hello there.", "How are you?"], + "", + id="ascii-terminator-at-delta-end", + ), + pytest.param( + ["Pi is 3.", "14 exactly. Done"], + ["Pi is 3.14 exactly."], + "Done", + id="decimal-point-not-a-boundary", + ), + pytest.param( + ["He said 「はい。」 then left."], + ["He said 「はい。」", "then left."], + "", + id="closing-quote-stays-with-sentence", + ), + pytest.param( + ["Line one\nLine two"], + ["Line one"], + "Line two", + id="newline-is-a-boundary", + ), + pytest.param( + [" ", "。"], + [], + "", + id="blank-sentence-dropped", + ), + ], +) +def test_push_releases_complete_sentences(deltas, released, pending): + buf = SentenceBuffer(soft_limit_chars=120) + got: list[str] = [] + for delta in deltas: + got.extend(buf.push(delta)) + assert got == released + assert buf.pending == pending + + +def test_flush_returns_remainder_once(): + buf = SentenceBuffer() + assert buf.push("未完の文") == [] + assert buf.flush() == "未完の文" + assert buf.flush() is None + assert buf.pending == "" + + +def test_soft_limit_cuts_at_comma(): + buf = SentenceBuffer(soft_limit_chars=10) + out = buf.push("一二三四五、六七八九十一二三四五六七八九十") + assert out == ["一二三四五、"] + assert buf.pending == "六七八九十一二三四五六七八九十" + + +def test_hard_limit_bounds_buffer_without_break_points(): + buf = SentenceBuffer(soft_limit_chars=5) + out = buf.push("a" * 23) + assert out == ["a" * 10, "a" * 10] + assert buf.pending == "aaa" + + +def test_hard_limit_prefers_space(): + buf = SentenceBuffer(soft_limit_chars=5) + out = buf.push("word another third") + assert out == ["word", "another"] + assert buf.pending == "third" + + +def test_punctuation_only_is_never_released(): + buf = SentenceBuffer() + assert buf.push("。。。") == [] + assert buf.push("...!?") == [] + assert buf.flush() is None + + +def test_rejects_invalid_soft_limit(): + with pytest.raises(ValueError): + SentenceBuffer(soft_limit_chars=0) diff --git a/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_state_machine.py b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_state_machine.py new file mode 100644 index 0000000000..64cd16195e --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/kotoba_tts_python/tests/test_state_machine.py @@ -0,0 +1,308 @@ +# +# Kotoba TTS extension for the TEN framework. +# Licensed under the Apache License, Version 2.0. +# +"""Appended text, sequential and interleaved requests, late input.""" +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from fake_kotoba_server import FakeKotobaServer, Scenario # noqa: E402 +from helpers import ThreadGate, make_config, run_tester # noqa: E402 + + +async def _wait_until(predicate, timeout_s: float = 5.0) -> None: + deadline = asyncio.get_running_loop().time() + timeout_s + while not predicate(): + if asyncio.get_running_loop().time() > deadline: + raise AssertionError("condition not met in time") + await asyncio.sleep(0.02) + + +def test_deltas_release_sentences_as_they_complete(): + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "r1", "今日は", end=False) + await tester.send_text(ten_env, "r1", "いい天気ですね。", end=False) + await tester.send_text(ten_env, "r1", "明日も晴れ", end=False) + await _wait_until( + lambda: server.recorded.texts == ["今日はいい天気ですね。"] + ) + await tester.wait_for_event("pcm_frame") + # The request stays open: no end event yet. + await asyncio.sleep(0.2) + assert tester.recording.named("tts_audio_end") == [] + await tester.send_text(ten_env, "r1", "でしょう。", end=True) + end = await tester.wait_for_audio_end("r1") + rec = tester.recording + assert server.recorded.texts == [ + "今日はいい天気ですね。", + "明日も晴れでしょう。", + ] + assert len(rec.named("tts_audio_start")) == 1 + assert rec.audio_bytes("r1") == server.expected_pcm(2) + assert end.payload["reason"] == 1 + assert rec.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.connections == 1 + finally: + server.stop() + + +def test_sequential_requests_keep_separate_counters(): + server = FakeKotobaServer(Scenario(chunks_per_response=3)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "a", "一文目。", end=True) + end_a = await tester.wait_for_audio_end("a") + await tester.send_text(ten_env, "b", "一文目。二文目。", end=True) + end_b = await tester.wait_for_audio_end("b") + rec = tester.recording + assert rec.audio_bytes("a") == server.expected_pcm(1) + assert rec.audio_bytes("b") == server.expected_pcm(2) + dur_a = end_a.payload["request_total_audio_duration_ms"] + dur_b = end_b.payload["request_total_audio_duration_ms"] + assert dur_b == 2 * dur_a + starts = [ + e.payload["request_id"] for e in rec.named("tts_audio_start") + ] + assert starts == ["a", "b"] + ends = [e.payload["request_id"] for e in rec.named("tts_audio_end")] + assert ends == ["a", "b"] + assert rec.sequence("a") == [ + "tts_audio_start", + "pcm_frame", + "tts_audio_end", + ] + assert rec.sequence("b") == [ + "tts_audio_start", + "pcm_frame", + "tts_audio_end", + ] + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["一文目。", "一文目。", "二文目。"] + finally: + server.stop() + + +def test_interleaved_request_ids_stay_isolated_and_ordered(): + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "x", "Xの前半、", end=False) + await tester.send_text(ten_env, "y", "Yの前半、", end=False) + await tester.send_text(ten_env, "x", "Xの後半。", end=True) + await tester.send_text(ten_env, "y", "Yの後半。", end=True) + await tester.wait_for_audio_end("x") + await tester.wait_for_audio_end("y") + rec = tester.recording + ends = [e.payload["request_id"] for e in rec.named("tts_audio_end")] + assert ends == ["x", "y"] + assert rec.audio_bytes("x") == server.expected_pcm(1) + assert rec.audio_bytes("y") == server.expected_pcm(1) + for e in rec.named("tts_audio_end"): + assert e.payload["reason"] == 1 + assert rec.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == [ + "Xの前半、Xの後半。", + "Yの前半、Yの後半。", + ] + finally: + server.stop() + + +def test_late_text_for_finished_request_is_ignored(): + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "done", "終わり。", end=True) + await tester.wait_for_audio_end("done") + frames_before = len(tester.recording.frames) + await tester.send_text(ten_env, "done", "遅れた文。", end=True) + await asyncio.sleep(0.4) + rec = tester.recording + assert len(rec.frames) == frames_before + assert len(rec.named("tts_audio_end")) == 1 + assert rec.named("error") == [] + await tester.send_text(ten_env, "next", "次の文。", end=True) + await tester.wait_for_audio_end("next") + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["終わり。", "次の文。"] + finally: + server.stop() + + +def test_unterminated_tail_is_sent_at_text_input_end(): + """Main control ends a turn with an empty final input; a tail without + a terminator must still be synthesized then.""" + server = FakeKotobaServer(Scenario(chunks_per_response=2)).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "tail", "今日は", end=False) + await tester.send_text(ten_env, "tail", "", end=True) + end = await tester.wait_for_audio_end("tail") + assert end.payload["reason"] == 1 + assert tester.recording.audio_bytes("tail") == server.expected_pcm( + 1 + ) + await tester.send_text(ten_env, "tail2", "明日は晴れ", end=True) + await tester.wait_for_audio_end("tail2") + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["今日は", "明日は晴れ"] + finally: + server.stop() + + +def test_completed_id_history_evicts_oldest(): + """With the history shortened to two ids, the first request id is + forgotten after two more requests and is treated as new again.""" + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer(Scenario(chunks_per_response=1)).start() + try: + with patch.object(ext_mod, "_COMPLETED_HISTORY", 2): + + async def script(tester, ten_env): + for rid in ("r1", "r2", "r3"): + await tester.send_text(ten_env, rid, f"{rid}。", end=True) + await tester.wait_for_audio_end(rid) + # r1 was evicted: its id is accepted as a fresh request. + await tester.send_text(ten_env, "r1", "再び。", end=True) + await tester.wait_for( + lambda r: sum( + 1 + for e in r.named("tts_audio_end") + if e.payload["request_id"] == "r1" + ) + == 2, + timeout_s=5.0, + ) + assert tester.recording.named("error") == [] + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["r1。", "r2。", "r3。", "再び。"] + finally: + server.stop() + + +def _usage(tester, request_id: str) -> list: + return [ + e + for e in tester.recording.named("metrics") + if "recv_audio_duration" in e.payload.get("metrics", {}) + and e.payload["metadata"].get("request_id") == request_id + ] + + +def test_late_input_while_usage_settles_does_not_release_the_next_request(): + """A's usage send is held after its counters were read. B waits in the + base queue, then a late delta for A arrives. That delta must not + release B early: B may start only after A's terminal sequence is done, + and B's usage must hold exactly its own text and audio.""" + from unittest.mock import patch + import kotoba_tts_python.extension as ext_mod + + server = FakeKotobaServer( + Scenario(chunks_per_response=20, chunk_delay_s=0.05) + ).start() + original = ext_mod.KotobaTTSExtension.send_metrics + entered, release = ThreadGate(), ThreadGate() + state = {"held": False} + + async def gated_send_metrics(self, metrics, request_id=""): + is_usage = "recv_audio_duration" in metrics.metrics + if is_usage and request_id == "A" and not state["held"]: + state["held"] = True + entered.set() + await release.wait() + return await original(self, metrics, request_id) + + try: + with patch.object( + ext_mod.KotobaTTSExtension, "send_metrics", gated_send_metrics + ): + + async def script(tester, ten_env): + await tester.send_text(ten_env, "A", "前。", end=True) + await tester.wait_for_audio_end("A", timeout_s=5.0) + await asyncio.wait_for(entered.wait(), 5.0) + # A's usage is in flight. B queues behind A, then a late + # delta for A arrives. + await tester.send_text(ten_env, "B", "後ろの文。", end=True) + await tester.send_text(ten_env, "A", "遅い。", end=True) + await asyncio.sleep(0.8) + b_before_release = len(tester.recording.audio_bytes("B")) + release.set() + await tester.wait_for_audio_end("B", timeout_s=8.0) + await tester.wait_for( + lambda r: len(_usage(tester, "B")) >= 1, timeout_s=2.0 + ) + assert ( + b_before_release == 0 + ), "B started while A's usage was still in flight" + usage_a, usage_b = _usage(tester, "A"), _usage(tester, "B") + assert len(usage_a) == 1 and len(usage_b) == 1 + assert usage_a[0].payload["metrics"]["input_characters"] == ( + len("前。") + ) + b = usage_b[0].payload["metrics"] + assert b["input_characters"] == len("後ろの文。"), b + assert b["recv_audio_duration"] == ( + len(tester.recording.audio_bytes("B")) * 1000 / 32000 + ), b + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == ["前。", "後ろの文。"] + finally: + release.set() + server.stop() + + +def test_late_input_after_input_end_is_not_spoken_or_counted(): + """A late delta for a request whose text_input_end already arrived, + while that request is still speaking: it is neither sent to the + service nor counted in the request's usage.""" + sentences = ["一つ目の文。", "二つ目の文。", "三つ目の文。"] + server = FakeKotobaServer( + Scenario(chunks_per_response=4, chunk_delay_s=0.05) + ).start() + try: + + async def script(tester, ten_env): + await tester.send_text(ten_env, "L", "".join(sentences), end=True) + await tester.wait_for_event("pcm_frame", count=1) + await tester.send_text(ten_env, "L", "余分な文。", end=True) + end = await tester.wait_for_audio_end("L", timeout_s=8.0) + assert end.payload["reason"] == 1 + await tester.wait_for( + lambda r: len(_usage(tester, "L")) >= 1, timeout_s=2.0 + ) + usage = _usage(tester, "L") + assert len(usage) == 1 + assert usage[0].payload["metrics"]["input_characters"] == len( + "".join(sentences) + ) + assert tester.recording.audio_bytes("L") == server.expected_pcm( + len(sentences) + ) + assert len(tester.recording.named("tts_audio_end")) == 1 + + run_tester(script, make_config(server.url)) + assert server.recorded.texts == sentences + finally: + server.stop()