Skip to content

feat(voice): forced-EOU rework and Smart Turn no-signal handling #123

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
sam-s10s wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(voice): forced-EOU rework and Smart Turn no-signal handling #123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 44 additions & 48 deletions sdk/voice/speechmatics/voice/_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,10 @@ def __init__(
preset_config = VoiceAgentConfigPreset.load(preset)
config = VoiceAgentConfigPreset._merge_configs(preset_config, config)

# Validate the final config (deferred to allow overlay/preset merging first)
if config is not None:
config.validate_config()

# Process the config
self._config, self._transcription_config, self._audio_format = self._prepare_config(config)

Expand Down Expand Up @@ -310,20 +314,18 @@ def __init__(
self._turn_handler: TurnTaskProcessor = TurnTaskProcessor(name="turn_handler", done_callback=self.finalize)
self._eot_calculation_task: Optional[asyncio.Task] = None

# Uses fixed EndOfUtterance message from STT
self._uses_fixed_eou: bool = (
self._eou_mode == EndOfUtteranceMode.FIXED
and not self._silero_detector
and not self._config.end_of_turn_config.use_forced_eou
)

# Uses ForceEndOfUtterance message
self._uses_forced_eou: bool = not self._uses_fixed_eou
# Forced end of utterance handling
# FEOU is not used in FIXED mode, unless VAD has been enabled. It can / should
# also be disabled during testing when not connected to an endpoint, as the
# waiting for FEOU response will block the test.
self._use_forced_eou: bool = self._eou_mode is not EndOfUtteranceMode.FIXED or self._uses_silero_vad
self._forced_eou_active: bool = False
self._last_forced_eou_latency: float = 0.0

# Emit EOT prediction (uses _uses_forced_eou)
self._uses_eot_prediction: bool = self._eou_mode not in [
# Emit EOT prediction
# EOT predictions are only relevant when not using the FIXED or EXTERNAL modes,
# as these use different triggers to finalize the turn.
self._emit_eot_predictions: bool = self._eou_mode not in [
EndOfUtteranceMode.FIXED,
EndOfUtteranceMode.EXTERNAL,
]
Expand Down Expand Up @@ -360,8 +362,8 @@ def __init__(
AudioEncoding.PCM_S16LE: 2,
}.get(self._audio_format.encoding, 1)

# Default audio buffer
if not self._config.audio_buffer_length and (self._uses_smart_turn or self._uses_silero_vad):
# Default audio buffer (used when Silero VAD is enabled and with Smart Turn)
if not self._config.audio_buffer_length and self._uses_silero_vad:
self._config.audio_buffer_length = 15.0

# Audio buffer
Expand Down Expand Up @@ -447,9 +449,7 @@ def _prepare_config(
)

# Fixed end of Utterance
if bool(
config.end_of_utterance_mode == EndOfUtteranceMode.FIXED and not config.end_of_turn_config.use_forced_eou
):
if config.end_of_utterance_mode == EndOfUtteranceMode.FIXED:
transcription_config.conversation_config = ConversationConfig(
end_of_utterance_silence_trigger=config.end_of_utterance_silence_trigger,
)
Expand Down Expand Up @@ -717,14 +717,11 @@ def update_diarization_config(self, config: SpeakerFocusConfig) -> None:
# PUBLIC UTTERANCE / TURN MANAGEMENT
# ============================================================================

def finalize(self, end_of_turn: bool = False) -> None:
def finalize(self) -> None:
"""Finalize segments.

This function will emit segments in the buffer without any further checks
on the contents of the segments.

Args:
end_of_turn: Whether to emit an end of turn message.
"""

# Clear smart turn cutoff
Expand All @@ -738,7 +735,7 @@ async def emit() -> None:
"""Wait for EndOfUtterance if needed, then emit segments."""

# Forced end of utterance message (only when no speaker is detected)
if self._config.end_of_turn_config.use_forced_eou:
if self._use_forced_eou:
await self._await_forced_eou()

# Check if the turn has changed
Expand All @@ -749,7 +746,7 @@ async def emit() -> None:
self._stt_message_queue.put_nowait(lambda: self._emit_segments(finalize=True, is_eou=True))

# Call async task (only if not already waiting for forced EOU)
if not (self._config.end_of_turn_config.use_forced_eou and self._forced_eou_active):
if not self._forced_eou_active:
asyncio.create_task(emit())

# ============================================================================
Expand Down Expand Up @@ -788,8 +785,8 @@ def _evt_on_final_transcript(message: dict[str, Any]) -> None:
return
self._stt_message_queue.put_nowait(lambda: self._handle_transcript(message, is_final=True))

# End of Utterance (FIXED mode only)
if self._uses_fixed_eou:
# End of Utterance - only when not using ForceEndOfUtterance messages
if not self._use_forced_eou:

@self.on(ServerMessageType.END_OF_UTTERANCE) # type: ignore[misc]
def _evt_on_end_of_utterance(message: dict[str, Any]) -> None:
Expand Down Expand Up @@ -1121,7 +1118,7 @@ async def _add_speech_fragments(self, message: dict[str, Any], is_final: bool =
self._last_fragment_end_time = max(self._last_fragment_end_time, fragment.end_time)

# Evaluate for VAD (only done on partials)
await self._vad_evaluation(fragments, is_final=is_final)
await self._speaker_start_stop_evaluation(fragments, is_final=is_final)

# Fragments to retain
retained_fragments = [
Expand Down Expand Up @@ -1205,18 +1202,8 @@ async def _process_speech_fragments(self, change_filter: Optional[list[Annotatio
if change_filter and not changes.any(*change_filter):
return

# Skip re-evaluation if transcripts are older than smart turn cutoff
if self._smart_turn_pending_cutoff is not None and self._current_view:
latest_end_time = max(
(f.end_time for f in self._current_view.fragments if f.end_time is not None), default=0.0
)

# If all fragments end before or at the cutoff, skip re-evaluation
if latest_end_time <= self._smart_turn_pending_cutoff:
return

# Turn prediction
if self._uses_eot_prediction and self._uses_forced_eou and not self._forced_eou_active:
if self._emit_eot_predictions and not self._forced_eou_active and self._use_forced_eou:

async def fn() -> None:
ttl = await self._calculate_finalize_delay()
Expand Down Expand Up @@ -1518,14 +1505,20 @@ async def _calculate_finalize_delay(
annotation = annotation or AnnotationResult()

# VAD enabled
if self._silero_detector:
if self._uses_silero_vad:
annotation.add(AnnotationFlags.VAD_ACTIVE)
else:
annotation.add(AnnotationFlags.VAD_INACTIVE)

# Smart Turn enabled
if self._smart_turn_detector:
annotation.add(AnnotationFlags.SMART_TURN_ACTIVE)
# If Smart Turn hasn't returned a result yet but is enabled, add NO_SIGNAL annotation.
# This covers the case where the TTL fires before VAD triggers Smart Turn inference.
if not annotation.has(AnnotationFlags.SMART_TURN_TRUE) and not annotation.has(
AnnotationFlags.SMART_TURN_FALSE
):
annotation.add(AnnotationFlags.SMART_TURN_NO_SIGNAL)
else:
annotation.add(AnnotationFlags.SMART_TURN_INACTIVE)

Expand All @@ -1551,8 +1544,7 @@ async def _calculate_finalize_delay(
delay = round(self._config.end_of_utterance_silence_trigger * multiplier, 3)

# Trim off the most recent forced EOU delay if we're in forced EOU mode
if self._uses_forced_eou:
delay -= self._last_forced_eou_latency
delay -= self._last_forced_eou_latency

# Clamp to max delay and adjust for TTFB
clamped_delay = min(delay, self._config.end_of_utterance_max_delay)
Expand Down Expand Up @@ -1586,7 +1578,10 @@ async def _eot_prediction(
# Wait for Smart Turn result
if self._smart_turn_detector and end_time is not None:
result = await self._smart_turn_prediction(end_time, self._config.language, speaker=speaker)
if result.prediction:
if result.error:
# No valid prediction — SMART_TURN_NO_SIGNAL will be applied by _calculate_finalize_delay
pass
elif result.prediction:
annotation.add(AnnotationFlags.SMART_TURN_TRUE)
else:
annotation.add(AnnotationFlags.SMART_TURN_FALSE)
Expand Down Expand Up @@ -1676,9 +1671,6 @@ async def _await_forced_eou(self, timeout: float = 1.0) -> None:
# Add listener
self.once(AgentServerMessageType.END_OF_UTTERANCE, lambda message: eou_received.set())

# Trigger EOU message
self._emit_diagnostic_message("ForceEndOfUtterance sent - waiting for EndOfUtterance")

# Wait for EOU
try:
# Track the start time
Expand All @@ -1687,6 +1679,9 @@ async def _await_forced_eou(self, timeout: float = 1.0) -> None:

# Send the force EOU and wait for the response
await self.force_end_of_utterance()
self._emit_diagnostic_message("ForceEndOfUtterance sent - waiting for EndOfUtterance")

# Wait for the response
await asyncio.wait_for(eou_received.wait(), timeout=timeout)

# Record the latency
Expand All @@ -1702,7 +1697,7 @@ async def _await_forced_eou(self, timeout: float = 1.0) -> None:
# VAD (VOICE ACTIVITY DETECTION) / SPEAKER DETECTION
# ============================================================================

async def _vad_evaluation(self, fragments: list[SpeechFragment], is_final: bool) -> None:
async def _speaker_start_stop_evaluation(self, fragments: list[SpeechFragment], is_final: bool) -> None:
"""Emit a VAD event.

This will emit `SPEAKER_STARTED` and `SPEAKER_ENDED` events to the client and is
Expand Down Expand Up @@ -1850,18 +1845,20 @@ def _handle_silero_vad_result(self, result: SileroVADResult) -> None:
annotation.add(AnnotationFlags.VAD_STARTED)

# If speech has ended, we need to predict the end of turn
if result.speech_ended and self._uses_eot_prediction:
if self._emit_eot_predictions and result.speech_ended:
"""VAD-based end of turn prediction."""

# Set cutoff to prevent late transcripts from cancelling finalization
self._smart_turn_pending_cutoff = event_time

# Async callback
async def fn() -> None:
ttl = await self._eot_prediction(
end_time=event_time, speaker=self._current_speaker, annotation=annotation
)
self._turn_handler.update_timer(ttl)

# Call the eot calculation asynchronously
self._run_background_eot_calculation(fn, "silero_vad")

async def _handle_speaker_started(self, speaker: Optional[str], event_time: float) -> None:
Expand All @@ -1878,8 +1875,7 @@ async def _handle_speaker_started(self, speaker: Optional[str], event_time: floa
await self._emit_start_of_turn(event_time)

# Update the turn handler
if self._uses_forced_eou:
self._turn_handler.reset()
self._turn_handler.reset()

# Emit the event
self._emit_message(
Expand All @@ -1902,7 +1898,7 @@ async def _handle_speaker_stopped(self, speaker: Optional[str], event_time: floa
self._last_speak_end_latency = self._total_time - event_time

# Turn prediction
if self._uses_eot_prediction and not self._forced_eou_active:
if self._emit_eot_predictions and not self._forced_eou_active:

async def fn() -> None:
ttl = await self._eot_prediction(event_time, speaker)
Expand Down
65 changes: 44 additions & 21 deletions sdk/voice/speechmatics/voice/_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from pydantic import BaseModel as PydanticBaseModel
from pydantic import ConfigDict
from pydantic import Field
from pydantic import model_validator
from typing_extensions import Self

from speechmatics.rt import AudioEncoding
Expand Down Expand Up @@ -261,6 +260,7 @@ class AnnotationFlags(str, Enum):
SMART_TURN_INACTIVE = "smart_turn_inactive"
SMART_TURN_TRUE = "smart_turn_true"
SMART_TURN_FALSE = "smart_turn_false"
SMART_TURN_NO_SIGNAL = "smart_turn_no_signal"


# ==============================================================================
Expand Down Expand Up @@ -410,35 +410,57 @@ class EndOfTurnConfig(BaseModel):
base_multiplier: Base multiplier for end of turn delay.
min_end_of_turn_delay: Minimum end of turn delay.
penalties: List of end of turn penalty items.
use_forced_eou: Whether to use forced end of utterance detection.
use_forced_eou: Whether to use forced end of utterance detection. (SHOULD ONLY EVER BE TRUE)
"""

base_multiplier: float = 1.0
min_end_of_turn_delay: float = 0.01
penalties: list[EndOfTurnPenaltyItem] = Field(
default_factory=lambda: [
# Increase delay
#
# Speaker rate increases expected TTL
EndOfTurnPenaltyItem(penalty=3.0, annotation=[AnnotationFlags.VERY_SLOW_SPEAKER]),
EndOfTurnPenaltyItem(penalty=2.0, annotation=[AnnotationFlags.SLOW_SPEAKER]),
#
# High / low rate of disfluencies
EndOfTurnPenaltyItem(penalty=2.5, annotation=[AnnotationFlags.ENDS_WITH_DISFLUENCY]),
EndOfTurnPenaltyItem(penalty=1.1, annotation=[AnnotationFlags.HAS_DISFLUENCY]),
#
# We do NOT have an end of sentence character
EndOfTurnPenaltyItem(
penalty=2.0,
annotation=[AnnotationFlags.ENDS_WITH_EOS],
is_not=True,
),
# Decrease delay
#
# We have finals and end of sentence
EndOfTurnPenaltyItem(
penalty=0.5, annotation=[AnnotationFlags.ENDS_WITH_FINAL, AnnotationFlags.ENDS_WITH_EOS]
),
# Smart Turn + VAD
EndOfTurnPenaltyItem(penalty=0.2, annotation=[AnnotationFlags.SMART_TURN_TRUE]),
#
# Smart Turn - when false, wait longer to prevent premature end of turn
EndOfTurnPenaltyItem(
penalty=0.2, annotation=[AnnotationFlags.VAD_STOPPED, AnnotationFlags.SMART_TURN_INACTIVE]
penalty=0.2, annotation=[AnnotationFlags.SMART_TURN_TRUE, AnnotationFlags.SMART_TURN_ACTIVE]
),
EndOfTurnPenaltyItem(
penalty=2.0, annotation=[AnnotationFlags.SMART_TURN_FALSE, AnnotationFlags.SMART_TURN_ACTIVE]
),
EndOfTurnPenaltyItem(
penalty=1.5, annotation=[AnnotationFlags.SMART_TURN_NO_SIGNAL, AnnotationFlags.SMART_TURN_ACTIVE]
),
#
# VAD - only applied when smart turn is not in use and on the speaker stopping
EndOfTurnPenaltyItem(
penalty=0.2,
annotation=[
AnnotationFlags.VAD_STOPPED,
AnnotationFlags.VAD_ACTIVE,
AnnotationFlags.SMART_TURN_INACTIVE,
],
),
]
)
use_forced_eou: bool = False
use_forced_eou: bool = True


class VoiceActivityConfig(BaseModel):
Expand Down Expand Up @@ -711,10 +733,16 @@ class VoiceAgentConfig(BaseModel):
audio_encoding: AudioEncoding = AudioEncoding.PCM_S16LE
chunk_size: int = 160

# Validation
@model_validator(mode="after") # type: ignore[misc]
def validate_config(self) -> Self:
"""Validate the configuration."""
def validate_config(self) -> None:
"""Validate the configuration.

Cross-field validation is deferred to this method so that configs can be
constructed as overlays (e.g. for presets) without triggering validation
on intermediate states. Call this once the final config is ready.

Raises:
ValueError: If any validation errors are found.
"""

# Validation errors
errors: list[str] = []
Expand All @@ -723,12 +751,6 @@ def validate_config(self) -> Self:
if self.end_of_utterance_mode == EndOfUtteranceMode.EXTERNAL and self.smart_turn_config:
errors.append("EXTERNAL mode cannot be used in conjunction with SmartTurnConfig")

# Cannot have FIXED and forced end of utterance enabled without VAD being enabled
if (self.end_of_utterance_mode == EndOfUtteranceMode.FIXED and self.end_of_turn_config.use_forced_eou) and not (
self.vad_config and self.vad_config.enabled
):
errors.append("FIXED mode cannot be used in conjunction with forced end of utterance without VAD enabled")

# Cannot use VAD with external end of utterance mode
if self.end_of_utterance_mode == EndOfUtteranceMode.EXTERNAL and (self.vad_config and self.vad_config.enabled):
errors.append("EXTERNAL mode cannot be used in conjunction with VAD being enabled")
Expand All @@ -751,13 +773,14 @@ def validate_config(self) -> Self:
if self.sample_rate not in [8000, 16000]:
errors.append("sample_rate must be 8000 or 16000")

# Check that forced end of utterance is set to True
if not self.end_of_turn_config.use_forced_eou:
errors.append("EndOfTurnConfig.use_forced_eou cannot be False")

# Raise error if any validation errors
if errors:
raise ValueError(f"{len(errors)} config error(s): {'; '.join(errors)}")

# Return validated config
return self


# ==============================================================================
# SESSION & INFO MODELS
Expand Down
Loading
Loading