diff --git a/.github/workflows/ci-android.yml b/.github/workflows/ci-android.yml index e6545547..999df0ad 100644 --- a/.github/workflows/ci-android.yml +++ b/.github/workflows/ci-android.yml @@ -188,6 +188,9 @@ jobs: --tests com.hermesandroid.relay.util.IssueReportAndDiagnosticsTest \ --tests com.hermesandroid.relay.data.AppLanguageTest \ --tests com.hermesandroid.relay.viewmodel.ChatStreamRecoveryTest \ + --tests com.hermesandroid.relay.viewmodel.ChatViewModelGatewayInboundTurnTest \ + --tests com.hermesandroid.relay.viewmodel.VoiceInboundCompletionTest \ + --tests com.hermesandroid.relay.voice.VoiceViewModelBargeInTest \ --tests com.hermesandroid.relay.viewmodel.ChatViewModelRealtimeTurnTest \ --tests com.hermesandroid.relay.network.relay.RealtimeVoiceEventParsingTest \ --tests com.hermesandroid.relay.voice.VoiceCommandInterpreterTest \ diff --git a/CHANGELOG.md b/CHANGELOG.md index d33ac7ce..eb9e2e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this ## [Unreleased] +### Fixed + +- Android Standard Voice speaks live background completions in its active conversation after the original reply finishes. Stop and conversation changes discard pending speech. (#545) + ## [Android 1.17.0] - 2026-09-13 ### Added diff --git a/app/src/androidTest/kotlin/com/hermesandroid/relay/viewmodel/GatewayExternalFixtureInstrumentedTest.kt b/app/src/androidTest/kotlin/com/hermesandroid/relay/viewmodel/GatewayExternalFixtureInstrumentedTest.kt index 02aec00f..2943b90b 100644 --- a/app/src/androidTest/kotlin/com/hermesandroid/relay/viewmodel/GatewayExternalFixtureInstrumentedTest.kt +++ b/app/src/androidTest/kotlin/com/hermesandroid/relay/viewmodel/GatewayExternalFixtureInstrumentedTest.kt @@ -62,9 +62,17 @@ class GatewayExternalFixtureInstrumentedTest { private var gatewayScope: CoroutineScope? = null private var gatewayClient: GatewayChatClient? = null private var viewModel: ChatViewModel? = null + private var voiceViewModel: VoiceViewModel? = null + private var voicePlayer: com.hermesandroid.relay.audio.VoicePlayer? = null + private var voiceSfx: com.hermesandroid.relay.audio.VoiceSfxPlayer? = null @After fun tearDown() { + compose.runOnUiThread { + voiceViewModel?.exitVoiceMode() + voicePlayer?.release() + voiceSfx?.release() + } viewModel?.updateGatewayClient(null) gatewayClient?.shutdown() gatewayScope?.cancel() @@ -250,6 +258,80 @@ class GatewayExternalFixtureInstrumentedTest { assertEquals("gateway", vm.streamingEndpoint) } + @Test + fun unsolicitedVoiceCompletions_surviveActivityPauseWithoutHistorySpeech() { + val base = InstrumentationRegistry.getArguments().getString(ARG_FIXTURE_BASE_URL) + ?.trim()?.trimEnd('/') + assumeTrue("Pass the unsolicited_voice_completions fixture URL", !base.isNullOrBlank()) + requireNotNull(base) + val http = OkHttpClient.Builder().callTimeout(10, TimeUnit.SECONDS).build() + assertEquals("unsolicited_voice_completions", readFixtureJson(http, "$base/__fixture__/state")["scenario"]?.jsonString()) + val dashboard = DashboardApiClient(base, http) + val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO).also { gatewayScope = it } + val gateway = GatewayChatClient( + initialDashboardClient = dashboard, okHttpClient = http, + callbackDispatcher = { Handler(Looper.getMainLooper()).post(it) }, scope = scope, + ).also { gatewayClient = it } + val handler = ChatHandler().also { it.setSessionId(STORED_SESSION_ID) } + val spoken = java.util.concurrent.CopyOnWriteArrayList() + lateinit var vm: ChatViewModel + compose.runOnUiThread { + val app = compose.activity.application + vm = ChatViewModel().also { + it.initialize(null, handler) + it.streamingEndpoint = "gateway" + it.setProfileMessageLoaderWithMode { profile, id, mode -> + dashboard.getSessionMessages(id, profile, mode) + } + it.updateGatewayClient(gateway) + viewModel = it + } + val audio = object : com.hermesandroid.relay.network.shared.VoiceAudioClient { + override val route = com.hermesandroid.relay.data.VoiceAudioRoute.Standard + override suspend fun transcribe(audioFile: java.io.File) = Result.success("") + override suspend fun synthesize(text: String): Result { + spoken.add(text) + // A short silent WAV exercises the production play/drain path without a provider. + val pcm = ByteArray(3200) + val header = java.nio.ByteBuffer.allocate(44).order(java.nio.ByteOrder.LITTLE_ENDIAN) + .put("RIFF".toByteArray()).putInt(36 + pcm.size).put("WAVEfmt ".toByteArray()) + .putInt(16).putShort(1).putShort(1).putInt(16000).putInt(32000) + .putShort(2).putShort(16).put("data".toByteArray()).putInt(pcm.size).array() + val file = java.io.File.createTempFile("fixture-voice", ".wav", app.cacheDir) + file.writeBytes(header + pcm) + return Result.success(file) + } + } + val player = com.hermesandroid.relay.audio.VoicePlayer(app).also { voicePlayer = it } + val sfx = com.hermesandroid.relay.audio.VoiceSfxPlayer(app).also { voiceSfx = it } + voiceViewModel = VoiceViewModel(app).also { + it.initialize( + voiceClient = com.hermesandroid.relay.network.relay.RelayVoiceClient(app, http, { null }, { null }), + voiceAudioClient = audio, chatViewModel = vm, + recorder = com.hermesandroid.relay.audio.VoiceRecorder(app, scope), + player = player, sfxPlayer = sfx, + ) + it.enterVoiceMode() + } + } + compose.setContent { + val messages by vm.messages.collectAsStateWithLifecycle() + Text(messages.joinToString("\n") { it.content }, Modifier.testTag("voice-fixture-history")) + } + assertTrue(runBlocking { gateway.prewarmAwait(STORED_SESSION_ID) }) + compose.runOnUiThread { vm.sendMessage("Start background work.") } + compose.waitUntil(10_000) { handler.messages.value.any { it.content == "Work started." } } + compose.activityRule.scenario.moveToState(androidx.lifecycle.Lifecycle.State.STARTED) + compose.waitUntil(15_000) { spoken.size == 3 } + compose.activityRule.scenario.moveToState(androidx.lifecycle.Lifecycle.State.RESUMED) + compose.runOnUiThread { voiceViewModel?.onAppResumed() } + compose.waitForIdle() + assertEquals(listOf("Process finished.", "Watch matched.", "Delegated work finished."), spoken.toList()) + assertEquals(1, readFixtureJson(http, "$base/__fixture__/evidence")["entries"].let { it as JsonArray }.rpcCount("prompt.submit")) + assertTrue(handler.messages.value.any { it.content == "Delegated work finished." }) + assertEquals("gateway", vm.streamingEndpoint) + } + private fun JsonArray.rpcCount(method: String): Int = count { element -> val entry = element as? JsonObject ?: return@count false entry["kind"]?.jsonString() == "rpc" && entry["method"]?.jsonString() == method diff --git a/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModel.kt b/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModel.kt index 9258a05d..d2e08023 100644 --- a/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModel.kt +++ b/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModel.kt @@ -780,6 +780,9 @@ class ChatViewModel : ViewModel() { /** Callback to persist session ID — set by RelayApp */ var onSessionChanged: ((String?) -> Unit)? = null + + /** Capture a voice-session receipt at live admission, never during history replay. */ + internal var gatewayInboundSpeechReceiver: (() -> ((String) -> Unit)?)? = null var onFreshDraftSelected: ((String?, SessionTransport) -> Unit)? = null /** @@ -3070,6 +3073,7 @@ class ChatViewModel : ViewModel() { var boundHandle: ActiveTurnHandle? = null var inputTokens: Int? = null var outputTokens: Int? = null + var speechReceiver: ((String) -> Unit)? = null fun ownsTranscriptSession(): Boolean = chatHandler === handler && handler.currentSessionId.value == storedSessionId @@ -3148,9 +3152,9 @@ class ChatViewModel : ViewModel() { onTurnComplete = { if (acceptsEvent()) handler.onTurnComplete(messageId) }, - // Server-initiated turns already take the bounded durable-history - // reconcile below on every completion. - onReconcileRequired = { }, + // Recovery can settle a partial live bubble before durable history + // arrives. That history repairs Chat, but is not a speech receipt. + onReconcileRequired = { speechReceiver = null }, onComplete = { val canWriteTranscript = acceptsEvent() val expectedText = handler.messages.value @@ -3168,7 +3172,9 @@ class ChatViewModel : ViewModel() { } else { finalizeTurnSideEffects(handler, messageId) AppAnalytics.onStreamComplete(inputTokens, outputTokens) + speechReceiver?.invoke(expectedText.orEmpty()) } + speechReceiver = null scheduleGatewayHistoryReconcile( storedSessionId = storedSessionId, expectedAssistantText = expectedText, @@ -3271,6 +3277,9 @@ class ChatViewModel : ViewModel() { baselineAssistantCount = handler.messages.value.count { it.role == MessageRole.ASSISTANT && !it.clientOnly } + if (queuedRecovery == null) { + speechReceiver = gatewayInboundSpeechReceiver?.invoke() + } boundHandle = handle accepted = true activeStream = handle diff --git a/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/VoiceViewModel.kt b/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/VoiceViewModel.kt index 86931e3b..bc7cd54b 100644 --- a/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/VoiceViewModel.kt +++ b/app/src/main/kotlin/com/hermesandroid/relay/viewmodel/VoiceViewModel.kt @@ -900,6 +900,11 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { * we don't re-process older turns when the history list updates. */ private var assistantSpeechCursor: AssistantSpeechCursor? = null private var voiceTurnSessionFence: VoiceTurnSessionFence? = null + private var inboundSpeechGeneration = 0L + private var inboundSpeechOwner: Pair? = null + private var inboundSpeechObserver: Job? = null + private val pendingInboundSpeech = ArrayDeque Boolean, String>>() + private var inboundSpeechPlaying = false private var sentenceBuffer: StringBuilder = StringBuilder() private val realtimeSpeechCoalescer = BalancedRealtimeTtsCoalescer() private val brokeredToolSpeechKeys = mutableSetOf() @@ -1199,9 +1204,12 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { voiceHandoffReporter: ((VoiceHandoffEvent) -> Unit)? = null, ) { cancelStandardSpeechStream("voice dependencies rewired") + retireInboundSpeech() + this.chatViewModel?.gatewayInboundSpeechReceiver = null this.voiceClient = voiceClient this.voiceAudioClient = voiceAudioClient ?: RelayVoiceAudioClientAdapter(voiceClient) this.chatViewModel = chatViewModel + chatViewModel.gatewayInboundSpeechReceiver = ::captureInboundSpeechReceiver this.recorder = recorder this.player = player this.realtimePcmPlayer = realtimePcmPlayer @@ -1537,6 +1545,10 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { */ private fun applyVoiceSettingsSnapshot(settings: com.hermesandroid.relay.data.VoiceSettings) { val nextEngineMode = VoiceEngineMode.fromStorage(settings.engineMode) + if (voiceEngineMode != nextEngineMode) { + if (inboundSpeechPlaying) interruptSpeaking(cancelActiveTurn = false) + else retireInboundSpeech() + } val finalAnswerPolicyChanged = finalAnswerOnly != settings.finalAnswerOnly val realtimeSelectionChanged = realtimeModel != settings.realtimeModel || realtimeVoice != settings.realtimeVoice @@ -1675,6 +1687,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { backgroundRun = if (orphanedRun != null) null else it.backgroundRun, ) } + if (freshEntry) bindInboundSpeechOwner() prewarmRealtimeSession() } @@ -1885,6 +1898,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { } fun exitVoiceMode() { + retireInboundSpeech() cancelPendingListeningStart() // Idempotence guard — added 2026-04-21 after logcat showed the voice- // exit chime playing on every Add-connection tap. @@ -2291,6 +2305,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { * listening turn; until then, idle queue-drain callbacks are ignored. */ fun pauseContinuousMode() { + retireInboundSpeech() cancelPendingListeningStart() continuousLoopArmed = false continuousListeningPaused = _uiState.value.interactionMode == InteractionMode.Continuous @@ -2427,6 +2442,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { * new turn on the next mic tap). */ fun interruptSpeaking(cancelActiveTurn: Boolean = true): Job? { + retireInboundSpeech() cancelPendingListeningStart() Log.i( TAG, @@ -3584,6 +3600,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { } } voiceTurnSessionFence?.bindSubmittedUser(submittedUserUiKey) + if (inboundSpeechOwner == null) bindInboundSpeechOwner() beginBargeInTurnIfEnabled() startStreamObserver(chatVm) } @@ -4738,6 +4755,99 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { standardSpeechStreamBargeInStarted.set(false) } + /** The voice overlay owns live completions only for the conversation it entered. */ + private fun bindInboundSpeechOwner() { + val chat = chatViewModel ?: return + retireInboundSpeech() + inboundSpeechOwner = chat.conversationBinding.value to chat.currentSessionId.value + inboundSpeechObserver = viewModelScope.launch { + combine(chat.conversationBinding, chat.currentSessionId, _uiState) { binding, id, state -> + Triple(binding, id, state) + }.collect { (binding, id, _) -> + val owner = inboundSpeechOwner ?: return@collect + if (owner != (binding to id)) { + // The first voice submission may create/adopt a durable session. + if (owner.second == null && owner.first.contextKey == binding.contextKey && + voiceTurnSessionFence?.accepts(id, chat.messages.value) == true + ) { + inboundSpeechOwner = binding to id + } else { + val wasPlaying = inboundSpeechPlaying + retireInboundSpeech() + if (wasPlaying) interruptSpeaking(cancelActiveTurn = false) + return@collect + } + } + drainInboundSpeech() + } + } + } + + private fun retireInboundSpeech() { + inboundSpeechGeneration++ + inboundSpeechOwner = null + inboundSpeechObserver?.cancel() + inboundSpeechObserver = null + pendingInboundSpeech.clear() + inboundSpeechPlaying = false + } + + /** Called on Main before Chat installs the new live assistant placeholder. */ + private fun captureInboundSpeechReceiver(): ((String) -> Unit)? { + val chat = chatViewModel ?: return null + val owner = inboundSpeechOwner ?: return null + val generation = inboundSpeechGeneration + fun current(): Boolean = + generation == inboundSpeechGeneration && _uiState.value.voiceMode && + voiceEngineMode == VoiceEngineMode.HermesVoiceOutput && + inboundSpeechOwner == owner && + owner == (chat.conversationBinding.value to chat.currentSessionId.value) + if (owner.second == null || !current()) return null + + // A fast unsolicited start can overtake combine's final local-turn snapshot. + // Consume that final snapshot before the new placeholder exists so the two + // speech paths cannot narrate the same assistant bubble. + if (streamObserverJob?.isActive == true && !chat.isStreaming.value) { + assistantSpeechCursor?.let { cursor -> + consumeAssistantSpeech(cursor.poll(chat.messages.value), runActive = false) + } + streamObserverJob?.cancel() + } + var consumed = false + return { text -> + if (!consumed) { + consumed = true + if (current() && sanitizeForTts(text).isNotBlank()) { + pendingInboundSpeech.addLast(::current to text) + drainInboundSpeech() + } + } + } + } + + /** Wait for a capture/earlier reply to settle; use the configured output renderer. */ + private fun drainInboundSpeech(): Boolean { + if (pendingInboundSpeech.isEmpty()) return false + if (_uiState.value.state != VoiceState.Idle || isMicCaptureActive() || + streamObserverJob?.isActive == true || + chatViewModel?.isStreaming?.value == true || + !agentAudioCompletionDecision().finishNow + ) return false + while (pendingInboundSpeech.isNotEmpty()) { + val (current, text) = pendingInboundSpeech.removeAt(0) + if (!current()) continue + cancelPendingListeningStart() + streamComplete = true + inboundSpeechPlaying = true + resetTtsTurnStats() + clearSpokenChunksState() + speakSettledFinalAnswer(text) + scheduleAgentAudioCompletionCheck() + return true + } + return false + } + /** * Observe every assistant bubble created by the active Hermes run. A tool * turn can finalize one bubble while the run is still active and later @@ -4770,37 +4880,40 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { return@collect } - val batch = cursor.poll(messages) - if (finalAnswerOnly) { - if (batch.deltas.isNotEmpty()) { - onVisualStreamDelta(batch.aggregateText) - } - } else { - batch.deltas.forEach { update -> - if (update.startsNewBubble) { - beginAssistantSpeechBubble() - } - onStreamDelta(update.text, batch.aggregateText) - } - } - // Tool state can change without text growth. - if (!finalAnswerOnly) { - batch.assistantMessages.forEach(::observeHermesToolLoopForSpeech) - } + consumeAssistantSpeech(cursor.poll(messages), runActive) + } + } + } - if (!runActive && batch.hasTurnAssistant) { - streamComplete = true - idleFlushJob?.cancel() - idleFlushJob = null - if (finalAnswerOnly) { - speakSettledFinalAnswer(batch.finalAnswerText) - } else if (!finishStandardSpeechStream()) { - flushRemainingBuffer() - } - streamObserverJob?.cancel() - scheduleAgentAudioCompletionCheck() - } + private fun consumeAssistantSpeech(batch: AssistantSpeechBatch, runActive: Boolean) { + if (finalAnswerOnly) { + if (batch.deltas.isNotEmpty()) { + onVisualStreamDelta(batch.aggregateText) + } + } else { + batch.deltas.forEach { update -> + if (update.startsNewBubble) { + beginAssistantSpeechBubble() } + onStreamDelta(update.text, batch.aggregateText) + } + } + // Tool state can change without text growth. + if (!finalAnswerOnly) { + batch.assistantMessages.forEach(::observeHermesToolLoopForSpeech) + } + + if (!runActive && batch.hasTurnAssistant) { + streamComplete = true + idleFlushJob?.cancel() + idleFlushJob = null + if (finalAnswerOnly) { + speakSettledFinalAnswer(batch.finalAnswerText) + } else if (!finishStandardSpeechStream()) { + flushRemainingBuffer() + } + streamObserverJob?.cancel() + scheduleAgentAudioCompletionCheck() } } @@ -5800,6 +5913,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { } private fun finishAgentAudioOutput() { + inboundSpeechPlaying = false continuousResumeJob = null _responseSpeechActive.value = false stopBargeInListener() @@ -5825,6 +5939,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { _uiState.update { it.copy(amplitude = 0f, outputAudioActive = false) } } + if (drainInboundSpeech()) return if (_uiState.value.interactionMode == InteractionMode.Continuous && continuousLoopArmed && _uiState.value.state == VoiceState.Idle @@ -6686,6 +6801,8 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) { } override fun onCleared() { + retireInboundSpeech() + chatViewModel?.gatewayInboundSpeechReceiver = null super.onCleared() voicePreviewJob?.cancel() voicePreviewJob = null diff --git a/app/src/test/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModelGatewayInboundTurnTest.kt b/app/src/test/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModelGatewayInboundTurnTest.kt index cc2046f1..44d681c1 100644 --- a/app/src/test/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModelGatewayInboundTurnTest.kt +++ b/app/src/test/kotlin/com/hermesandroid/relay/viewmodel/ChatViewModelGatewayInboundTurnTest.kt @@ -642,7 +642,7 @@ class ChatViewModelGatewayInboundTurnTest { awaitCondition { loadedProfile == owner.name } assertEquals(owner.name, viewModel.conversationBinding.value.profileName) assertEquals(owner.name, gatewayClient.sessionProfileProvider()) - assertEquals("X-bot", handler.activeAgentName) + assertEquals("x-bot", handler.activeAgentName) assertEquals("x-bot-session", persistedSession) viewModel.switchProfileContext( @@ -694,7 +694,7 @@ class ChatViewModelGatewayInboundTurnTest { ) assertEquals(alpha, selected) assertEquals(alpha.name, viewModel.conversationBinding.value.profileName) - assertEquals("Alpha", handler.activeAgentName) + assertEquals("alpha", handler.activeAgentName) viewModel.openProfileSession( profileName = beta.name, @@ -706,7 +706,7 @@ class ChatViewModelGatewayInboundTurnTest { assertEquals(beta, selected) assertEquals(beta.name, viewModel.conversationBinding.value.profileName) assertEquals(beta.name, gatewayClient.sessionProfileProvider()) - assertEquals("Beta", handler.activeAgentName) + assertEquals("beta", handler.activeAgentName) assertEquals("beta-session", handler.currentSessionId.value) } @@ -1462,7 +1462,7 @@ class ChatViewModelGatewayInboundTurnTest { assertEquals("default", viewModel.conversationBinding.value.profileName) assertEquals("default", gatewayClient.sessionProfileProvider()) assertEquals(null, handler.currentSessionId.value) - assertEquals("Hermes", handler.activeAgentName) + assertEquals("default", handler.activeAgentName) assertEquals("cleared", persistedSession) } @@ -2000,7 +2000,7 @@ class ChatViewModelGatewayInboundTurnTest { }) } viewModel.setDashboardConfigLoader { Result.success(config) } - shadowOf(Looper.getMainLooper()).idle() + awaitCondition { viewModel.personalityNames.value == listOf("private-a") } assertEquals(listOf("private-a"), viewModel.personalityNames.value) viewModel.resetConnectionCatalogs() @@ -2013,6 +2013,12 @@ class ChatViewModelGatewayInboundTurnTest { @Test fun unsolicitedGatewayCompletionAppearsAsOneAssistantTurnAndSettles() { + val spoken = mutableListOf() + var admissions = 0 + viewModel.gatewayInboundSpeechReceiver = { + admissions++ + { text -> spoken.add(text); Unit } + } // Upstream's process-completion poller currently emits this adjacent // duplicate pair; it must still create exactly one placeholder. serverWs.send(gatewayHarness.eventFrame("message.start", null, "live-resumed")) @@ -2048,6 +2054,13 @@ class ChatViewModelGatewayInboundTurnTest { } assertFalse(handler.messages.value.single().isStreaming) assertFalse(gatewayHarness.rpcLog.any { it.first == "prompt.submit" }) + assertEquals(1, admissions) + assertEquals(listOf(BACKGROUND_ANSWER), spoken) + serverWs.send(gatewayHarness.eventFrame("message.complete", buildJsonObject { + put("text", BACKGROUND_ANSWER) + }, "live-resumed")) + shadowOf(Looper.getMainLooper()).idle() + assertEquals(listOf(BACKGROUND_ANSWER), spoken) } @Test @@ -2077,6 +2090,42 @@ class ChatViewModelGatewayInboundTurnTest { assertTrue(activeOwner.content.isBlank()) } + @Test + fun inboundSpeechIgnoresForeignUnscopedAndFailedTurns() { + val spoken = mutableListOf() + viewModel.gatewayInboundSpeechReceiver = { { text -> spoken.add(text); Unit } } + for (session in listOf("foreign-session", null)) { + serverWs.send(gatewayHarness.eventFrame("message.start", null, session)) + serverWs.send(gatewayHarness.eventFrame("message.complete", buildJsonObject { + put("text", "Foreign answer") + }, session)) + } + serverWs.send(gatewayHarness.eventFrame("message.start", null, "live-resumed")) + serverWs.send(gatewayHarness.eventFrame("message.complete", buildJsonObject { + put("status", "error") + put("text", "Failed answer") + put("error", "Synthetic failure") + }, "live-resumed")) + awaitCondition { handler.messages.value.any { "Error" in it.badges } } + assertTrue(spoken.isEmpty()) + } + + @Test + fun inboundSpeechUsesFinalAnswerAfterToolInterim() { + val spoken = mutableListOf() + viewModel.gatewayInboundSpeechReceiver = { { text -> spoken.add(text); Unit } } + serverWs.send(gatewayHarness.eventFrame("message.start", null, "live-resumed")) + serverWs.send(gatewayHarness.eventFrame("message.interim", buildJsonObject { + put("text", "Checking the completed work.") + put("already_streamed", false) + }, "live-resumed")) + serverWs.send(gatewayHarness.eventFrame("message.complete", buildJsonObject { + put("text", "The completed work passed.") + }, "live-resumed")) + awaitCondition { spoken.isNotEmpty() } + assertEquals(listOf("The completed work passed."), spoken) + } + @Test fun queuedMainDispatchAdmitsBackgroundStartAfterLocalCompletion() { viewModel.sendMessage("Local gateway turn") @@ -2499,6 +2548,12 @@ class ChatViewModelGatewayInboundTurnTest { gatewayHarness.awaitRpc("approval.respond") awaitCondition { !handler.isStreaming.value } awaitCondition { checkpointStore.checkpoint == null } + // The RPC log records request receipt, before its acknowledgement is + // dispatched back to Main. Checkpoint retirement is independent too. + awaitCondition { + handler.messages.value.singleOrNull { it.id == "ask-approval-1" } + ?.cardDispatches?.isNotEmpty() == true + } assertEquals( "once", handler.messages.value.single { it.id == "ask-approval-1" } diff --git a/app/src/test/kotlin/com/hermesandroid/relay/viewmodel/VoiceInboundCompletionTest.kt b/app/src/test/kotlin/com/hermesandroid/relay/viewmodel/VoiceInboundCompletionTest.kt new file mode 100644 index 00000000..16c6cda6 --- /dev/null +++ b/app/src/test/kotlin/com/hermesandroid/relay/viewmodel/VoiceInboundCompletionTest.kt @@ -0,0 +1,269 @@ +package com.hermesandroid.relay.viewmodel + +import android.app.Application +import androidx.test.core.app.ApplicationProvider +import com.hermesandroid.relay.audio.VoicePlayer +import com.hermesandroid.relay.audio.VoiceRecorder +import com.hermesandroid.relay.data.ChatMessage +import com.hermesandroid.relay.data.MessageRole +import com.hermesandroid.relay.data.VoiceAudioRoute +import com.hermesandroid.relay.data.VoiceEngineMode +import com.hermesandroid.relay.network.shared.VoiceAudioClient +import com.hermesandroid.relay.network.upstream.ChatHandler +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.resetMain +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.test.setMain +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import java.io.File + +@OptIn(ExperimentalCoroutinesApi::class) +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class VoiceInboundCompletionTest { + private val dispatcher = StandardTestDispatcher() + private lateinit var chat: ChatViewModel + private lateinit var handler: ChatHandler + private lateinit var voice: VoiceViewModel + private lateinit var recorder: VoiceRecorder + private lateinit var player: VoicePlayer + private val synthesis = mutableListOf() + + @Before + fun setup() { + Dispatchers.setMain(dispatcher) + handler = ChatHandler().also { it.setSessionId("session-a") } + chat = ChatViewModel().also { + it.initialize(null, handler) + it.streamingEndpoint = "gateway" + } + recorder = mockk(relaxed = true) { + every { amplitude } returns MutableStateFlow(0f) + every { isRecording() } returns false + } + player = mockk(relaxed = true) { + every { amplitude } returns MutableStateFlow(0f) + } + val app = ApplicationProvider.getApplicationContext() + val audio = object : VoiceAudioClient { + override val route = VoiceAudioRoute.Standard + override suspend fun transcribe(audioFile: File) = Result.success("") + override suspend fun synthesize(text: String): Result { + synthesis.add(text) + return Result.success(File.createTempFile("inbound-voice", ".wav", app.cacheDir)) + } + } + voice = VoiceViewModel(app).also { + it.initialize( + voiceClient = mockk(relaxed = true), voiceAudioClient = audio, + chatViewModel = chat, recorder = recorder, player = player, + sfxPlayer = mockk(relaxed = true), + ) + it.enterVoiceMode() + } + } + + @After + fun teardown() { + voice.exitVoiceMode() + Dispatchers.resetMain() + } + + private fun admission(): (String) -> Unit = requireNotNull(chat.gatewayInboundSpeechReceiver?.invoke()) + + @Test + fun settledCompletionUsesConfiguredTtsExactlyOnce() = runTest(dispatcher) { + runCurrent() + val receipt = admission() + receipt("The timer finished.") + receipt("The timer finished.") + runCurrent() + assertEquals(listOf("The timer finished."), synthesis) + verify(exactly = 1) { player.play(any()) } + } + + @Test + fun fastInboundStartCannotBeConsumedByPreviousVoiceObserver() = runTest(dispatcher) { + runCurrent() + // Exercise the production observer with the Main dispatcher held between + // the old terminal and the next inbound admission, as OkHttp can do. + VoiceViewModel::class.java.getDeclaredMethod("startStreamObserver", ChatViewModel::class.java) + .apply { isAccessible = true }.invoke(voice, chat) + handler.addPlaceholderMessage(ChatMessage( + id = "ordinary", role = MessageRole.ASSISTANT, content = "Work started.", + timestamp = 1L, isStreaming = true, + )) + handler.onStreamComplete("ordinary") + val receipt = admission() + handler.addPlaceholderMessage(ChatMessage( + id = "inbound", role = MessageRole.ASSISTANT, content = "Work finished.", + timestamp = 2L, isStreaming = true, + )) + handler.onStreamComplete("inbound") + receipt("Work finished.") + runCurrent() + assertEquals(listOf("Work started.", "Work finished."), synthesis) + } + + @Test + fun completionWaitsForEarlierSpeechAndPreservesOrder() = runTest(dispatcher) { + runCurrent() + voice.seedSpeakingStateForTest(listOf("Earlier answer"), 0) + admission()("Process finished.") + admission()("Delegated work finished.") + runCurrent() + assertTrue(synthesis.isEmpty()) + voice.finishAgentAudioOutputForTest() + runCurrent() + assertEquals(listOf("Process finished.", "Delegated work finished."), synthesis) + } + + @Test + fun activeCaptureIsNeverCancelledForCompletionSpeech() = runTest(dispatcher) { + runCurrent() + every { recorder.isRecording() } returns true + admission()("Watch matched.") + runCurrent() + assertTrue(synthesis.isEmpty()) + verify(exactly = 0) { recorder.cancel() } + every { recorder.isRecording() } returns false + voice.finishAgentAudioOutputForTest() + runCurrent() + assertEquals(listOf("Watch matched."), synthesis) + } + + @Test + fun stopInvalidatesAdmittedAndQueuedCompletions() = runTest(dispatcher) { + runCurrent() + val late = admission() + voice.seedSpeakingStateForTest(emptyList(), 0) + admission()("Queued answer.") + voice.interruptSpeaking() + late("Late answer.") + runCurrent() + assertTrue(synthesis.isEmpty()) + assertNull(chat.gatewayInboundSpeechReceiver?.invoke()) + } + + @Test + fun exitAndReentryRejectsOldReceiptButAcceptsNewTurn() = runTest(dispatcher) { + runCurrent() + val old = admission() + voice.exitVoiceMode() + voice.enterVoiceMode() + old("Old answer.") + admission()("New answer.") + runCurrent() + assertEquals(listOf("New answer."), synthesis) + } + + @Test + fun sessionSwitchRejectsOldReceiptAndDoesNotAdoptNewSession() = runTest(dispatcher) { + runCurrent() + val old = admission() + handler.setSessionId("session-b") + old("Wrong session.") + runCurrent() + assertNull(chat.gatewayInboundSpeechReceiver?.invoke()) + handler.setSessionId("session-a") + old("Stale return.") + runCurrent() + assertTrue(synthesis.isEmpty()) + } + + @Test + fun sameSessionIdInAnotherProfileCannotSpeak() = runTest(dispatcher) { + runCurrent() + val old = admission() + chat.switchProfileContext("connection-b::profile-b", "session-a") + old("Wrong profile.") + runCurrent() + assertTrue(synthesis.isEmpty()) + assertNull(chat.gatewayInboundSpeechReceiver?.invoke()) + } + + @Test + fun realtimeEngineDoesNotConsumeGatewaySpeech() = runTest(dispatcher) { + runCurrent() + val old = admission() + voice.setVoiceEngineModeForTest(VoiceEngineMode.RealtimeAgent) + old("Wrong engine.") + assertNull(chat.gatewayInboundSpeechReceiver?.invoke()) + runCurrent() + assertTrue(synthesis.isEmpty()) + } + + @Test + fun switchingEnginesBackCannotReviveAnOldReceipt() = runTest(dispatcher) { + runCurrent() + val old = admission() + val applySettings = VoiceViewModel::class.java.getDeclaredMethod( + "applyVoiceSettingsSnapshot", com.hermesandroid.relay.data.VoiceSettings::class.java, + ).apply { isAccessible = true } + applySettings.invoke(voice, com.hermesandroid.relay.data.VoiceSettings( + engineMode = VoiceEngineMode.RealtimeAgent.storageValue, + )) + applySettings.invoke(voice, com.hermesandroid.relay.data.VoiceSettings()) + old("Stale engine receipt.") + runCurrent() + assertTrue(synthesis.isEmpty()) + } + + @Test + fun newVoiceConversationAdoptsOnlyItsSubmittedSession() = runTest(dispatcher) { + runCurrent() + voice.exitVoiceMode() + handler.setSessionId(null) + voice.enterVoiceMode() + val fence = VoiceTurnSessionFence(null).also { it.bindSubmittedUser("voice-user") } + VoiceViewModel::class.java.getDeclaredField("voiceTurnSessionFence") + .apply { isAccessible = true }.set(voice, fence) + handler.addUserMessage(ChatMessage( + id = "voice-user", role = MessageRole.USER, content = "Start work.", timestamp = 1L, + )) + handler.setSessionId("new-voice-session") + runCurrent() + admission()("New conversation completion.") + runCurrent() + assertEquals(listOf("New conversation completion."), synthesis) + } + + @Test + fun conversationChangeCancelsPendingSynthesisWithoutCancellingChat() = runTest(dispatcher) { + runCurrent() + voice.stopTtsConsumerForTest() + admission()("Old profile output.") + chat.switchProfileContext("connection-b::profile-b", "session-b") + runCurrent() + assertTrue(voice.drainTtsQueueForTest().isEmpty()) + assertTrue(synthesis.isEmpty()) + verify(atLeast = 1) { player.stop() } + } + + @Test + fun historyAndForegroundReplayNeverCreateSpeech() = runTest(dispatcher) { + runCurrent() + handler.addPlaceholderMessage(ChatMessage( + id = "history-a", role = MessageRole.ASSISTANT, content = "Historical answer.", + timestamp = 1L, isStreaming = false, + )) + voice.onAppResumed() + runCurrent() + assertTrue(synthesis.isEmpty()) + } +} diff --git a/docs/gateway-contract-testing.md b/docs/gateway-contract-testing.md index 2791ff1e..a22adc4e 100644 --- a/docs/gateway-contract-testing.md +++ b/docs/gateway-contract-testing.md @@ -68,6 +68,7 @@ the upstream contract identifiers it depends on. |---|---| | `initial_history_bind` | Durable, profile-scoped history is already available when the client resumes and first binds its rendered transcript | | `ordinary_turn` | Normal message start, deltas, completion, and persisted history | +| `unsolicited_voice_completions` | One submitted turn followed by live same-session process, watch, and delegation answers, including duplicate start/terminal frames; Standard Voice receives each admitted answer once | | `clarify_legacy` | Top-level single question and unkeyed `clarify.respond` | | `clarify_normalized_single` | One normalized `questions[]` entry still requires its exact `qid` | | `clarify_batch` | Independent qid responses, partial acknowledgement, and answered-question replay on reconnect | @@ -179,6 +180,35 @@ redacted. ## Current-upstream conformance +Standard Voice receives successful unsolicited assistant answers from live Chat +admission, with a receipt captured before the new assistant placeholder exists. +The receipt belongs to the active voice generation and conversation binding; +history reads, passive Desktop observation, unmatched terminal recovery, and +queued-checkpoint restoration do not create speech receipts. Stop, voice exit, +engine changes, and conversation changes invalidate pending receipts. An active +microphone capture or earlier spoken answer finishes before queued speech starts. +The existing Continuous microphone release barrier still owns rearming. + +Process completion/watch notifications and async delegation wakes enter upstream's +ordinary prompt runner (`tui_gateway/session_notifications.py` and `prompt_turn.py` +in current split upstream sources). The resulting assistant answer uses the same +live admission contract, regardless of its trigger. Raw process output, child +previews, and `background.complete` side-agent events are not assistant answers +and do not independently trigger narration. Reconnect history remains silent; +new live turns after reconnect can receive new receipts for the same owner. + +`VoiceInboundCompletionTest` exercises the voice receipt and configured synthesis +path; `ChatViewModelGatewayInboundTurnTest` exercises real WebSocket admission. +The `unsolicited_voice_completions` manifest certifies the upstream terminal +contract without making provider or physical-audio claims. + +For emulator lifecycle coverage, start that fixture on host loopback and run +`GatewayExternalFixtureInstrumentedTest#unsolicitedVoiceCompletions_surviveActivityPauseWithoutHistorySpeech` +on `standardPhoneApi36`, passing its emulator-accessible URL through +`gatewayFixtureBaseUrl`. The test uses production Chat/Voice view models and a +synthetic Standard audio client returning silent WAVs; it asserts three synthesis +requests across Activity pause/resume, with no provider calls or microphone capture. + Run against a clean checkout of `NousResearch/hermes-agent`: ```powershell diff --git a/docs/spec.md b/docs/spec.md index 31321b8b..30382ddd 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1199,7 +1199,7 @@ utilities. transfers ownership so assistant-process cleanup cannot cancel the main-app flow. While keyguard is active, the surface keeps only generic phase and retry copy; transcript, response, route-specific errors, and screen context remain hidden. -- Stable voice integrates with `ChatViewModel` by **observing** `messages: StateFlow`; transcribed text goes through normal `chatVm.sendMessage(text)` so voice utterances appear as regular user messages in chat history. Experimental Realtime Agent creates a mirrored chat turn and applies broker events directly so tool state, transcript text, assistant deltas, and final responses appear without leaving voice mode. +- Stable voice observes the submitted run through `messages: StateFlow`; transcribed text uses the normal Chat pipeline. While voice remains active, successful live unsolicited Gateway turns in that exact conversation also deliver their settled answer once through the configured voice output. Delivery waits for current capture/playback; Stop, exit, engine changes, and conversation changes invalidate pending speech. History/reconnect replay and passive observation never create speech. Experimental Realtime Agent retains its separate mirrored chat turn and broker events. - `VoiceModeOverlay` — full-screen UI with the MorphingSphere at 60% height in `voiceMode=true`, transcribed + response text, mic button supporting Tap / Hold / Continuous interaction modes. - The optional `SYSTEM_ALERT_WINDOW` Voice control is user-invoked from an active in-app turn. It starts as a wide compact bar, expands for transcript, diff --git a/scripts/android-prepush.py b/scripts/android-prepush.py index 1d238f23..c2e2c6c0 100644 --- a/scripts/android-prepush.py +++ b/scripts/android-prepush.py @@ -23,7 +23,9 @@ FOCUSED_TESTS = ( "com.hermesandroid.relay.viewmodel.InjectedContextTest", "com.hermesandroid.relay.screenshots.InjectedContextSheetTest", - "com.hermesandroid.relay.viewmodel.ChatViewModelGatewayInboundTurnTest.injectedContextPreviewMatchesBareGatewayPayload", + "com.hermesandroid.relay.viewmodel.ChatViewModelGatewayInboundTurnTest", + "com.hermesandroid.relay.viewmodel.VoiceInboundCompletionTest", + "com.hermesandroid.relay.voice.VoiceViewModelBargeInTest", "com.hermesandroid.relay.voice.VoiceOverlayLifecycleTest", "com.hermesandroid.relay.voice.VoiceOverlayForegroundServiceTest", "com.hermesandroid.relay.voice.VoiceOverlayPresentationTest", diff --git a/test-fixtures/vanilla-gateway/tests/test_fixture.py b/test-fixtures/vanilla-gateway/tests/test_fixture.py index 53142ac5..3b8e1804 100644 --- a/test-fixtures/vanilla-gateway/tests/test_fixture.py +++ b/test-fixtures/vanilla-gateway/tests/test_fixture.py @@ -218,6 +218,21 @@ async def test_child_activity_outlives_parent_before_completion_wake(self) -> No self.assertNotIn("child_session_id", receipt["display_metadata"]) self.assertEqual("Delegation complete.", history[3]["content"]) + async def test_unsolicited_voice_turns_follow_one_submit(self) -> None: + _, base_url = await self.start("unsolicited_voice_completions") + ws, _ = await self.connect(base_url) + await self.rpc(ws, 1, "prompt.submit", {"text": "fixture"}) + frames = await self.frames_until(ws, lambda f: ( + f.get("params", {}).get("type") == "message.complete" + and f.get("params", {}).get("payload", {}).get("text") == "Delegated work finished." + )) + answers = [f["params"]["payload"]["text"] for f in frames + if f.get("params", {}).get("type") == "message.complete"] + self.assertEqual([ + "Work started.", "Process finished.", "Process finished.", + "Watch matched.", "Delegated work finished.", + ], answers) + async def test_ownership_rejection_is_terminal_without_persisted_turn(self) -> None: fixture, base_url = await self.start("ownership_rejection") ws, _ = await self.connect(base_url) @@ -537,6 +552,7 @@ def test_all_bundled_scenarios_validate(self) -> None: "cross_client_observation", "initial_history_bind", "ordinary_turn", + "unsolicited_voice_completions", "ownership_rejection", "rapid_tools_interims", "subagent_child_preview", diff --git a/test-fixtures/vanilla-gateway/vanilla_gateway/scenarios/unsolicited_voice_completions.json b/test-fixtures/vanilla-gateway/vanilla_gateway/scenarios/unsolicited_voice_completions.json new file mode 100644 index 00000000..0a4324f4 --- /dev/null +++ b/test-fixtures/vanilla-gateway/vanilla_gateway/scenarios/unsolicited_voice_completions.json @@ -0,0 +1,37 @@ +{ + "name": "unsolicited_voice_completions", + "live_session_id": "fixture-live-1", + "stored_session_id": "20260821_120000_fixture", + "profile": "default", + "contract_requirements": ["gateway.message_complete"], + "turns": [{ + "steps": [ + {"op": "set_running", "value": true}, + {"op": "event", "type": "message.start"}, + {"op": "persist", "messages": [{"id": 1, "role": "user", "content": "Start background work.", "timestamp": 1.0}, {"id": 2, "role": "assistant", "content": "Work started.", "timestamp": 2.0}]}, + {"op": "set_running", "value": false}, + {"op": "event", "type": "message.complete", "payload": {"text": "Work started.", "status": "complete"}}, + {"op": "sleep", "milliseconds": 500}, + {"op": "set_running", "value": true}, + {"op": "event", "type": "message.start"}, + {"op": "event", "type": "message.start"}, + {"op": "event", "type": "message.delta", "payload": {"text": "Process finished."}}, + {"op": "persist", "messages": [{"id": 1, "role": "user", "content": "Start background work.", "timestamp": 1.0}, {"id": 2, "role": "assistant", "content": "Work started.", "timestamp": 2.0}, {"id": 3, "role": "assistant", "content": "Process finished.", "timestamp": 3.0}]}, + {"op": "set_running", "value": false}, + {"op": "event", "type": "message.complete", "payload": {"text": "Process finished.", "status": "complete"}}, + {"op": "event", "type": "message.complete", "payload": {"text": "Process finished.", "status": "complete"}}, + {"op": "sleep", "milliseconds": 500}, + {"op": "set_running", "value": true}, + {"op": "event", "type": "message.start"}, + {"op": "persist", "messages": [{"id": 1, "role": "user", "content": "Start background work.", "timestamp": 1.0}, {"id": 2, "role": "assistant", "content": "Work started.", "timestamp": 2.0}, {"id": 3, "role": "assistant", "content": "Process finished.", "timestamp": 3.0}, {"id": 4, "role": "assistant", "content": "Watch matched.", "timestamp": 4.0}]}, + {"op": "set_running", "value": false}, + {"op": "event", "type": "message.complete", "payload": {"text": "Watch matched.", "status": "complete"}}, + {"op": "sleep", "milliseconds": 500}, + {"op": "set_running", "value": true}, + {"op": "event", "type": "message.start"}, + {"op": "persist", "messages": [{"id": 1, "role": "user", "content": "Start background work.", "timestamp": 1.0}, {"id": 2, "role": "assistant", "content": "Work started.", "timestamp": 2.0}, {"id": 3, "role": "assistant", "content": "Process finished.", "timestamp": 3.0}, {"id": 4, "role": "assistant", "content": "Watch matched.", "timestamp": 4.0}, {"id": 5, "role": "assistant", "content": "Delegated work finished.", "timestamp": 5.0}]}, + {"op": "set_running", "value": false}, + {"op": "event", "type": "message.complete", "payload": {"text": "Delegated work finished.", "status": "complete"}} + ] + }] +}