Fix for speaker identification playback (#1627)

Author: raivisdejus

buzz/audio_devices.py

@@ -0,0 +1,71 @@
+import logging
+import threading
+import weakref
+from contextlib import contextmanager
+
+import sounddevice as sd
+
+
+# PortAudio reads the list of audio devices once, when it initializes, and keeps
+# using that snapshot. A device that appears or disappears later - Bluetooth
+# headphones, a dock, or simply a change of the default output device in Windows
+# - is invisible to us until PortAudio is re-initialized, which shows up as
+# playback going to a device that is no longer there, i.e. silence.
+#
+# Re-initializing while a stream is open crashes PortAudio, so every stream we
+# open is registered here and the refresh is skipped while any of them is alive.
+
+# Weak, so a stream that is dropped without being closed cannot block refreshes forever.
+_active_streams = weakref.WeakSet()
+_lock = threading.RLock()
+
+
+@contextmanager
+def registered_stream(stream):
+    """Mark a sounddevice stream as open, so devices are not refreshed under it."""
+    with _lock:
+        _active_streams.add(stream)
+    try:
+        yield stream
+    finally:
+        unregister_stream(stream)
+
+
+def register_stream(stream) -> None:
+    with _lock:
+        _active_streams.add(stream)
+
+
+def unregister_stream(stream) -> None:
+    with _lock:
+        _active_streams.discard(stream)
+
+
+def refresh_audio_devices() -> bool:
+    """Make PortAudio re-read the system's audio devices.
+
+    Returns True when the device list was actually refreshed, False when it was
+    skipped because a stream is still open or PortAudio refused to restart.
+    """
+    with _lock:
+        if _active_streams:
+            logging.debug("Skipping audio device refresh, %s stream(s) open", len(_active_streams))
+            return False
+        try:
+            sd._terminate()
+        except Exception:
+            logging.warning("Could not shut down audio devices for a refresh", exc_info=True)
+            return False
+
+        try:
+            sd._initialize()
+        except Exception:
+            # Leaving PortAudio down would break all playback, so try once more.
+            logging.warning("Could not re-initialize audio devices, retrying", exc_info=True)
+            try:
+                sd._initialize()
+            except Exception:
+                logging.error("Audio devices are unavailable", exc_info=True)
+            return False
+    logging.debug("Refreshed audio devices")
+    return True

buzz/recording.py

@@ -5,6 +5,8 @@
 import sounddevice
 from PyQt6.QtCore import QObject, pyqtSignal
 
+from buzz.audio_devices import register_stream, unregister_stream
+
 
 class RecordingAmplitudeListener(QObject):
     stream: Optional[sounddevice.InputStream] = None
@@ -32,6 +34,7 @@ def start_recording(self):
                 channels=1,
                 callback=self.stream_callback,
             )
+            register_stream(self.stream)
             self.stream.start()
             self.accumulation_size = int(self.stream.samplerate * self.ACCUMULATION_SECONDS)
         except Exception as e:
@@ -43,6 +46,7 @@ def stop_recording(self):
         if self.stream is not None:
             self.stream.stop()
             self.stream.close()
+            unregister_stream(self.stream)
 
     def stream_callback(self, in_data: np.ndarray, frame_count, time_info, status):
         if not self._active:

buzz/sounddevice_player.py

@@ -9,6 +9,11 @@
 import numpy as np
 import sounddevice as sd
 
+from buzz.audio_devices import (
+    refresh_audio_devices,
+    register_stream,
+    unregister_stream,
+)
 from buzz.ffmpeg_utils import find_ffmpeg
 from buzz.pip_utils import subprocess_hide_window_kwargs
 
@@ -93,6 +98,7 @@ def _open_stream(self):
             callback=self._callback,
             finished_callback=self._on_finished,
         )
+        register_stream(self._stream)
         try:
             self._latency_frames = int(self._stream.latency * self.samplerate)
         except Exception:
@@ -103,6 +109,24 @@ def _on_finished(self):
 
     def play(self):
         self._stop_stream()
+
+        # Pick up devices that were connected, removed, or made default since
+        # the last time we played, so we open the output the user is actually
+        # listening to instead of a stale one.
+        refresh_audio_devices()
+
+        try:
+            self._start_stream()
+        except Exception:
+            # The device can disappear between opening and starting it, e.g.
+            # headphones dropping off. Re-read the devices and try once more
+            # against whatever is the default output now.
+            logging.warning("Could not start audio stream, retrying", exc_info=True)
+            self._stop_stream()
+            refresh_audio_devices()
+            self._start_stream()
+
+    def _start_stream(self):
         self._open_stream()
         self._playing = True
         self._stream.start()
@@ -144,6 +168,7 @@ def _stop_stream(self):
                 self._stream.close()
             except Exception:
                 logging.debug("Error stopping sounddevice stream", exc_info=True)
+            unregister_stream(self._stream)
             self._stream = None
 
     def close(self):

buzz/transcriber/recording_transcriber.py

@@ -20,6 +20,8 @@
 from buzz.transcriber.cuda_device import cuda_works
 import sounddevice
 from sounddevice import PortAudioError
+
+from buzz.audio_devices import registered_stream
 from openai import OpenAI
 from PyQt6.QtCore import QObject, pyqtSignal
 
@@ -102,13 +104,14 @@ def start(self):
         )
 
         try:
-            with self.sounddevice.InputStream(
+            stream = self.sounddevice.InputStream(
                 samplerate=self.sample_rate,
                 device=self.input_device_index,
                 dtype="float32",
                 channels=1,
                 callback=self.stream_callback,
-            ):
+            )
+            with stream, registered_stream(stream):
                 while self.is_running:
                     if self.queue.size >= self.n_batch_samples:
                         self.mutex.acquire()

buzz/widgets/audio_player.py

@@ -290,7 +290,12 @@ def update_time_label(self):
 
     def on_audio_outputs_changed(self):
         if self._use_sd:
-            # sounddevice always uses system default; nothing to reconnect
+            # sounddevice holds on to the device for the life of the stream, so
+            # a stream that is already running keeps feeding the old output.
+            # Reopen it to move playback to whatever the default output is now.
+            if self._sd_player is not None and self._sd_player.is_playing:
+                self._sd_player.pause()
+                self._sd_player.play()
             return
         was_playing = self.media_player.playbackState() == QMediaPlayer.PlaybackState.PlayingState
         position = self.media_player.position()

buzz/widgets/transcription_viewer/speaker_identification_widget.py

@@ -21,8 +21,7 @@
 
 import faster_whisper
 import torch
-from PyQt6.QtMultimedia import QMediaPlayer, QAudioOutput
-from PyQt6.QtCore import Qt, QThread, QObject, pyqtSignal, QUrl, QTimer
+from PyQt6.QtCore import Qt, QThread, QObject, pyqtSignal, QTimer
 from PyQt6.QtGui import QFont
 from PyQt6.QtWidgets import (
     QWidget,
@@ -46,12 +45,17 @@
 from buzz.db.service.transcription_service import TranscriptionService
 from buzz.paths import file_path_as_title
 from buzz.settings.settings import Settings
+from buzz.sounddevice_player import AudioFilePlayer
 from buzz.widgets.line_edit import LineEdit
 from buzz.transcriber.transcriber import Segment
 from buzz.transcriber.speaker_identifier import build_speaker_segments
 
 
 
+# Longest speaker sample played by the "Play sample" button
+MAX_PREVIEW_DURATION_MS = 10 * 1000
+
+
 def process_in_batches(
     items,
     process_func,
@@ -695,13 +699,35 @@ def _create_save_section(self, layout: QFormLayout) -> None:
         layout.addRow(self.save_button)
 
     def _setup_audio_player(self) -> None:
-        url = QUrl.fromLocalFile(self.transcription.file)
-        self.player = QMediaPlayer()
-        self.audio_output = QAudioOutput()
-        self.player.setAudioOutput(self.audio_output)
-        self.player.setSource(url)
+        # Created on first playback: decoding the file takes a moment and the
+        # dialog is often closed without any sample being played.
+        self.player: Optional[AudioFilePlayer] = None
         self.player_timer = None
 
+    def _get_audio_player(self) -> Optional["AudioFilePlayer"]:
+        """Returns the sample player, creating it on first use, or None if the file cannot be played."""
+        if self.player is not None:
+            return self.player
+
+        if not os.path.isfile(self.transcription.file):
+            logging.warning(
+                "Speaker identification: cannot play sample, file not found %s",
+                self.transcription.file,
+            )
+            return None
+
+        player = AudioFilePlayer(self.transcription.file)
+        if not player.ready:
+            player.close()
+            logging.error(
+                "Speaker identification: could not open %s for playback",
+                self.transcription.file,
+            )
+            return None
+
+        self.player = player
+        return self.player
+
     def on_identify_button_clicked(self):
         self.step_1_button.setEnabled(False)
         self.step_1_button.setVisible(False)
@@ -841,24 +867,40 @@ def on_identification_finished(self, result):
         self.needs_layout_update = True
 
     def on_speaker_preview(self, speaker_id):
-        if self.player_timer:
-            self.player_timer.stop()
+        self._stop_playback()
 
         speaker_records = [record for record in self.identification_result if record['speaker'] == speaker_id]
 
-        if speaker_records:
-            random_record = random.choice(speaker_records)
+        if not speaker_records:
+            return
 
-            start_time = random_record['start_time']
-            end_time = random_record['end_time']
+        player = self._get_audio_player()
+        if player is None:
+            return
 
-            self.player.setPosition(int(start_time))
-            self.player.play()
+        random_record = random.choice(speaker_records)
 
-            self.player_timer = QTimer(self)
-            self.player_timer.setSingleShot(True)
-            self.player_timer.timeout.connect(self.player.stop)
-            self.player_timer.start(min(end_time, 10 * 1000))  # 10 seconds
+        start_time = int(random_record['start_time'])
+        end_time = int(random_record['end_time'])
+        # Stop at the end of the sampled segment, capped so a long monologue
+        # does not keep playing after the button was clicked.
+        duration = min(max(end_time - start_time, 0), MAX_PREVIEW_DURATION_MS)
+        if duration <= 0:
+            return
+
+        player.seek(start_time)
+        player.play()
+
+        self.player_timer = QTimer(self)
+        self.player_timer.setSingleShot(True)
+        self.player_timer.timeout.connect(self._stop_playback)
+        self.player_timer.start(duration)
+
+    def _stop_playback(self) -> None:
+        if self.player_timer:
+            self.player_timer.stop()
+        if self.player is not None:
+            self.player.stop()
 
     def on_save_button_clicked(self):
         speaker_names = []
@@ -892,10 +934,7 @@ def on_save_button_clicked(self):
         if self.transcriptions_updated_signal:
             self.transcriptions_updated_signal.emit(new_transcript_id)
 
-        self.player.stop()
-
-        if self.player_timer:
-            self.player_timer.stop()
+        self._stop_playback()
 
         self.close()
 
@@ -911,10 +950,11 @@ def changeEvent(self, event):
     def closeEvent(self, event):
         self.hide()
 
-        # Stop media player
-        self.player.stop()
-        if self.player_timer:
-            self.player_timer.stop()
+        # Stop playback and release the decoded audio
+        self._stop_playback()
+        if self.player is not None:
+            self.player.close()
+            self.player = None
 
         # Clean up thread if running
         self._cleanup_thread()

tests/mock_sounddevice.py

@@ -157,6 +157,7 @@ def close(self):
 
     def __enter__(self):
         self.start()
+        return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.stop()