From 911fa64264309bf043e9a3f6c78ed73bdd6192c2 Mon Sep 17 00:00:00 2001 From: Raivis Dejus Date: Sat, 5 Sep 2026 15:55:21 +0300 Subject: [PATCH] Fix for parakeet / MP4/M4A issue --- buzz/transformers_whisper.py | 55 +++++++++---------- buzz/whisper_audio.py | 10 ++-- .../transcriber/transformers_whisper_test.py | 40 +++++++++++++- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/buzz/transformers_whisper.py b/buzz/transformers_whisper.py index f1e545f874..7543000155 100644 --- a/buzz/transformers_whisper.py +++ b/buzz/transformers_whisper.py @@ -15,6 +15,7 @@ from transformers.pipelines.audio_utils import ffmpeg_read from transformers.pipelines.automatic_speech_recognition import is_torchaudio_available +from buzz import whisper_audio from buzz.model_loader import ( is_mms_model, is_parakeet_model, @@ -24,6 +25,21 @@ ) +def load_audio_input(audio: Union[str, np.ndarray], sampling_rate: int) -> np.ndarray: + """Decode an audio/video file into a mono float32 array at `sampling_rate`. + + transformers' `ffmpeg_read` pipes the whole file to ffmpeg over stdin, which + ffmpeg cannot seek. MP4/M4A files that keep their `moov` atom at the end of + the file (anything not written with `-movflags +faststart`) are therefore + rejected as "not in the correct format or is malformed", even though they are + valid. Decoding from the file path instead lets ffmpeg seek and handles them. + """ + if not isinstance(audio, str): + return audio + + return whisper_audio.load_audio(audio, sampling_rate) + + def is_intel_mac() -> bool: """Check if running on Intel Mac (x86_64).""" return sys.platform == 'darwin' and platform.machine() == 'x86_64' @@ -75,8 +91,9 @@ def _load_audio_from_source(self, inputs): if inputs.startswith("http://") or inputs.startswith("https://"): inputs = requests.get(inputs).content else: - with open(inputs, "rb") as f: - inputs = f.read() + # Decode from the path, not from bytes over stdin, so that + # non-faststart MP4/M4A files load. See `load_audio_input`. + return load_audio_input(inputs, self.feature_extractor.sampling_rate) if isinstance(inputs, bytes): inputs = ffmpeg_read(inputs, self.feature_extractor.sampling_rate) @@ -506,7 +523,6 @@ def _transcribe_parakeet( AutoModelForRNNT, AutoModelForTDT, ) - from transformers.pipelines.audio_utils import ffmpeg_read as pk_ffmpeg_read force_cpu = os.getenv("BUZZ_FORCE_CPU", "false") use_cuda = torch.cuda.is_available() and force_cpu == "false" @@ -530,12 +546,7 @@ def _transcribe_parakeet( sampling_rate = processor.feature_extractor.sampling_rate # Load audio into a mono float32 array at the model's sampling rate - if isinstance(audio, str): - with open(audio, "rb") as f: - audio_bytes = f.read() - audio_array = pk_ffmpeg_read(audio_bytes, sampling_rate) - else: - audio_array = audio + audio_array = load_audio_input(audio, sampling_rate) pipe = pipeline( task="automatic-speech-recognition", @@ -599,7 +610,6 @@ def _transcribe_vibevoice( the audio ourselves. """ from transformers import VibeVoiceAsrForConditionalGeneration - from transformers.pipelines.audio_utils import ffmpeg_read as vv_ffmpeg_read force_cpu = os.getenv("BUZZ_FORCE_CPU", "false") use_cuda = torch.cuda.is_available() and force_cpu == "false" @@ -627,12 +637,7 @@ def _transcribe_vibevoice( # transformers' load_audio(), which imports torchcodec and can hard-crash # (std::bad_alloc / SIGABRT) on some systems. Passing a numpy array skips # that path entirely. - if isinstance(audio, str): - with open(audio, "rb") as f: - audio_bytes = f.read() - audio_array = vv_ffmpeg_read(audio_bytes, sampling_rate) - else: - audio_array = audio + audio_array = load_audio_input(audio, sampling_rate) # An optional prompt provides extra context for proper nouns. inputs = processor.apply_transcription_request( @@ -697,7 +702,6 @@ def _transcribe_qwen( longer files and to report progress. """ from transformers import AutoModelForMultimodalLM - from transformers.pipelines.audio_utils import ffmpeg_read as qw_ffmpeg_read force_cpu = os.getenv("BUZZ_FORCE_CPU", "false") use_cuda = torch.cuda.is_available() and force_cpu == "false" @@ -733,12 +737,7 @@ def _transcribe_qwen( # Load audio into a mono float32 array with ffmpeg. The processor accepts # numpy arrays directly, which also avoids the torchcodec-based load_audio() # path that can crash on some systems. - if isinstance(audio, str): - with open(audio, "rb") as f: - audio_bytes = f.read() - audio_array = qw_ffmpeg_read(audio_bytes, sampling_rate) - else: - audio_array = audio + audio_array = load_audio_input(audio, sampling_rate) chunk_seconds = 30 chunk_size = int(chunk_seconds * sampling_rate) @@ -792,7 +791,6 @@ def _transcribe_mms( ): """Transcribe using MMS (Massively Multilingual Speech) model.""" from transformers import Wav2Vec2ForCTC, AutoProcessor as MMSAutoProcessor - from transformers.pipelines.audio_utils import ffmpeg_read as mms_ffmpeg_read force_cpu = os.getenv("BUZZ_FORCE_CPU", "false") use_cuda = torch.cuda.is_available() and force_cpu == "false" @@ -826,12 +824,9 @@ def _transcribe_mms( sys.stderr.write("25%\n") # Load and process audio - if isinstance(audio, str): - with open(audio, "rb") as f: - audio_data = f.read() - audio_array = mms_ffmpeg_read(audio_data, processor.feature_extractor.sampling_rate) - else: - audio_array = audio + audio_array = load_audio_input( + audio, processor.feature_extractor.sampling_rate + ) # Ensure audio is the right sample rate sampling_rate = processor.feature_extractor.sampling_rate diff --git a/buzz/whisper_audio.py b/buzz/whisper_audio.py index a46c469caf..fc285d4192 100644 --- a/buzz/whisper_audio.py +++ b/buzz/whisper_audio.py @@ -45,7 +45,7 @@ def load_audio(file: str, sr: int = SAMPLE_RATE): "-ac", "1", "-acodec", "pcm_s16le", "-ar", str(sr), - "-loglevel", "panic", + "-loglevel", "error", "-" ] # fmt: on @@ -63,8 +63,10 @@ def load_audio(file: str, sr: int = SAMPLE_RATE): else: result = subprocess.run(cmd, capture_output=True) - if result.returncode != 0 and len(result.stderr): - error_text = result.stderr.decode("utf-8", errors="replace") - raise RuntimeError(f"FFMPEG Failed to load audio: {error_text}") + if result.returncode != 0: + error_text = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError( + f"FFMPEG Failed to load audio: {error_text or result.returncode}" + ) return np.frombuffer(result.stdout, np.int16).flatten().astype(np.float32) / 32768.0 diff --git a/tests/transcriber/transformers_whisper_test.py b/tests/transcriber/transformers_whisper_test.py index ec09034e0f..ff8c00834b 100644 --- a/tests/transcriber/transformers_whisper_test.py +++ b/tests/transcriber/transformers_whisper_test.py @@ -1,11 +1,18 @@ import os import sys import platform +import subprocess from unittest.mock import patch +import numpy as np import pytest -from buzz.transformers_whisper import TransformersTranscriber, is_intel_mac, is_peft_model +from buzz.transformers_whisper import ( + TransformersTranscriber, + is_intel_mac, + is_peft_model, + load_audio_input, +) class TestIsIntelMac: @@ -104,3 +111,34 @@ def test_nested_org_name(self): with patch("os.path.exists", return_value=True), \ patch("buzz.transformers_whisper.os.sep", "/"): assert transcriber._get_mms_repo_id() == "openai/whisper-large-v3" + + +class TestLoadAudioInput: + def test_passes_through_arrays(self): + array = np.zeros(10, dtype=np.float32) + assert load_audio_input(array, 16000) is array + + def test_loads_mp4_with_moov_atom_at_end(self, tmp_path): + """Non-faststart MP4s must load (transformers' ffmpeg_read rejects them). + + Regression test for https://github.com/chidiwilliams/buzz/issues/1603 + """ + path = str(tmp_path / "moov-last.mp4") + subprocess.run( + [ + "ffmpeg", "-y", "-v", "error", + "-f", "lavfi", "-i", "sine=frequency=440:duration=60", + "-f", "lavfi", "-i", "testsrc=s=640x480:d=60:r=25", + "-c:a", "aac", "-c:v", "libx264", "-preset", "ultrafast", "-qp", "0", + "-shortest", path, + ], + check=True, + ) + + # Guard the premise: moov must sit after mdat for this to be a real test + data = open(path, "rb").read() + assert data.find(b"moov") > data.find(b"mdat") + + audio = load_audio_input(path, 16000) + assert audio.dtype == np.float32 + assert len(audio) == pytest.approx(60 * 16000, rel=0.01)