Fix for parakeet / MP4/M4A issue

buzz/transformers_whisper.py

@@ -15,6 +15,7 @@
 from transformers.pipelines.audio_utils import ffmpeg_read
 from transformers.pipelines.automatic_speech_recognition import is_torchaudio_available
 
+from buzz import whisper_audio
 from buzz.model_loader import (
     is_mms_model,
     is_parakeet_model,
@@ -24,6 +25,21 @@
 )
 
 
+def load_audio_input(audio: Union[str, np.ndarray], sampling_rate: int) -> np.ndarray:
+    """Decode an audio/video file into a mono float32 array at `sampling_rate`.
+
+    transformers' `ffmpeg_read` pipes the whole file to ffmpeg over stdin, which
+    ffmpeg cannot seek. MP4/M4A files that keep their `moov` atom at the end of
+    the file (anything not written with `-movflags +faststart`) are therefore
+    rejected as "not in the correct format or is malformed", even though they are
+    valid. Decoding from the file path instead lets ffmpeg seek and handles them.
+    """
+    if not isinstance(audio, str):
+        return audio
+
+    return whisper_audio.load_audio(audio, sampling_rate)
+
+
 def is_intel_mac() -> bool:
     """Check if running on Intel Mac (x86_64)."""
     return sys.platform == 'darwin' and platform.machine() == 'x86_64'
@@ -75,8 +91,9 @@ def _load_audio_from_source(self, inputs):
             if inputs.startswith("http://") or inputs.startswith("https://"):
                 inputs = requests.get(inputs).content
             else:
-                with open(inputs, "rb") as f:
-                    inputs = f.read()
+                # Decode from the path, not from bytes over stdin, so that
+                # non-faststart MP4/M4A files load. See `load_audio_input`.
+                return load_audio_input(inputs, self.feature_extractor.sampling_rate)
 
         if isinstance(inputs, bytes):
             inputs = ffmpeg_read(inputs, self.feature_extractor.sampling_rate)
@@ -506,7 +523,6 @@ def _transcribe_parakeet(
             AutoModelForRNNT,
             AutoModelForTDT,
         )
-        from transformers.pipelines.audio_utils import ffmpeg_read as pk_ffmpeg_read
 
         force_cpu = os.getenv("BUZZ_FORCE_CPU", "false")
         use_cuda = torch.cuda.is_available() and force_cpu == "false"
@@ -530,12 +546,7 @@ def _transcribe_parakeet(
         sampling_rate = processor.feature_extractor.sampling_rate
 
         # Load audio into a mono float32 array at the model's sampling rate
-        if isinstance(audio, str):
-            with open(audio, "rb") as f:
-                audio_bytes = f.read()
-            audio_array = pk_ffmpeg_read(audio_bytes, sampling_rate)
-        else:
-            audio_array = audio
+        audio_array = load_audio_input(audio, sampling_rate)
 
         pipe = pipeline(
             task="automatic-speech-recognition",
@@ -599,7 +610,6 @@ def _transcribe_vibevoice(
         the audio ourselves.
         """
         from transformers import VibeVoiceAsrForConditionalGeneration
-        from transformers.pipelines.audio_utils import ffmpeg_read as vv_ffmpeg_read
 
         force_cpu = os.getenv("BUZZ_FORCE_CPU", "false")
         use_cuda = torch.cuda.is_available() and force_cpu == "false"
@@ -627,12 +637,7 @@ def _transcribe_vibevoice(
         # transformers' load_audio(), which imports torchcodec and can hard-crash
         # (std::bad_alloc / SIGABRT) on some systems. Passing a numpy array skips
         # that path entirely.
-        if isinstance(audio, str):
-            with open(audio, "rb") as f:
-                audio_bytes = f.read()
-            audio_array = vv_ffmpeg_read(audio_bytes, sampling_rate)
-        else:
-            audio_array = audio
+        audio_array = load_audio_input(audio, sampling_rate)
 
         # An optional prompt provides extra context for proper nouns.
         inputs = processor.apply_transcription_request(
@@ -697,7 +702,6 @@ def _transcribe_qwen(
         longer files and to report progress.
         """
         from transformers import AutoModelForMultimodalLM
-        from transformers.pipelines.audio_utils import ffmpeg_read as qw_ffmpeg_read
 
         force_cpu = os.getenv("BUZZ_FORCE_CPU", "false")
         use_cuda = torch.cuda.is_available() and force_cpu == "false"
@@ -733,12 +737,7 @@ def _transcribe_qwen(
         # Load audio into a mono float32 array with ffmpeg. The processor accepts
         # numpy arrays directly, which also avoids the torchcodec-based load_audio()
         # path that can crash on some systems.
-        if isinstance(audio, str):
-            with open(audio, "rb") as f:
-                audio_bytes = f.read()
-            audio_array = qw_ffmpeg_read(audio_bytes, sampling_rate)
-        else:
-            audio_array = audio
+        audio_array = load_audio_input(audio, sampling_rate)
 
         chunk_seconds = 30
         chunk_size = int(chunk_seconds * sampling_rate)
@@ -792,7 +791,6 @@ def _transcribe_mms(
     ):
         """Transcribe using MMS (Massively Multilingual Speech) model."""
         from transformers import Wav2Vec2ForCTC, AutoProcessor as MMSAutoProcessor
-        from transformers.pipelines.audio_utils import ffmpeg_read as mms_ffmpeg_read
 
         force_cpu = os.getenv("BUZZ_FORCE_CPU", "false")
         use_cuda = torch.cuda.is_available() and force_cpu == "false"
@@ -826,12 +824,9 @@ def _transcribe_mms(
         sys.stderr.write("25%\n")
 
         # Load and process audio
-        if isinstance(audio, str):
-            with open(audio, "rb") as f:
-                audio_data = f.read()
-            audio_array = mms_ffmpeg_read(audio_data, processor.feature_extractor.sampling_rate)
-        else:
-            audio_array = audio
+        audio_array = load_audio_input(
+            audio, processor.feature_extractor.sampling_rate
+        )
 
         # Ensure audio is the right sample rate
         sampling_rate = processor.feature_extractor.sampling_rate

buzz/whisper_audio.py

@@ -45,7 +45,7 @@ def load_audio(file: str, sr: int = SAMPLE_RATE):
         "-ac", "1",
         "-acodec", "pcm_s16le",
         "-ar", str(sr),
-        "-loglevel", "panic",
+        "-loglevel", "error",
         "-"
     ]
     # fmt: on
@@ -63,8 +63,10 @@ def load_audio(file: str, sr: int = SAMPLE_RATE):
     else:
         result = subprocess.run(cmd, capture_output=True)
 
-    if result.returncode != 0 and len(result.stderr):
-        error_text = result.stderr.decode("utf-8", errors="replace")
-        raise RuntimeError(f"FFMPEG Failed to load audio: {error_text}")
+    if result.returncode != 0:
+        error_text = result.stderr.decode("utf-8", errors="replace").strip()
+        raise RuntimeError(
+            f"FFMPEG Failed to load audio: {error_text or result.returncode}"
+        )
 
     return np.frombuffer(result.stdout, np.int16).flatten().astype(np.float32) / 32768.0

tests/transcriber/transformers_whisper_test.py

@@ -1,11 +1,18 @@
 import os
 import sys
 import platform
+import subprocess
 from unittest.mock import patch
 
+import numpy as np
 import pytest
 
-from buzz.transformers_whisper import TransformersTranscriber, is_intel_mac, is_peft_model
+from buzz.transformers_whisper import (
+    TransformersTranscriber,
+    is_intel_mac,
+    is_peft_model,
+    load_audio_input,
+)
 
 
 class TestIsIntelMac:
@@ -104,3 +111,34 @@ def test_nested_org_name(self):
         with patch("os.path.exists", return_value=True), \
              patch("buzz.transformers_whisper.os.sep", "/"):
             assert transcriber._get_mms_repo_id() == "openai/whisper-large-v3"
+
+
+class TestLoadAudioInput:
+    def test_passes_through_arrays(self):
+        array = np.zeros(10, dtype=np.float32)
+        assert load_audio_input(array, 16000) is array
+
+    def test_loads_mp4_with_moov_atom_at_end(self, tmp_path):
+        """Non-faststart MP4s must load (transformers' ffmpeg_read rejects them).
+
+        Regression test for https://github.com/chidiwilliams/buzz/issues/1603
+        """
+        path = str(tmp_path / "moov-last.mp4")
+        subprocess.run(
+            [
+                "ffmpeg", "-y", "-v", "error",
+                "-f", "lavfi", "-i", "sine=frequency=440:duration=60",
+                "-f", "lavfi", "-i", "testsrc=s=640x480:d=60:r=25",
+                "-c:a", "aac", "-c:v", "libx264", "-preset", "ultrafast", "-qp", "0",
+                "-shortest", path,
+            ],
+            check=True,
+        )
+
+        # Guard the premise: moov must sit after mdat for this to be a real test
+        data = open(path, "rb").read()
+        assert data.find(b"moov") > data.find(b"mdat")
+
+        audio = load_audio_input(path, 16000)
+        assert audio.dtype == np.float32
+        assert len(audio) == pytest.approx(60 * 16000, rel=0.01)