Add FunASR recognizer (local multilingual ASR: SenseVoice/Paraformer)

README.rst

@@ -65,6 +65,7 @@ Speech recognition engine/API support:
     * OpenAI compatible self-hosted endpoints (e.g. vLLM, Ollama)
 * `Groq Whisper API <https://console.groq.com/docs/speech-to-text>`__
 * `Cohere Transcribe API <https://docs.cohere.com/docs/transcribe>`__
+* `FunASR <https://github.com/modelscope/FunASR>`__ (works offline)
 
 **Quickstart:** ``pip install SpeechRecognition``. See the "Installing" section for more details.
 
@@ -124,6 +125,7 @@ To use all of the functionality of the library, you should have:
     * includes OpenAI compatible self-hosted endpoints (e.g. vLLM, Ollama)
 * **groq** (required only if you need to use Groq Whisper API speech recognition ``recognizer_instance.recognize_groq``)
 * **cohere** (required only if you need to use Cohere Transcribe API speech recognition ``recognizer_instance.recognize_cohere_api``; install with ``pip install SpeechRecognition[cohere-api]``. Set ``CO_API_KEY`` as documented by the Cohere SDK.)
+* **funasr** (required only if you need to use FunASR speech recognition ``recognizer_instance.recognize_funasr``)
 
 The following requirements are optional, but can improve or extend functionality in some situations:
 
@@ -208,6 +210,15 @@ The library `faster-whisper <https://pypi.org/project/faster-whisper/>`__ is **r
 
 You can install it with ``python3 -m pip install SpeechRecognition[faster-whisper]``.
 
+FunASR (for FunASR users)
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The library `funasr <https://pypi.org/project/funasr/>`__ is **required if and only if you want to use FunASR** (``recognizer_instance.recognize_funasr``).
+
+You can install it with ``python3 -m pip install SpeechRecognition[funasr]``.
+
+FunASR runs locally and supports models such as SenseVoice and Paraformer. The default model is ``iic/SenseVoiceSmall``; set ``model`` and ``device`` when calling ``recognizer_instance.recognize_funasr`` to choose another FunASR model or a CUDA device.
+
 OpenAI Whisper API (for OpenAI Whisper API users) 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

pyproject.toml

@@ -83,6 +83,10 @@ cohere-api = [
 ]
 assemblyai = ["requests"]
 vosk = ["vosk"]
+funasr = [
+  "funasr",
+  "numpy",
+]
 audio-split = [
   "librosa>=0.10.2,<1.0",
   "numpy>=1.26.0",

reference/library-reference.rst

@@ -266,6 +266,11 @@ Raises a ``speech_recognition.UnknownValueError`` exception if the speech is uni
 
 .. autofunction:: speech_recognition.recognizers.whisper_local.faster_whisper.recognize
 
+``recognizer_instance.recognize_funasr(audio_data: AudioData, model: str = "iic/SenseVoiceSmall", device: str = "cpu", language: str = "auto", use_itn: bool = True)``
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+.. autofunction:: speech_recognition.recognizers.funasr.recognize
+
 ``recognizer_instance.recognize_openai(audio_data: AudioData, model = "whisper-1", **kwargs)``
 ----------------------------------------------------------------------------------------------
 

speech_recognition/__init__.py

@@ -1189,7 +1189,7 @@ def flush(self, *args, **kwargs):
 # At this time, the dependencies are not yet installed, resulting in a ModuleNotFoundError.
 # This is a workaround to resolve this issue
 try:
-    from .recognizers import cohere_api, google, google_cloud, pocketsphinx, vosk
+    from .recognizers import cohere_api, funasr, google, google_cloud, pocketsphinx, vosk
     from .recognizers.whisper_api import groq, openai
     from .recognizers.whisper_local import faster_whisper, whisper
 except (ModuleNotFoundError, ImportError):
@@ -1204,6 +1204,7 @@ def flush(self, *args, **kwargs):
     Recognizer.recognize_cohere_api = cohere_api.recognize  # type: ignore[attr-defined]
     Recognizer.recognize_sphinx = pocketsphinx.recognize  # type: ignore[attr-defined]
     Recognizer.recognize_vosk = vosk.recognize  # type: ignore[attr-defined]
+    Recognizer.recognize_funasr = funasr.recognize  # type: ignore[attr-defined]
 
 
 # ===============================

speech_recognition/recognizers/funasr.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from speech_recognition.exceptions import SetupError
+
+if TYPE_CHECKING:
+    from speech_recognition.audio import AudioData
+
+# Cache loaded models by (model, device) so repeated calls do not reload from disk.
+_MODEL_CACHE: dict = {}
+
+# Languages SenseVoice understands; anything else falls back to auto-detection.
+_SENSEVOICE_LANGUAGES = {"auto", "zh", "en", "yue", "ja", "ko", "nospeech"}
+
+SAMPLE_RATE = 16_000
+
+
+def recognize(
+    _recognizer,
+    audio_data: "AudioData",
+    model: str = "iic/SenseVoiceSmall",
+    *,
+    device: str = "cpu",
+    language: str = "auto",
+    use_itn: bool = True,
+) -> str:
+    """Perform speech recognition on ``audio_data`` using FunASR (runs locally, no API key).
+
+    `FunASR <https://github.com/modelscope/FunASR>`__ is an open-source speech toolkit
+    providing models such as SenseVoice (multilingual: Chinese, Cantonese, English,
+    Japanese, Korean ...), Paraformer and Fun-ASR-Nano, with strong Chinese accuracy.
+
+    ``model`` is a FunASR / ModelScope / Hugging Face model id
+    (default ``"iic/SenseVoiceSmall"``). ``device`` is ``"cpu"`` or ``"cuda"``.
+    ``language`` is the spoken language (``"auto"`` to auto-detect; SenseVoice supports
+    ``zh`` / ``en`` / ``yue`` / ``ja`` / ``ko``). ``use_itn`` applies inverse text
+    normalization (e.g. "nine" -> "9").
+
+    The model is downloaded on first use and cached for subsequent calls.
+
+    Requires `funasr` to be installed (``pip install funasr``).
+    """
+    try:
+        from funasr import AutoModel
+        from funasr.utils.postprocess_utils import rich_transcription_postprocess
+    except ImportError:
+        raise SetupError("missing funasr, please `pip install funasr`")
+
+    import numpy as np
+
+    cache_key = (model, device)
+    if cache_key not in _MODEL_CACHE:
+        _MODEL_CACHE[cache_key] = AutoModel(model=model, device=device, disable_update=True)
+    funasr_model = _MODEL_CACHE[cache_key]
+
+    spoken_language = language if language in _SENSEVOICE_LANGUAGES else "auto"
+    raw_data = audio_data.get_raw_data(convert_rate=SAMPLE_RATE, convert_width=2)
+    samples = np.frombuffer(raw_data, dtype=np.int16).astype(np.float32) / 32768.0
+
+    result = funasr_model.generate(
+        input=samples, cache={}, language=spoken_language, use_itn=use_itn
+    )
+    text = result[0]["text"] if result else ""
+    return rich_transcription_postprocess(text).strip()

tests/recognizers/test_funasr.py

@@ -0,0 +1,96 @@
+import sys
+import types
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+
+from speech_recognition import AudioData, Recognizer
+from speech_recognition.exceptions import SetupError
+from speech_recognition.recognizers import funasr
+
+
+@pytest.fixture(autouse=True)
+def clear_model_cache():
+    funasr._MODEL_CACHE.clear()
+    yield
+    funasr._MODEL_CACHE.clear()
+
+
+def install_fake_funasr(monkeypatch):
+    model_instance = MagicMock()
+    model_instance.generate.return_value = [{"text": "<|zh|><|NEUTRAL|> 你好世界 "}]
+
+    funasr_module = types.ModuleType("funasr")
+    funasr_module.AutoModel = MagicMock(return_value=model_instance)
+
+    utils_module = types.ModuleType("funasr.utils")
+    postprocess_module = types.ModuleType("funasr.utils.postprocess_utils")
+    postprocess_module.rich_transcription_postprocess = MagicMock(return_value="你好世界")
+
+    monkeypatch.setitem(sys.modules, "funasr", funasr_module)
+    monkeypatch.setitem(sys.modules, "funasr.utils", utils_module)
+    monkeypatch.setitem(sys.modules, "funasr.utils.postprocess_utils", postprocess_module)
+
+    return funasr_module, postprocess_module, model_instance
+
+
+def test_recognize_funasr_missing_dependency_raises_setup_error(monkeypatch):
+    monkeypatch.setitem(sys.modules, "funasr", None)
+
+    audio_data = MagicMock(spec=AudioData)
+
+    with pytest.raises(SetupError, match="missing funasr"):
+        funasr.recognize(MagicMock(spec=Recognizer), audio_data)
+
+
+def test_recognize_funasr_uses_cached_model_and_converts_audio(monkeypatch):
+    funasr_module, postprocess_module, model_instance = install_fake_funasr(monkeypatch)
+    raw_pcm = np.array([0, 32767, -32768], dtype=np.int16).tobytes()
+    audio_data = MagicMock(spec=AudioData)
+    audio_data.get_raw_data.return_value = raw_pcm
+
+    first = funasr.recognize(
+        MagicMock(spec=Recognizer),
+        audio_data,
+        model="iic/SenseVoiceSmall",
+        device="cpu",
+        language="zh",
+        use_itn=False,
+    )
+    second = funasr.recognize(
+        MagicMock(spec=Recognizer),
+        audio_data,
+        model="iic/SenseVoiceSmall",
+        device="cpu",
+        language="zh",
+    )
+
+    assert first == "你好世界"
+    assert second == "你好世界"
+    funasr_module.AutoModel.assert_called_once_with(
+        model="iic/SenseVoiceSmall", device="cpu", disable_update=True
+    )
+    assert audio_data.get_raw_data.call_count == 2
+    audio_data.get_raw_data.assert_called_with(convert_rate=16000, convert_width=2)
+    assert model_instance.generate.call_count == 2
+    first_call = model_instance.generate.call_args_list[0].kwargs
+    np.testing.assert_allclose(first_call["input"], np.array([0.0, 32767 / 32768, -1.0], dtype=np.float32))
+    assert first_call["cache"] == {}
+    assert first_call["language"] == "zh"
+    assert first_call["use_itn"] is False
+    postprocess_module.rich_transcription_postprocess.assert_called_with("<|zh|><|NEUTRAL|> 你好世界 ")
+
+
+def test_recognize_funasr_falls_back_to_auto_for_unknown_language(monkeypatch):
+    _, _, model_instance = install_fake_funasr(monkeypatch)
+    audio_data = MagicMock(spec=AudioData)
+    audio_data.get_raw_data.return_value = np.array([0], dtype=np.int16).tobytes()
+
+    funasr.recognize(MagicMock(spec=Recognizer), audio_data, language="fr")
+
+    assert model_instance.generate.call_args.kwargs["language"] == "auto"
+
+
+def test_recognizer_has_recognize_funasr_method():
+    assert Recognizer.recognize_funasr is funasr.recognize