Skip to content

Commit f5504e9

Browse files
doqhuclauderaivisdejus
authored
Run whisper.cpp on the discrete GPU on hybrid graphics laptops (#1676)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Raivis Dejus <orvils@gmail.com> Co-authored-by: Raivis Dejus <raivisd@scandiweb.com>
1 parent 50c6bce commit f5504e9

4 files changed

Lines changed: 145 additions & 3 deletions

File tree

‎buzz/transcriber/recording_transcriber.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -519,13 +519,18 @@ def start_local_whisper_server(self):
519519

520520
logging.debug(f"Starting Whisper server with command: {' '.join(cmd)}")
521521

522+
# Imported here, as whisper_cpp probes Vulkan devices on import
523+
from buzz.transcriber.whisper_cpp import WhisperCpp
524+
525+
env = WhisperCpp._whisper_env()
522526
try:
523527
if sys.platform == "win32":
524528
self.process = subprocess.Popen(
525529
cmd,
526530
stdout=subprocess.DEVNULL,
527531
stderr=subprocess.PIPE,
528532
shell=False,
533+
env=env,
529534
creationflags=subprocess.CREATE_NO_WINDOW
530535
)
531536
else:
@@ -534,6 +539,7 @@ def start_local_whisper_server(self):
534539
stdout=subprocess.DEVNULL,
535540
stderr=subprocess.PIPE,
536541
shell=False,
542+
env=env,
537543
)
538544
except Exception as e:
539545
error_msg = f"Failed to start whisper-server subprocess: {str(e)}"

‎buzz/transcriber/whisper_cpp.py‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,40 @@
1414

1515

1616
IS_VULKAN_SUPPORTED = False
17+
# Index of the first discrete GPU in Vulkan's physical device list, if any.
18+
# whisper-cli defaults to device 0, which on hybrid laptops is usually the
19+
# integrated GPU, so the discrete one is selected via GGML_VK_VISIBLE_DEVICES.
20+
DISCRETE_GPU_INDEX: Optional[int] = None
21+
22+
23+
def _create_vulkan_instance(vk):
24+
"""Create a Vulkan instance that lists physical devices in ggml-vulkan's order."""
25+
# The loader can order physical devices differently for 1.0 and 1.1+ instances
26+
# (e.g. discrete GPU first for 1.0, integrated first for 1.1+). ggml-vulkan creates
27+
# its instance at the loader's highest version, so the same version is used here,
28+
# otherwise the index would point at a different GPU in GGML_VK_VISIBLE_DEVICES.
29+
app_info = vk.VkApplicationInfo(apiVersion=vk.vkEnumerateInstanceVersion())
30+
return vk.vkCreateInstance(vk.VkInstanceCreateInfo(pApplicationInfo=app_info), None)
31+
32+
33+
def _find_discrete_gpu_index(vk, instance) -> Optional[int]:
34+
"""Index of the first discrete GPU in the instance's physical device list."""
35+
for index, device in enumerate(vk.vkEnumeratePhysicalDevices(instance)):
36+
properties = vk.vkGetPhysicalDeviceProperties(device)
37+
if properties.deviceType == vk.VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU:
38+
logging.debug("Discrete Vulkan GPU %s: %s", index, properties.deviceName)
39+
return index
40+
return None
41+
42+
1743
try:
1844
import vulkan
1945

20-
instance = vulkan.vkCreateInstance(vulkan.VkInstanceCreateInfo(), None)
46+
instance = _create_vulkan_instance(vulkan)
47+
try:
48+
DISCRETE_GPU_INDEX = _find_discrete_gpu_index(vulkan, instance)
49+
except Exception as e:
50+
logging.debug(f"Vulkan device enumeration error: {e}")
2151
vulkan.vkDestroyInstance(instance, None)
2252
vulkan_version = vulkan.vkEnumerateInstanceVersion()
2353
major = (vulkan_version >> 22) & 0x3FF
@@ -252,9 +282,18 @@ def _build_command(task, file_to_process, language, vad_enabled) -> list:
252282
print(f"Running Whisper CLI: {' '.join(cmd)}")
253283
return cmd
254284

285+
@staticmethod
286+
def _whisper_env() -> dict:
287+
"""Environment for whisper-cli, steering ggml-vulkan to the discrete GPU."""
288+
env = (app_env if sys.platform == "win32" else os.environ).copy()
289+
if DISCRETE_GPU_INDEX is not None and "GGML_VK_VISIBLE_DEVICES" not in env:
290+
env["GGML_VK_VISIBLE_DEVICES"] = str(DISCRETE_GPU_INDEX)
291+
return env
292+
255293
@staticmethod
256294
def _run_whisper(cmd) -> tuple:
257295
"""Run whisper-cli subprocess, returning (return code, tail of stderr)."""
296+
env = WhisperCpp._whisper_env()
258297
if sys.platform == "win32":
259298
si = subprocess.STARTUPINFO()
260299
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
@@ -267,7 +306,7 @@ def _run_whisper(cmd) -> tuple:
267306
encoding="utf-8",
268307
errors="replace",
269308
startupinfo=si,
270-
env=app_env,
309+
env=env,
271310
creationflags=subprocess.CREATE_NO_WINDOW
272311
)
273312
else:
@@ -278,6 +317,7 @@ def _run_whisper(cmd) -> tuple:
278317
text=True,
279318
encoding="utf-8",
280319
errors="replace",
320+
env=env,
281321
)
282322

283323
stderr_tail = []
@@ -522,6 +562,7 @@ def detect_language(file_path: str, model_path: str) -> Optional[str]:
522562

523563
print(f"Running Whisper CLI language detection: {' '.join(cmd)}")
524564

565+
env = WhisperCpp._whisper_env()
525566
try:
526567
if sys.platform == "win32":
527568
si = subprocess.STARTUPINFO()
@@ -534,7 +575,7 @@ def detect_language(file_path: str, model_path: str) -> Optional[str]:
534575
encoding="utf-8",
535576
errors="replace",
536577
startupinfo=si,
537-
env=app_env,
578+
env=env,
538579
creationflags=subprocess.CREATE_NO_WINDOW,
539580
)
540581
else:
@@ -544,6 +585,7 @@ def detect_language(file_path: str, model_path: str) -> Optional[str]:
544585
text=True,
545586
encoding="utf-8",
546587
errors="replace",
588+
env=env,
547589
)
548590

549591
# whisper-cli writes the detection line to stderr, e.g.:

‎tests/recording_transcriber_test.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
import threading
23
from unittest.mock import MagicMock, patch, PropertyMock
34

@@ -7,6 +8,7 @@
78

89
from buzz.model_loader import TranscriptionModel, ModelType, WhisperModelSize
910
from buzz.settings.recording_transcriber_mode import RecordingTranscriberMode
11+
from buzz.transcriber import whisper_cpp
1012
from buzz.transcriber.recording_transcriber import RecordingTranscriber
1113
from buzz.transcriber.transcriber import TranscriptionOptions, Task
1214

@@ -417,6 +419,25 @@ def test_success_creates_openai_client(self):
417419
assert "--language" in cmd and "auto" in cmd
418420
assert any("transcription" in m.lower() for m in emitted)
419421

422+
def test_server_runs_on_discrete_gpu(self, monkeypatch):
423+
monkeypatch.delitem(os.environ, "GGML_VK_VISIBLE_DEVICES", raising=False)
424+
monkeypatch.delitem(whisper_cpp.app_env, "GGML_VK_VISIBLE_DEVICES", raising=False)
425+
t = make_transcriber()
426+
427+
process = MagicMock()
428+
process.poll.return_value = None
429+
process.stderr = []
430+
431+
with patch("buzz.transcriber.recording_transcriber.subprocess.Popen", return_value=process) as popen, \
432+
patch("buzz.transcriber.recording_transcriber.time.sleep"), \
433+
patch("buzz.transcriber.recording_transcriber._", lambda s: s), \
434+
patch("buzz.transcriber.whisper_cpp.DISCRETE_GPU_INDEX", 1), \
435+
patch("openai.OpenAI"):
436+
t.is_running = True
437+
t.start_local_whisper_server()
438+
439+
assert popen.call_args.kwargs["env"]["GGML_VK_VISIBLE_DEVICES"] == "1"
440+
420441
def test_failure_emits_error_message(self):
421442
t = make_transcriber()
422443
emitted = []

‎tests/transcriber/whisper_cpp_test.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import json
33
import os
44
import subprocess
5+
import sys
56

67
import pytest
78

@@ -12,11 +13,17 @@
1213
FileTranscriptionTask,
1314
FileTranscriptionOptions,
1415
)
16+
from buzz.transcriber import whisper_cpp
1517
from buzz.transcriber.whisper_cpp import WhisperCpp, _is_cli_safe_path
1618
from tests.audio import test_audio_path, test_multibyte_utf8_audio_path
1719
from tests.model_loader import get_model_path
1820

1921

22+
def _whisper_env_source() -> dict:
23+
# On Windows _whisper_env() copies the app_env snapshot, not os.environ
24+
return whisper_cpp.app_env if sys.platform == "win32" else os.environ
25+
26+
2027
class TestWhisperCpp:
2128
def test_transcribe(self):
2229
transcription_options = TranscriptionOptions(
@@ -617,3 +624,69 @@ def test_parse_word_level_timings_filters_aborted_superseded_fragments(self):
617624
segments = WhisperCpp._parse_word_level_timings(mock_result, language="en", vad_enabled=False)
618625
words = [s.text for s in segments]
619626
assert words == ["Hello", "world", "Goodbye", "all"]
627+
628+
def test_whisper_env_selects_discrete_gpu(self, monkeypatch):
629+
monkeypatch.delitem(_whisper_env_source(), "GGML_VK_VISIBLE_DEVICES", raising=False)
630+
with patch("buzz.transcriber.whisper_cpp.DISCRETE_GPU_INDEX", 1):
631+
assert WhisperCpp._whisper_env()["GGML_VK_VISIBLE_DEVICES"] == "1"
632+
633+
def test_whisper_env_keeps_user_device_choice(self, monkeypatch):
634+
monkeypatch.setitem(_whisper_env_source(), "GGML_VK_VISIBLE_DEVICES", "0")
635+
with patch("buzz.transcriber.whisper_cpp.DISCRETE_GPU_INDEX", 1):
636+
assert WhisperCpp._whisper_env()["GGML_VK_VISIBLE_DEVICES"] == "0"
637+
638+
def test_detect_language_uses_discrete_gpu(self, monkeypatch):
639+
monkeypatch.delitem(_whisper_env_source(), "GGML_VK_VISIBLE_DEVICES", raising=False)
640+
completed = MagicMock(stderr="auto-detected language: en (p = 0.9)\n", stdout="")
641+
with patch(
642+
"buzz.transcriber.whisper_cpp.subprocess.run", return_value=completed
643+
) as run_mock, patch(
644+
"buzz.transcriber.whisper_cpp.get_whisper_cli_path",
645+
return_value="/fake/whisper-cli",
646+
), patch("buzz.transcriber.whisper_cpp.DISCRETE_GPU_INDEX", 1):
647+
WhisperCpp.detect_language("/fake/audio.wav", "/fake/model.bin")
648+
649+
assert run_mock.call_args.kwargs["env"]["GGML_VK_VISIBLE_DEVICES"] == "1"
650+
651+
def test_whisper_env_without_discrete_gpu(self, monkeypatch):
652+
monkeypatch.delitem(_whisper_env_source(), "GGML_VK_VISIBLE_DEVICES", raising=False)
653+
with patch("buzz.transcriber.whisper_cpp.DISCRETE_GPU_INDEX", None):
654+
assert "GGML_VK_VISIBLE_DEVICES" not in WhisperCpp._whisper_env()
655+
656+
def test_vulkan_instance_uses_loader_api_version(self):
657+
# ggml-vulkan creates its instance at the loader's highest API version, and the
658+
# device order can differ between versions, so the probe must use the same one
659+
vk = MagicMock()
660+
vk.vkEnumerateInstanceVersion.return_value = 4206592 # 1.3.0
661+
662+
instance = whisper_cpp._create_vulkan_instance(vk)
663+
664+
vk.VkApplicationInfo.assert_called_once_with(apiVersion=4206592)
665+
vk.VkInstanceCreateInfo.assert_called_once_with(
666+
pApplicationInfo=vk.VkApplicationInfo.return_value
667+
)
668+
vk.vkCreateInstance.assert_called_once_with(vk.VkInstanceCreateInfo.return_value, None)
669+
assert instance is vk.vkCreateInstance.return_value
670+
671+
@staticmethod
672+
def _fake_vulkan(device_types):
673+
vk = MagicMock()
674+
vk.VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU = 1
675+
vk.VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU = 2
676+
vk.VK_PHYSICAL_DEVICE_TYPE_CPU = 4
677+
devices = [MagicMock(deviceType=t, deviceName=f"gpu{i}") for i, t in enumerate(device_types)]
678+
vk.vkEnumeratePhysicalDevices.return_value = list(range(len(devices)))
679+
vk.vkGetPhysicalDeviceProperties.side_effect = lambda index: devices[index]
680+
return vk
681+
682+
def test_find_discrete_gpu_index_after_integrated(self):
683+
vk = self._fake_vulkan([1, 2, 4])
684+
assert whisper_cpp._find_discrete_gpu_index(vk, MagicMock()) == 1
685+
686+
def test_find_discrete_gpu_index_returns_first_discrete(self):
687+
vk = self._fake_vulkan([2, 1, 2])
688+
assert whisper_cpp._find_discrete_gpu_index(vk, MagicMock()) == 0
689+
690+
def test_find_discrete_gpu_index_without_discrete(self):
691+
vk = self._fake_vulkan([1, 4])
692+
assert whisper_cpp._find_discrete_gpu_index(vk, MagicMock()) is None

0 commit comments

Comments
 (0)