diff --git a/README.md b/README.md index 4feefb9a5..f5327a459 100644 --- a/README.md +++ b/README.md @@ -77,13 +77,15 @@ MLX-Audio provides a modern web interface with real-time audio visualization cap #### Key Features -- **Voice Customization**: Select from multiple voice presets including AF Heart, AF Nova, AF Bella, and BF Emma +- **Voice Customization**: Auto-matched Kokoro voices per language (e.g., `jf_alpha` for Japanese) with presets like AF Heart, AF Nova, AF Bella, and BF Emma +- **Language Selector**: Choose Kokoro language codes (English, Japanese, Chinese, etc.) directly in the UI; options automatically adjust per-model and “Auto detect” only appears when the selected model supports multiple languages - **Speech Rate Control**: Fine-tune speech generation speed using an intuitive slider (range: 0.5x - 2.0x) - **Dynamic 3D Visualization**: Experience audio through an interactive 3D orb that responds to frequency changes - **Audio Management**: Upload, play, and visualize custom audio files - **Smart Playback**: Optional automatic playback of generated audio - **File Management**: Quick access to the output directory through an integrated file explorer button - **Speech Recognition**: Convert speech to text with support for multiple languages and models +- **Auto Language Detection (STT)**: File uploads automatically detect spoken language (including Japanese/Chinese) before transcription—no manual dropdown needed To start the web interface and API server: UI: @@ -113,10 +115,10 @@ Available command line arguments: - `--host`: Host address to bind the server to (default: 127.0.0.1) - `--port`: Port to bind the server to (default: 8000) -Then open your browser and navigate to: -``` -http://127.0.0.1:8000 -``` +Once both processes are running: + +- Visit `http://localhost:3000` to use the graphical interface. +- The FastAPI backend stays on `http://localhost:8000`; interactive docs are available at `/docs` and the root URL now shows a short status page with setup instructions. #### API Endpoints @@ -142,6 +144,8 @@ The server provides the following REST API endpoints: > Note: Generated audio files are stored in `~/.mlx_audio/outputs` by default, or in a fallback directory if that location is not writable. +> **Kokoro language extras:** Japanese synthesis uses `misaki[ja]` (installs `pyopenjtalk` and depends on `fugashi[unidic-lite]`) and Mandarin Chinese uses `misaki[zh]`. These packages are listed in `requirements.txt`, so `pip install -r requirements.txt` pulls them in, but if you installed earlier re-run the install (or add the extras manually) before selecting those languages in the UI or API. The server automatically points fugashi to the bundled UniDic-lite dictionary, so no additional MeCab setup is needed. + ## Models ### Kokoro diff --git a/mlx_audio/server.py b/mlx_audio/server.py index 6a7f38e58..eb7a1a401 100644 --- a/mlx_audio/server.py +++ b/mlx_audio/server.py @@ -7,21 +7,73 @@ import argparse import asyncio +import importlib import io +import math import os +import sys +import shutil import time +from numbers import Real +from textwrap import dedent from typing import Any, Dict, List, Optional from urllib.parse import unquote import soundfile as sf import uvicorn from fastapi import FastAPI, File, Form, HTTPException, Response, UploadFile +from fastapi.encoders import jsonable_encoder from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse +from fastapi.responses import HTMLResponse, StreamingResponse from pydantic import BaseModel from mlx_audio.utils import load_model + +def configure_fugashi_dictionary(): + """ + Ensure fugashi (used by misaki Cutlet) always has a dictionary. + + Prefer unidic_lite, fall back to unidic if present. + """ + + dicdir = None + chosen_module = None + for module_name in ("unidic_lite", "unidic"): + try: + module = __import__(module_name) + dicdir = getattr(module, "DICDIR", None) + if dicdir and os.path.isdir(dicdir): + mecabrc_path = os.path.join(dicdir, "mecabrc") + if os.path.exists(mecabrc_path): + chosen_module = module + break + dicdir = None + except ImportError: + continue + + if not dicdir: + return + + # Ensure any code importing `unidic` sees a module whose DICDIR actually exists. + if chosen_module.__name__ == "unidic_lite": + try: + import unidic # type: ignore + + unidic.DICDIR = dicdir # type: ignore[attr-defined] + except ImportError: + sys.modules["unidic"] = chosen_module + + mecabrc_path = os.path.join(dicdir, "mecabrc") + + os.environ["FUGASHI_DICDIR"] = dicdir + os.environ["FUGASHI_ARGS"] = f"-d {dicdir} -r {mecabrc_path}" + if os.path.exists(mecabrc_path): + os.environ["MECABRC"] = mecabrc_path + + +configure_fugashi_dictionary() + MLX_AUDIO_NUM_WORKERS = os.getenv("MLX_AUDIO_NUM_WORKERS", "2") @@ -51,6 +103,80 @@ async def get_available_models(self): app = FastAPI() +@app.get("/", response_class=HTMLResponse) +async def root(): + """ + Simple landing page so the root URL no longer returns a 404. The full GUI + lives in the Next.js project under ``mlx_audio/ui``. + """ + + html = dedent( + """ + + + + + MLX Audio Server + + + +
+

MLX Audio API server is running

+

+ This endpoint exposes the FastAPI backend (OpenAI-compatible TTS / STT APIs). + Open /docs for interactive API documentation + or /redoc for the ReDoc view. +

+

+ Looking for the graphical interface? Start the Next.js app inside + mlx_audio/ui: +

+ 
cd mlx_audio/ui
+npm install
+export NEXT_PUBLIC_API_BASE_URL=http://localhost
+export NEXT_PUBLIC_API_PORT=8000
+npm run dev
+

+ Then open + http://localhost:3000. +

+
+ + + """ + ) + + return HTMLResponse(content=html) + + def int_or_float(value): try: @@ -104,6 +230,51 @@ def setup_cors(app: FastAPI, allowed_origins: List[str]): setup_cors(app, default_origins) +LANGUAGE_DEPENDENCIES = { + "j": ("misaki.ja", "pip install misaki[ja]"), + "z": ("misaki.zh", "pip install misaki[zh]"), +} + + +def ensure_tts_language_support(lang_code: Optional[str]): + """Ensure extra dependencies for specific languages are installed.""" + code = (lang_code or "").lower() + requirement = LANGUAGE_DEPENDENCIES.get(code) + if requirement is None: + return + + module_name, install_hint = requirement + try: + importlib.import_module(module_name) + except ModuleNotFoundError as exc: + missing = module_name.split(".")[0] + raise HTTPException( + status_code=503, + detail=( + f"Language '{code}' requires `{install_hint}` " + f"(missing dependency: {missing})." + ), + ) from exc + + +def sanitize_jsonable(value: Any): + """ + Recursively sanitize any jsonable structure by replacing NaN/inf values + with ``None`` so the response can be serialized by the JSON encoder. + """ + + if isinstance(value, dict): + return {k: sanitize_jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [sanitize_jsonable(v) for v in value] + if isinstance(value, Real) and not isinstance(value, bool): + number = float(value) + if math.isnan(number) or math.isinf(number): + return None + return number + return value + + # Request schemas for OpenAI-compatible endpoints class SpeechRequest(BaseModel): model: str @@ -207,6 +378,7 @@ async def generate_audio(model, payload: SpeechRequest, verbose: bool = False): @app.post("/v1/audio/speech") async def tts_speech(payload: SpeechRequest): """Generate speech audio following the OpenAI text-to-speech API.""" + ensure_tts_language_support(payload.lang_code) model = model_provider.load_model(payload.model) return StreamingResponse( generate_audio(model, payload), @@ -230,9 +402,13 @@ async def stt_transcriptions( sf.write(tmp_path, audio, sr) stt_model = model_provider.load_model(model) - result = stt_model.generate(tmp_path) + generate_kwargs = {} + if language: + generate_kwargs["language"] = language + result = stt_model.generate(tmp_path, **generate_kwargs) os.remove(tmp_path) - return result + response = sanitize_jsonable(jsonable_encoder(result)) + return response def main(): diff --git a/mlx_audio/ui/app/speech-to-text/[id]/page.tsx b/mlx_audio/ui/app/speech-to-text/[id]/page.tsx index 38f23ed9a..a744f688c 100644 --- a/mlx_audio/ui/app/speech-to-text/[id]/page.tsx +++ b/mlx_audio/ui/app/speech-to-text/[id]/page.tsx @@ -52,6 +52,7 @@ export default function TranscriptViewerPage() { const [language, setLanguage] = useState("English") const [date, setDate] = useState("yesterday") const [isExportDropdownOpen, setIsExportDropdownOpen] = useState(false) + const [audioSource, setAudioSource] = useState(null) useEffect(() => { // Set up audio player @@ -91,16 +92,53 @@ export default function TranscriptViewerPage() { } }, [isExportDropdownOpen]) - const togglePlayPause = () => { + useEffect(() => { + const audio = audioRef.current + if (audio && audioSource) { + audio.src = audioSource + const handleLoadedMetadata = () => { + const metaDuration = audio.duration + if (!Number.isNaN(metaDuration) && metaDuration > 0) { + setDuration(metaDuration) + } + } + audio.addEventListener("loadedmetadata", handleLoadedMetadata) + return () => { + audio.removeEventListener("loadedmetadata", handleLoadedMetadata) + } + } else if (audio && !audioSource) { + audio.src = "" + } + return undefined + }, [audioSource]) + + const playAudio = async () => { const audio = audioRef.current if (!audio) return + try { + await audio.play() + setIsPlaying(true) + } catch (error) { + if (error instanceof DOMException && error.name === "AbortError") { + return + } + console.error("Error playing audio:", error) + } + } + + const togglePlayPause = () => { + const audio = audioRef.current + if (!audio || !audioSource) { + console.warn("No audio available for playback.") + return + } if (isPlaying) { audio.pause() + setIsPlaying(false) } else { - audio.play().catch((e) => console.error("Error playing audio:", e)) + void playAudio() } - setIsPlaying(!isPlaying) } const handleSeek = (e: React.ChangeEvent) => { @@ -155,6 +193,11 @@ export default function TranscriptViewerPage() { setLanguage(data.language ?? "English") setDate(data.date ?? "yesterday") setDuration(data.duration ?? 8) + if (data.audioDataUrl) { + setAudioSource(data.audioDataUrl) + } else { + setAudioSource(null) + } if (data.segments?.length) { const segments = data.segments.map(seg => ({ @@ -414,7 +457,12 @@ export default function TranscriptViewerPage() { diff --git a/mlx_audio/ui/app/speech-to-text/page.tsx b/mlx_audio/ui/app/speech-to-text/page.tsx index 0761d99e4..39ccf517d 100644 --- a/mlx_audio/ui/app/speech-to-text/page.tsx +++ b/mlx_audio/ui/app/speech-to-text/page.tsx @@ -4,7 +4,7 @@ import type React from "react" import { useState, useRef, useEffect } from "react" import { LayoutWrapper } from "@/components/layout-wrapper" -import { FileText, Upload, MoreVertical, X, ChevronDown } from "lucide-react" +import { FileText, Upload, MoreVertical, X } from "lucide-react" import Link from "next/link" interface TranscriptionFile { @@ -22,7 +22,6 @@ export default function SpeechToTextPage() { }, ]) const [isModalOpen, setIsModalOpen] = useState(false) - const [primaryLanguage, setPrimaryLanguage] = useState("Detect") const [tagAudioEvents, setTagAudioEvents] = useState(false) const [selectedModel, setSelectedModel] = useState("mlx-community/whisper-large-v3-turbo") const fileInputRef = useRef(null) @@ -70,11 +69,19 @@ export default function SpeechToTextPage() { loadStoredTranscriptions() }, []) + const fileToDataUrl = (file: File) => + new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.onerror = () => reject(new Error("Failed to read file")) + reader.readAsDataURL(file) + }) + const uploadAndTranscribe = async (file: File, id: string) => { const formData = new FormData() formData.append("file", file) formData.append("model", selectedModel) - formData.append("language", primaryLanguage === "Detect" ? "en" : primaryLanguage.toLowerCase()) + formData.append("language", "") formData.append("response_format", "verbose_json") formData.append("temperature", "0") @@ -84,6 +91,7 @@ export default function SpeechToTextPage() { const API_PORT = process.env.NEXT_PUBLIC_API_PORT || '8000'; try { + const dataUrlPromise = fileToDataUrl(file) const res = await fetch(`${API_BASE_URL}:${API_PORT}/v1/audio/transcriptions`, { method: "POST", body: formData, @@ -92,6 +100,7 @@ export default function SpeechToTextPage() { // Save the transcription to a JSON file via API data.fileName = fileName + data.audioDataUrl = await dataUrlPromise localStorage.setItem(`mlx-audio-transcription-${id}`, JSON.stringify(data)) setFiles((prev) => prev.map((f) => @@ -307,27 +316,11 @@ export default function SpeechToTextPage() {
- -
-
- - +
+

+ Automatically detects the spoken language and transcribes accordingly (supports Japanese, Chinese, and multilingual audio). +

diff --git a/mlx_audio/ui/app/text-to-speech/page.tsx b/mlx_audio/ui/app/text-to-speech/page.tsx index 985b36149..3e4cddef1 100644 --- a/mlx_audio/ui/app/text-to-speech/page.tsx +++ b/mlx_audio/ui/app/text-to-speech/page.tsx @@ -2,10 +2,9 @@ import type React from "react" -import { useState, useRef } from "react" +import { useState, useRef, useMemo, useEffect } from "react" import { ChevronDown, Download, ThumbsUp, ThumbsDown, Play, Pause, RefreshCw } from "lucide-react" import { LayoutWrapper } from "@/components/layout-wrapper" -import { VoiceSelection } from "@/components/voice-selection" // Custom range input component with colored progress function RangeInput({ @@ -48,6 +47,75 @@ function RangeInput({ ) } +const LANGUAGE_OPTIONS = [ + { value: "auto", label: "Auto detect (match text)", defaultVoice: "af_heart" }, + { value: "a", label: "English (American)", defaultVoice: "af_heart" }, + { value: "b", label: "English (British)", defaultVoice: "bf_emma" }, + { value: "e", label: "Spanish", defaultVoice: "ef_dora" }, + { value: "f", label: "French", defaultVoice: "ff_siwis" }, + { value: "h", label: "Hindi", defaultVoice: "hf_alpha" }, + { value: "i", label: "Italian", defaultVoice: "if_sara" }, + { value: "p", label: "Portuguese (Brazil)", defaultVoice: "pf_dora" }, + { value: "j", label: "Japanese", defaultVoice: "jf_alpha" }, + { value: "z", label: "Mandarin Chinese", defaultVoice: "zf_xiaobei" }, +] + +const MODEL_LANGUAGE_SUPPORT: Record = { + marvis: ["a"], + kokoro: LANGUAGE_OPTIONS.filter((option) => option.value !== "auto").map( + (option) => option.value + ), + spark: ["a", "b"], + default: ["a"], +} + +const getModelFamily = (modelId: string) => { + const normalized = modelId.toLowerCase() + if (normalized.includes("marvis")) return "marvis" + if (normalized.includes("kokoro")) return "kokoro" + if (normalized.includes("spark")) return "spark" + return "default" +} + +const isMarvisModel = (modelId: string) => modelId.toLowerCase().includes("marvis") + +const getDefaultVoice = (modelId: string, langCode: string) => { + if (isMarvisModel(modelId)) { + return "conversational_a" + } + const option = LANGUAGE_OPTIONS.find((entry) => entry.value === langCode) + return option?.defaultVoice ?? "af_heart" +} + +const detectLanguageFromText = (text: string): string | null => { + const content = text.trim() + if (!content) { + return null + } + + const hasHiragana = /[\u3040-\u309F]/.test(content) + const hasKatakana = /[\u30A0-\u30FF]/.test(content) + const hasCJK = /[\u4E00-\u9FFF]/.test(content) + if (hasHiragana || hasKatakana) { + return "j" + } + if (hasCJK) { + return "z" + } + + const accentedSpanish = /[áéíóúñüÁÉÍÓÚÑÜ]/.test(content) + if (accentedSpanish) { + return "e" + } + + const accentedFrench = /[àâçéèêëîïôûùüÿÀÂÇÉÈÊËÎÏÔÛÙÜŸ]/.test(content) + if (accentedFrench) { + return "f" + } + + return null +} + export default function SpeechSynthesis() { const [text, setText] = useState("But I also have other interests, such as playing tic-tac-toe.") const [isPlaying, setIsPlaying] = useState(false) @@ -59,11 +127,76 @@ export default function SpeechSynthesis() { const [duration, setDuration] = useState("00:04") const [activeTab, setActiveTab] = useState<"settings" | "history">("settings") const [model, setModel] = useState("Marvis-AI/marvis-tts-100m-v0.2-MLX-6bit") - const [language, setLanguage] = useState("English-detected") + const [language, setLanguage] = useState("auto") const [liked, setLiked] = useState(null) - const [selectedVoice, setSelectedVoice] = useState("conversational_a") const audioRef = useRef(null) + const [currentAudioUrl, setCurrentAudioUrl] = useState(null) + const modelFamily = getModelFamily(model) + const allowedLanguages = useMemo( + () => MODEL_LANGUAGE_SUPPORT[modelFamily] ?? MODEL_LANGUAGE_SUPPORT.default, + [modelFamily] + ) + const autoAllowed = allowedLanguages.length > 1 + + useEffect(() => { + if (!autoAllowed && language === "auto") { + setLanguage(allowedLanguages[0]) + } else if ( + language !== "auto" && + !allowedLanguages.includes(language) + ) { + setLanguage(autoAllowed ? "auto" : allowedLanguages[0]) + } + }, [autoAllowed, allowedLanguages, language]) + + const languageOptionsForModel = useMemo( + () => + LANGUAGE_OPTIONS.filter((option) => + option.value === "auto" + ? autoAllowed + : allowedLanguages.includes(option.value) + ), + [allowedLanguages, autoAllowed] + ) + + const languageLabel = + language === "auto" + ? "Auto detect (match text)" + : LANGUAGE_OPTIONS.find((option) => option.value === language)?.label ?? + "Auto detect (match text)" + + const detectedLanguage = + language === "auto" && autoAllowed ? detectLanguageFromText(text) : null + const resolvedLanguage = + language === "auto" + ? detectedLanguage ?? allowedLanguages[0] + : language + const selectedVoice = getDefaultVoice(model, resolvedLanguage) + + const cleanupAudioUrl = () => { + if (audioRef.current?.src && audioRef.current.src.startsWith("blob:")) { + URL.revokeObjectURL(audioRef.current.src) + } + setCurrentAudioUrl(null) + } + + const handlePlaybackError = (error: unknown) => { + if (error instanceof DOMException && error.name === "AbortError") { + return + } + console.error("Audio playback failed:", error) + } + + const playAudio = async () => { + if (!audioRef.current) return + try { + await audioRef.current.play() + setIsPlaying(true) + } catch (error) { + handlePlaybackError(error) + } + } const handleTextChange = (e: React.ChangeEvent) => { setText(e.target.value) @@ -78,21 +211,24 @@ export default function SpeechSynthesis() { } if (isPlaying) { - audioRef.current?.pause() + audioRef.current.pause() + setIsPlaying(false) } else { - audioRef.current?.play() + void playAudio() } - setIsPlaying(!isPlaying) } const handleGenerate = async () => { if (!audioRef.current) return setIsGenerating(true) + audioRef.current.pause() + setIsPlaying(false) + cleanupAudioUrl() const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || 'http://localhost'; const API_PORT = process.env.NEXT_PUBLIC_API_PORT || '8000'; - const voice = (model.includes("marvis") ? "conversational_a" : "af_heart"); + const voice = selectedVoice.trim() || getDefaultVoice(model, resolvedLanguage) try { const response = await fetch(`${API_BASE_URL}:${API_PORT}/v1/audio/speech`, { @@ -105,6 +241,7 @@ export default function SpeechSynthesis() { input: text, voice: voice, speed: speed, + ...(resolvedLanguage ? { lang_code: resolvedLanguage } : {}), // pitch and other parameters can be added here if supported by the backend }), }) @@ -116,12 +253,12 @@ export default function SpeechSynthesis() { const blob = await response.blob() const audioUrl = URL.createObjectURL(blob) audioRef.current.src = audioUrl + setCurrentAudioUrl(audioUrl) audioRef.current.onloadedmetadata = () => { setDuration(formatTime(Math.floor(audioRef.current?.duration || 0))) setCurrentTime("00:00") - setIsPlaying(true) - audioRef.current?.play() + void playAudio() } audioRef.current.ontimeupdate = () => { @@ -132,9 +269,7 @@ export default function SpeechSynthesis() { setIsPlaying(false) setCurrentTime("00:00") // Revoke the object URL to free up resources - if (audioRef.current?.src.startsWith("blob:")) { - URL.revokeObjectURL(audioRef.current.src) - } + cleanupAudioUrl() } } catch (error) { console.error("Error generating speech:", error) @@ -144,9 +279,39 @@ export default function SpeechSynthesis() { } } - const handleDownload = () => { - // In a real app, this would download the audio file - alert("Downloading audio...") + const handleDownload = async () => { + const src = audioRef.current?.src + if (!src || src === window.location.href) { + alert("Generate audio before downloading.") + return + } + + let downloadUrl = src + let revokeAfter = false + + if (!src.startsWith("blob:")) { + try { + const response = await fetch(src) + const blob = await response.blob() + downloadUrl = URL.createObjectURL(blob) + revokeAfter = true + } catch (error) { + console.error("Failed to download audio:", error) + alert("Unable to download audio. Please try regenerating.") + return + } + } + + const link = document.createElement("a") + link.href = downloadUrl + link.download = "mlx-audio.wav" + document.body.appendChild(link) + link.click() + document.body.removeChild(link) + + if (revokeAfter) { + URL.revokeObjectURL(downloadUrl) + } } const handleFeedback = (isPositive: boolean) => { @@ -161,10 +326,6 @@ export default function SpeechSynthesis() { return `00:${seconds.toString().padStart(2, "0")}` } - const handleVoiceChange = (voice: string) => { - setSelectedVoice(voice) - } - return (
@@ -190,11 +351,30 @@ export default function SpeechSynthesis() {
- + +
+

+ {autoAllowed + ? "Auto-detect is available for this model." + : `This model supports: ${allowedLanguages + .map( + (code) => + LANGUAGE_OPTIONS.find((opt) => opt.value === code)?.label ?? + code + ) + .join(", ")}`} +

- -
Speed @@ -409,7 +587,7 @@ export default function SpeechSynthesis() {
- {selectedVoice}: {text.length > 20 ? text.substring(0, 20) + "..." : text} + {languageLabel}: {text.length > 20 ? text.substring(0, 20) + "..." : text}
How did this sound?
diff --git a/requirements.txt b/requirements.txt index 931562203..bfbd0dd7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,8 @@ misaki[en]>=0.8.2 +misaki[ja]>=0.8.2 +misaki[zh]>=0.8.2 +pyopenjtalk>=0.3.3 +fugashi[unidic-lite]>=1.3.2 loguru>=0.7.3 num2words>=0.5.14 spacy>=3.8.4 @@ -25,4 +29,4 @@ fastrtc[vad, stt] webrtcvad>=2.0.10 dacite>=1.9.2 pytest-asyncio>=1.0.0 -mistral-common[audio] \ No newline at end of file +mistral-common[audio]