diff --git a/OpenUtau.Core/Analysis/Game.cs b/OpenUtau.Core/Analysis/Game.cs index cebb17227..0713f2e88 100644 --- a/OpenUtau.Core/Analysis/Game.cs +++ b/OpenUtau.Core/Analysis/Game.cs @@ -1,11 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; -using Microsoft.ML.OnnxRuntime; -using Microsoft.ML.OnnxRuntime.Tensors; using OpenUtau.Core.Util; using Serilog; @@ -39,37 +36,49 @@ public class GameOptions { /// Note presence threshold (--est-threshold). Default: 0.2 public float ScoreThreshold { get; set; } = 0.2f; + + /// + /// RNG seed driving the D3PM stochastic boundary removal. 0 = random per + /// inference. Honored by the GGML backend for reproducible runs; the ONNX + /// path reads its own RNG stream and ignores this. + /// + public ulong Seed { get; set; } = 0; } +/// +/// GAME MIDI extractor. This class is a thin +/// orchestrator over a pluggable (ONNX or GGML). +/// Audio chunking, resampling, batching and note→tick mapping live in the base +/// class; the inference contract is delegated entirely to the active backend, +/// selected via from the user's preferences. +/// public class Game : MidiExtractor { private const string PackageId = "game"; public const string DownloadUrl = "https://github.com/openvpi/GAME/releases/tag/oudep"; - InferenceSession? encoderSession; - InferenceSession? segmenterSession; - InferenceSession? estimatorSession; - InferenceSession? bd2durSession; - RunOptions? runOptions; - bool sessionsLoaded = false; + readonly IGameBackend backend; + readonly GameConfig config; bool disposed = false; volatile bool stopping = false; - GameConfig config; - string Location; protected override int ExpectedSampleRate => config.SampleRate; public float Timestep => config.Timestep; public IReadOnlyDictionary? Languages => config.Languages; + /// The resolved backend's display name (e.g. "GGML"). + public string BackendName => backend.Name; + /// - /// Check if GAME is installed (config.json is present) without loading models. + /// Check if any GAME backend is installed (ONNX or GGML) without loading models. /// public static bool IsInstalled(string? location = null) { - location ??= PackageManager.Inst.GetInstalledPath(PackageId); - return location != null && File.Exists(Path.Combine(location, "config.json")); + return location != null + ? GameOnnxBackend.IsInstalled(location) + : GameBackendFactory.IsAnyInstalled(); } /// - /// Load only the config (no ONNX sessions). Safe to call before showing a UI dialog. + /// Load only the config (no model sessions). Safe to call before showing a UI dialog. /// Throws if config.json is missing. /// public static GameConfig LoadConfig(string? modelPath = null) { @@ -81,263 +90,43 @@ public static GameConfig LoadConfig(string? modelPath = null) { } /// + /// Create a GAME instance using the user's preferred backend. /// public Game() : this(null) { } - /// - /// Create GAME instance with specified model path and parameters. - /// Sessions are loaded lazily on first Transcribe call. - /// - /// Path to model directory, or null for default (Dependencies/game) + /// Create a GAME instance with an explicit ONNX model directory override. public Game(string? location) { - Location = location ?? PackageManager.Inst.GetInstalledPath(PackageId)!; - Log.Information("GAME: Model location = {Location}", Location); - config = LoadConfig(location); - } - - /// - /// Ensure ONNX sessions are loaded. Called lazily before inference. - /// - private void EnsureSessionsLoaded() { - if (sessionsLoaded) return; - if (stopping) { - throw new OperationCanceledException(); - } - runOptions = new RunOptions(); - if (stopping) { - runOptions.Terminate = true; - throw new OperationCanceledException(); - } - encoderSession = CreateSession("encoder.onnx", OnnxRunnerChoice.CPUForCoreML); - segmenterSession = CreateSession("segmenter.onnx", OnnxRunnerChoice.Default); - estimatorSession = CreateSession("estimator.onnx", OnnxRunnerChoice.Default); - bd2durSession = CreateSession("bd2dur.onnx", OnnxRunnerChoice.Default); - sessionsLoaded = true; - if (stopping) { - runOptions.Terminate = true; - throw new OperationCanceledException(); + if (!string.IsNullOrEmpty(location) && GameOnnxBackend.IsInstalled(location)) { + config = LoadConfig(location); + backend = new GameOnnxBackend(config, location); + } else { + backend = GameBackendFactory.Create(); + config = backend.Config; } + Log.Information("GAME: active backend = {Backend}", backend.Name); } - protected override bool SupportsBatch => true; + protected override bool SupportsBatch => + backend is GameOnnxBackend; // only ONNX has native batching today protected override List> TranscribeWaveformBatch(List batch, GameOptions options) { - EnsureSessionsLoaded(); - return RunPipeline(batch, options); + if (stopping) throw new OperationCanceledException(); + return backend.RunInferenceBatch(batch, options); } protected override List TranscribeWaveform(float[] samples, GameOptions options) { - EnsureSessionsLoaded(); - return RunPipeline(new List { samples }, options)[0]; - } - - private List> RunPipeline(List batch, GameOptions options) { - int B = batch.Count; - int maxLen = batch.Max(s => s.Length); - - var waveformData = new float[B * maxLen]; - var durationData = new float[B]; - for (int b = 0; b < B; b++) { - var s = batch[b]; - s.CopyTo(waveformData, b * maxLen); - durationData[b] = (float)s.Length / config.SampleRate; - } - - var waveform = new DenseTensor(waveformData, new[] { B, maxLen }); - var duration = new DenseTensor(durationData, new[] { B }); - - try { - // 1. Encoder - var (xSeg, xEst, maskT) = RunEncoder(waveform, duration); - - // 2. Segmentation (D3PM loop) - int T = xSeg.Dimensions[1]; - Tensor knownBoundaries = new DenseTensor(new[] { B, T }); - Tensor boundaries = new DenseTensor(new[] { B, T }); - - Tensor? language = null; - if (config.Languages != null) { - int languageId = ResolveLanguageId(options.LanguageCode); - language = new DenseTensor( - Enumerable.Repeat((long)languageId, B).ToArray(), new[] { B }); - } - - var segThreshold = new DenseTensor(new[] { options.BoundaryThreshold }, Array.Empty()); - var radius = new DenseTensor(new long[] { options.BoundaryRadius }, Array.Empty()); - - if (config.Loop) { - float step = 1.0f / options.SamplingSteps; - for (int i = 0; i < options.SamplingSteps; i++) { - var t = new DenseTensor( - Enumerable.Repeat(i * step, B).ToArray(), new[] { B }); - boundaries = RunSegmenter(xSeg, knownBoundaries, boundaries, t, maskT, language, segThreshold, radius); - } - } else { - boundaries = RunSegmenter(xSeg, knownBoundaries, null, null, maskT, language, segThreshold, radius); - } - - // 3. Boundaries to durations - var (durations, maskN) = RunBd2Dur(boundaries, maskT); - int N = maskN.Dimensions[1]; - - // 4. Estimation - var scoreThreshold = new DenseTensor(new[] { options.ScoreThreshold }, Array.Empty()); - var (presence, scores) = RunEstimator(xEst, boundaries, maskT, maskN, scoreThreshold); - - // 5. Split results per batch item - var results = new List>(B); - for (int b = 0; b < B; b++) { - var notes = new List(N); - for (int i = 0; i < N; i++) { - if (!maskN[b, i]) break; - notes.Add(new TranscribedNote(durations[b, i], scores[b, i], presence[b, i])); - } - - results.Add(notes); - } - - return results; - } catch (OnnxRuntimeException) { - if (runOptions != null && runOptions.Terminate) { - throw new OperationCanceledException(); - } - throw; - } + if (stopping) throw new OperationCanceledException(); + return backend.RunInference(samples, options); } public override void Interrupt() { stopping = true; - if (!disposed && runOptions != null) { - runOptions.Terminate = true; - } + backend.Interrupt(); } protected override void DisposeManaged() { if (disposed) return; disposed = true; - runOptions?.Dispose(); - encoderSession?.Dispose(); - segmenterSession?.Dispose(); - estimatorSession?.Dispose(); - bd2durSession?.Dispose(); - sessionsLoaded = false; - } - - // ------------------------------------------------------------------------- - // Implementation details: session creation and low-level ONNX runners - // ------------------------------------------------------------------------- - - /// - /// Create an ONNX session for the given model file. - /// - private InferenceSession CreateSession(string modelFile, OnnxRunnerChoice runnerChoice) { - string modelPath = Path.Combine(Location, modelFile); - Log.Information("GAME: Loading model {ModelPath} (exists={Exists})", - modelPath, File.Exists(modelPath)); - return Onnx.getInferenceSession(modelPath, runnerChoice); - } - - /// - /// Resolve a language code string to an integer ID using the config's language map. - /// Returns 0 (universal) if the code is null or not found. - /// - private int ResolveLanguageId(string? languageCode) { - if (languageCode != null && config.Languages != null && - config.Languages.TryGetValue(languageCode, out int id)) { - return id; - } - - return 0; - } - - /// - /// Run encoder: waveform -> x_seg, x_est, maskT - /// - private (Tensor x_seg, Tensor x_est, Tensor maskT) - RunEncoder(Tensor waveform, Tensor duration) { - var inputs = new List { - NamedOnnxValue.CreateFromTensor("waveform", waveform), - NamedOnnxValue.CreateFromTensor("duration", duration), - }; - - using var outputs = encoderSession!.Run(inputs, encoderSession.OutputNames, runOptions); - - var xSeg = outputs.First(o => o.Name == "x_seg").AsTensor().ToDenseTensor(); - var xEst = outputs.First(o => o.Name == "x_est").AsTensor().ToDenseTensor(); - var maskT = outputs.First(o => o.Name == "maskT").AsTensor().ToDenseTensor(); - - return (xSeg, xEst, maskT); - } - - /// - /// Run a single segmenter step (D3PM sampling iteration) - /// - private Tensor RunSegmenter( - Tensor xSeg, - Tensor knownBoundaries, Tensor? prevBoundaries, - Tensor? t, Tensor maskT, - Tensor? language, - Tensor threshold, Tensor radius) { - var inputs = new List(); - inputs.Add(NamedOnnxValue.CreateFromTensor("x_seg", xSeg)); - - if (language != null) { - inputs.Add(NamedOnnxValue.CreateFromTensor("language", language)); - } - - inputs.Add(NamedOnnxValue.CreateFromTensor("known_boundaries", knownBoundaries)); - - if (prevBoundaries != null) { - inputs.Add(NamedOnnxValue.CreateFromTensor("prev_boundaries", prevBoundaries)); - } - - if (t != null) { - inputs.Add(NamedOnnxValue.CreateFromTensor("t", t)); - } - - inputs.Add(NamedOnnxValue.CreateFromTensor("maskT", maskT)); - inputs.Add(NamedOnnxValue.CreateFromTensor("threshold", threshold)); - inputs.Add(NamedOnnxValue.CreateFromTensor("radius", radius)); - - using var outputs = segmenterSession!.Run(inputs, segmenterSession.OutputNames, runOptions); - var boundaries = outputs.First(o => o.Name == "boundaries").AsTensor().ToDenseTensor(); - return boundaries; - } - - /// - /// Run bd2dur: boundaries -> durations (seconds) + maskN - /// - private (Tensor durations, Tensor maskN) - RunBd2Dur(Tensor boundaries, Tensor maskT) { - var inputs = new List { - NamedOnnxValue.CreateFromTensor("boundaries", boundaries), - NamedOnnxValue.CreateFromTensor("maskT", maskT), - }; - - using var outputs = bd2durSession!.Run(inputs, bd2durSession.OutputNames, runOptions); - var durations = outputs.First(o => o.Name == "durations").AsTensor().ToDenseTensor(); - var maskN = outputs.First(o => o.Name == "maskN").AsTensor().ToDenseTensor(); - - return (durations, maskN); - } - - /// - /// Run estimator: predict note presence and pitch scores - /// - private (Tensor presence, Tensor scores) - RunEstimator(Tensor xEst, Tensor boundaries, Tensor maskT, - Tensor maskN, Tensor threshold) { - var inputs = new List { - NamedOnnxValue.CreateFromTensor("x_est", xEst), - NamedOnnxValue.CreateFromTensor("boundaries", boundaries), - NamedOnnxValue.CreateFromTensor("maskT", maskT), - NamedOnnxValue.CreateFromTensor("maskN", maskN), - NamedOnnxValue.CreateFromTensor("threshold", threshold), - }; - - using var outputs = estimatorSession!.Run(inputs, estimatorSession.OutputNames, runOptions); - var presence = outputs.First(o => o.Name == "presence").AsTensor().ToDenseTensor(); - var scores = outputs.First(o => o.Name == "scores").AsTensor().ToDenseTensor(); - return (presence, scores); + backend.Dispose(); } } diff --git a/OpenUtau.Core/Analysis/GameBackendFactory.cs b/OpenUtau.Core/Analysis/GameBackendFactory.cs new file mode 100644 index 000000000..79a9c0ea9 --- /dev/null +++ b/OpenUtau.Core/Analysis/GameBackendFactory.cs @@ -0,0 +1,82 @@ +using System; +using System.IO; +using OpenUtau.Core.Util; +using Serilog; + +namespace OpenUtau.Core.Analysis; + +/// +/// Selects and constructs the active for a GAME +/// inference request, based on user preferences and which backend is installed. +/// +/// Preference resolution order: +/// 1. (the user's explicit choice) — +/// used if that backend is installed. +/// 2. Fall back to whichever backend *is* installed (ONNX preferred over GGML, +/// since ONNX is the trusted baseline). +/// 3. If neither is installed, throw — callers should have guarded with +/// . +/// +public static class GameBackendFactory { + public const string OnnxValue = "onnx"; + public const string GgmlValue = "ggml"; + + /// True when at least one backend's weights + binaries are in place. + public static bool IsAnyInstalled() { + return GameOnnxBackend.IsInstalled() || GameGgmlBackend.IsInstalled(); + } + + /// The backend id that will actually be used given prefs + installs. + public static string ResolveChoice() { + string pref = Preferences.Default.GameBackend ?? ""; + if (pref == GgmlValue && GameGgmlBackend.IsInstalled()) return GgmlValue; + if (pref == OnnxValue && GameOnnxBackend.IsInstalled()) return OnnxValue; + // Fall back to whatever is installed, preferring ONNX. + if (GameOnnxBackend.IsInstalled()) return OnnxValue; + if (GameGgmlBackend.IsInstalled()) return GgmlValue; + return ""; // none + } + + /// The display name ("ONNX" / "GGML") for the resolved backend. + public static string ResolvedBackendName() => ResolveChoice() switch { + GgmlValue => "GGML", + OnnxValue => "ONNX", + _ => "(none)", + }; + + /// + /// Load the configuration belonging to the backend that will actually be used. + /// This must not fall back to the ONNX package when only GGML is installed. + /// + public static GameConfig LoadResolvedConfig() { + string choice = ResolveChoice(); + if (choice == OnnxValue) { + string location = PackageManager.Inst.GetInstalledPath("game")!; + return Game.LoadConfig(location); + } + if (choice == GgmlValue) { + return GameGgmlBackend.LoadConfig(); + } + throw new InvalidOperationException( + "No GAME backend is installed. Install the GAME ONNX or GGML weights via the Package Manager."); + } + + /// + /// Construct the resolved backend and load its config from the installed + /// GAME weights directory. + /// + public static IGameBackend Create() { + string choice = ResolveChoice(); + Log.Information("GAME: backend choice resolved to {Choice}", choice); + if (choice == OnnxValue) { + string location = PackageManager.Inst.GetInstalledPath("game")!; + var config = Game.LoadConfig(location); + return new GameOnnxBackend(config, location); + } + if (choice == GgmlValue) { + return GameGgmlBackend.Create(); + } + throw new InvalidOperationException( + "No GAME backend is installed. Install the GAME ONNX or GGML weights via the Package Manager."); + } +} diff --git a/OpenUtau.Core/Analysis/GameGgmlBackend.cs b/OpenUtau.Core/Analysis/GameGgmlBackend.cs new file mode 100644 index 000000000..65396f01f --- /dev/null +++ b/OpenUtau.Core/Analysis/GameGgmlBackend.cs @@ -0,0 +1,355 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using Serilog; + +namespace OpenUtau.Core.Analysis; + +/// +/// GAME inference backend that drives the native GAME-ggml CLI in long-lived +/// serve mode. Communication is a binary stdin → JSON stdout protocol: +/// +/// On : +/// 1. Resolve the CLI binary and GGUF weights under the OpenUtau Dependencies folder. +/// 2. Spawn game_ggml_cli serve with redirected stdin/stdout/stderr. +/// 3. Wait for the {"type":"ready"} line on stdout. +/// +/// Per inference (): +/// • Write a 36-byte request header + the float32 waveform to stdin. +/// • Read one JSON object back describing the transcribed notes for this chunk. +/// +/// Cancelation () writes the quit magic and reaps the process. +/// One instance owns exactly one subprocess; the +/// subclass disposes it after the whole transcription. +/// +/// +/// Recommended GGML configuration (7-channel 60 s benchmark, nsteps=8, +/// frame-level metrics vs torch-CUDA fp32 no-cache baseline — all EPS ≥ 0.97 RPA): +/// +/// | platform | GPU | weights | EP offered by oudep pkg | +/// |---------------|---------------|---------|--------------------------| +/// | Windows | NVIDIA | F32 | CUDA (fallback Vulkan) | +/// | Windows | Intel/AMD/etc | F32 | Vulkan | +/// | Windows | integrated | F32/Q8 | CPU (+ DBCache): 0.44x wall | +/// | Linux | NVIDIA | F32 | CUDA (fallback Vulkan) | +/// | Linux | Nouveau/AMD | F32 | Vulkan | +/// | macOS | Apple Silicon | F32 | Metal (only EP) | +/// | macOS | Intel | F32 | Metal (cross-compiled) | +/// +/// Rules of thumb (no user config required — CLI picks backend from GGUF/EP): +/// * GPU present → F32 weights; CPU-only → CPU EP with DBCache on by default +/// * on GPU, Vulkan is our smallest-VRAM (≈+0.3 GiB) and CUDA the fastest +/// * Q8 saves VRAM/RAM (~3.4x smaller weights) at near-lossless quality but may +/// flip a boundary note on Vulkan — prefer the -full package if you need +/// bit-consistent output; both are equally valid in practice. +public class GameGgmlBackend : IGameBackend { + private const string PackageId = "game"; + // Single oudep package contains both the CLI binary and the GGUF weights. + private const string GgmlPackageId = "game-ggml-medium"; + + // Must match serv_proto::MAGIC_INFERENCE / MAGIC_QUIT in src/cli/main.cpp. + private const uint MAGIC_INFERENCE = 0x53455256u; // "VRES" + private const uint MAGIC_QUIT = 0x54495155u; // "UQIT" + + const int RequestHeaderBytes = 36; + + readonly GameConfig config; + readonly string cliPath; + readonly string ggufPath; + + Process? process; + BinaryWriter? stdinWriter; + StreamReader? stdoutReader; + volatile bool disposed = false; + volatile bool stopping = false; + + public string Name => "GGML"; + public GameConfig Config => config; + + private GameGgmlBackend(GameConfig config, string cliPath, string ggufPath) { + this.config = config; + this.cliPath = cliPath; + this.ggufPath = ggufPath; + } + + /// Locate the shipped CLI binary, platform-aware. + public static string? ResolveCliPath() { + string dep = PathManager.Inst.DependencyPath; + string exeName = OS.IsWindows() ? "game_ggml_cli.exe" : "game_ggml_cli"; + // The game-ggml-medium oudep package ships the CLI under its root. + string cliDir = Path.Combine(dep, GgmlPackageId); + string exe = Path.Combine(cliDir, exeName); + if (File.Exists(exe)) return exe; + return null; + } + + /// Locate the largest .gguf weight file. An explicit location is + /// honored first; otherwise checks the dedicated game-ggml-medium package, + /// then the shared game package for backward compatibility. + public static string? ResolveGgufPath(string? location = null) { + if (location != null) { + return FindLargestGguf(location); + } + string ggmlDir = Path.Combine(PathManager.Inst.DependencyPath, GgmlPackageId); + string? gguf = FindLargestGguf(ggmlDir); + if (gguf != null) return gguf; + + string? gameLoc = PackageManager.Inst.GetInstalledPath(PackageId); + return gameLoc == null ? null : FindLargestGguf(gameLoc); + } + + private static string? FindLargestGguf(string directory) { + if (!Directory.Exists(directory)) return null; + return Directory.GetFiles(directory, "*.gguf") + .OrderByDescending(f => new FileInfo(f).Length) + .FirstOrDefault(); + } + + /// True when the CLI, GGUF weights and backend config are installed. + public static bool IsInstalled(string? location = null) { + string? gguf = ResolveGgufPath(location); + return ResolveCliPath() != null && + gguf != null && + File.Exists(Path.Combine(Path.GetDirectoryName(gguf)!, "config.json")); + } + + /// Load config.json next to the GGUF weights. + public static GameConfig LoadConfig(string? location = null) { + string? gguf = ResolveGgufPath(location); + if (gguf == null) { + throw new InvalidOperationException("GAME GGML backend is missing .gguf weights."); + } + string configPath = Path.Combine(Path.GetDirectoryName(gguf)!, "config.json"); + if (!File.Exists(configPath)) { + throw new InvalidOperationException( + $"GAME GGML backend is missing config.json at {configPath}"); + } + var jsonText = File.ReadAllText(configPath, Encoding.UTF8); + return JsonSerializer.Deserialize(jsonText) + ?? throw new InvalidOperationException("Failed to parse GAME config.json"); + } + + /// + /// Construct a ready-to-serve backend if installed; otherwise throws so the + /// factory can fall back. Model loading is lazy (deferred to EnsureLoaded). + /// + public static GameGgmlBackend Create() { + string? cli = ResolveCliPath(); + string? gguf = ResolveGgufPath(); + if (cli == null || gguf == null) { + throw new InvalidOperationException( + "GAME GGML backend is not installed: missing CLI binary or .gguf weights."); + } + return new GameGgmlBackend(LoadConfig(), cli, gguf); + } + + public bool EnsureLoaded() { + if (process != null && !process.HasExited) return true; + + Log.Information("GAME(GGML): launching serve subprocess cli={Cli} gguf={Gguf}", cliPath, ggufPath); + EnsureExecutable(cliPath); + var psi = new ProcessStartInfo { + FileName = cliPath, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + StandardOutputEncoding = Encoding.UTF8, + }; + psi.ArgumentList.Add("serve"); + psi.ArgumentList.Add(ggufPath); + process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start game_ggml_cli."); + // Binary stdin: avoid a text writer wrapper to control framing exactly. + stdinWriter = new BinaryWriter(process.StandardInput.BaseStream, Encoding.ASCII, leaveOpen: false); + stdoutReader = process.StandardOutput; + + // Drain stderr on a background thread so it can't deadlock the pipe. + StartStderrPump(); + + // Wait for {"type":"ready"} (or {"type":"error",...}) on stdout. + string? first = ReadJsonObject(); + if (first == null || ParseType(first) == "error") { + string msg = first ?? "subprocess closed stdout before ready"; + if (first != null) msg = ParseError(first) ?? msg; + throw new InvalidOperationException($"GAME GGML backend failed to initialize: {msg}"); + } + return true; + } + + public List RunInference(float[] samples, GameOptions options) { + EnsureLoaded(); + if (stopping || process == null || process.HasExited) { + throw new OperationCanceledException(); + } + + // Resolve language id. + int languageId = 0; + if (config.Languages != null && options.LanguageCode != null && + config.Languages.TryGetValue(options.LanguageCode, out int id)) { + languageId = id; + } + + // Build the request header + waveform, write to stdin. + Span header = stackalloc byte[RequestHeaderBytes]; + BinaryPrimitivesWriteU32Little(header.Slice(0, 4), MAGIC_INFERENCE); + BinaryPrimitivesWriteI32Little(header.Slice(4, 4), languageId); + BinaryPrimitivesWriteU64Little(header.Slice(8, 8), (ulong)options.Seed); + BinaryPrimitivesWriteI32Little(header.Slice(16, 4), options.SamplingSteps); + BinaryPrimitivesWriteFloatLittle(header.Slice(20, 4), options.BoundaryThreshold); + BinaryPrimitivesWriteI32Little(header.Slice(24, 4), options.BoundaryRadius); + BinaryPrimitivesWriteFloatLittle(header.Slice(28, 4), options.ScoreThreshold); + BinaryPrimitivesWriteU32Little(header.Slice(32, 4), (uint)samples.Length); + + var stdin = stdinWriter!; + stdin.Write((ReadOnlySpan)header); + // Float32 waveform, little-endian. On x86/x64 .NET float layout is LE. + ReadOnlySpan waveBytes = System.Runtime.InteropServices.MemoryMarshal.AsBytes(samples.AsSpan()); + stdin.Write(waveBytes); + stdin.Flush(); + + // Read one JSON object back: {"type":"notes","count":N,"notes":[...]} + string? line = ReadJsonObject(); + if (line == null) throw new OperationCanceledException("GGML subprocess closed stdout during inference."); + string type = ParseType(line); + if (type == "error") { + throw new InvalidOperationException($"GGML inference error: {ParseError(line)}"); + } + if (type != "notes") { + throw new InvalidOperationException($"Unexpected GGML response: {line}"); + } + return ParseNotes(line); + } + + public void Interrupt() { + stopping = true; + // Send the quit magic; the subprocess exits cleanly. Fall through to + // a hard kill if it hangs. + try { + if (process != null && !process.HasExited && stdinWriter != null) { + Span quit = stackalloc byte[4]; + BinaryPrimitivesWriteU32Little(quit, MAGIC_QUIT); + stdinWriter.Write(quit); + stdinWriter.Flush(); + if (!process.WaitForExit(3000)) { + try { process.Kill(entireProcessTree: true); } catch { /* best effort */ } + } + } + } catch { + // Cancellation must never throw; swallow IO errors on a dying pipe. + } + } + + public void Dispose() { + if (disposed) return; + disposed = true; + try { + if (process != null && !process.HasExited) { + try { + if (stdinWriter != null) { + Span quit = stackalloc byte[4]; + BinaryPrimitivesWriteU32Little(quit, MAGIC_QUIT); + stdinWriter.Write(quit); + stdinWriter.Flush(); + } + } catch { /* pipe may already be broken */ } + if (!process.WaitForExit(2000)) { + try { process.Kill(entireProcessTree: true); } catch { } + } + } + } finally { + stdinWriter?.Dispose(); + stdoutReader?.Dispose(); + process?.Dispose(); + process = null; + } + } + + // ------------------------------------------------------------------------- + // stdout framing: read exactly one JSON object terminated by '\n' + // ------------------------------------------------------------------------- + + private string? ReadJsonObject() { + // ReadLine blocks until '\n' or EOF; JSON objects are single-line. + string? line = stdoutReader!.ReadLine(); + return string.IsNullOrEmpty(line) ? null : line; + } + + private static string? ParseType(string json) { + using var doc = JsonDocument.Parse(json); + return doc.RootElement.TryGetProperty("type", out var t) ? t.GetString() : null; + } + + private static string? ParseError(string json) { + using var doc = JsonDocument.Parse(json); + return doc.RootElement.TryGetProperty("message", out var m) ? m.GetString() : null; + } + + private static List ParseNotes(string json) { + var notes = new List(); + using var doc = JsonDocument.Parse(json); + if (!doc.RootElement.TryGetProperty("notes", out var arr)) return notes; + // The GGML backend emitted absolute offset_seconds per note, but the + // segmenter partitions the chunk into contiguous regions whose lengths + // sum exactly to the chunk length — each note already carries its own + // duration_seconds. The MidiExtractor base class positions the chunk by + // its audio offset and accumulates note durations sequentially, so we + // pass duration_seconds through directly and ignore offset_seconds. + foreach (var n in arr.EnumerateArray()) { + float duration = n.TryGetProperty("d", out var de) ? de.GetSingle() : 0f; + float pitch = n.TryGetProperty("p", out var pe) ? pe.GetSingle() : 0f; + bool voiced = n.TryGetProperty("v", out var ve) && ve.GetInt32() != 0; + if (duration < 0) duration = 0; + notes.Add(new TranscribedNote(duration, pitch, voiced)); + } + return notes; + } + + // ------------------------------------------------------------------------- + // stderr pump (keeps the subprocess from blocking on a full error buffer) + // ------------------------------------------------------------------------- + private void StartStderrPump() { + var err = process!.StandardError; + System.Threading.Tasks.Task.Run(() => { + string? l; + while ((l = err.ReadLine()) != null) { + Log.Information("GAME(GGML)[stderr] {Line}", l); + } + }); + } + + private static void EnsureExecutable(string path) { + if (OS.IsWindows()) return; + const UnixFileMode executableMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + try { + File.SetUnixFileMode(path, executableMode); + } catch (Exception e) { + Log.Warning(e, "GAME(GGML): failed to set executable permission on {Cli}", path); + } + } + + // ------------------------------------------------------------------------- + // Little-endian binary writers (no BitConverter pinned-buffer dance needed) + // ------------------------------------------------------------------------- + private static void BinaryPrimitivesWriteU32Little(Span dst, uint v) { + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(dst, v); + } + private static void BinaryPrimitivesWriteI32Little(Span dst, int v) { + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(dst, v); + } + private static void BinaryPrimitivesWriteU64Little(Span dst, ulong v) { + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(dst, v); + } + private static unsafe void BinaryPrimitivesWriteFloatLittle(Span dst, float v) { + int bits = *(int*)&v; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(dst, bits); + } +} diff --git a/OpenUtau.Core/Analysis/GameOnnxBackend.cs b/OpenUtau.Core/Analysis/GameOnnxBackend.cs new file mode 100644 index 000000000..66c1ea872 --- /dev/null +++ b/OpenUtau.Core/Analysis/GameOnnxBackend.cs @@ -0,0 +1,265 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using Serilog; + +namespace OpenUtau.Core.Analysis; + +/// +/// ONNX Runtime backend for GAME. Runs the four-model pipeline +/// (encoder → segmenter (D3PM loop) → bd2dur → estimator) directly in-process. +/// This is a behavior-preserving extraction of the original single-backend +/// implementation; ONNX is always available when the +/// game dependency package with the .onnx files is installed. +/// +public class GameOnnxBackend : IGameBackend { + private const string PackageId = "game"; + + InferenceSession? encoderSession; + InferenceSession? segmenterSession; + InferenceSession? estimatorSession; + InferenceSession? bd2durSession; + RunOptions? runOptions; + bool sessionsLoaded = false; + bool disposed = false; + volatile bool stopping = false; + + readonly GameConfig config; + readonly string Location; + + public string Name => "ONNX"; + public GameConfig Config => config; + + public GameOnnxBackend(GameConfig config, string location) { + this.config = config; + this.Location = location; + } + + /// Check if the ONNX GAME weights are installed without loading models. + public static bool IsInstalled(string? location = null) { + location ??= PackageManager.Inst.GetInstalledPath(PackageId); + if (location == null) return false; + if (!System.IO.File.Exists(System.IO.Path.Combine(location, "config.json"))) return false; + // All four ONNX models must be present. + return new[] { "encoder.onnx", "segmenter.onnx", "estimator.onnx", "bd2dur.onnx" } + .All(f => System.IO.File.Exists(System.IO.Path.Combine(location, f))); + } + + public bool EnsureLoaded() { + if (sessionsLoaded) return true; + runOptions = new RunOptions(); + encoderSession = CreateSession("encoder.onnx", OnnxRunnerChoice.CPUForCoreML); + segmenterSession = CreateSession("segmenter.onnx", OnnxRunnerChoice.Default); + estimatorSession = CreateSession("estimator.onnx", OnnxRunnerChoice.Default); + bd2durSession = CreateSession("bd2dur.onnx", OnnxRunnerChoice.Default); + sessionsLoaded = true; + if (stopping) { + runOptions.Terminate = true; + throw new OperationCanceledException(); + } + return true; + } + + public List RunInference(float[] samples, GameOptions options) { + EnsureLoaded(); + return RunPipeline(new List { samples }, options)[0]; + } + + public List> RunInferenceBatch(List batch, GameOptions options) { + EnsureLoaded(); + return RunPipeline(batch, options); + } + + private List> RunPipeline(List batch, GameOptions options) { + int B = batch.Count; + int maxLen = batch.Max(s => s.Length); + + var waveformData = new float[B * maxLen]; + var durationData = new float[B]; + for (int b = 0; b < B; b++) { + var s = batch[b]; + s.CopyTo(waveformData, b * maxLen); + durationData[b] = (float)s.Length / config.SampleRate; + } + + var waveform = new DenseTensor(waveformData, new[] { B, maxLen }); + var duration = new DenseTensor(durationData, new[] { B }); + + try { + // 1. Encoder + var (xSeg, xEst, maskT) = RunEncoder(waveform, duration); + + // 2. Segmentation (D3PM loop) + int T = xSeg.Dimensions[1]; + Tensor knownBoundaries = new DenseTensor(new[] { B, T }); + Tensor boundaries = new DenseTensor(new[] { B, T }); + + Tensor? language = null; + if (config.Languages != null) { + int languageId = ResolveLanguageId(options.LanguageCode); + language = new DenseTensor( + Enumerable.Repeat((long)languageId, B).ToArray(), new[] { B }); + } + + var segThreshold = new DenseTensor(new[] { options.BoundaryThreshold }, Array.Empty()); + var radius = new DenseTensor(new long[] { options.BoundaryRadius }, Array.Empty()); + + if (config.Loop) { + float step = 1.0f / options.SamplingSteps; + for (int i = 0; i < options.SamplingSteps; i++) { + var t = new DenseTensor( + Enumerable.Repeat(i * step, B).ToArray(), new[] { B }); + boundaries = RunSegmenter(xSeg, knownBoundaries, boundaries, t, maskT, language, segThreshold, radius); + } + } else { + boundaries = RunSegmenter(xSeg, knownBoundaries, null, null, maskT, language, segThreshold, radius); + } + + // 3. Boundaries to durations + var (durations, maskN) = RunBd2Dur(boundaries, maskT); + int N = maskN.Dimensions[1]; + + // 4. Estimation + var scoreThreshold = new DenseTensor(new[] { options.ScoreThreshold }, Array.Empty()); + var (presence, scores) = RunEstimator(xEst, boundaries, maskT, maskN, scoreThreshold); + + // 5. Split results per batch item + var results = new List>(B); + for (int b = 0; b < B; b++) { + var notes = new List(N); + for (int i = 0; i < N; i++) { + if (!maskN[b, i]) break; + notes.Add(new TranscribedNote(durations[b, i], scores[b, i], presence[b, i])); + } + + results.Add(notes); + } + + return results; + } catch (OnnxRuntimeException) { + if (runOptions != null && runOptions.Terminate) { + throw new OperationCanceledException(); + } + throw; + } + } + + public void Interrupt() { + stopping = true; + if (!disposed && runOptions != null) { + runOptions.Terminate = true; + } + } + + public void Dispose() { + if (disposed) return; + disposed = true; + runOptions?.Dispose(); + encoderSession?.Dispose(); + segmenterSession?.Dispose(); + estimatorSession?.Dispose(); + bd2durSession?.Dispose(); + sessionsLoaded = false; + } + + // ------------------------------------------------------------------------- + // Implementation details: session creation and low-level ONNX runners + // ------------------------------------------------------------------------- + + private InferenceSession CreateSession(string modelFile, OnnxRunnerChoice runnerChoice) { + string modelPath = System.IO.Path.Combine(Location, modelFile); + Log.Information("GAME(ONNX): Loading model {ModelPath} (exists={Exists})", + modelPath, System.IO.File.Exists(modelPath)); + return Onnx.getInferenceSession(modelPath, runnerChoice); + } + + private int ResolveLanguageId(string? languageCode) { + if (languageCode != null && config.Languages != null && + config.Languages.TryGetValue(languageCode, out int id)) { + return id; + } + + return 0; + } + + private (Tensor x_seg, Tensor x_est, Tensor maskT) + RunEncoder(Tensor waveform, Tensor duration) { + var inputs = new List { + NamedOnnxValue.CreateFromTensor("waveform", waveform), + NamedOnnxValue.CreateFromTensor("duration", duration), + }; + + using var outputs = encoderSession!.Run(inputs, encoderSession.OutputNames, runOptions); + + var xSeg = outputs.First(o => o.Name == "x_seg").AsTensor().ToDenseTensor(); + var xEst = outputs.First(o => o.Name == "x_est").AsTensor().ToDenseTensor(); + var maskT = outputs.First(o => o.Name == "maskT").AsTensor().ToDenseTensor(); + + return (xSeg, xEst, maskT); + } + + private Tensor RunSegmenter( + Tensor xSeg, + Tensor knownBoundaries, Tensor? prevBoundaries, + Tensor? t, Tensor maskT, + Tensor? language, + Tensor threshold, Tensor radius) { + var inputs = new List(); + inputs.Add(NamedOnnxValue.CreateFromTensor("x_seg", xSeg)); + + if (language != null) { + inputs.Add(NamedOnnxValue.CreateFromTensor("language", language)); + } + + inputs.Add(NamedOnnxValue.CreateFromTensor("known_boundaries", knownBoundaries)); + + if (prevBoundaries != null) { + inputs.Add(NamedOnnxValue.CreateFromTensor("prev_boundaries", prevBoundaries)); + } + + if (t != null) { + inputs.Add(NamedOnnxValue.CreateFromTensor("t", t)); + } + + inputs.Add(NamedOnnxValue.CreateFromTensor("maskT", maskT)); + inputs.Add(NamedOnnxValue.CreateFromTensor("threshold", threshold)); + inputs.Add(NamedOnnxValue.CreateFromTensor("radius", radius)); + + using var outputs = segmenterSession!.Run(inputs, segmenterSession.OutputNames, runOptions); + var boundaries = outputs.First(o => o.Name == "boundaries").AsTensor().ToDenseTensor(); + return boundaries; + } + + private (Tensor durations, Tensor maskN) + RunBd2Dur(Tensor boundaries, Tensor maskT) { + var inputs = new List { + NamedOnnxValue.CreateFromTensor("boundaries", boundaries), + NamedOnnxValue.CreateFromTensor("maskT", maskT), + }; + + using var outputs = bd2durSession!.Run(inputs, bd2durSession.OutputNames, runOptions); + var durations = outputs.First(o => o.Name == "durations").AsTensor().ToDenseTensor(); + var maskN = outputs.First(o => o.Name == "maskN").AsTensor().ToDenseTensor(); + + return (durations, maskN); + } + + private (Tensor presence, Tensor scores) + RunEstimator(Tensor xEst, Tensor boundaries, Tensor maskT, + Tensor maskN, Tensor threshold) { + var inputs = new List { + NamedOnnxValue.CreateFromTensor("x_est", xEst), + NamedOnnxValue.CreateFromTensor("boundaries", boundaries), + NamedOnnxValue.CreateFromTensor("maskT", maskT), + NamedOnnxValue.CreateFromTensor("maskN", maskN), + NamedOnnxValue.CreateFromTensor("threshold", threshold), + }; + + using var outputs = estimatorSession!.Run(inputs, estimatorSession.OutputNames, runOptions); + var presence = outputs.First(o => o.Name == "presence").AsTensor().ToDenseTensor(); + var scores = outputs.First(o => o.Name == "scores").AsTensor().ToDenseTensor(); + return (presence, scores); + } +} diff --git a/OpenUtau.Core/Analysis/IGameBackend.cs b/OpenUtau.Core/Analysis/IGameBackend.cs new file mode 100644 index 000000000..7c28e96ec --- /dev/null +++ b/OpenUtau.Core/Analysis/IGameBackend.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; + +namespace OpenUtau.Core.Analysis; + +/// +/// Pluggable inference backend for the GAME MIDI extractor. +/// +/// wraps the ONNX Runtime four-model pipeline. +/// drives the native GAME-ggml CLI (serve mode). +/// +/// Implementations are responsible for lazily loading their model lifecycle and +/// for honoring . Audio chunking, resampling, batching and +/// the final note→tick mapping are handled by , +/// so backends only need to transcribe a single preprocessed waveform (or a batch). +/// +public interface IGameBackend : IDisposable { + /// Short display name for diagnostics ("ONNX", "GGML"). + string Name { get; } + + /// Configuration loaded from this backend's own package. + GameConfig Config { get; } + + /// + /// Load the model if not already loaded. Called lazily before the first + /// inference. Returns true if the weights/executable are present and ready. + /// + bool EnsureLoaded(); + + /// Transcribe a single preprocessed (mono, expected-sample-rate) waveform. + List RunInference(float[] samples, GameOptions options); + + /// + /// Transcribe a batch of waveforms. Default implementation falls back to + /// per-chunk calls; backends with native batching override it. + /// + List> RunInferenceBatch(List batch, GameOptions options) { + var results = new List>(batch.Count); + foreach (var samples in batch) { + results.Add(RunInference(samples, options)); + } + return results; + } + + /// Request cancellation of any in-flight inference. + void Interrupt(); +} diff --git a/OpenUtau.Core/Util/Preferences.cs b/OpenUtau.Core/Util/Preferences.cs index 79de981f3..178912b84 100644 --- a/OpenUtau.Core/Util/Preferences.cs +++ b/OpenUtau.Core/Util/Preferences.cs @@ -159,6 +159,11 @@ public class SerializablePreferences { public int WorldlineR = 0; public string OnnxRunner = string.Empty; public int OnnxGpu = 0; + /// + /// GAME MIDI extractor backend preference: "onnx" (default) or "ggml". + /// Affects which inference engine Game uses; see GameBackendFactory. + /// + public string GameBackend = "onnx"; public double DiffSingerDepth = 1.0; public int DiffSingerSteps = 20; public int DiffSingerStepsVariance = 20; diff --git a/OpenUtau.Test/Core/Analysis/GameGgmlBackendTest.cs b/OpenUtau.Test/Core/Analysis/GameGgmlBackendTest.cs new file mode 100644 index 000000000..ace83d887 --- /dev/null +++ b/OpenUtau.Test/Core/Analysis/GameGgmlBackendTest.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using OpenUtau.Core.Analysis; +using Xunit; + +namespace OpenUtau.Core { + public class GameGgmlBackendTest { + [Fact] + public void ResolveGgufPathHonorsExplicitLocation() { + string directory = CreateTempDirectory(); + try { + string small = Path.Combine(directory, "small.gguf"); + string medium = Path.Combine(directory, "medium.gguf"); + File.WriteAllBytes(small, new byte[1]); + File.WriteAllBytes(medium, new byte[2]); + + Assert.Equal(medium, GameGgmlBackend.ResolveGgufPath(directory)); + } finally { + Directory.Delete(directory, true); + } + } + + [Fact] + public void LoadConfigReadsConfigNextToGgufWithoutOnnxPackage() { + string directory = CreateTempDirectory(); + try { + File.WriteAllBytes(Path.Combine(directory, "model.gguf"), new byte[1]); + File.WriteAllText( + Path.Combine(directory, "config.json"), + "{\"samplerate\":32000,\"timestep\":0.02,\"languages\":{\"zh\":1}}"); + + GameConfig config = GameGgmlBackend.LoadConfig(directory); + + Assert.Equal(32000, config.SampleRate); + Assert.Equal(0.02f, config.Timestep); + Assert.Equal(1, config.Languages!["zh"]); + } finally { + Directory.Delete(directory, true); + } + } + + [Fact] + public void LoadConfigRejectsMissingConfigNextToGguf() { + string directory = CreateTempDirectory(); + try { + File.WriteAllBytes(Path.Combine(directory, "model.gguf"), new byte[1]); + + var error = Assert.Throws( + () => GameGgmlBackend.LoadConfig(directory)); + + Assert.Contains("config.json", error.Message); + } finally { + Directory.Delete(directory, true); + } + } + + private static string CreateTempDirectory() { + string directory = Path.Combine(Path.GetTempPath(), $"OpenUtau.GameGgmlBackendTest.{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + return directory; + } + } +} diff --git a/OpenUtau/Strings/Strings.axaml b/OpenUtau/Strings/Strings.axaml index fe9830797..fd21315d0 100644 --- a/OpenUtau/Strings/Strings.axaml +++ b/OpenUtau/Strings/Strings.axaml @@ -619,6 +619,7 @@ Warning: this option removes custom presets. Use system default device Test Rendering + GAME Inference Backend Default renderer (for classic voicebanks) DiffSinger Render Depth DiffSinger Render Steps for Acoustic diff --git a/OpenUtau/ViewModels/PreferencesViewModel.cs b/OpenUtau/ViewModels/PreferencesViewModel.cs index 30033cc38..7b74cf3b3 100644 --- a/OpenUtau/ViewModels/PreferencesViewModel.cs +++ b/OpenUtau/ViewModels/PreferencesViewModel.cs @@ -93,6 +93,10 @@ public int SafeMaxThreadCount { [Reactive] public GpuInfo OnnxGpu { get; set; } [Reactive] public bool ShowOnnxGpu { get; set; } + // GAME backend (onnx / ggml) + public List GameBackendOptions { get; } = new() { "ONNX", "GGML" }; + [Reactive] public string GameBackend { get; set; } + // Appearance [Reactive] public string ThemeName { get; set; } [Reactive] public int DegreeStyle { get; set; } @@ -170,6 +174,12 @@ public PreferencesViewModel() { OnnxGpuOptions = Onnx.getGpuInfo(); OnnxGpu = OnnxGpuOptions.FirstOrDefault(x => x.deviceId == Preferences.Default.OnnxGpu, OnnxGpuOptions[0]); ShowOnnxGpu = OnnxRunner == "DirectML"; + // GAME backend: ONNX is the default, GGML is available when installed. + // The options list always includes both so the ComboBox UX is stable. + GameBackend = Preferences.Default.GameBackend switch { + "ggml" => "GGML", + _ => "ONNX", // default / empty / unrecognized all map to ONNX + }; DiffSingerDepth = Preferences.Default.DiffSingerDepth * 100; DiffSingerSteps = Preferences.Default.DiffSingerSteps; DiffSingerStepsVariance = Preferences.Default.DiffSingerStepsVariance; @@ -344,6 +354,11 @@ public PreferencesViewModel() { Preferences.Default.OnnxGpu = index.deviceId; Preferences.Save(); }); + this.WhenAnyValue(vm => vm.GameBackend) + .Subscribe(index => { + Preferences.Default.GameBackend = index == "GGML" ? "ggml" : "onnx"; + Preferences.Save(); + }); this.WhenAnyValue(vm => vm.RememberMid) .Subscribe(index => { Preferences.Default.RememberMid = index; diff --git a/OpenUtau/ViewModels/TranscribeViewModel.cs b/OpenUtau/ViewModels/TranscribeViewModel.cs index 1b10f6eff..7503aaf2e 100644 --- a/OpenUtau/ViewModels/TranscribeViewModel.cs +++ b/OpenUtau/ViewModels/TranscribeViewModel.cs @@ -100,10 +100,15 @@ public TranscribeViewModel() { SelectedAlgorithm = TranscribeAlgorithm.SOME; } - // Load GAME config (no model sessions) to populate options + // Load config from the backend that will actually run (no model sessions). + // In particular, GGML-only installs must not probe the ONNX package. GameConfig? gameConfig = null; if (GameAvailable) { - gameConfig = Game.LoadConfig(); + try { + gameConfig = GameBackendFactory.LoadResolvedConfig(); + } catch { + GameAvailable = false; + } } GameHasLanguages = (gameConfig?.Languages?.Count ?? 0) > 0; diff --git a/OpenUtau/Views/PreferencesDialog.axaml b/OpenUtau/Views/PreferencesDialog.axaml index 661cff5b5..365b80049 100644 --- a/OpenUtau/Views/PreferencesDialog.axaml +++ b/OpenUtau/Views/PreferencesDialog.axaml @@ -207,6 +207,8 @@ + +