diff --git a/acestep/streaming/ace_backend.py b/acestep/streaming/ace_backend.py index 75225010..e2f93f89 100644 --- a/acestep/streaming/ace_backend.py +++ b/acestep/streaming/ace_backend.py @@ -42,6 +42,7 @@ from acestep.streaming.knobs import ( CHANNEL_GROUPS, KEYSTONE_CHANNELS, + KnobSlewLimiter, knob_specs as registry_knob_specs, manual_slot_specs, steering_axis_spec, @@ -244,6 +245,19 @@ def __init__( # ``None`` on the first tick just seeds the baseline. self._last_rebuild_keys = None + # Continuous-knob slew limiter. Sits at the once-per-tick knob + # read so a fast/erratic sweep ramps toward its target instead of + # snapping — the single engine-side guard against control + # discontinuities driving the audio into distortion/clipping + # (protects every transport: web, VST, MCP, headless). Discrete + # knobs and non-registry keys pass through untouched; see + # :class:`acestep.streaming.knobs.KnobSlewLimiter`. The per-knob + # ceilings come from the session's live spec map (so runtime LoRA + # / steering knobs are covered); a static fallback keeps bare + # test fixtures (no session manifest) working. + self._slew = KnobSlewLimiter() + self._slew_fallback_specs_by_name = None + # Activation steering. The controller is the source of truth for # the slot count and vector catalog; the session mirrors its # slot ops into KnobState / the knob manifest. ``None`` (e.g. a @@ -544,21 +558,47 @@ def sync_source(self, ctx: TickContext) -> None: self._walk_w1 = walk_w1 self._walk_chunk_start_s = walk_chunk_start_s + def _slew_specs_by_name(self) -> dict: + """The live ``{name: KnobSpec}`` map the slew limiter reads its + per-knob ceilings from. + + Prefers the session's manifest (``_knob_specs_by_name``), which + is reassigned wholesale when the LoRA / steering knob set changes + — so runtime knobs are covered and the limiter rebuilds its + ceilings exactly when the universe shifts. Falls back to this + backend's own static manifest for bare construction (test + fixtures with no session manifest).""" + sbn = getattr(self.session, "_knob_specs_by_name", None) + if sbn: + return sbn + if self._slew_fallback_specs_by_name is None: + self._slew_fallback_specs_by_name = { + s.name: s for s in self.knob_specs() + } + return self._slew_fallback_specs_by_name + def read_knobs(self) -> dict: if self.use_midi: - return self.midi_knobs.get_all_values() - with self.state._lock: - m = self.state.motion_val - raw = { - self.k1_name: m, - "seed": 0.0, - "feedback": 0.0, - "feedback_depth": 1.0, - "shift": 3.5, - } - if self.use_sde: - raw["periodicity"] = 0.0 - return raw + target = self.midi_knobs.get_all_values() + else: + with self.state._lock: + m = self.state.motion_val + target = { + self.k1_name: m, + "seed": 0.0, + "feedback": 0.0, + "feedback_depth": 1.0, + "shift": 3.5, + } + if self.use_sde: + target["periodicity"] = 0.0 + # Rate-limit continuous knobs at this once-per-tick boundary so a + # fast sweep ramps instead of stepping. Discrete knobs and + # non-registry keys (curves, the playback clock) ride through + # unchanged. ``_prepare_tick`` and the channel/steering sync all + # read from this returned dict, so the whole translation path + # sees the slewed values. + return self._slew.apply(target, self._slew_specs_by_name()) def has_pending_refit(self) -> bool: """True when ``before_tick`` is about to apply LoRA commands. @@ -696,7 +736,12 @@ def _prepare_tick(self, knobs: dict, ctx: TickContext) -> dict: if abs(lora_str - self.state.params.get(key, -1)) > 0.02: self.engine_obj.set_lora_strength(desc.id, lora_str) - hint_str = self.midi_knobs.get_param("hint_strength") if self.use_midi else 1.0 + # Read the (already-slewed) value from ``raw`` rather than the + # backing KnobState so the hint-strength blend honors the slew + # limiter. ``raw`` carries hint_strength as a seeded bank knob in + # midi mode; the non-midi path has no such knob, so 1.0 (the + # historical default) applies. + hint_str = float(raw.get("hint_strength", 1.0)) # Silence latent must match the T of the latent it's blended # against. walk_active can flip mid-session if a swap drops # the source below the window — rebuild on demand here so @@ -797,7 +842,9 @@ def _prepare_tick(self, knobs: dict, ctx: TickContext) -> dict: if x0_target_curve is not None: x0_str = 0.0 else: - x0_str = self.midi_knobs.get_param("x0_target") if self.use_midi else 0.0 + # Slewed value via ``raw`` (a seeded bank knob in midi mode); + # 0.0 when absent, matching the historical non-midi default. + x0_str = float(raw.get("x0_target", 0.0)) # Use the live (possibly sliced) source as the x0_target so # the per-frame curve / strength scalar lines up with the # latent the DiT actually denoises against. diff --git a/acestep/streaming/knobs.py b/acestep/streaming/knobs.py index df8c0eff..53691e97 100644 --- a/acestep/streaming/knobs.py +++ b/acestep/streaming/knobs.py @@ -9,7 +9,9 @@ the operator's hardware controller. """ +import math import threading +import time from dataclasses import dataclass from typing import Any, Optional @@ -18,7 +20,24 @@ # way a frontend/re-skin/agent must notice (knob added/removed/retyped, # bounds semantics changed). Served alongside the catalog at /api/knobs and # by the MCP list_knobs tool so a consumer can detect a stale build. -KNOB_SCHEMA_VERSION = 1 +# +# v2: added per-knob ``slew_max_per_s`` to the catalog (the server-side +# slew/rate-limit ceiling applied to continuous knobs; see ``KnobSpec. +# slew_max_per_s`` / :class:`KnobSlewLimiter`). +KNOB_SCHEMA_VERSION = 2 + + +# Default continuous-knob slew ceiling, expressed as a fraction of the +# knob's full ``[min, max]`` range traversable per second. A float knob +# without an explicit ``slew_max_per_s`` is rate-limited to this many +# range-widths/second when its target jumps; discrete knobs (int / enum / +# bool) are never slewed. 3.0 == a full-range sweep ramps over ~0.33 s: +# below any deliberate human knob turn (so normal adjustments are NOT +# delayed — a slew limiter only caps the *rate*, it adds zero lag below +# the ceiling), but fast/erratic sweeps are broken into bounded per-tick +# deltas so the diffusion stream never sees a control discontinuity that +# drives the audio into distortion/clipping. +DEFAULT_SLEW_FRACTION_PER_S = 3.0 # Channel groups / keystones used by the server-side pipeline for @@ -62,6 +81,14 @@ class KnobSpec: options: tuple = () # allowed values for enum / bool description: str = "" bank: bool = True + # Per-knob server-side slew ceiling, in knob units per second. Caps how + # fast the value applied to the stream may move toward a freshly-set + # target so a fast/erratic sweep ramps instead of jumping (see + # :class:`KnobSlewLimiter`). Only meaningful for ``float`` knobs; + # ``int`` / ``enum`` / ``bool`` are always applied verbatim. ``None`` + # selects the range-relative default (:data:`DEFAULT_SLEW_FRACTION_PER_S`); + # ``0.0`` (or negative) opts a continuous knob OUT of slewing entirely. + slew_max_per_s: Optional[float] = None def knob_specs(sde: bool, loras=None) -> list: @@ -323,6 +350,33 @@ def knob_catalog(sde: bool, loras=None) -> dict: return catalog_from_specs(knob_specs(sde, loras)) +def effective_slew_max_per_s( + spec, fraction_per_s: float = DEFAULT_SLEW_FRACTION_PER_S, +) -> Optional[float]: + """Resolve the slew ceiling (knob units / second) for one spec, or + ``None`` when the knob must NOT be slewed. + + Only ``float`` knobs are continuous; ``int`` / ``enum`` / ``bool`` + return ``None`` so discrete controls (seed, step count, modes, + toggles) pass through untouched. For a float knob: + + * an explicit positive ``spec.slew_max_per_s`` is honored verbatim; + * an explicit ``0.0`` / negative value opts the knob OUT (``None``); + * ``None`` falls back to ``fraction_per_s`` range-widths/second over + the knob's ``[min, max]`` span (a zero-width span yields ``None``). + """ + if spec.type != "float": + return None + explicit = spec.slew_max_per_s + if explicit is not None: + return float(explicit) if explicit > 0.0 else None + lo = spec.min_val if spec.min_val is not None else 0.0 + span = float(spec.max_val) - float(lo) + if span <= 0.0: + return None + return fraction_per_s * span + + def catalog_from_specs(specs) -> dict: """Project an arbitrary :class:`KnobSpec` list into the catalog shape (the same projection :func:`knob_catalog` serves). Used by @@ -348,6 +402,13 @@ def catalog_from_specs(specs) -> dict: entry["min"] = spec.min_val if spec.min_val is not None else 0.0 if spec.options: entry["options"] = list(spec.options) + # Continuous knobs publish their server-side slew ceiling so a + # contract-built client can mirror the ramp in its own UI tween + # (and so the behavior is discoverable, not hidden in the engine). + # Discrete knobs omit it entirely — they are never slewed. + slew = effective_slew_max_per_s(spec) + if slew is not None: + entry["slew_max_per_s"] = slew if spec.description: entry["description"] = spec.description out[spec.name] = entry @@ -467,3 +528,126 @@ def get_param(self, name: str) -> float: def get_all_values(self) -> dict: with self._lock: return dict(self._values) + + +class KnobSlewLimiter: + """Per-knob slew-rate limiter for continuous control changes. + + The streaming runner reads the knob target once per tick and the + backend turns it into the per-inference-step values the diffusion + stream consumes. A raw target dict carries whatever the operator last + set, so a fast / erratic knob sweep lands as a step discontinuity on + consecutive generations — the source of the "single fast sweep -> + chaotic, unusable audio" distortion. This limiter sits at that + once-per-tick boundary and rate-limits the *applied* value toward the + target so a large jump ramps over a few ticks instead of snapping. + + Key properties: + + * **Transport-agnostic.** Applied engine-side (in the backend's + per-tick knob read), so every client — web, VST, MCP, headless — + is protected by one implementation rather than each UI's own tween. + * **Zero added lag below the ceiling.** A slew limiter only caps the + rate of change; any adjustment slower than the per-knob ceiling + passes through verbatim, so normal/slow knob turns are untouched. + * **Discrete knobs pass through.** Only knobs with a positive + :func:`effective_slew_max_per_s` are limited; int / enum / bool + knobs (seed, step count, modes, toggles) and any non-registry keys + (curve specs, the playback clock) ride through unchanged. + * **Wall-clock paced.** The per-call delta is ``ceiling * dt`` using + the real elapsed time between ticks, so the ramp duration is + independent of the loop's tick rate. ``dt`` is capped + (``max_dt_s``) so resuming after an idle pause can't release one + giant un-ramped jump. + + Not thread-safe: call :meth:`apply` from a single thread (the runner + thread, where the backend's ``read_knobs`` already runs). + """ + + def __init__( + self, + *, + fraction_per_s: float = DEFAULT_SLEW_FRACTION_PER_S, + max_dt_s: float = 0.2, + clock=time.monotonic, + ): + self._fraction = float(fraction_per_s) + self._max_dt_s = float(max_dt_s) + self._clock = clock + self._last_t: Optional[float] = None + # name -> current applied (slewed) value. + self._cur: dict = {} + # name -> slew ceiling (units/s), rebuilt only when the spec map + # identity changes (the session swaps it wholesale on LoRA / + # steering knob add/remove), so the steady path is allocation-free. + self._limit: dict = {} + self._limit_src = None + + def reset(self) -> None: + """Forget all accumulated state so the next :meth:`apply` snaps + every knob straight to its target (used on session re-seed).""" + self._cur.clear() + self._last_t = None + + def _ensure_limits(self, specs_by_name: dict) -> None: + # Identity compare: the session reassigns ``_knob_specs_by_name`` + # to a fresh dict whenever the knob set changes, so this rebuilds + # exactly when (and only when) the universe shifts. + if specs_by_name is self._limit_src: + return + limits: dict = {} + for name, spec in specs_by_name.items(): + lim = effective_slew_max_per_s(spec, self._fraction) + if lim is not None: + limits[name] = lim + self._limit = limits + self._limit_src = specs_by_name + # Drop carried state for knobs that no longer slew so a future + # re-add starts cleanly from the (then-current) target. + for name in [n for n in self._cur if n not in limits]: + self._cur.pop(name, None) + + def apply(self, target: dict, specs_by_name: dict) -> dict: + """Return a copy of ``target`` with every continuous knob moved + toward its target by at most its per-knob ceiling for the elapsed + tick. ``specs_by_name`` is the live ``{name: KnobSpec}`` map + (the session's per-session manifest); keys absent from it, and + keys whose spec is discrete, pass through unchanged. + """ + self._ensure_limits(specs_by_name) + now = self._clock() + if self._last_t is None: + dt = 0.0 + else: + dt = now - self._last_t + if dt < 0.0: + dt = 0.0 + elif dt > self._max_dt_s: + dt = self._max_dt_s + self._last_t = now + + out = dict(target) + for name, lim in self._limit.items(): + if name not in target: + continue + raw_val = target[name] + try: + tgt = float(raw_val) + except (TypeError, ValueError): + continue # not a number this tick (shouldn't happen): skip + cur = self._cur.get(name) + # First sighting, or no measurable time passed: adopt the + # target as-is (no startup ramp from a stale/zero baseline). + if cur is None or dt <= 0.0: + self._cur[name] = tgt + out[name] = tgt + continue + max_delta = lim * dt + diff = tgt - cur + if -max_delta <= diff <= max_delta: + cur = tgt + else: + cur += math.copysign(max_delta, diff) + self._cur[name] = cur + out[name] = cur + return out diff --git a/demos/realtime_motion_graph_web/web/engine/lora/dispatcher.ts b/demos/realtime_motion_graph_web/web/engine/lora/dispatcher.ts index e1f9889e..e91cfc2b 100644 --- a/demos/realtime_motion_graph_web/web/engine/lora/dispatcher.ts +++ b/demos/realtime_motion_graph_web/web/engine/lora/dispatcher.ts @@ -103,3 +103,37 @@ export function seedLoraSliderValue(id: string, strength: number): void { const value = clamp(strength); usePerformanceStore.getState().setSliderDirect(param, value); } + +/** Reconcile the engine-facing slider value of EVERY enabled LoRA with + * the store's current ``lora.strengths``. Unlike the per-enable seed + * (``seedLoraSliderValue`` fired once when a LoRA flips on), this runs + * for ALREADY-enabled LoRAs too. + * + * Why it exists — the session-restore desync: a saved-session resume + * writes the persisted strength into ``useLoraStore.strengths`` (which + * the fader UI reads, so it shows 1.0) but NOT into + * ``perf.sliderValues``, the value ``useParamSync`` actually ships each + * tick. ``useParamSync`` prefers ``sliderValues`` and only falls back + * to ``lora.strengths`` when the ``lora_str_`` key is ABSENT. The + * boot-time catalog seed already wrote a ``sliderValues`` entry for the + * default-on LoRAs, so on restore the key is present-but-stale: the + * engine stays pinned to the old strength while the slider reads the + * restored one. The LoRA is inaudible until a fader drag finally writes + * ``sliderValues`` through the dispatcher — exactly the reported bug + * ("shows 1.0 but you only hear it once I wiggle the fader"). + * + * Idempotent: on a fresh (non-restored) session ``sliderValues`` and + * ``lora.strengths`` already agree (both seeded from the same defaults + * / config), so every write lands the same value — the server only + * refits on a > 0.02 delta, so re-seeding the matching value triggers + * no extra refit. Call exactly once per session-ready edge (initial + * connect + reconnect), never mid-drag — the dispatcher owns + * ``sliderValues`` while a fader is in flight, and ready edges can't + * overlap a pointer gesture. */ +export function reconcileEnabledLoraStrengths(): void { + const { enabled, strengths } = useLoraStore.getState(); + for (const id of enabled) { + const value = strengths[id]; + if (typeof value === "number") seedLoraSliderValue(id, value); + } +} diff --git a/demos/realtime_motion_graph_web/web/hooks/useFixtureSwap.ts b/demos/realtime_motion_graph_web/web/hooks/useFixtureSwap.ts index d19298a6..c7588243 100644 --- a/demos/realtime_motion_graph_web/web/hooks/useFixtureSwap.ts +++ b/demos/realtime_motion_graph_web/web/hooks/useFixtureSwap.ts @@ -13,7 +13,7 @@ import { } from "@/lib/config"; import { useCustomTracksStore } from "@/store/useCustomTracksStore"; import { usePerformanceStore } from "@/store/usePerformanceStore"; -import { useSessionStore } from "@/store/useSessionStore"; +import { useSessionStore, type SessionStatus } from "@/store/useSessionStore"; import { isTimeSignature } from "@/types/engine"; // In-place fixture swap. Mirrors swapToFixture() in DEMON's app.js: when the @@ -26,6 +26,36 @@ import { isTimeSignature } from "@/types/engine"; // state where it can't accept a new source). The full restart is delegated // back to useStartSession via the same fixture name. +/** + * Decide whether a recovered session must swap to the user's current + * track selection. + * + * The reconnect path (useStartSession) rebinds the fixture snapshotted + * at session start. If the user switched tracks while the socket was + * down, the perf store's `fixture` (what the UI shows) diverges from the + * session store's `boundFixture` (what the recovered backend + player + * actually play). Mid-outage the perf-store swap subscription fires but + * `run()` bails on `status !== "ready"`, so the change is dropped and + * never re-applied — the recovered session keeps playing the stale + * track. This guard re-applies it exactly once, the instant the + * reconnect completes (status "reconnecting" → "ready"). + * + * Returns false for every other transition so it never double-swaps a + * fresh Play (idle/connecting → ready) or a normal reconnect where the + * selection never changed. + */ +export function needsFixtureReconcile(args: { + prevStatus: SessionStatus; + status: SessionStatus; + selectedFixture: string; + boundFixture: string | null; +}): boolean { + const { prevStatus, status, selectedFixture, boundFixture } = args; + if (prevStatus !== "reconnecting" || status !== "ready") return false; + if (!selectedFixture) return false; + return selectedFixture !== boundFixture; +} + export function useFixtureSwap() { // Skip the very first fixture write (which fires when the catalog populates // and writes the default name into the store). @@ -241,6 +271,11 @@ export function useFixtureSwap() { return; } lastSwappedTo.current = name; + // The live session is now bound to this track server-side. Keep + // the session store in sync so the reconnect reconcile compares + // against the truth (and a later reconnect doesn't re-swap a track + // that's already loaded). + useSessionStore.getState().setBoundFixture(name); // A source-mode hotswap (force) is the same song with a different // stem feeding inference — skip the new-track gate entirely so the // performer's denoise / remix-started state is left untouched. @@ -305,6 +340,30 @@ export function useFixtureSwap() { void run(fixture, true); }); + // Reconnect reconcile: when a recovered session reaches "ready", the + // backend + player are bound to the fixture snapshotted at session + // start. If the user switched tracks while the socket was down, that + // mid-outage change was dropped by run()'s `status !== "ready"` + // bail. Re-apply it here exactly once on the reconnecting → ready + // edge so the recovered session swaps to the live selection instead + // of playing the stale track. `force` re-runs the swap even when the + // name matches lastSwappedTo (which still points at the stale bound + // fixture until the swap below updates it). + const unsubReconnect = useSessionStore.subscribe((s, prev) => { + const selectedFixture = usePerformanceStore.getState().fixture; + if ( + !needsFixtureReconcile({ + prevStatus: prev.status, + status: s.status, + selectedFixture, + boundFixture: s.boundFixture, + }) + ) { + return; + } + void run(selectedFixture, true); + }); + // Seed lastSwappedTo with the current fixture so the initial population // (catalog → default fixture write) doesn't trigger a no-op swap, and // seed lastSwappedMode so the first stem_assets echo is recognised. @@ -318,6 +377,7 @@ export function useFixtureSwap() { cancelled = true; unsub(); unsubSource(); + unsubReconnect(); }; }, []); } diff --git a/demos/realtime_motion_graph_web/web/hooks/useStartSession.ts b/demos/realtime_motion_graph_web/web/hooks/useStartSession.ts index 5cc79e99..ce2fb1fd 100644 --- a/demos/realtime_motion_graph_web/web/hooks/useStartSession.ts +++ b/demos/realtime_motion_graph_web/web/hooks/useStartSession.ts @@ -15,6 +15,7 @@ import { resolveLoraCapForSource, } from "@/lib/config"; import { wirePromptTransform } from "@/lib/loraTriggers"; +import { reconcileEnabledLoraStrengths } from "@/engine/lora/dispatcher"; import { useCustomTracksStore } from "@/store/useCustomTracksStore"; import { useLoraStore } from "@/store/useLoraStore"; import { usePerformanceStore, type RefSource } from "@/store/usePerformanceStore"; @@ -617,7 +618,26 @@ export function useStartSession() { // ghost-LoRA leak when the reconnected source's tier is // tighter than what was previously enabled). applyLoraCapWithServerSync(resolveLoraCapForSource(remote.duration)); + // Re-sync the engine-facing LoRA strengths from the store now + // that the (post-cap) enabled set is final. A reconnect rebuilds + // the server session from buildConfig, so the materialized + // strengths are correct — but useParamSync ships sliderValues, + // which may hold a stale per-LoRA entry from before a restore. + // Reconciling here keeps the streamed value equal to the UI + // strength (no-op when they already agree). See + // reconcileEnabledLoraStrengths. + reconcileEnabledLoraStrengths(); useSessionStore.getState().setSession(remote, player); + // Record what this recovered session is actually bound to. The + // reconnect rebinds the fixture snapshotted at session start, + // which can lag the user's current selection if they switched + // tracks while the socket was down. Set this BEFORE onSuccess + // flips status to "ready" so useFixtureSwap's reconnect + // reconcile observes the (possibly stale) bound fixture and + // heals the divergence by swapping to the live selection. + useSessionStore + .getState() + .setBoundFixture(sessionFixture.fixtureName); // Rebuild the network-quality monitor against the new // remote — the old one was bound to the dropped backend's // `slice` events and is dead now. @@ -724,6 +744,18 @@ export function useStartSession() { // server-side after they vanish from the UI. applyLoraCapWithServerSync(resolveLoraCapForSource(remote.duration)); + // Re-sync the engine-facing LoRA strengths from the store before the + // first param tick ships. On a fresh session this is a no-op (the + // catalog seed already wrote matching sliderValues), but on a + // saved-session resume the host restores the persisted strength into + // useLoraStore.strengths WITHOUT touching perf.sliderValues — so + // useParamSync would otherwise keep streaming the stale value the + // catalog seed left behind (the LoRA shows its restored strength but + // is inaudible until the fader is moved). Runs once here, gated by + // this hook only reaching "ready" with a live remote, so it can't + // fire before the engine is connected or spam per-tick dispatches. + reconcileEnabledLoraStrengths(); + // "Hear the source first" gate: when enabled in config.json, every // session start snaps engine denoise to 0 and plays a visual-only // glide from the slider's prior value down to 0 over glide_ms. The @@ -753,6 +785,10 @@ export function useStartSession() { perfState.setRemixStarted(false); setSession(remote, player); + // The live session is now bound to this fixture. Mirrored into the + // session store so the reconnect path can tell whether the user + // switched tracks during an outage (see useFixtureSwap). + useSessionStore.getState().setBoundFixture(resolved.fixtureName); setStatus("ready", "Playing"); // Start the network-quality monitor now that the WS is "ready". diff --git a/demos/realtime_motion_graph_web/web/store/useSessionStore.ts b/demos/realtime_motion_graph_web/web/store/useSessionStore.ts index 3567def3..a88e4bf2 100644 --- a/demos/realtime_motion_graph_web/web/store/useSessionStore.ts +++ b/demos/realtime_motion_graph_web/web/store/useSessionStore.ts @@ -37,6 +37,15 @@ interface SessionState { /** Server-issued WS URL (from /api/queue/join). Null when no queue is in * use — useStartSession falls back to defaultWsUrl(). */ wsUrl: string | null; + /** Fixture name the LIVE session is actually bound to server-side. + * Distinct from usePerformanceStore.fixture (what the UI shows + * selected): the two diverge when the user picks a different track + * while the socket is down. The reconnect path rebinds to the + * fixture snapshotted at session start, so without reconciling + * against the current selection the recovered session keeps playing + * the stale track. useFixtureSwap reads this to detect and heal that + * divergence on reconnect. Null until the first session binds. */ + boundFixture: string | null; /** Active checkpoint's model-scale label ("2B" | "5B" | null). Set * from the WS ready message and from /api/loras. Null when unknown. * The LoRA library uses this to hide LoRAs whose trained @@ -78,6 +87,7 @@ interface SessionState { setMonitor: (monitor: NetworkMonitor | null) => void; setReconnector: (reconnector: WsReconnector | null) => void; setWsUrl: (wsUrl: string | null) => void; + setBoundFixture: (fixture: string | null) => void; setCheckpointScale: (scale: string | null) => void; setPipelineDepth: (depth: number | null) => void; setMaxPipelineDepth: (max: number | null) => void; @@ -99,6 +109,7 @@ export const useSessionStore = create((set, get) => ({ monitor: null, reconnector: null, wsUrl: null, + boundFixture: null, checkpointScale: null, pipelineDepth: null, maxPipelineDepth: null, @@ -115,6 +126,7 @@ export const useSessionStore = create((set, get) => ({ setMonitor: (monitor) => set({ monitor }), setReconnector: (reconnector) => set({ reconnector }), setWsUrl: (wsUrl) => set({ wsUrl }), + setBoundFixture: (fixture) => set({ boundFixture: fixture }), setCheckpointScale: (scale) => set({ checkpointScale: scale }), setPipelineDepth: (depth) => set({ pipelineDepth: depth }), setMaxPipelineDepth: (max) => set({ maxPipelineDepth: max }), @@ -139,6 +151,8 @@ export const useSessionStore = create((set, get) => ({ player: null, monitor: null, reconnector: null, + // Per-session truth: the next session rebinds its own fixture. + boundFixture: null, pipelineDepth: null, maxPipelineDepth: null, // Capability mask is per-session truth (the next session may pick diff --git a/demos/realtime_motion_graph_web/web/tests/unit/fixtureReconnectReconcile.test.ts b/demos/realtime_motion_graph_web/web/tests/unit/fixtureReconnectReconcile.test.ts new file mode 100644 index 00000000..07630450 --- /dev/null +++ b/demos/realtime_motion_graph_web/web/tests/unit/fixtureReconnectReconcile.test.ts @@ -0,0 +1,116 @@ +// Regression guard for: "Audio source mode reconnect keeps playing the +// stale track." +// +// Repro from the tracker: in audio-source mode with a track loaded, the +// socket drops; the user quickly switches to a different source while the +// "Reconnecting…" placard is up; when the session recovers it keeps +// playing the PREVIOUS track even though the UI now shows the new one. +// +// Root cause: the reconnect path (useStartSession) rebinds the fixture +// snapshotted at session start. The mid-outage track change is dropped +// because useFixtureSwap.run() bails while `status !== "ready"`, and +// nothing re-applies it once the session recovers. So the recovered +// backend + AudioPlayer stay bound to the stale track (session store's +// `boundFixture`) while the perf store's `fixture` shows the new pick. +// +// The fix adds a reconnect reconcile: on the "reconnecting" → "ready" +// edge, if the live `boundFixture` diverges from the selected `fixture`, +// re-run the swap exactly once. `needsFixtureReconcile` is that decision; +// these tests pin its matrix so the heal fires for the bug scenario and +// for nothing else (no double-swap on fresh Play or a clean reconnect). + +import { describe, expect, it } from "vitest"; + +import { needsFixtureReconcile } from "@/hooks/useFixtureSwap"; +import type { SessionStatus } from "@/store/useSessionStore"; + +const A = "inside_confusion_loop_60s_gsm.wav"; // track loaded at start +const B = "low_fi_Gm_loop_60s_gnm.wav"; // track picked during the outage + +describe("needsFixtureReconcile", () => { + it("heals the reported bug: track switched during the outage", () => { + // Recovered session is bound to A (the snapshot); the UI selection is + // B (chosen while the socket was down). The reconnect completed + // (reconnecting → ready) → reconcile must fire. + expect( + needsFixtureReconcile({ + prevStatus: "reconnecting", + status: "ready", + selectedFixture: B, + boundFixture: A, + }), + ).toBe(true); + }); + + it("does NOT fire on a clean reconnect (selection unchanged)", () => { + // The common case: the socket blipped, the user touched nothing. The + // bound track still matches the selection — no redundant swap. + expect( + needsFixtureReconcile({ + prevStatus: "reconnecting", + status: "ready", + selectedFixture: A, + boundFixture: A, + }), + ).toBe(false); + }); + + it("does NOT fire on a fresh Play (idle/connecting → ready)", () => { + // useStartSession already resolves the CURRENT selection and records + // it as boundFixture, so a fresh start never needs a reconcile — even + // if boundFixture hasn't been written yet at the instant of the edge. + for (const prevStatus of [ + "idle", + "loading-fixture", + "connecting", + ] as const satisfies readonly SessionStatus[]) { + expect( + needsFixtureReconcile({ + prevStatus, + status: "ready", + selectedFixture: B, + boundFixture: null, + }), + ).toBe(false); + expect( + needsFixtureReconcile({ + prevStatus, + status: "ready", + selectedFixture: B, + boundFixture: A, + }), + ).toBe(false); + } + }); + + it("ignores non-ready destinations of the reconnect edge", () => { + // Entering reconnect, giving up, and re-render churn must not swap. + for (const status of [ + "reconnecting", + "error", + "closed", + "idle", + ] as const satisfies readonly SessionStatus[]) { + expect( + needsFixtureReconcile({ + prevStatus: "reconnecting", + status, + selectedFixture: B, + boundFixture: A, + }), + ).toBe(false); + } + }); + + it("never fires with an empty selection", () => { + // No track selected yet → nothing to reconcile to. + expect( + needsFixtureReconcile({ + prevStatus: "reconnecting", + status: "ready", + selectedFixture: "", + boundFixture: A, + }), + ).toBe(false); + }); +}); diff --git a/demos/realtime_motion_graph_web/web/tests/unit/loraStrengthReconcile.test.ts b/demos/realtime_motion_graph_web/web/tests/unit/loraStrengthReconcile.test.ts new file mode 100644 index 00000000..fb0c5e4d --- /dev/null +++ b/demos/realtime_motion_graph_web/web/tests/unit/loraStrengthReconcile.test.ts @@ -0,0 +1,99 @@ +// Regression: "LoRAs not loading upon refresh / session reload." +// +// A saved-session resume restores the persisted LoRA strength into +// useLoraStore.strengths (which the fader UI reads, so it shows 1.0) but +// NOT into perf.sliderValues — the value useParamSync ships to the +// engine each tick. useParamSync prefers sliderValues and only falls +// back to lora.strengths when the lora_str_ key is ABSENT, so a +// stale entry seeded for a default-on LoRA at boot pins the engine to +// the old strength: the LoRA is inaudible until a fader drag finally +// writes sliderValues. reconcileEnabledLoraStrengths() closes that gap +// on every session-ready edge. + +import { beforeEach, describe, expect, it } from "vitest"; + +import { reconcileEnabledLoraStrengths } from "@/engine/lora/dispatcher"; +import { useLoraStore } from "@/store/useLoraStore"; +import { usePerformanceStore } from "@/store/usePerformanceStore"; + +/** + * Mirror of the LoRA-strength selection useParamSync performs each tick + * (hooks/useParamSync.ts): start from the sliderValues snapshot, then + * fall back to lora.strengths only for enabled keys that are absent. + * Returns the value the engine would actually receive for `id`. + */ +function engineShippedStrength(id: string): number | undefined { + const perf = usePerformanceStore.getState(); + const lora = useLoraStore.getState(); + const raw: Record = { ...perf.sliderValues }; + for (const lid of lora.enabled) { + const k = `lora_str_${lid}`; + if (k in raw) continue; + const v = lora.strengths[lid]; + if (typeof v === "number") raw[k] = v; + } + return raw[`lora_str_${id}`]; +} + +beforeEach(() => { + useLoraStore.setState({ enabled: new Set(), strengths: {} }); + usePerformanceStore.setState({ sliderValues: {}, sliderTargets: {} }); +}); + +describe("reconcileEnabledLoraStrengths", () => { + it("pushes a restored strength to the engine when sliderValues is stale", () => { + // Boot seed left the default-on LoRA at 0.8 in the engine-facing + // sliderValues; the restore then wrote 1.0 into lora.strengths only. + usePerformanceStore.getState().setSliderDirect("lora_str_deathstep", 0.8); + useLoraStore.setState({ + enabled: new Set(["deathstep"]), + strengths: { deathstep: 1.0 }, + }); + + // Before the fix: the engine keeps streaming the stale 0.8. + expect(engineShippedStrength("deathstep")).toBe(0.8); + + // On session-ready, the reconcile runs. + reconcileEnabledLoraStrengths(); + + // The engine now receives the restored 1.0 — no fader drag needed. + expect(usePerformanceStore.getState().sliderValues["lora_str_deathstep"]).toBe( + 1.0, + ); + expect(engineShippedStrength("deathstep")).toBe(1.0); + }); + + it("is a no-op on a fresh session where the values already agree", () => { + usePerformanceStore.getState().setSliderDirect("lora_str_synthpop", 0.8); + useLoraStore.setState({ + enabled: new Set(["synthpop"]), + strengths: { synthpop: 0.8 }, + }); + + reconcileEnabledLoraStrengths(); + + expect(usePerformanceStore.getState().sliderValues["lora_str_synthpop"]).toBe( + 0.8, + ); + expect(engineShippedStrength("synthpop")).toBe(0.8); + }); + + it("only reconciles enabled LoRAs, leaving disabled ones untouched", () => { + usePerformanceStore.getState().setSliderDirect("lora_str_ambient", 0.5); + useLoraStore.setState({ + // ambient is NOT enabled — its persisted strength must not be + // pushed to the engine. + enabled: new Set(["deathstep"]), + strengths: { deathstep: 1.0, ambient: 0.9 }, + }); + + reconcileEnabledLoraStrengths(); + + expect(usePerformanceStore.getState().sliderValues["lora_str_deathstep"]).toBe( + 1.0, + ); + expect(usePerformanceStore.getState().sliderValues["lora_str_ambient"]).toBe( + 0.5, + ); + }); +}); diff --git a/demos/realtime_motion_graph_web/web/tests/unit/sessionStore.test.ts b/demos/realtime_motion_graph_web/web/tests/unit/sessionStore.test.ts index 3e20ab7b..ee066278 100644 --- a/demos/realtime_motion_graph_web/web/tests/unit/sessionStore.test.ts +++ b/demos/realtime_motion_graph_web/web/tests/unit/sessionStore.test.ts @@ -51,6 +51,7 @@ describe("useSessionStore", () => { const s = useSessionStore.getState(); s.setMonitor({ stop } as never); s.setReconnector({ cancel } as never); + s.setBoundFixture("track.wav"); s.setStatus("ready", "Playing"); s.reset(); expect(stop).toHaveBeenCalledTimes(1); @@ -61,6 +62,7 @@ describe("useSessionStore", () => { expect(after.reconnector).toBeNull(); expect(after.remote).toBeNull(); expect(after.player).toBeNull(); + expect(after.boundFixture).toBeNull(); expect(after.pipelineDepth).toBeNull(); expect(after.maxPipelineDepth).toBeNull(); expect(after.lastWsTrace).toBeNull(); diff --git a/packages/demon-client/types/knobs.ts b/packages/demon-client/types/knobs.ts index 9d6b3d60..77cc83d5 100644 --- a/packages/demon-client/types/knobs.ts +++ b/packages/demon-client/types/knobs.ts @@ -25,6 +25,13 @@ export interface KnobManifestEntry { max?: number; /** Allowed values for enum/bool knobs. */ options?: Array; + /** Server-side slew ceiling in knob units per second (continuous knobs + * only). The backend rate-limits the applied value toward a freshly + * set target at this rate so a fast sweep ramps instead of jumping; + * a contract-built UI can mirror the same ramp in its own tween. + * Absent for discrete (int/enum/bool) knobs — they are never slewed — + * and for backends predating KNOB_SCHEMA_VERSION 2. */ + slew_max_per_s?: number; /** Agent/human-facing one-liner. */ description?: string; } diff --git a/packages/demon-client/types/wireContract.gen.ts b/packages/demon-client/types/wireContract.gen.ts index 9c6d62c5..7e933b7f 100644 --- a/packages/demon-client/types/wireContract.gen.ts +++ b/packages/demon-client/types/wireContract.gen.ts @@ -18,7 +18,7 @@ export const PROTOCOL_VERSION = 1; // Knob-manifest schema version (the `version` field served by GET // /api/knobs and the MCP list_knobs tool). Compare against the live // manifest to detect a stale build, exactly like PROTOCOL_VERSION. -export const KNOB_SCHEMA_VERSION = 1; +export const KNOB_SCHEMA_VERSION = 2; export type CommandName = | "params" diff --git a/tests/unit/test_knob_slew.py b/tests/unit/test_knob_slew.py new file mode 100644 index 00000000..4d4988d9 --- /dev/null +++ b/tests/unit/test_knob_slew.py @@ -0,0 +1,289 @@ +"""Unit tests for continuous-knob slew / rate-limiting (knobs.py). + +The bug: a fast knob sweep applied raw discontinuous values to the +running stream pipeline with no smoothing, so a single sweep could drive +the audio into distortion/clipping. The fix rate-limits continuous knobs +at the engine's once-per-tick knob read (:class:`KnobSlewLimiter`), +keyed by per-knob ceilings in the registry +(:func:`effective_slew_max_per_s`). Discrete knobs are never touched. + +These are pure (torch-free, no GPU) tests over the registry + limiter: +they pin the two regression-critical properties — a large instantaneous +jump is broken into bounded per-tick deltas, and discrete / non-registry +keys pass through unchanged — plus the supporting metadata behavior. +""" + +import math + +from acestep.streaming.knobs import ( + DEFAULT_SLEW_FRACTION_PER_S, + KnobSlewLimiter, + KnobSpec, + catalog_from_specs, + effective_slew_max_per_s, + knob_catalog, + knob_specs, +) + + +class _FakeClock: + """A hand-advanced monotonic clock so the slew math is deterministic + and independent of wall time.""" + + def __init__(self, t: float = 0.0): + self.t = float(t) + + def __call__(self) -> float: + return self.t + + def advance(self, dt: float) -> None: + self.t += float(dt) + + +def _specs_by_name(*specs) -> dict: + return {s.name: s for s in specs} + + +# -------------------------------------------------------------------------- +# effective_slew_max_per_s: which knobs are continuous, and at what rate +# -------------------------------------------------------------------------- + +def test_effective_slew_default_is_range_relative(): + # A float knob with no explicit ceiling slews at the default fraction + # of its full [min, max] span per second. + spec = KnobSpec("denoise", min_val=0.0, max_val=1.0) # span 1.0 + assert effective_slew_max_per_s(spec) == DEFAULT_SLEW_FRACTION_PER_S * 1.0 + + bipolar = KnobSpec("steer", min_val=-30.0, max_val=30.0) # span 60.0 + assert effective_slew_max_per_s(bipolar) == DEFAULT_SLEW_FRACTION_PER_S * 60.0 + + +def test_effective_slew_explicit_override_and_optout(): + explicit = KnobSpec("x", min_val=0.0, max_val=1.0, slew_max_per_s=0.5) + assert effective_slew_max_per_s(explicit) == 0.5 + + # 0 / negative opts a continuous knob OUT of slewing. + assert effective_slew_max_per_s( + KnobSpec("y", max_val=1.0, slew_max_per_s=0.0) + ) is None + assert effective_slew_max_per_s( + KnobSpec("z", max_val=1.0, slew_max_per_s=-1.0) + ) is None + + +def test_effective_slew_none_for_discrete_and_degenerate(): + # Discrete knob types are never slewed. + assert effective_slew_max_per_s(KnobSpec("seed", type="int", max_val=9.0)) is None + assert effective_slew_max_per_s( + KnobSpec("mode", type="enum", options=("a", "b")) + ) is None + assert effective_slew_max_per_s( + KnobSpec("flag", type="bool", options=(False, True)) + ) is None + # A zero-width float range has no meaningful rate. + assert effective_slew_max_per_s( + KnobSpec("pin", min_val=1.0, max_val=1.0) + ) is None + + +def test_registry_knobs_slew_only_continuous_floats(): + # Tie the registry to expected behavior: the load-bearing continuous + # knobs slew; the discrete control knobs do not. + by_name = {s.name: s for s in knob_specs(sde=False)} + slewed = {n for n, s in by_name.items() + if effective_slew_max_per_s(s) is not None} + + for cont in ("denoise", "feedback", "shift", "hint_strength", + "x0_target", "guidance_scale", "cfg_rescale", + "dcw_scaler", "ch_g0", "ch13"): + assert cont in slewed, cont + for discrete in ("seed", "feedback_depth", "steps_override", + "rcfg_mode", "dcw_enabled", "dcw_mode"): + assert discrete not in slewed, discrete + + +# -------------------------------------------------------------------------- +# KnobSlewLimiter: the actual rate limiting +# -------------------------------------------------------------------------- + +def test_large_jump_is_broken_into_bounded_per_tick_deltas(): + # The core regression guard: an instantaneous full-range jump must be + # ramped, never applied as one step. + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + specs = _specs_by_name(KnobSpec("denoise", min_val=0.0, max_val=1.0)) + ceiling = effective_slew_max_per_s(specs["denoise"]) + + # Seed at 0.0 (first sighting snaps, no startup ramp). + assert lim.apply({"denoise": 0.0}, specs)["denoise"] == 0.0 + + dt = 1.0 / 60.0 # ~60 Hz tick + prev = 0.0 + seen = [prev] + for _ in range(200): + clock.advance(dt) + cur = lim.apply({"denoise": 1.0}, specs)["denoise"] + step = abs(cur - prev) + # No tick may move more than the ceiling allows (+ float slop). + assert step <= ceiling * dt + 1e-9, (step, ceiling * dt) + prev = cur + seen.append(cur) + if math.isclose(cur, 1.0): + break + + # It did eventually converge, and it took more than one tick (i.e. it + # actually ramped instead of snapping). + assert math.isclose(prev, 1.0) + assert len([v for v in seen if 0.0 < v < 1.0]) >= 1 + + +def test_slew_converges_and_is_monotonic_toward_target(): + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + specs = _specs_by_name(KnobSpec("ch_g0", default=1.0, max_val=3.0)) + lim.apply({"ch_g0": 1.0}, specs) # seed + last = 1.0 + for _ in range(500): + clock.advance(0.01) + cur = lim.apply({"ch_g0": 3.0}, specs)["ch_g0"] + assert cur >= last - 1e-9 # never overshoots backward + assert cur <= 3.0 + 1e-9 # never overshoots the target + last = cur + if math.isclose(cur, 3.0): + break + assert math.isclose(last, 3.0) + + +def test_negative_direction_slews_symmetrically(): + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + specs = _specs_by_name(KnobSpec("denoise", min_val=0.0, max_val=1.0)) + ceiling = effective_slew_max_per_s(specs["denoise"]) + lim.apply({"denoise": 1.0}, specs) # seed high + clock.advance(0.05) + cur = lim.apply({"denoise": 0.0}, specs)["denoise"] + assert math.isclose(cur, 1.0 - ceiling * 0.05) + assert cur < 1.0 + + +def test_slow_changes_pass_through_without_lag(): + # A change slower than the ceiling is applied in full on the same tick + # — a slew limiter adds zero latency below its rate. + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + specs = _specs_by_name(KnobSpec("denoise", min_val=0.0, max_val=1.0)) + ceiling = effective_slew_max_per_s(specs["denoise"]) + lim.apply({"denoise": 0.0}, specs) # seed + clock.advance(0.1) + small = ceiling * 0.1 * 0.5 # half of what the tick budget allows + cur = lim.apply({"denoise": small}, specs)["denoise"] + assert math.isclose(cur, small) + + +def test_discrete_and_unknown_keys_pass_through_unchanged(): + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + specs = _specs_by_name( + KnobSpec("denoise", min_val=0.0, max_val=1.0), + KnobSpec("seed", type="int", max_val=9999.0), + KnobSpec("steps_override", type="int", min_val=1.0, max_val=16.0), + KnobSpec("rcfg_mode", type="enum", options=("off", "full")), + ) + lim.apply( + {"denoise": 0.0, "seed": 0, "steps_override": 8, "rcfg_mode": "off"}, + specs, + ) + clock.advance(1.0 / 60.0) + out = lim.apply( + { + "denoise": 1.0, # slewed + "seed": 4242, # int: verbatim + "steps_override": 16, # int: verbatim + "rcfg_mode": "full", # enum: verbatim + "playback_pos": 12.5, # not in registry: verbatim + "sde_denoise_curve": {"type": "constant"}, # curve spec: verbatim + }, + specs, + ) + assert out["denoise"] < 1.0 # ramping + assert out["seed"] == 4242 # instant + assert out["steps_override"] == 16 # instant (no rebuild-detection drift) + assert out["rcfg_mode"] == "full" + assert out["playback_pos"] == 12.5 + assert out["sde_denoise_curve"] == {"type": "constant"} + + +def test_first_sighting_snaps_no_startup_ramp(): + # A session that starts with a non-zero default must not ramp up from + # 0 on the first tick. + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + specs = _specs_by_name(KnobSpec("ch_g0", default=1.0, max_val=3.0)) + assert lim.apply({"ch_g0": 1.0}, specs)["ch_g0"] == 1.0 + + +def test_dt_is_capped_so_idle_resume_cannot_release_a_full_jump(): + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock, max_dt_s=0.2) + specs = _specs_by_name(KnobSpec("denoise", min_val=0.0, max_val=1.0)) + ceiling = effective_slew_max_per_s(specs["denoise"]) + lim.apply({"denoise": 0.0}, specs) # seed + clock.advance(100.0) # simulate a long idle gap + cur = lim.apply({"denoise": 1.0}, specs)["denoise"] + # Bounded by the capped dt, NOT a full snap to 1.0. + assert math.isclose(cur, ceiling * 0.2) + assert cur < 1.0 + + +def test_limits_rebuild_when_spec_map_identity_changes(): + # The session swaps its spec map wholesale when the LoRA / steering + # knob set changes; the limiter must pick up the new continuous knobs. + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + base = _specs_by_name(KnobSpec("denoise", min_val=0.0, max_val=1.0)) + lim.apply({"denoise": 0.0}, base) + + # A new map (new dict object) that adds a runtime LoRA strength knob. + extended = dict(base) + extended["lora_str_x"] = KnobSpec("lora_str_x", max_val=2.0) + extended_map = dict(extended) + lim.apply({"denoise": 0.0, "lora_str_x": 0.0}, extended_map) + clock.advance(1.0 / 60.0) + out = lim.apply({"denoise": 0.0, "lora_str_x": 2.0}, extended_map) + assert out["lora_str_x"] < 2.0 # the runtime knob is now slewed too + + +def test_reset_clears_accumulated_state(): + clock = _FakeClock() + lim = KnobSlewLimiter(clock=clock) + specs = _specs_by_name(KnobSpec("denoise", min_val=0.0, max_val=1.0)) + lim.apply({"denoise": 0.0}, specs) + clock.advance(1.0 / 60.0) + lim.apply({"denoise": 1.0}, specs) # mid-ramp + lim.reset() + # After reset the next sighting snaps straight to the target again. + clock.advance(1.0 / 60.0) + assert lim.apply({"denoise": 0.9}, specs)["denoise"] == 0.9 + + +# -------------------------------------------------------------------------- +# Catalog projection: the slew ceiling is discoverable on the wire +# -------------------------------------------------------------------------- + +def test_catalog_exposes_slew_for_continuous_knobs_only(): + cat = catalog_from_specs([ + KnobSpec("denoise", min_val=0.0, max_val=1.0), + KnobSpec("seed", type="int", max_val=9999.0), + KnobSpec("rcfg_mode", type="enum", options=("off", "full")), + ]) + assert cat["denoise"]["slew_max_per_s"] == DEFAULT_SLEW_FRACTION_PER_S * 1.0 + assert "slew_max_per_s" not in cat["seed"] + assert "slew_max_per_s" not in cat["rcfg_mode"] + + +def test_full_registry_catalog_has_slew_on_floats(): + cat = knob_catalog(sde=False) + assert "slew_max_per_s" in cat["denoise"] + assert "slew_max_per_s" in cat["guidance_scale"] + assert "slew_max_per_s" not in cat["seed"] + assert "slew_max_per_s" not in cat["steps_override"]