From 63aab79263b9947a958a618c7fec168d0fa87792 Mon Sep 17 00:00:00 2001 From: Hilton Vidigal Jr Date: Tue, 15 Sep 2026 15:12:44 -0300 Subject: [PATCH 1/2] fix(zoom): fall back to cursor dwell when a recording has no clicks buildInteractionZoomSuggestions only ever considers explicit click candidates, so a recording whose telemetry carries no click events at all returns "no-interactions" and automatic zooms silently never appear. On Windows this is the common case rather than an edge case: the global uiohook hook starts without throwing but delivers no events, leaving telemetry that contains nothing but "move" samples. Add an opt-in allowDwellFallback flag. When a recording has no explicit click telemetry whatsoever, derive zoom windows from the dwell candidates that detectInteractionCandidates already computes and currently discards. Recordings that do contain clicks keep the existing click-only behaviour, so the deliberate tightening from 676d0182 is preserved. Verified against four real recordings (33s, 64s, 5m24s, 37s), all with zero click samples: every one went from "no-interactions" with 0 suggestions to "ok" with 4, 4, 16 and 4 suggestions respectively. Co-Authored-By: Claude Opus 5 (1M context) --- .../hooks/actions/useTimelineZoomActions.ts | 11 ++- .../timeline/zoomSuggestionUtils.test.ts | 69 +++++++++++++++++++ .../timeline/zoomSuggestionUtils.ts | 27 +++++++- 3 files changed, 104 insertions(+), 3 deletions(-) diff --git a/src/components/video-editor/timeline/hooks/actions/useTimelineZoomActions.ts b/src/components/video-editor/timeline/hooks/actions/useTimelineZoomActions.ts index 4f85751e3..1dcf3db21 100644 --- a/src/components/video-editor/timeline/hooks/actions/useTimelineZoomActions.ts +++ b/src/components/video-editor/timeline/hooks/actions/useTimelineZoomActions.ts @@ -1,7 +1,10 @@ import type { Span } from "dnd-timeline"; import { useCallback, useEffect, useMemo } from "react"; import type { CursorTelemetryPoint, ZoomFocus, ZoomRegion } from "../../../types"; -import { buildInteractionZoomSuggestions } from "../../zoomSuggestionUtils"; +import { + buildInteractionZoomSuggestions, + hasExplicitClickTelemetry, +} from "../../zoomSuggestionUtils"; import { timelineNotifications } from "../utils/timelineNotifications"; interface UseTimelineZoomActionsParams { @@ -137,6 +140,11 @@ export function useTimelineZoomActions({ return; } + // Recordings captured without a working global interaction hook contain no + // click samples at all. Rather than silently producing nothing, fall back to + // cursor dwell points so auto-zoom still follows the cursor, as advertised. + const allowDwellFallback = !hasExplicitClickTelemetry(cursorTelemetry); + const result = buildInteractionZoomSuggestions({ cursorTelemetry, totalMs, @@ -144,6 +152,7 @@ export function useTimelineZoomActions({ reservedSpans: zoomRegions .map((region) => ({ start: region.startMs, end: region.endMs })) .sort((a, b) => a.start - b.start), + allowDwellFallback, }); if (result.status === "no-telemetry") { diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts index 5f237d5eb..124031326 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts @@ -4,6 +4,7 @@ import { buildInteractionZoomSuggestions, CLICK_CLUSTER_MERGE_GAP_MS, CLICK_CLUSTER_PAD_MS, + hasExplicitClickTelemetry, shouldAutoApplyFreshRecordingZoomsForSource, } from "./zoomSuggestionUtils"; @@ -47,6 +48,26 @@ describe("shouldAutoApplyFreshRecordingZoomsForSource", () => { }); }); +describe("hasExplicitClickTelemetry", () => { + it("reports false for recordings that only contain move samples", () => { + expect(hasExplicitClickTelemetry([makeMove(0), makeMove(500), makeMove(1_000)])).toBe( + false, + ); + }); + + it("reports true as soon as one explicit click sample is present", () => { + expect(hasExplicitClickTelemetry([makeMove(0), makeClick(500), makeMove(1_000)])).toBe( + true, + ); + }); + + it("does not treat mouseup alone as click telemetry", () => { + expect(hasExplicitClickTelemetry([makeMove(0), makeClick(500, 0.5, 0.5, "mouseup")])).toBe( + false, + ); + }); +}); + describe("buildInteractionZoomSuggestions (click-cluster logic)", () => { it("creates one zoom track for a single isolated click with 500ms padding", () => { const telemetry = withMoves([makeClick(5_000)], TOTAL_MS); @@ -192,6 +213,54 @@ describe("buildInteractionZoomSuggestions (click-cluster logic)", () => { expect(result.suggestions).toHaveLength(0); }); + it("falls back to dwell heuristics when allowed and no explicit clicks exist", () => { + const telemetry: CursorTelemetryPoint[] = [ + makeMove(0, 0.1, 0.1), + makeMove(1_000, 0.5, 0.5), + makeMove(5_000, 0.6, 0.6), + makeMove(5_300, 0.6, 0.6), + makeMove(5_700, 0.6, 0.6), + makeMove(6_200, 0.6, 0.6), + makeMove(8_000, 0.9, 0.9), + ]; + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + allowDwellFallback: true, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(1); + + const [suggestion] = result.suggestions; + expect(suggestion.focus.cx).toBeCloseTo(0.6, 2); + expect(suggestion.start).toBeLessThan(5_700); + expect(suggestion.end).toBeGreaterThan(5_700); + }); + + it("still ignores dwell heuristics when the fallback is not enabled", () => { + const telemetry: CursorTelemetryPoint[] = [ + makeMove(0, 0.1, 0.1), + makeMove(1_000, 0.5, 0.5), + makeMove(5_000, 0.6, 0.6), + makeMove(5_300, 0.6, 0.6), + makeMove(5_700, 0.6, 0.6), + makeMove(6_200, 0.6, 0.6), + makeMove(8_000, 0.9, 0.9), + ]; + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("no-interactions"); + expect(result.suggestions).toHaveLength(0); + }); + it("skips clusters that overlap reserved spans", () => { const click = 5_000; diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.ts index fbdc5a68b..4e5ed0140 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.ts @@ -84,6 +84,15 @@ function isExplicitClickType( return typeof interactionType === "string" && EXPLICIT_CLICK_TYPES.has(interactionType); } +/** + * True when the recording carries at least one real click event. Recordings made + * while the global interaction hook is unavailable contain only `move` samples, + * and callers use this to decide whether dwell heuristics should stand in. + */ +export function hasExplicitClickTelemetry(samples: CursorTelemetryPoint[]): boolean { + return samples.some((sample) => isExplicitClickType(sample.interactionType)); +} + function normalizeTelemetrySample( sample: CursorTelemetryPoint, totalMs: number, @@ -372,6 +381,13 @@ export function buildInteractionZoomSuggestions(params: { spacingMs?: number; mergeGapMs?: number; padMs?: number; + /** + * When no explicit click telemetry exists at all, derive zoom windows from + * cursor dwell heuristics instead of giving up. Recordings made without a + * working global interaction hook contain only `move` samples, and without + * this fallback they would never receive automatic zooms. + */ + allowDwellFallback?: boolean; }): InteractionZoomSuggestionResult { const { cursorTelemetry, @@ -379,6 +395,7 @@ export function buildInteractionZoomSuggestions(params: { reservedSpans = [], mergeGapMs = CLICK_CLUSTER_MERGE_GAP_MS, padMs = CLICK_CLUSTER_PAD_MS, + allowDwellFallback = false, } = params; if (totalMs <= 0) { @@ -397,10 +414,16 @@ export function buildInteractionZoomSuggestions(params: { return { status: "no-telemetry", suggestions: [] }; } - // Only use explicit click events (uiohook telemetry) – ignore dwell heuristics - const clickCandidates = detectInteractionCandidates(normalizedSamples).filter( + // Prefer explicit click events (uiohook telemetry). Dwell heuristics are only + // consulted when the recording carries no clicks at all and the caller opted in. + const interactionCandidates = detectInteractionCandidates(normalizedSamples); + const explicitCandidates = interactionCandidates.filter( (candidate) => candidate.source === "explicit", ); + const clickCandidates = + explicitCandidates.length === 0 && allowDwellFallback + ? interactionCandidates.filter((candidate) => candidate.source === "heuristic") + : explicitCandidates; if (clickCandidates.length === 0) { return { status: "no-interactions", suggestions: [] }; From c2a5661bc434f54e772a2a50b29abc13f451a6f9 Mon Sep 17 00:00:00 2001 From: Hilton Vidigal Jr Date: Tue, 15 Sep 2026 15:12:56 -0300 Subject: [PATCH 2/2] fix(windows): report mouse clicks from the native cursor monitor handleCursorMonitorStdout parses INTERACTION:mousedown/mouseup lines and feeds them to recordCursorMouseDown, but the Windows cursor-monitor helper never emitted them: main.cpp only polled the cursor shape and printed STATE: lines. That parser has therefore been dead code, leaving uiohook as the sole click source on Windows. That single source is unreliable. A WH_MOUSE_LL hook lives inside the process that installs it, and Windows silently unhooks it whenever the callback overruns LowLevelHooksTimeout -- which is what happens while the recorder's main process is busy encoding. The symptom is silent: the hook loads, start() does not throw, and no event ever arrives, so nothing is logged and every recording ends up with move-only telemetry. Sample the three mouse buttons with GetAsyncKeyState every 8ms and emit the INTERACTION: protocol the main process already understands. A separate process performing a cheap state read cannot be unhooked, so click telemetry survives a loaded main process. Cursor-shape sampling keeps its original ~50ms cadence. Also cover the INTERACTION: contract with tests, which it never had. NOTE FOR REVIEWERS: main.cpp could not be compiled here (no MSVC/CMake available), so the bundled electron/native/bin/win32-x64/cursor-monitor.exe and its helpers-manifest.json fingerprint are stale and must be rebuilt. The C++ change itself is unverified at runtime; the TypeScript side of the protocol is covered by the new tests. Co-Authored-By: Claude Opus 5 (1M context) --- electron/ipc/cursor/monitor.test.ts | 91 +++++++++++++++++++++ electron/native/cursor-monitor/src/main.cpp | 73 +++++++++++++++-- 2 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 electron/ipc/cursor/monitor.test.ts diff --git a/electron/ipc/cursor/monitor.test.ts b/electron/ipc/cursor/monitor.test.ts new file mode 100644 index 000000000..735941a73 --- /dev/null +++ b/electron/ipc/cursor/monitor.test.ts @@ -0,0 +1,91 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { recordCursorMouseDown, recordCursorMouseUp } = vi.hoisted(() => ({ + recordCursorMouseDown: vi.fn(), + recordCursorMouseUp: vi.fn(), +})); + +vi.mock("./interaction", () => ({ + recordCursorMouseDown, + recordCursorMouseUp, +})); + +vi.mock("electron", () => ({ + BrowserWindow: { + getAllWindows: () => [], + }, +})); + +vi.mock("../paths/binaries", () => ({ + ensureNativeCursorMonitorBinary: vi.fn(), + getCursorMonitorExePath: vi.fn(() => "/tmp/cursor-monitor.exe"), +})); + +import { setNativeCursorMonitorOutputBuffer } from "../state"; +import { handleCursorMonitorStdout } from "./monitor"; + +/** + * The native cursor-monitor helper is the click source on Windows: the global + * uiohook hook gets silently unhooked by Windows while the recorder's main + * process is busy, so these `INTERACTION:` lines are what keeps click telemetry + * alive. They are the contract between main.cpp and the main process. + */ +describe("handleCursorMonitorStdout interaction protocol", () => { + beforeEach(() => { + recordCursorMouseDown.mockClear(); + recordCursorMouseUp.mockClear(); + setNativeCursorMonitorOutputBuffer(""); + }); + + it("records a left click from INTERACTION:mousedown:1", () => { + handleCursorMonitorStdout(Buffer.from("INTERACTION:mousedown:1\n")); + + expect(recordCursorMouseDown).toHaveBeenCalledTimes(1); + expect(recordCursorMouseDown).toHaveBeenCalledWith(1); + }); + + it.each([ + ["2", 2], + ["3", 3], + ])("maps button %s to recordCursorMouseDown(%i)", (reported, expected) => { + handleCursorMonitorStdout(Buffer.from(`INTERACTION:mousedown:${reported}\n`)); + + expect(recordCursorMouseDown).toHaveBeenCalledWith(expected); + }); + + it("defaults to the left button when no button is reported", () => { + handleCursorMonitorStdout(Buffer.from("INTERACTION:mousedown\n")); + + expect(recordCursorMouseDown).toHaveBeenCalledWith(1); + }); + + it("records mouse releases from INTERACTION:mouseup", () => { + handleCursorMonitorStdout(Buffer.from("INTERACTION:mouseup\n")); + + expect(recordCursorMouseUp).toHaveBeenCalledTimes(1); + }); + + it("reassembles an interaction line split across two stdout chunks", () => { + handleCursorMonitorStdout(Buffer.from("INTERACTION:mouse")); + expect(recordCursorMouseDown).not.toHaveBeenCalled(); + + handleCursorMonitorStdout(Buffer.from("down:1\n")); + expect(recordCursorMouseDown).toHaveBeenCalledWith(1); + }); + + it("keeps handling cursor STATE lines interleaved with interactions", () => { + handleCursorMonitorStdout( + Buffer.from("STATE:pointer\nINTERACTION:mousedown:1\nINTERACTION:mouseup\n"), + ); + + expect(recordCursorMouseDown).toHaveBeenCalledWith(1); + expect(recordCursorMouseUp).toHaveBeenCalledTimes(1); + }); + + it("ignores malformed interaction lines", () => { + handleCursorMonitorStdout(Buffer.from("INTERACTION:mousedown:9\nINTERACTION:wheel\n")); + + expect(recordCursorMouseDown).not.toHaveBeenCalled(); + expect(recordCursorMouseUp).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/native/cursor-monitor/src/main.cpp b/electron/native/cursor-monitor/src/main.cpp index 9a811f7be..665c4bedc 100644 --- a/electron/native/cursor-monitor/src/main.cpp +++ b/electron/native/cursor-monitor/src/main.cpp @@ -19,6 +19,43 @@ static void stdinListener() { g_running.store(false); } +namespace { + +struct MouseButtonWatch { + int virtualKey; + int reportedButton; // 1 = left, 2 = right, 3 = middle + bool wasDown; +}; + +// Polling GetAsyncKeyState instead of installing a WH_MOUSE_LL hook: a low-level +// hook lives inside the hooking process and Windows silently unhooks it whenever +// the callback misses LowLevelHooksTimeout, which is exactly what happens while +// the recorder's main process is busy encoding. This helper is a separate +// process doing a cheap state read, so it cannot be unhooked. +bool isButtonDown(int virtualKey) { + return (GetAsyncKeyState(virtualKey) & 0x8000) != 0; +} + +void reportMouseButtonEdges(MouseButtonWatch buttons[], size_t count) { + for (size_t i = 0; i < count; ++i) { + MouseButtonWatch& button = buttons[i]; + const bool isDown = isButtonDown(button.virtualKey); + + if (isDown == button.wasDown) { + continue; + } + + button.wasDown = isDown; + if (isDown) { + std::cout << "INTERACTION:mousedown:" << button.reportedButton << std::endl; + } else { + std::cout << "INTERACTION:mouseup" << std::endl; + } + } +} + +} // namespace + int main() { std::setvbuf(stdout, nullptr, _IONBF, 0); @@ -39,21 +76,39 @@ int main() { std::string lastType; + MouseButtonWatch buttons[] = { + {VK_LBUTTON, 1, false}, + {VK_RBUTTON, 2, false}, + {VK_MBUTTON, 3, false}, + }; + const size_t buttonCount = sizeof(buttons) / sizeof(buttons[0]); + + // Buttons are sampled every 8ms so short clicks are not missed; the cursor + // shape only needs the original ~50ms cadence. + const int buttonPollMs = 8; + const int cursorPollEvery = 50 / buttonPollMs; + int tick = 0; + while (g_running.load()) { - CURSORINFO ci = {}; - ci.cbSize = sizeof(ci); + reportMouseButtonEdges(buttons, buttonCount); + + if (tick == 0) { + CURSORINFO ci = {}; + ci.cbSize = sizeof(ci); - if (GetCursorInfo(&ci) && (ci.flags & CURSOR_SHOWING)) { - auto it = cursorMap.find(ci.hCursor); - std::string type = (it != cursorMap.end()) ? it->second : "arrow"; + if (GetCursorInfo(&ci) && (ci.flags & CURSOR_SHOWING)) { + auto it = cursorMap.find(ci.hCursor); + std::string type = (it != cursorMap.end()) ? it->second : "arrow"; - if (type != lastType) { - lastType = type; - std::cout << "STATE:" << type << std::endl; + if (type != lastType) { + lastType = type; + std::cout << "STATE:" << type << std::endl; + } } } - Sleep(50); + tick = (tick + 1) % cursorPollEvery; + Sleep(buttonPollMs); } return 0;