fix(windows): capture mouse clicks so auto-zoom works

electron/ipc/cursor/monitor.test.ts

@@ -0,0 +1,91 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { recordCursorMouseDown, recordCursorMouseUp } = vi.hoisted(() => ({
+	recordCursorMouseDown: vi.fn(),
+	recordCursorMouseUp: vi.fn(),
+}));
+
+vi.mock("./interaction", () => ({
+	recordCursorMouseDown,
+	recordCursorMouseUp,
+}));
+
+vi.mock("electron", () => ({
+	BrowserWindow: {
+		getAllWindows: () => [],
+	},
+}));
+
+vi.mock("../paths/binaries", () => ({
+	ensureNativeCursorMonitorBinary: vi.fn(),
+	getCursorMonitorExePath: vi.fn(() => "/tmp/cursor-monitor.exe"),
+}));
+
+import { setNativeCursorMonitorOutputBuffer } from "../state";
+import { handleCursorMonitorStdout } from "./monitor";
+
+/**
+ * The native cursor-monitor helper is the click source on Windows: the global
+ * uiohook hook gets silently unhooked by Windows while the recorder's main
+ * process is busy, so these `INTERACTION:` lines are what keeps click telemetry
+ * alive. They are the contract between main.cpp and the main process.
+ */
+describe("handleCursorMonitorStdout interaction protocol", () => {
+	beforeEach(() => {
+		recordCursorMouseDown.mockClear();
+		recordCursorMouseUp.mockClear();
+		setNativeCursorMonitorOutputBuffer("");
+	});
+
+	it("records a left click from INTERACTION:mousedown:1", () => {
+		handleCursorMonitorStdout(Buffer.from("INTERACTION:mousedown:1\n"));
+
+		expect(recordCursorMouseDown).toHaveBeenCalledTimes(1);
+		expect(recordCursorMouseDown).toHaveBeenCalledWith(1);
+	});
+
+	it.each([
+		["2", 2],
+		["3", 3],
+	])("maps button %s to recordCursorMouseDown(%i)", (reported, expected) => {
+		handleCursorMonitorStdout(Buffer.from(`INTERACTION:mousedown:${reported}\n`));
+
+		expect(recordCursorMouseDown).toHaveBeenCalledWith(expected);
+	});
+
+	it("defaults to the left button when no button is reported", () => {
+		handleCursorMonitorStdout(Buffer.from("INTERACTION:mousedown\n"));
+
+		expect(recordCursorMouseDown).toHaveBeenCalledWith(1);
+	});
+
+	it("records mouse releases from INTERACTION:mouseup", () => {
+		handleCursorMonitorStdout(Buffer.from("INTERACTION:mouseup\n"));
+
+		expect(recordCursorMouseUp).toHaveBeenCalledTimes(1);
+	});
+
+	it("reassembles an interaction line split across two stdout chunks", () => {
+		handleCursorMonitorStdout(Buffer.from("INTERACTION:mouse"));
+		expect(recordCursorMouseDown).not.toHaveBeenCalled();
+
+		handleCursorMonitorStdout(Buffer.from("down:1\n"));
+		expect(recordCursorMouseDown).toHaveBeenCalledWith(1);
+	});
+
+	it("keeps handling cursor STATE lines interleaved with interactions", () => {
+		handleCursorMonitorStdout(
+			Buffer.from("STATE:pointer\nINTERACTION:mousedown:1\nINTERACTION:mouseup\n"),
+		);
+
+		expect(recordCursorMouseDown).toHaveBeenCalledWith(1);
+		expect(recordCursorMouseUp).toHaveBeenCalledTimes(1);
+	});
+
+	it("ignores malformed interaction lines", () => {
+		handleCursorMonitorStdout(Buffer.from("INTERACTION:mousedown:9\nINTERACTION:wheel\n"));
+
+		expect(recordCursorMouseDown).not.toHaveBeenCalled();
+		expect(recordCursorMouseUp).not.toHaveBeenCalled();
+	});
+});

electron/native/cursor-monitor/src/main.cpp

@@ -19,6 +19,43 @@ static void stdinListener() {
     g_running.store(false);
 }
 
+namespace {
+
+struct MouseButtonWatch {
+    int virtualKey;
+    int reportedButton;  // 1 = left, 2 = right, 3 = middle
+    bool wasDown;
+};
+
+// Polling GetAsyncKeyState instead of installing a WH_MOUSE_LL hook: a low-level
+// hook lives inside the hooking process and Windows silently unhooks it whenever
+// the callback misses LowLevelHooksTimeout, which is exactly what happens while
+// the recorder's main process is busy encoding. This helper is a separate
+// process doing a cheap state read, so it cannot be unhooked.
+bool isButtonDown(int virtualKey) {
+    return (GetAsyncKeyState(virtualKey) & 0x8000) != 0;
+}
+
+void reportMouseButtonEdges(MouseButtonWatch buttons[], size_t count) {
+    for (size_t i = 0; i < count; ++i) {
+        MouseButtonWatch& button = buttons[i];
+        const bool isDown = isButtonDown(button.virtualKey);
+
+        if (isDown == button.wasDown) {
+            continue;
+        }
+
+        button.wasDown = isDown;
+        if (isDown) {
+            std::cout << "INTERACTION:mousedown:" << button.reportedButton << std::endl;
+        } else {
+            std::cout << "INTERACTION:mouseup" << std::endl;
+        }
+    }
+}
+
+}  // namespace
+
 int main() {
     std::setvbuf(stdout, nullptr, _IONBF, 0);
 
@@ -39,21 +76,39 @@ int main() {
 
     std::string lastType;
 
+    MouseButtonWatch buttons[] = {
+        {VK_LBUTTON, 1, false},
+        {VK_RBUTTON, 2, false},
+        {VK_MBUTTON, 3, false},
+    };
+    const size_t buttonCount = sizeof(buttons) / sizeof(buttons[0]);
+
+    // Buttons are sampled every 8ms so short clicks are not missed; the cursor
+    // shape only needs the original ~50ms cadence.
+    const int buttonPollMs = 8;
+    const int cursorPollEvery = 50 / buttonPollMs;
+    int tick = 0;
+
     while (g_running.load()) {
-        CURSORINFO ci = {};
-        ci.cbSize = sizeof(ci);
+        reportMouseButtonEdges(buttons, buttonCount);
+
+        if (tick == 0) {
+            CURSORINFO ci = {};
+            ci.cbSize = sizeof(ci);
 
-        if (GetCursorInfo(&ci) && (ci.flags & CURSOR_SHOWING)) {
-            auto it = cursorMap.find(ci.hCursor);
-            std::string type = (it != cursorMap.end()) ? it->second : "arrow";
+            if (GetCursorInfo(&ci) && (ci.flags & CURSOR_SHOWING)) {
+                auto it = cursorMap.find(ci.hCursor);
+                std::string type = (it != cursorMap.end()) ? it->second : "arrow";
 
-            if (type != lastType) {
-                lastType = type;
-                std::cout << "STATE:" << type << std::endl;
+                if (type != lastType) {
+                    lastType = type;
+                    std::cout << "STATE:" << type << std::endl;
+                }
             }
         }
 
-        Sleep(50);
+        tick = (tick + 1) % cursorPollEvery;
+        Sleep(buttonPollMs);
     }
 
     return 0;

src/components/video-editor/timeline/hooks/actions/useTimelineZoomActions.ts

@@ -1,7 +1,10 @@
 import type { Span } from "dnd-timeline";
 import { useCallback, useEffect, useMemo } from "react";
 import type { CursorTelemetryPoint, ZoomFocus, ZoomRegion } from "../../../types";
-import { buildInteractionZoomSuggestions } from "../../zoomSuggestionUtils";
+import {
+	buildInteractionZoomSuggestions,
+	hasExplicitClickTelemetry,
+} from "../../zoomSuggestionUtils";
 import { timelineNotifications } from "../utils/timelineNotifications";
 
 interface UseTimelineZoomActionsParams {
@@ -137,13 +140,19 @@ export function useTimelineZoomActions({
 			return;
 		}
 
+		// Recordings captured without a working global interaction hook contain no
+		// click samples at all. Rather than silently producing nothing, fall back to
+		// cursor dwell points so auto-zoom still follows the cursor, as advertised.
+		const allowDwellFallback = !hasExplicitClickTelemetry(cursorTelemetry);
+
 		const result = buildInteractionZoomSuggestions({
 			cursorTelemetry,
 			totalMs,
 			defaultDurationMs: defaultDuration,
 			reservedSpans: zoomRegions
 				.map((region) => ({ start: region.startMs, end: region.endMs }))
 				.sort((a, b) => a.start - b.start),
+			allowDwellFallback,
 		});
 
 		if (result.status === "no-telemetry") {

src/components/video-editor/timeline/zoomSuggestionUtils.test.ts

@@ -4,6 +4,7 @@ import {
 	buildInteractionZoomSuggestions,
 	CLICK_CLUSTER_MERGE_GAP_MS,
 	CLICK_CLUSTER_PAD_MS,
+	hasExplicitClickTelemetry,
 	shouldAutoApplyFreshRecordingZoomsForSource,
 } from "./zoomSuggestionUtils";
 
@@ -47,6 +48,26 @@ describe("shouldAutoApplyFreshRecordingZoomsForSource", () => {
 	});
 });
 
+describe("hasExplicitClickTelemetry", () => {
+	it("reports false for recordings that only contain move samples", () => {
+		expect(hasExplicitClickTelemetry([makeMove(0), makeMove(500), makeMove(1_000)])).toBe(
+			false,
+		);
+	});
+
+	it("reports true as soon as one explicit click sample is present", () => {
+		expect(hasExplicitClickTelemetry([makeMove(0), makeClick(500), makeMove(1_000)])).toBe(
+			true,
+		);
+	});
+
+	it("does not treat mouseup alone as click telemetry", () => {
+		expect(hasExplicitClickTelemetry([makeMove(0), makeClick(500, 0.5, 0.5, "mouseup")])).toBe(
+			false,
+		);
+	});
+});
+
 describe("buildInteractionZoomSuggestions (click-cluster logic)", () => {
 	it("creates one zoom track for a single isolated click with 500ms padding", () => {
 		const telemetry = withMoves([makeClick(5_000)], TOTAL_MS);
@@ -192,6 +213,54 @@ describe("buildInteractionZoomSuggestions (click-cluster logic)", () => {
 		expect(result.suggestions).toHaveLength(0);
 	});
 
+	it("falls back to dwell heuristics when allowed and no explicit clicks exist", () => {
+		const telemetry: CursorTelemetryPoint[] = [
+			makeMove(0, 0.1, 0.1),
+			makeMove(1_000, 0.5, 0.5),
+			makeMove(5_000, 0.6, 0.6),
+			makeMove(5_300, 0.6, 0.6),
+			makeMove(5_700, 0.6, 0.6),
+			makeMove(6_200, 0.6, 0.6),
+			makeMove(8_000, 0.9, 0.9),
+		];
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+			allowDwellFallback: true,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toHaveLength(1);
+
+		const [suggestion] = result.suggestions;
+		expect(suggestion.focus.cx).toBeCloseTo(0.6, 2);
+		expect(suggestion.start).toBeLessThan(5_700);
+		expect(suggestion.end).toBeGreaterThan(5_700);
+	});
+
+	it("still ignores dwell heuristics when the fallback is not enabled", () => {
+		const telemetry: CursorTelemetryPoint[] = [
+			makeMove(0, 0.1, 0.1),
+			makeMove(1_000, 0.5, 0.5),
+			makeMove(5_000, 0.6, 0.6),
+			makeMove(5_300, 0.6, 0.6),
+			makeMove(5_700, 0.6, 0.6),
+			makeMove(6_200, 0.6, 0.6),
+			makeMove(8_000, 0.9, 0.9),
+		];
+
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: telemetry,
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("no-interactions");
+		expect(result.suggestions).toHaveLength(0);
+	});
+
 	it("skips clusters that overlap reserved spans", () => {
 		const click = 5_000;
 

src/components/video-editor/timeline/zoomSuggestionUtils.ts

@@ -84,6 +84,15 @@ function isExplicitClickType(
 	return typeof interactionType === "string" && EXPLICIT_CLICK_TYPES.has(interactionType);
 }
 
+/**
+ * True when the recording carries at least one real click event. Recordings made
+ * while the global interaction hook is unavailable contain only `move` samples,
+ * and callers use this to decide whether dwell heuristics should stand in.
+ */
+export function hasExplicitClickTelemetry(samples: CursorTelemetryPoint[]): boolean {
+	return samples.some((sample) => isExplicitClickType(sample.interactionType));
+}
+
 function normalizeTelemetrySample(
 	sample: CursorTelemetryPoint,
 	totalMs: number,
@@ -372,13 +381,21 @@ export function buildInteractionZoomSuggestions(params: {
 	spacingMs?: number;
 	mergeGapMs?: number;
 	padMs?: number;
+	/**
+	 * When no explicit click telemetry exists at all, derive zoom windows from
+	 * cursor dwell heuristics instead of giving up. Recordings made without a
+	 * working global interaction hook contain only `move` samples, and without
+	 * this fallback they would never receive automatic zooms.
+	 */
+	allowDwellFallback?: boolean;
 }): InteractionZoomSuggestionResult {
 	const {
 		cursorTelemetry,
 		totalMs,
 		reservedSpans = [],
 		mergeGapMs = CLICK_CLUSTER_MERGE_GAP_MS,
 		padMs = CLICK_CLUSTER_PAD_MS,
+		allowDwellFallback = false,
 	} = params;
 
 	if (totalMs <= 0) {
@@ -397,10 +414,16 @@ export function buildInteractionZoomSuggestions(params: {
 		return { status: "no-telemetry", suggestions: [] };
 	}
 
-	// Only use explicit click events (uiohook telemetry) – ignore dwell heuristics
-	const clickCandidates = detectInteractionCandidates(normalizedSamples).filter(
+	// Prefer explicit click events (uiohook telemetry). Dwell heuristics are only
+	// consulted when the recording carries no clicks at all and the caller opted in.
+	const interactionCandidates = detectInteractionCandidates(normalizedSamples);
+	const explicitCandidates = interactionCandidates.filter(
 		(candidate) => candidate.source === "explicit",
 	);
+	const clickCandidates =
+		explicitCandidates.length === 0 && allowDwellFallback
+			? interactionCandidates.filter((candidate) => candidate.source === "heuristic")
+			: explicitCandidates;
 
 	if (clickCandidates.length === 0) {
 		return { status: "no-interactions", suggestions: [] };