diff --git a/.github/workflows/ios-capture.yml b/.github/workflows/ios-capture.yml new file mode 100644 index 000000000..d6f943bfe --- /dev/null +++ b/.github/workflows/ios-capture.yml @@ -0,0 +1,39 @@ +name: iOS capture software checks +on: + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ios-capture-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + macos: + runs-on: macos-15 + timeout-minutes: 30 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: '22' + cache: npm + - name: Assert macOS and install locked test dependencies + run: | + test "$(uname -s)" = Darwin + npm ci --ignore-scripts + node node_modules/ffmpeg-static/install.js + - run: npm run build:ios-helper + - run: npm run test:ios-native + - run: npx tsc --noEmit + - run: npm test + - run: npm run lint + - run: npm run i18n:check + - name: Helper self-test without capture or TCC + run: | + if [ "$(uname -m)" = arm64 ]; then + electron/native/bin/darwin-arm64/recordly-ios-device-helper --self-test + else + electron/native/bin/darwin-x64/recordly-ios-device-helper --self-test + fi diff --git a/docs/ios-usb-capture.md b/docs/ios-usb-capture.md new file mode 100644 index 000000000..45f66fc9e --- /dev/null +++ b/docs/ios-usb-capture.md @@ -0,0 +1,74 @@ +# iPhone and iPad USB capture + +Recordly contains a development-preview capture path for recording the screen stream that macOS receives from a connected iPhone or iPad. The feature remains disabled by default in release packaged builds while physical-device and release gates are incomplete. An isolated local test build enables it for development validation. + +## Development preview + +The preview requires macOS 14 or later, Xcode/Swift build tools and Recordly's normal development dependencies. Build the helper before starting the app: + +```bash +npm run build:ios-helper +RECORDLY_ENABLE_IOS_CAPTURE=1 npm run dev +``` + +`RECORDLY_ENABLE_IOS_CAPTURE` is read by the Electron main process and is honored only in an unpackaged development run. Setting it does not enable the feature in a packaged application. + +Connect one iPhone or iPad using a data-capable USB cable, unlock it and approve the system Trust prompt if one appears. Open the source picker and choose **iPhone / iPad**, then select the device. Discovery stays active while the launcher is open, including when the picker is closed. **Refresh** rebuilds idle discovery and shows progress while waiting for devices. Close QuickTime or another application if it already owns the device. An empty list does not prove that the device is locked or untrusted; use Refresh after checking the cable and system prompts. + +Recordly prepares one device at a time and waits for real video samples before showing Ready. The preview is an aspect-fit, low-bandwidth framing preview. Its longest edge is limited to 480 pixels and it updates at no more than five frames per second; recording uses the native device stream rather than the preview image. Hiding the picker stops preview work without stopping a prepared or active recording. + +If Camera or Microphone permission was denied, review Recordly's access in macOS **System Settings → Privacy & Security**, grant the permission needed for the selected inputs, and relaunch before preparing again. Device-screen capture does not require granting desktop Screen Recording permission. Installed-app prompt identity still requires the physical release checks below. + +Keep the phone in one orientation during a take. A detected format or transform change stops before incompatible samples are appended and preserves the preceding valid media as an interrupted take. Some physical rotations may change only the displayed content and cannot be detected reliably. + +### Status bar for showcases + +The familiar showcase status bar uses **9:41** with full signal and battery indicators. On a physical iPhone, this is controlled by iOS during USB screen capture; Recordly has no documented public API for forcing those values. The user confirmed this appearance on the connected iPhone during the 8 September 2026 Recordly test. Check the phone's status bar after preparation; this single-device result is not a guarantee for other devices or OS versions. + +QuickTime's **File → New Movie Recording** preview is a known way to trigger this appearance: choose your iPhone in the capture-device selection menu, without pressing Record. The override lasts while that preview is open; close it before selecting the phone in Recordly if the device is busy. See [Apple's USB capture instructions](https://support.apple.com/en-gb/guide/quicktime-player/qtp356b55534/mac) and the [physical-device status bar walkthrough](https://shareshot.app/blog/Clean-9-41-Status-Bar-In-Screenshots.html). Simulator status-bar overrides apply only to simulators. + +## Video and audio behavior + +The helper accepts a compatible baseline H.264 stream for passthrough, or supported 8-bit 4:2:0 SDR input for H.264 encoding. Rec.709 primaries and matrix are required; the transfer function can be Rec.709, or sRGB on macOS 15 and later. The source's transfer metadata is preserved. Unsupported codecs, color formats and clock mappings fail before or during capture with a specific status. Recordly does not promise lossless framebuffer capture, a particular frame rate, HDR, Display P3, 4K, or 60/120 fps. The displayed geometry and format come from samples actually received; observed frame rate is omitted when it has not been measured. + +**Device audio** and **Mac narration** are separate controls. Device audio defaults on when available; narration is optional and uses a microphone identity supplied by the native helper. These settings do not reuse desktop system-audio or browser microphone IDs. If requested audio cannot be opened before recording, Recordly asks for a settings change instead of silently substituting a source. If audio disappears after video begins, the valid video is retained and the result reports the interruption. + +Device audio and narration are stored as native sidecars and aligned to the video using common-clock timing. A completed take has one mixed editor soundtrack: one source uses unity gain; two sources use 0.5 linear gain each. The video stream is copied during normal audio assembly. Original source media remains available for recovery under the normal session retention policy. + +The mobile source does not support webcam capture, pause/resume, touch control, wireless capture or simultaneous desktop capture. Keyboard, menu and HUD actions reject unsupported operations rather than presenting a false state. Desktop capture preferences are preserved when switching sources. + +## Recording and editor workflow + +The existing countdown runs only after the source is Ready. Cancelling the countdown keeps the prepared preview and creates no media. The recording timer starts only after the helper accepts the first eligible video sample. Stop enters **Saving recording…** until native finalization, validation, optional audio assembly, session commit and editor handoff finish. + +A fresh mobile take opens with its transformed native aspect, no crop, zero corner radius, no cursor overlay or cursor telemetry, and no automatic mouse-driven zooms. These are initial defaults only. Saved crop, aspect, annotations, manual zooms and cursor choices remain authoritative when a project is reopened. Mobile provenance survives raw-session reopen, project save/load and Save As even when no webcam exists. + +Interrupted recordings show the recorded outcome in the capture UI and editor. Requested audio is not described as recorded unless committed metadata confirms it. + +### iPhone mockup frames + +In the editor, open **Scene → Frame → Device frame** and choose **Black** or +**White**. The iPhone 16 Pro artwork includes the bezel, metallic edges, side +buttons and Dynamic Island. **None** restores the unframed recording. Use +**Padding** for space around the phone and **Shadow** to adjust its depth on the +background. + +The frame follows the original recording orientation and keeps the phone and +video proportions intact. A cropped recording with a different aspect fits inside +the phone with black letterboxing. Frame selection is stored in projects, +preferences and presets, and is included in thumbnails, MP4 and GIF output. +The artwork is bundled locally with its [MIT license](../public/third-party/Maya-LICENSE.txt). + +## Recovery and privacy + +The main process owns session directories and validates every media path. Native media, timing checkpoints and an atomic journal are retained when finalization fails or the helper, renderer or parent process exits unexpectedly. The launcher no longer displays a device recovery section. Existing saved or interrupted media is retained; a damaged final fragment, power loss or filesystem failure may still be unrecoverable. + +Capture, preview, audio and diagnostics remain local. Diagnostics are opt-in and omit device names, raw device identifiers, home-directory paths and media content. Notifications and sensitive screen content can appear in a recording. Recordly does not automatically suppress or redact them, and it does not bypass protected content. + +## Current verification status + +Software tests cover protocol validation, native writers and inspection, preview bounds, controller lifecycle, IPC authorization, storage/finalization, persistence, recovery and renderer routing. Synthetic media verifies geometry, timing failures, audio offsets, final video packet preservation and manifest reopen. These checks do not prove physical iPhone/iPad behavior. + +G1 and G2 have **Partial** evidence from one USB iPhone on the Apple Silicon development host. Its 1206 × 2622 H.264 High-profile stream negotiates to `420v` with Rec.709 primaries/matrix and sRGB transfer. After the transfer-policy fix, Ready and a live preview were directly observed; the user confirmed that recording completed and opened in the editor. Direct editor inspection then showed a saved recording, recorded device-audio metadata and approximately 20.2 seconds of clip duration. A synthetic native round trip separately verifies the same geometry, color tags and decoded flat patches. + +G3 and G4 remain **Not tested**. Audible device-audio playback, narration, audio sync, export fidelity, five- and thirty-minute runs, physical unplug/rotation behavior, Intel runtime, signed/notarized clean-account permission and installation checks, manual keyboard/screen-reader/reduced-motion review and the full platform matrix remain unverified. No release gate has passed. See [implementation evidence](testing/ios-usb-capture-implementation.md), [feasibility evidence](testing/ios-usb-capture-feasibility.md), and the [acceptance matrix](testing/ios-usb-capture-matrix.md). diff --git a/docs/superpowers/plans/2026-09-08-ios-usb-capture.md b/docs/superpowers/plans/2026-09-08-ios-usb-capture.md new file mode 100644 index 000000000..2c5f4c80b --- /dev/null +++ b/docs/superpowers/plans/2026-09-08-ios-usb-capture.md @@ -0,0 +1,853 @@ +# Recordly USB iPhone / iPad Capture Implementation Plan + +> **For agentic workers:** Execute this plan task-by-task using the installed `superpowers:subagent-driven-development` or `superpowers:executing-plans` workflow. Read the specification first. The checkboxes are implementation work to be performed, not claims of completed work. Review each task's contracts and tests before merging it. + +**Goal:** Add reliable, native USB device-screen recording to Recordly on macOS, including device audio, optional Mac narration, recovery and existing-editor integration. + +**Architecture:** A dedicated Swift capture helper writes native media; Electron owns its lifecycle, an exclusive recording lease, validated IPC, finalisation and persistence. React presents source selection, compressed previews and authoritative recording state. Existing desktop backends and the editor remain in place. + +**Tech stack:** The repository's existing Electron/React/TypeScript/Vite/Vitest/Biome stack; Swift, AVFoundation, CoreMediaIO, CoreMedia and VideoToolbox; the existing bundled FFmpeg resolver. No new server, driver, browser-recording dependency or required third-party Swift package. + +**Spec:** `01-feature-spec.md`, version 1.0, 8 September 2026. Suggested repository location: `docs/superpowers/specs/2026-09-08-ios-usb-capture.md`. Suggested repository location for this plan: `docs/superpowers/plans/2026-09-08-ios-usb-capture.md`. + +**Baseline:** `4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1` (`v1.4.0` release commit). All existing paths named below were surfaced through repository inspection or are directly identified existing integration paths. Paths marked **Create** are proposed. Do not assume they already exist. Line numbers are intentionally not used because the implementation spans multiple commits; function and type names are the edit anchors. + +## Global constraints + +macOS 14 is the existing native deployment floor. Build for `darwin-arm64` and `darwin-x64`; advertise only physically validated combinations. iPhone and iPad share the backend; iPad advertising requires its tests. + +The source discriminator is exactly `ios-device`. Use device-screen media, not QuickTime/iPhone Mirroring windows or a Continuity Camera lens feed. Do not fall back to Mac desktop recording when this source fails. + +No pause/resume, webcam recording, touch telemetry, wireless transport, remote device control, HDR promise or multi-device take in v1. Device audio and optional native Mac narration are in scope. Do not use browser microphone timestamps for the native recording timeline. + +Full-resolution capture stays native. Preview is capped at 480 pixels on its longest edge, 5 frames/second and 128 KiB per JPEG. Record control uses version-1 NDJSON; preview uses the separate RLIP pipe described in the spec. + +Complete means media validated, finalisation committed and editor handoff performed once—not merely “helper exited” or “file exists.” Do not delete recoverable source files after an error. Do not report native-quality, frame-rate or recovery guarantees unsupported by evidence. + +All TDD steps must first fail for the intended missing behaviour, then pass. Fixtures and fake helpers test orchestration without a phone; real hardware gates test platform behaviour. Mocks cannot satisfy those gates. + +## 1. File map and ownership + +### New shared contracts + +- **Create** `src/shared/iosCapture.ts`: portable types, runtime validators, capability constants, protocol names and type guards. No Node, Electron or DOM imports. +- **Create** `src/shared/iosCapture.test.ts`: source discrimination, runtime parsing, time-value validation and capability tests. + +### Native package + +**Create** `electron/native/ios-device-capture/Package.swift`, a macOS-14 package with library target `IOSCaptureCore`, executable target `IOSDeviceCaptureHelper` producing `recordly-ios-device-helper`, and test target `IOSCaptureCoreTests`. + +**Create** the following under `electron/native/ios-device-capture/`: + +``` +Resources/Info.plist +Sources/IOSDeviceCaptureHelper/main.swift +Sources/IOSCaptureCore/Protocol.swift +Sources/IOSCaptureCore/DeviceClassifier.swift +Sources/IOSCaptureCore/DeviceDiscovery.swift +Sources/IOSCaptureCore/CaptureEngine.swift +Sources/IOSCaptureCore/CaptureClock.swift +Sources/IOSCaptureCore/VideoWriter.swift +Sources/IOSCaptureCore/AudioWriter.swift +Sources/IOSCaptureCore/PreviewEncoder.swift +Sources/IOSCaptureCore/MediaInspector.swift +Sources/IOSCaptureCore/NativeTimingStore.swift +Tests/IOSCaptureCoreTests/ProtocolTests.swift +Tests/IOSCaptureCoreTests/DeviceClassifierTests.swift +Tests/IOSCaptureCoreTests/CaptureClockTests.swift +Tests/IOSCaptureCoreTests/VideoWriterTests.swift +Tests/IOSCaptureCoreTests/AudioWriterTests.swift +Tests/IOSCaptureCoreTests/PreviewEncoderTests.swift +Tests/IOSCaptureCoreTests/MediaInspectorTests.swift +Tests/IOSCaptureCoreTests/CaptureEngineTests.swift +``` + +A fixture generator belongs under **Create** `scripts/fixtures/ios-capture/`; synthetic media must have known timing, geometry and colour. Do not check personal recordings into the repository. + +### Electron backend + +**Create**: + +``` +electron/ipc/recording/ios/protocol.ts +electron/ipc/recording/ios/helperProcess.ts +electron/ipc/recording/ios/preview.ts +electron/ipc/recording/ios/controller.ts +electron/ipc/recording/ios/permissions.ts +electron/ipc/recording/ios/featurePolicy.ts +electron/ipc/recording/ios/storage.ts +electron/ipc/recording/ios/finalize.ts +electron/ipc/recording/ios/recovery.ts +electron/ipc/recording/recordingLease.ts +electron/ipc/register/iosCapture.ts +``` + +Create adjacent `.test.ts` files for every pure or orchestration module above, and fake-process fixtures under `electron/ipc/recording/ios/__fixtures__/`. The fake helper is a small Node program used only by tests, never shipped as the native backend. + +**Modify** existing `electron/ipc/handlers.ts`, `electron/ipc/types.ts`, `electron/ipc/state.ts`, `electron/ipc/paths/binaries.ts`, `electron/ipc/register/sources.ts`, `electron/ipc/register/recording.ts`, `electron/ipc/register/project.ts`, `electron/ipc/project/session.ts`, `electron/preload.ts`, `electron/electron-env.d.ts` and `electron/main.ts` only at their registration, type, lifecycle and handoff boundaries. Add/update adjacent tests as appropriate. Do not transplant the new implementation into the large existing `recording.ts`. + +### Renderer and persistence + +**Create** `src/hooks/useIOSDeviceRecorder.ts` and its `.test.ts`, `src/components/launch/ios/IOSDevicePanel.tsx`, `src/components/launch/ios/IOSCaptureStatus.tsx`, and `src/lib/iosCapturePresentation.ts` with its `.test.ts`. + +**Modify** existing `src/hooks/useScreenRecorder.ts` and `useScreenRecorder.test.ts`; `src/components/launch/SourceSelector.tsx`, `LaunchWindow.tsx`, `HudWindow.tsx`, `popovers/SourcePopover.tsx`, `popovers/launchPopoverTypes.ts`; `src/components/video-editor/projectPersistence.ts` and its tests; `project/useInitialEditorSource.ts`, `project/useProjectSaveActions.ts`, `state/useEditorUiState.ts`; and relevant `src/i18n/locales/*/launch.json` entries following the existing locale policy. + +### Build, verification and documentation + +**Create** `scripts/build-ios-device-helper.mjs`, `scripts/test-ios-device-helper.mjs`, `scripts/verify-ios-capture-fixture.mjs`, `docs/ios-usb-capture.md`, `docs/testing/ios-usb-capture-matrix.md`, `docs/testing/ios-usb-capture-feasibility.md` and `.github/workflows/ios-capture.yml`. + +**Modify** existing `scripts/build-native-helpers.mjs`, `scripts/smoke-packaged-binaries.mjs`, `scripts/verify-macos-distribution.mjs`, `electron-builder.json5`, `build/entitlements.mac.plist`, `build/entitlements.mac.inherit.plist` and `package.json`. Do not broaden hardened-runtime entitlements beyond what testing establishes is necessary. + +## 2. Fixed interfaces + +Implement these names once in Task 03 and use them consistently. Add fields only through an explicit contract change and test, not local aliases that silently diverge. + +```ts +export type IOSSessionId = string; +export type IOSDeviceToken = string; +export type IOSCaptureMode = 'passthrough' | 'h264-encode'; +export type AudioAvailability = 'unknown' | 'available' | 'unavailable'; +export interface NativeTime { value: string; timescale: number } +export interface IOSRecordingOptions { + deviceAudio: boolean; + microphoneToken: IOSDeviceToken | null; +} +export interface IOSVideoFormat { + codedWidth: number; + codedHeight: number; + displayWidth: number; + displayHeight: number; + codec: string; + colorPrimaries: string | null; + transferFunction: string | null; + ycbcrMatrix: string | null; + fullRange: boolean | null; + transform: readonly [number, number, number, number, number, number]; + observedFrameRate: number | null; + fingerprint: string; +} +export interface IOSDeviceSource { + sourceType: 'ios-device'; + id: string; // validated ios-device: + deviceToken: IOSDeviceToken; + displayName: string; + generation: number; + deviceAudio: AudioAvailability; +} +export interface IOSMicrophoneOption { + token: IOSDeviceToken; + label: string; +} +export type CaptureSource = TDesktop | IOSDeviceSource; +export type IOSCapturePhase = + | 'unavailable' | 'idle' | 'discovering' | 'preparing' | 'ready' + | 'starting' | 'recording' | 'stopping' | 'finalising' + | 'completed' | 'cancelled' | 'failed' | 'interrupted' | 'recoveryAvailable'; +export interface IOSCaptureFailure { + code: string; // runtime validation restricts to the spec's error-code union + recoverable: boolean; +} +export interface IOSCaptureSnapshot { + sequence: number; + generation: number; + sessionId: IOSSessionId | null; + phase: IOSCapturePhase; + devices: readonly IOSDeviceSource[]; + microphones: readonly IOSMicrophoneOption[]; + source: IOSDeviceSource | null; + options: IOSRecordingOptions | null; + format: IOSVideoFormat | null; + mode: IOSCaptureMode | null; + elapsedMs: number; + acceptedVideoSamples: number; + warningCodes: readonly string[]; + error: IOSCaptureFailure | null; +} +export interface CaptureMetadata { + version: 1; + sourceKind: 'ios-device'; + mode: IOSCaptureMode; + format: IOSVideoFormat; + deviceAudioRecorded: boolean; + narrationRecorded: boolean; + stopReason: string; + interrupted: boolean; +} +export interface CommittedIOSRecording { + sessionId: IOSSessionId; + videoPath: string; + hideOverlayCursorByDefault: true; + captureMetadata: CaptureMetadata; +} +``` + +Device-source `generation` is the inventory revision used to validate selection. Snapshot/preview `generation` is the active preparation generation, incremented on every new preparation and helper restart; these are distinct counters despite sharing the property name in separate payload types. A changed inventory revision never invalidates accepted recording media from an already prepared source. Route preview frames only for the active preparation generation. Opaque tokens are nonempty, bounded to 128 characters, and restricted to an agreed ASCII token alphabet. Session IDs are UUIDs allocated by main. Time scales must be positive integers in a bounded supported range, and tick strings must parse as signed 64-bit integers. A valid scalar field is not enough to establish that a renderer may use the referenced device/session. + +The helper's `NativeCaptureResult` consists of `sessionId`, `stopReason`, `format`, `mode`, `video: NativeMediaArtifact`, optional `deviceAudio`/`microphone` artifacts, and `timingFile: "native-timing.json"`. `NativeMediaArtifact` has `relativeName`, `mediaKind: "video" | "device-audio" | "microphone"`, `firstHostTime: NativeTime`, `duration: NativeTime`, `sampleCount` and its actual media-format metadata. No arbitrary absolute file paths originate in the renderer. + +Expose an `IOSCaptureController` with: + +```ts +getSnapshot(): IOSCaptureSnapshot; +discover(): Promise; +prepare(input: { + deviceToken: IOSDeviceToken; + generation: number; + options: IOSRecordingOptions; +}): Promise; +start(sessionId: IOSSessionId): Promise; +stop(sessionId: IOSSessionId): Promise; +cancel(sessionId: IOSSessionId, discardAcceptedMedia: boolean): Promise; +release(sessionId: IOSSessionId): Promise; +subscribe(listener: (state: IOSCaptureSnapshot) => void): () => void; +shutdown(): Promise; +``` + +Countdown is a UI substate of Ready, not an additional native/controller phase. Use the existing launcher countdown mechanism; it must not create media or duplicate main's recorder state. `start` resolves at `recordingStarted`, not the command's accepted response. `stop` resolves after commit. Automatic interruption uses the same memoised stop/finalisation path and publishes a retained `CommittedIOSRecording` through a typed controller completion callback; this prevents the result from depending on a surviving renderer promise. `cancel(..., true)` is used only after explicit discard confirmation. Completion delivery is consumed exactly once by the main-process editor-handoff callback. + +## 3. Task sequence + +### Task 01 — Establish the baseline and test evidence format + +**Files:** Create `docs/testing/ios-usb-capture-matrix.md` and `docs/testing/ios-usb-capture-feasibility.md`; use the existing package scripts unchanged. + +**Consumes:** pinned baseline and specification. **Produces:** baseline test/build results, reproducible device-matrix template and an isolated execution branch. + +- [ ] Check the current worktree before modifying it. Use a dedicated branch/worktree through the installed worktree workflow; do not reset a dirty checkout. Fetch and compare the inspected commit to the intended implementation base. + +```bash +git status --short +git rev-parse HEAD +git show --no-patch --format='%H %s' 4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1 +npm ci +npm test +npx tsc --noEmit +npm run lint +npm run i18n:check +``` + +- [ ] Record existing failures separately from regressions. Do not “fix” unrelated export, captions or dependency issues to make this feature's PR look clean. +- [ ] On macOS record `sw_vers`, `uname -m`, `xcodebuild -version`, `swift --version`, app signing identity category, device family, iOS/iPadOS version and cable/hub. Do not record serials or personal device names. +- [ ] Create evidence rows with test ID, exact command/actions, expected result, observed result, artifact location and pass/fail. Initialise them as **Not run**, not Pass. +- [ ] Commit only the templates and baseline findings: `docs: establish ios capture baseline and hardware test matrix`. + +**Review gate:** An executor can distinguish baseline failure, new test failure and hardware not tested. No feature code yet. + +### Task 02 — Prove the native path and signed-app permissions before UI work + +**Files:** Create an explicitly disposable local probe under `scripts/spikes/ios-usb-capture/` for the feasibility experiment; record results in `docs/testing/ios-usb-capture-feasibility.md`. The probe is not a shipped helper or a replacement for the tested implementation in later tasks. + +**Consumes:** Apple APIs and target hardware. **Produces:** gates G1/G2 findings and fixtures defining actual screen-device identity, format, colour, clock and sample-delivery behaviour. + +- [ ] Build a minimal Swift probe that enables the CoreMediaIO property, prints sanitised discovery attributes, selects only an eligible screen candidate, and receives video plus optional device audio. The opt-in must check its return status: + +```swift +var address = CMIOObjectPropertyAddress( + mSelector: CMIOObjectPropertySelector(kCMIOHardwarePropertyAllowScreenCaptureDevices), + mScope: kCMIOObjectPropertyScopeGlobal, + mElement: kCMIOObjectPropertyElementMain +) +var enabled: UInt32 = 1 +let status = CMIOObjectSetPropertyData( + CMIOObjectID(kCMIOObjectSystemObject), &address, + 0, nil, UInt32(MemoryLayout.size), &enabled +) +guard status == noErr else { throw ProbeError.cmio(status) } +``` + +Declare `ProbeError.cmio(OSStatus)` in the disposable probe; do not hide a failed property set. Check the SDK's public element constant spelling and use its supported equivalent when targeting an older SDK. + +- [ ] Capture a short moving-UI take using `videoSettings = [:]`; record whether samples contain compressed H.264 or pixel buffers. Capture the same fixture with an explicitly uncompressed request and compare. Inspect media descriptions, sync-sample flags, clock availability and audio-delivery timing. +- [ ] Exercise a five-minute static/dark screen, rotation in both directions, unplug, and a quiet phone that later starts playing audio. Record what is directly observable and what cannot be diagnosed. +- [ ] Package the probe inside a signed Recordly development distribution with the intended helper privacy metadata and entitlements. Test on a clean privacy state: Camera deny/allow, audio deny/allow and relaunch. Test from the installed app, not only Terminal. Do not instruct users to open QuickTime to initialise capture. +- [ ] Measure QuickTime/native source/editor-export colour and geometry on a deliberately non-personal test screen. Include compressed passthrough beginning at a keyframe and a sparse-stream duration test. +- [ ] Write exact findings and a candidate classifier fixture. The expected result is an eligible iOS screen source that works through public APIs and ordinary permissions, with a known supported recording path. If it fails, mark that combination unsupported and do not proceed with a false universal support claim. +- [ ] Remove the disposable probe from production packaging; commit sanitised evidence/fixtures: `test: document ios usb capture feasibility gates`. + +**Review gate:** G1 passes on at least the primary development combination, and G2 has a demonstrated supported path. This task is not satisfied by a README claiming the APIs work. + +### Task 03 — Lock source, capability, protocol and metadata contracts + +**Files:** Create `src/shared/iosCapture.ts` and `.test.ts`; modify `electron/ipc/types.ts` and `electron/electron-env.d.ts` to introduce `CaptureSource` at selection boundaries without making all desktop APIs accept phones. + +**Consumes:** Task 02 findings. **Produces:** the fixed interfaces in section 2 plus `isIOSDeviceSource`, `parseIOSCaptureCommand`, `parseIOSCaptureEvent`, `validateNativeTime` and `IOS_CAPTURE_CAPABILITIES`. + +- [ ] Write a failing discrimination test. It must distinguish a mobile source from both a legacy desktop source without `sourceType` and a malformed mobile object. + +```ts +it('does not mistake an ios source for a desktop source', () => { + expect(isIOSDeviceSource({ + sourceType: 'ios-device', id: 'ios-device:device_a', + deviceToken: 'device_a', displayName: 'Phone', generation: 1, + deviceAudio: 'unknown', + })).toBe(true); + expect(isIOSDeviceSource({ id: 'screen:1', name: 'Screen' })).toBe(false); + expect(isIOSDeviceSource({ sourceType: 'ios-device', id: 'screen:1' })).toBe(false); +}); +``` + +- [ ] Add failing tests for unknown protocol versions/commands, negative or noninteger generations, oversized strings, invalid rational time, stale session structure and unknown metadata fields. Unknown optional metadata may be dropped; executable-like fields must never gain meaning. +- [ ] Run `npx vitest run src/shared/iosCapture.test.ts`; confirm failures are due to missing validation. +- [ ] Implement portable types and explicit runtime validators using existing dependencies. Do not introduce a schema library solely for this feature. Define the complete error-code literal union from spec section 9, rather than leaving arbitrary strings at runtime. +- [ ] Run the targeted tests and `npx tsc --noEmit`. Expected: legacy desktop shapes still compile; mobile sources cannot enter desktop-only functions without deliberate narrowing. +- [ ] Commit: `feat: define ios capture contracts and source capabilities`. + +### Task 04 — Build the native package and deterministic control loop + +**Files:** Create the package, `Protocol.swift`, `main.swift`, `Resources/Info.plist`, `ProtocolTests.swift`, `scripts/build-ios-device-helper.mjs`, `scripts/test-ios-device-helper.mjs`; modify `scripts/build-native-helpers.mjs`, `electron/ipc/paths/binaries.ts` and package scripts. + +**Consumes:** shared wire schema. **Produces:** `recordly-ios-device-helper`, `--self-test`, and development/packaged path resolution through `getIOSDeviceCaptureHelperBinaryPath` and `ensureIOSDeviceCaptureHelperBinary`. + +- [ ] Write failing native parser tests for a valid `hello`, split lines, invalid UTF-8, a 65-KiB line, unknown version, duplicate request ID and stdin EOF. `hello` and `--self-test` must never touch camera discovery. +- [ ] Run `swift test --package-path electron/native/ios-device-capture --filter ProtocolTests`; verify the intended failures. +- [ ] Implement the CLI protocol loop with bounded input, explicit flush after output, stdout reserved for JSON, redacted stderr, and typed acceptance/error/completion events. Define `ProtocolCommand` and `ProtocolEvent` as Codable equivalents of Task 03. Keep unknown-command rejection independent of capture state. +- [ ] Add the build script: compile the Swift package in release mode separately for arm64 and x86_64 using the macOS-14 deployment target and selected Xcode SDK; pass linker options to embed the helper Info.plist; stage each product into the matching `electron/native/bin//` directory. Verify each output's architecture and deployment target rather than trusting a directory name. +- [ ] Add `test:ios-native` to invoke the new test runner. On non-macOS it prints a clear skipped-native-tests result; the macOS workflow must assert its platform so an accidental skip cannot pass that job. +- [ ] Resolve packaged binaries before source compilation. Packaged missing-helper tests must fail with `HELPER_UNAVAILABLE`, not invoke `swiftc` or SwiftPM. Development builds may compile on demand through the explicit build script. + +```bash +npm run build:native-helpers +npm run test:ios-native +./electron/native/bin/darwin-arm64/recordly-ios-device-helper --self-test +``` + +Run the final command only on its matching architecture (or use the x64 path on an Intel test machine). Expected: self-test exits zero without permissions; both architecture products are present after a cross-target build. + +- [ ] Commit: `feat: add packaged ios capture helper and protocol loop`. + +### Task 05 — Implement discovery, stable identities and native permissions + +**Files:** Create `DeviceClassifier.swift`, `DeviceDiscovery.swift`, `DeviceClassifierTests.swift`, `electron/ipc/recording/ios/permissions.ts` and `.test.ts`; modify `CaptureEngine.swift` as it is introduced to own prepared device inputs. + +**Consumes:** Task 02 identity fixtures and Task 04 protocol. **Produces:** `DeviceInventorySnapshot`, session-stable opaque tokens, and `prepare` completion only when a selected eligible screen produces valid media. + +- [ ] Write native classifier tests for the verified muxed `iOS Device` signature, ordinary external webcam, lens-only Continuity Camera, duplicate names, changed native identifier and unrelated sole-camera fallback. Use a `DeviceFacts` struct with model ID, category and advertised media types, not hardware objects, so the rules are independently testable. +- [ ] Write permission-policy tests: desktop source does not invoke this path; denied camera maps to `PERMISSION_DENIED`; audio-off does not request unnecessary audio access; denial is not automatically re-prompted. +- [ ] Implement positive classifier matching, token-to-native-ID mapping, connect/disconnect invalidation and bounded inventory reconciliation. Do not log raw native identifiers or match selected devices by label. +- [ ] Implement `DeviceDiscovery.start()`, `snapshot()` and `stop()` with the CoreMediaIO opt-in, notification handling, 250-ms coalescing, optional 2-second reconciliation and observer cleanup. +- [ ] Implement native microphone enumeration separately. Exclude the selected device audio source; return opaque tokens rather than browser IDs. Prove two same-label microphones can be selected unambiguously. +- [ ] Exercise a signed installed build: selecting a device prompts correctly, selecting/deselecting 20 times does not leak sessions, and changing the picker does not affect an active take. + +```bash +swift test --package-path electron/native/ios-device-capture --filter DeviceClassifierTests +npx vitest run electron/ipc/recording/ios/permissions.test.ts +``` + +- [ ] Commit: `feat: discover trusted ios screen sources and native microphones`. + +### Task 06 — Implement the native video writer and preparation policy + +**Files:** Create/complete `CaptureEngine.swift`, `VideoWriter.swift`, `MediaInspector.swift`, `VideoWriterTests.swift`, `MediaInspectorTests.swift`, `CaptureEngineTests.swift`; create deterministic fixture-generation recipes under `scripts/fixtures/ios-capture/`. + +**Consumes:** selected native device, observed sample format and recording boundary. **Produces:** `source-video.mov`, `IOSVideoFormat`, `IOSCaptureMode`, validated native media inspection and writer completion records. + +- [ ] Write writer tests that reject recording before preparation, wait for a post-boundary sync sample, preserve B-frame decode/presentation relationships, reject an incompatible format before appending it, and never label a zero-frame result complete. +- [ ] Write tests for source geometry/clean aperture, odd dimensions, native/uncompressed mode selection, bounded buffers, append failure, sparse/static duration, terminal tail timing and stop during start. +- [ ] Implement format selection explicitly: + +```swift +// Policy, not a hardware-model inference: +// compatible compressed H.264 + tested SDR metadata -> passthrough +// supported uncompressed SDR input -> native H.264 encode +// everything else -> unsupported format before the take begins +``` + +The implementation must inspect `CMFormatDescription` and sample payload kind. `videoSettings = [:]` is the native-sample request; a nil writer output-settings dictionary is used only after a compatible passthrough format is established. + +- [ ] Implement serial session configuration, writer/sample queues, source-format fingerprints and recording boundary handling. If passthrough fails the pre-recording gates, explicitly negotiate supported uncompressed output and revalidate it before choosing H.264 encode; otherwise reject that format. Never silently change modes during a take. Preserve timestamp relationships and valid sample durations; do not demand increasing PTS for B-frame decode order. Stop safely if compressed continuity is lost. +- [ ] Configure `.mov` fragments with initial 1 second / subsequent 10 seconds; test truncated files after different boundaries. Implement encode mode with validated H.264 settings and the spec's initial bitrate formula; pad rather than crop an odd coded size. +- [ ] Implement `MediaInspector.inspect(url:)` using AVURLAsset/track loading and a first decodable-frame check. Expose `inspectMedia` without starting capture or requesting TCC. Inspect geometry, transform, duration, codecs and audio tracks. Do not make packaged operation depend on a Darwin `ffprobe-static` binary. +- [ ] Run all native writer/inspection tests and the physical moving/static fixture. Compare compressed payloads and decoded frames for passthrough fixtures; expected: no additional encode through normal finalisation, no damaged first frames, correct duration. Record colour comparisons rather than declaring H.264 inherently colour-correct. +- [ ] Commit: `feat: record ios video with passthrough and validated h264 encoding`. + +### Task 07 — Implement common-clock audio and durable timing metadata + +**Files:** Create/complete `CaptureClock.swift`, `AudioWriter.swift`, `NativeTimingStore.swift`, `CaptureClockTests.swift`, `AudioWriterTests.swift`; extend `CaptureEngine.swift` and native protocol result encoding. + +**Consumes:** video T0, capture-session clocks and selected native audio sources. **Produces:** `device-audio.mov`, `microphone.mov`, `native-timing.json`, per-stream offsets and internal timing/gap metadata. + +- [ ] Write clock tests for a phone stream beginning at host time 100 s and a microphone beginning at 100.25 s: expected narration offset is +250 ms, not zero. Test a -120 ms pre-video start, different time scales, invalid clocks, drift, and a sample-time discontinuity. + +```swift +func testDelayedMicrophoneRetainsItsOffset() throws { + let videoStart = CMTime(value: 100_000, timescale: 1_000) + let microphoneStart = CMTime(value: 100_250, timescale: 1_000) + let result = CaptureClock.offset(firstHostTime: microphoneStart, + videoStartHostTime: videoStart) + XCTAssertEqual(CMTimeGetSeconds(result), 0.25, accuracy: 0.000_001) +} +``` + +Define `CaptureClock.offset(firstHostTime:videoStartHostTime:) -> CMTime`; clock-domain conversion is performed before calling it. Tests with equal fake clocks do not prove real mapping. + +- [ ] Implement a deployment-compatible synchronisation-clock adapter and rational native time serialization. Convert to a common host clock, preserve rates/anchors and reject unavailable mappings. Never substitute IPC arrival timestamps. +- [ ] Implement independent audio-only PCM MOV writers. Start each when actual samples arrive, retain its video-relative offset, and represent internal gaps explicitly rather than flattening them. Preserve negative offsets for trim at finalisation. Capture the selected Mac microphone natively. +- [ ] Write `native-timing.json` atomically at first stream samples, timebase/format/gap changes, periodic checkpoints and finalisation. Keep this helper-owned file separate from main's journal. Make stdin EOF finish media and this file even if stdout has already broken. +- [ ] Add tests for no audio samples, valid silence, later-starting device audio, microphone removal, duplicate audio endpoints and partial sidecar recovery. A missing optional audio file must not invalidate a valid video. +- [ ] Run native tests plus the 30-minute physical flash/click test with phone audio, narration and both. Measure alignment at both ends; expected absolute error ≤80 ms. A failed clock/audio combination remains unshipped until corrected. +- [ ] Commit: `feat: align native ios audio and persist recovery timing`. + +### Task 08 — Implement disposable low-bandwidth previews + +**Files:** Create `PreviewEncoder.swift`, `PreviewEncoderTests.swift`, `electron/ipc/recording/ios/preview.ts`, `.test.ts`; extend native command handling for `setPreviewEnabled`. + +**Consumes:** native video samples and current prepared generation. **Produces:** bounded RLIP JPEG frames independent of recording writes. + +- [ ] Write tests for the 24-byte big-endian header, split headers/payloads, invalid magic/version, oversized JPEG, stale generation, and two frames arriving while the consumer remains blocked. Expected retained queue length: one latest preview. +- [ ] Implement `PreviewFrameDecoder.push(chunk: Uint8Array)` returning zero or more validated `{generation, sequence, jpeg}` records, with an internal bounded partial-record buffer. Export the type and parser from `preview.ts`. +- [ ] Implement native preview decode/resize/JPEG conversion on its own queue. Use source format and orientation; cap longest edge 480, frequency 5 Hz and JPEG payload 128 KiB. Failed/slow preview conversion must drop work rather than back up the writer. +- [ ] Implement fd-3 asynchronous/non-blocking delivery. Closing the pipe or withholding reads cannot block capture. Stop conversion while preview is hidden. +- [ ] Run parser/native tests and record with the preview consumer deliberately stalled. Expected: media counters continue, writer queues remain bounded, no extra capture session is opened. + +```bash +npx vitest run electron/ipc/recording/ios/preview.test.ts +swift test --package-path electron/native/ios-device-capture --filter PreviewEncoderTests +``` + +- [ ] Commit: `feat: stream bounded ios source previews outside capture control`. + +### Task 09 — Implement the main-process helper client and controller + +**Files:** Create `electron/ipc/recording/ios/protocol.ts`, `helperProcess.ts`, `controller.ts`, adjacent tests and `__fixtures__/fake-ios-helper.mjs`. + +**Consumes:** Tasks 03–08 wire contracts and bundled binary resolver. **Produces:** the `IOSCaptureController` interface from section 2, process lifecycle guarantees and authoritative snapshots. + +- [ ] Write failing fake-helper tests: delayed hello; command accepted but no recordingStarted event; malformed output; partial UTF-8/JSON chunks; stderr flood; helper crash; duplicate Stop; stale session event; unexpected EOF; and native-finalised event after renderer cancellation. + +```ts +it('does not mark recording on command acceptance alone', async () => { + const harness = await createIOSControllerHarness({ start: 'accept-only' }); + await harness.prepareReadySource(); + const pending = harness.controller.start(harness.sessionId); + await harness.flushProtocol(); + expect(harness.controller.getSnapshot().phase).toBe('starting'); + harness.emitRecordingStarted(); + await pending; + expect(harness.controller.getSnapshot().phase).toBe('recording'); +}); +``` + +`createIOSControllerHarness` is a test utility created in this task, exposing a deterministic fake clock, fake helper transport, injected storage/finaliser and the methods shown. Never export it in production bundles. + +- [ ] Implement `spawn` with exact executable path, argument arrays, `shell: false`, continuous stdout/stderr drains and fd-3 preview drain. Put process parsing in the client, not in React or a giant IPC handler. +- [ ] Implement bounded pending requests, version handshake, idempotent command cache and ordered session-scoped events. Start resolves on accepted media, Stop resolves on committed media via injected finalisation. +- [ ] Implement a single controller transition function. Reject invalid phase operations, cancel pending timeouts and use a memoised finalisation promise per session so Stop/disconnect/quit cannot race into multiple writers or editor opens. +- [ ] Inject storage, permission, helper and finaliser interfaces for tests. Production binds them explicitly; do not hide dependencies behind mutable module globals. +- [ ] Run `npx vitest run electron/ipc/recording/ios/controller.test.ts electron/ipc/recording/ios/helperProcess.test.ts electron/ipc/recording/ios/protocol.test.ts`. Expected: no unresolved promises, open handles or duplicate terminal notifications after each fault. +- [ ] Commit: `feat: orchestrate ios capture from the electron main process`. + +### Task 10 — Implement storage, audio assembly and the commit boundary + +**Files:** Create `electron/ipc/recording/ios/storage.ts`, `finalize.ts` and their tests. Extend `controller.ts` to bind the production implementations. Use the existing `electron/ipc/ffmpeg/binary.ts` resolver rather than adding a second FFmpeg lookup policy. + +**Consumes:** `NativeCaptureResult`, the helper-owned timing file, source media and an approved recordings root. **Produces:** a validated `CommittedIOSRecording`, or an incomplete journal retaining recoverable media. + +Define these module contracts: + +```ts +export interface IOSSessionStorage { + sessionId: IOSSessionId; + directory: string; + journalPath: string; +} +export async function allocateIOSSessionStorage( + recordingsRoot: string, sessionId: IOSSessionId +): Promise; +export async function finalizeIOSRecording(input: { + storage: IOSSessionStorage; + nativeResult: NativeCaptureResult; +}): Promise; +export function buildAudioAlignment(offsetMs: number): { + trimStartMs: number; + delayMs: number; +}; +``` + +`offsetMs` here is a bounded duration computed by subtracting rational common-clock values, not a raw 64-bit host-clock value converted to an imprecise JavaScript number. Keep all native clock arithmetic rational until that subtraction is complete. + +- [ ] Write failing tests for writable-root validation, UUID directory creation, pre-existing directory collision, path traversal, symlink escape, low disk space, journal atomic replacement and no overwrite of a committed source. Test capacity for a second video-sized output when mixing audio; video-only commit must not unnecessarily require that duplicate. +- [ ] Write alignment tests before constructing FFmpeg arguments: + +```ts +it('preserves delayed narration and trims pre-video audio', () => { + expect(buildAudioAlignment(250)).toEqual({ trimStartMs: 0, delayMs: 250 }); + expect(buildAudioAlignment(-120)).toEqual({ trimStartMs: 120, delayMs: 0 }); +}); +``` + +- [ ] Implement fixed session-relative filenames and atomic journal updates. Periodically check available bytes with the supported Node filesystem API or a narrowly scoped native query; do not parse a shell-formatted `df` command. Estimate rates from observed bytes and elapsed media time. Start at ≥1 GiB free; reserve final-copy capacity plus 60 seconds of incoming data and at least 256 MiB safety space during capture. +- [ ] Implement native inspection of every candidate through `inspectMedia`. Check decodability, duration, geometry and expected tracks. Do not equate nonzero file size with validity. Inspection must not trigger camera discovery or permissions. +- [ ] Build FFmpeg arguments from validated internal data. Map the native video explicitly and use `-c:v copy`; never accidentally apply a video filter while calling the result passthrough. Audio processing trims pre-video samples, inserts positive offsets and internal gaps, maps rate corrections from recorded timing, resamples to 48 kHz and encodes the final soundtrack once. Use unity gain for one stream, 0.5 per stream with normalisation disabled for two streams. Encode AAC at the spec's mono/stereo rate. +- [ ] Set the output timeline from the validated video interval, not the shortest audio stream. Pad missing audio to that interval and trim excess audio. Do not use `-shortest` as a shortcut that truncates a valid video. Verify channel layouts and filter support against the bundled FFmpeg in the fixture tests. +- [ ] With no audio, commit the validated native video directly. With audio, write `recording.pending.mov`; inspect it; atomically rename to `recording.mov`; persist the session manifest; mark the journal committed; then emit the completion callback. If any later step fails, retain enough journal state to retry without another video encode or a duplicate editor open. +- [ ] Inject failures after native validation, during FFmpeg, after rename and before manifest commit. Expected: no corrupt file opened as complete, no source deletion, retry either resumes or detects the existing committed result. Drain FFmpeg progress/stderr continuously and use a stall watchdog; do not impose a tiny fixed processing limit on long takes. + +```bash +npx vitest run electron/ipc/recording/ios/storage.test.ts electron/ipc/recording/ios/finalize.test.ts +``` + +- [ ] Commit: `feat: commit ios recordings with aligned audio and recoverable storage`. + +### Task 11 — Bind validated IPC, source identity and recording exclusivity + +**Files:** Create `electron/ipc/register/iosCapture.ts`, `electron/ipc/recording/recordingLease.ts`, `electron/ipc/recording/ios/featurePolicy.ts` and adjacent tests. Modify `electron/ipc/handlers.ts`, `electron/ipc/types.ts`, `electron/ipc/state.ts`, `electron/ipc/register/sources.ts`, `electron/ipc/register/recording.ts`, `electron/preload.ts`, `electron/electron-env.d.ts` and the desktop-browser start/stop boundary in `electron/main.ts` where applicable. + +**Consumes:** the controller interface, shared source contracts, existing desktop recording entry points. **Produces:** `window.electronAPI.iosCapture`, explicit device-source dispatch and one recording owner across all backends. + +The build policy is `IOS_CAPTURE_ENABLED_BY_DEFAULT = false` until Task 18's release gate. A development-only `RECORDLY_ENABLE_IOS_CAPTURE=1` override is evaluated by main, never trusted from renderer local storage. Expose the resulting boolean in a main-owned capability response. Keep recovery/import code available independently of whether new capture is enabled. + +Define a small exclusive lease: + +```ts +export interface RecordingLease { + owner: 'desktop' | 'ios-device'; + token: string; +} +export function acquireRecordingLease(owner: RecordingLease['owner']): RecordingLease; +export function releaseRecordingLease(lease: RecordingLease): void; +export function getRecordingLease(): RecordingLease | null; +``` + +An unsuccessful acquisition throws the typed `RECORDING_BUSY` error. Releasing an already released token is idempotent; a stale token cannot release another owner's lease. Desktop retains the lease from recording start through finalisation; iOS reserves it when preparation opens its capture inputs and releases it on deselection, cancellation, failure or committed completion. + +- [ ] Write failing lease tests for competing desktop/mobile starts, duplicate release and stale release after a new owner acquires. Test existing browser, ScreenCaptureKit and Windows start entry points, not just the new helper. +- [ ] Write IPC tests for a renderer-supplied path, unknown device token, stale generation, unknown session, oversized payload, untrusted sender and attempted FFmpeg filter injection. Expected: rejection before spawning a process or opening a file. + +```ts +it('rejects renderer output paths', () => { + expect(() => parseIOSCaptureCommand({ + protocolVersion: 1, + requestId: 'r1', + command: 'prepare', + payload: { + deviceToken: 'current-token', generation: 1, + options: { deviceAudio: false, microphoneToken: null }, + outputPath: '/tmp/untrusted.mov', + }, + })).toThrow(); +}); +``` + +This example is the renderer request shape: unknown properties must be rejected, rather than copied into a native helper request. Main adds its own approved storage capability when constructing the internal native command. + +- [ ] Bind narrow preload methods for `getSnapshot`, `discover`, `prepare`, `start`, `stop`, `cancel`, `release`, state/preview subscriptions and unsubscribe. Bind the discard confirmation result to the current session. Never expose a general helper-command or filesystem-execution method to renderer code. +- [ ] Register handlers once, guard allowed application WebContents and validate session ownership in main. Native inspection and shutdown remain internal. Do not let an unrelated editor webview issue capture-control requests. +- [ ] Add the `ios-device` branch to selected-source storage, broadcast and validation. Preserve the device token and discriminator. Explicitly bypass desktop window raising/highlighting, display-bound lookups, cursor monitoring and `desktopCapturer` for this branch. +- [ ] Make the existing desktop-only `ProcessedDesktopSource` remain desktop-only where possible; introduce the shared union at selection/dispatch boundaries. Do not satisfy TypeScript by assigning the phone a fake `screen:` ID or forcing every desktop API to accept mobile sources. +- [ ] Run the IPC/lease suites plus `npx tsc --noEmit`. Expected: all desktop source branches remain exhaustive, and the disabled feature creates no helper/discovery process on launch. +- [ ] Commit: `feat: expose guarded ios capture IPC and exclusive recording ownership`. + +### Task 12 — Connect recorder controls, countdown and HUD + +**Files:** Create `src/hooks/useIOSDeviceRecorder.ts` and its tests. Modify `src/hooks/useScreenRecorder.ts`, `src/hooks/useScreenRecorder.test.ts`, `src/components/launch/LaunchWindow.tsx`, `HudWindow.tsx` and recorder capability wiring. + +**Consumes:** namespaced preload API, controller snapshots and selected capture source. **Produces:** existing Record/Stop/Discard controls backed by the correct recorder without desktop fallthrough. + +Define `useIOSDeviceRecorder()` to expose `snapshot`, `prepare(source, options)`, `startPrepared()`, `stop()`, `cancel(discardAcceptedMedia)`, `release()` and source capabilities. `prepare` takes the shared `IOSDeviceSource` and `IOSRecordingOptions`; it uses main's prepared session ID for every later operation. UI busy flags are derived from snapshots, not independent copies of native state. + +- [ ] Add tests for a device source being routed before desktop permission preparation or capture acquisition. Spies for `getDisplayMedia`, `desktopCapturer`, desktop microphone fallback and cursor hooks must remain unused throughout an iOS take. +- [ ] Test the launcher's configured countdown choices, including no countdown, against a prepared session. Cancelling only the countdown returns to Ready and keeps the preview; changing source releases the prepared session, and unplugging prevents the eventual start. Countdown is renderer presentation over a prepared source; main remains Ready until start is requested. Recording state begins only after main acknowledges accepted video. +- [ ] Test stop during `starting`, double Record, double Stop, unmount/re-mount of the HUD, delayed finalisation and renderer reload. The main snapshot restores status without starting another session or opening the editor twice. + +```ts +it('has no desktop fallback for a failed device source', async () => { + const harness = createRecorderRoutingHarness({ sourceKind: 'ios-device' }); + harness.ios.prepare.mockRejectedValue(new Error('DEVICE_NOT_FOUND')); + await harness.pressRecord(); + expect(harness.desktop.start).not.toHaveBeenCalled(); + expect(harness.desktop.preparePermissions).not.toHaveBeenCalled(); +}); +``` + +Create `createRecorderRoutingHarness` in the existing hook test suite using its mocking pattern. Its `ios` and `desktop` interfaces wrap the injected recorder actions; it must not replace real production routing with a test-only implementation. + +- [ ] Implement source-aware permission preparation. Generic startup checks must not block access to the iPhone picker behind a screen-recording permission dialog. When the user returns to a desktop source, preserve that source's existing permission flow. +- [ ] Disable pause/resume and webcam for device capture at both the UI capability layer and command dispatch layer, including keyboard shortcuts and any menu/tray actions. Unsupported actions return `UNSUPPORTED_OPERATION`, not a fake paused state. +- [ ] Keep device-audio/narration preferences separate from desktop system-audio/webcam preferences. Switching sources restores prior desktop preferences without accidentally opening a phone Continuity Camera lens feed. +- [ ] Do not start browser audio-level monitoring against a microphone currently owned by the native session. Use native level/status events for mobile narration or show selection/status without a competing meter session. +- [ ] Run both recorder hook suites and TypeScript. Expected: the existing desktop flows pass unchanged and UI never shows “Recording” merely because a command was sent. +- [ ] Commit: `feat: integrate ios recording into launcher and hud controls`. + +### Task 13 — Build the device picker and accessible preparation experience + +**Files:** Create `src/components/launch/ios/IOSDevicePanel.tsx`, `IOSCaptureStatus.tsx`, `src/lib/iosCapturePresentation.ts` and its tests. Modify `SourceSelector.tsx`, `popovers/SourcePopover.tsx`, `popovers/launchPopoverTypes.ts`, `LaunchWindow.tsx` and locale strings. + +**Consumes:** supported-platform capability, device snapshots, bounded previews and native microphone options. **Produces:** the spec's device selection, help, preview and recording-status UI within Recordly's existing design system. + +`IOSDevicePanel` accepts `{snapshot, previewUrl, onSelectDevice, onOptionsChange, onRetry, onRelease}`. Device selection passes the source token and generation, not a name match. `IOSCaptureStatus` accepts `{snapshot, onStop, onCancel}`. The pure `getIOSCapturePresentation(snapshot)` function returns translation keys, status tone and enabled actions so state mapping can be tested without introducing a new React testing framework. + +- [ ] Write presentation tests for unsupported, empty, permission denied, discovered, preparing, ready, starting, recording, interrupted and finalising. “Ready” requires accepted preview samples; “Connected” must not imply ready. A black preview never maps to “Phone locked.” +- [ ] Write source-mapping tests that preserve `ios-device` and its token and never reinterpret it as a screen. Change `mapRawSource` callers to dispatch mobile sources through an explicit branch; do not let its current default-to-screen behaviour consume a new source type. +- [ ] Add the macOS-only **iPhone / iPad** category without eagerly calling desktop `getSources` to reach it. Keep Screens/Windows behaviour intact. Cover both the embedded source popover and standalone source-selector window where those modes are used. +- [ ] Implement empty/help copy exactly around observable facts: connect a data-capable cable; unlock the device; approve Trust if prompted; retry discovery. Never claim the app knows the trust/lock state when it merely has no samples. Provide permission-specific Settings guidance only for an actual denied permission. +- [ ] Render native aspect without cropping; show delivered width × height and observed rate only when measured. Show recording-mode details without a complicated quality selector. Include fixed-orientation guidance before Record. +- [ ] Render device audio as supported/unknown/unavailable. Optional narration selects from helper-provided native microphones. Do not resolve microphones by a Chromium device ID or take the first matching display name. +- [ ] Convert each preview into a bounded object URL, revoke the previous URL on replacement/unmount, and unsubscribe on hiding/deselection. Send `setPreviewEnabled(false)` when hidden; do not stop the prepared capture session just because the popover closes. +- [ ] Use existing motion, typography, spacing, Radix interactions and icons. Keep status announcements in an appropriate live region; keyboard focus moves predictably after denial, disconnect or recovery. Reduce nonessential motion when requested. No always-running shimmer or large fake phone bezel. +- [ ] Add English strings and update other locales according to the repository's translation/fallback policy. Run `npm run i18n:check`; manually verify keyboard-only use, a screen reader and reduced motion. Run `npx vitest run src/lib/iosCapturePresentation.test.ts`. +- [ ] Commit: `feat: add accessible ios device source selection and preview`. + +### Task 14 — Persist mobile sessions and initialise the editor correctly + +**Files:** Modify `electron/ipc/types.ts`, `electron/ipc/project/session.ts`, `electron/ipc/register/project.ts`, `src/components/video-editor/projectPersistence.ts`, `project/useInitialEditorSource.ts`, `project/useProjectSaveActions.ts`, `state/useEditorUiState.ts`, and their tests. Include project snapshot/dirty-state helpers when they reconstruct `EditorProjectData`. + +**Consumes:** validated `CaptureMetadata` and a committed media path. **Produces:** session-manifest v3, backwards-compatible project provenance and fresh-import mobile defaults that survive save/reopen. + +- [ ] Write a failing regression test for a mobile recording with no webcam. Use the existing `persistRecordingSessionManifest` and `resolveRecordingSessionManifest` entry points: + +```ts +it('keeps device metadata without a webcam', async () => { + const session = createCommittedMobileSessionFixture(); + await persistRecordingSessionManifest(session); + const reopened = await resolveRecordingSessionManifest(session.videoPath); + expect(reopened?.captureMetadata?.sourceKind).toBe('ios-device'); + expect(reopened?.hideOverlayCursorByDefault).toBe(true); + expect(reopened?.webcamPath).toBeNull(); +}); +``` + +Create `createCommittedMobileSessionFixture` inside the test using a temporary session directory and minimal valid `CaptureMetadata`. Clean the directory in the test teardown. + +- [ ] Add manifest v3 readers/writers. Persist meaningful mobile metadata even without webcam media; preserve reading versions 1 and 2. Validate all linked filenames against the session directory, including symlink escape. Missing optional provenance/audio sidecars must not prevent a valid committed movie opening. +- [ ] Extend `EditorProjectData` with optional validated `captureMetadata`. Keep the existing project version 2 unless a genuinely incompatible schema change is introduced. Preserve the field in save, load, snapshots, dirty checks and Save As; do not serialize recovery paths or raw device identifiers into it. +- [ ] Apply fresh-import defaults once: `showCursor = false`, no cursor telemetry or generated mouse zooms, zero crop and border radius, source-native aspect and existing non-cropping wallpaper/padding/shadow. Use actual transformed display dimensions, not coded dimensions alone or an assumed 9:16 screen. +- [ ] Persist normal editor choices immediately through the existing project model. On reopen, explicit saved crop, aspect, annotations, manual zooms and user cursor settings win over mobile defaults. Source provenance must not silently reset them. +- [ ] Test old desktop project, old webcam manifest, new mobile raw-session reopen, saved mobile project, Save As, malformed optional metadata and missing sidecars. Verify a stale desktop cursor sidecar is not loaded into an iOS take. +- [ ] Render/export portrait and landscape fixtures through the existing editor with native aspect, 9:16 and 16:9 compositions. Compare source edges, orientation, timing and audio. Fix only source-contract/geometry integration issues needed here; do not rewrite the exporter. +- [ ] Run persistence and initial-source suites plus all project tests. Expected: no version-1/2 regression, metadata retained, and fresh defaults do not override reopened user edits. +- [ ] Commit: `feat: persist ios capture provenance and mobile-safe editor defaults`. + +### Task 15 — Implement interruption recovery and complete lifecycle cleanup + +**Files:** Create `electron/ipc/recording/ios/recovery.ts` and tests. Extend `controller.ts`, `storage.ts`, `finalize.ts`, `electron/main.ts`, the recovery IPC/UI boundary in `electron/ipc/register/iosCapture.ts`, and the native `CaptureEngine.swift`/`NativeTimingStore.swift` tests. + +**Consumes:** incomplete journals, native timing checkpoints, media inspection, controller terminal states and application lifecycle events. **Produces:** honest interrupted results, recoverable candidates and no orphaned capture resources. + +Define internal recovery contracts: + +```ts +export interface IOSRecoveryCandidate { + sessionId: IOSSessionId; + status: 'recoverable-av' | 'recoverable-video' | 'unrecoverable'; + durationMs: number | null; + reasonCode: string; +} +export async function scanIOSRecoveryCandidates( + recordingsRoot: string +): Promise; +export async function recoverIOSRecording(input: { + sessionId: IOSSessionId; + mode: 'with-audio' | 'video-only'; +}): Promise; +``` + +The implementation resolves session IDs against its own scanned, validated directory registry. No recovery request accepts an absolute directory or arbitrary file from a renderer. Candidates expose only the metadata needed for the user to choose an action. + +- [ ] Write failing process-level tests for unplug before/after first frame, format change, native append failure, microphone removal, low disk reserve, helper crash, renderer crash, parent stdin EOF, system sleep and quit. Include simultaneous Stop/disconnect and Stop/quit. Expected: one terminal workflow, no stale recording state, no silently substituted audio source. +- [ ] After a detected incompatible format change, reject the new-format sample before appending it. Finish the preceding valid take and label it interrupted. A same-sized orientation/transform change must not be missed by comparing only width and height. Do not automatically restart or splice the rotated take. +- [ ] Add disk/checkpoint interruption fixtures: intact fragment, truncated final fragment, zero-frame file, missing native timing, partially written main journal, committed movie before journal update, and missing optional audio. Expected: inspect before offering recovery; unknown A/V timing offers video-only recovery, never guessed alignment. + +```ts +it('offers video-only recovery when timing was lost', async () => { + const fixture = await createRecoveryFixture({ + video: 'valid-fragmented', timing: 'missing', journal: 'recording', + }); + const candidates = await scanIOSRecoveryCandidates(fixture.recordingsRoot); + expect(candidates[0]?.status).toBe('recoverable-video'); +}); +``` + +`createRecoveryFixture` creates synthetic files in a temporary root and injects the inspector where unit tests do not run AVFoundation. The physical recovery suite must also exercise real fragments; mocks cannot prove damaged-media recovery. + +- [ ] Route spontaneous native finalisation to the same controller completion/commit path as a user Stop. Keep the completed result in main when the renderer is gone; restore it on reload without another recording or duplicate editor handoff. +- [ ] Attach the appropriate existing Electron lifecycle hooks for renderer loss, power suspend and application quit. Acquire a sleep-prevention assertion only while actively capturing and release it on all terminal paths. Safe quit waits for bounded native finalisation; after the 10-second grace period, escalate termination while preserving files. Do not block application exit forever or promise recording across forced sleep. +- [ ] Native stdin EOF must finish writers and save native timing without depending on an alive stdout consumer. Catch broken-pipe output errors separately from file finalisation. Test parent termination while each audio stream has a nonzero offset. +- [ ] Offer Recover recording, Open folder and Discard from validated candidates. Confirm destructive discard; do not auto-delete incomplete sessions by age. Recovery must work with the capture feature disabled and without camera permission. +- [ ] Add opt-in diagnostic export containing versions, generic capabilities, redacted state transitions, timing summaries and error codes. Strip names, raw device IDs, home-directory paths and media contents. Default journal retention serves local recovery, not remote telemetry. +- [ ] Run controller/recovery/native engine tests and a physical unplug/crash cycle. Expected: each terminal path releases capture inputs, preview queues, lease, power assertion, listeners, timers and child-process handles. +- [ ] Commit: `feat: recover interrupted ios recordings and close native lifecycle gaps`. + +### Task 16 — Build deterministic end-to-end fixtures and regression evidence + +**Files:** Create `scripts/fixtures/ios-capture/generate.mjs`, `scripts/verify-ios-capture-fixture.mjs`, fixture metadata JSON and `docs/testing/ios-usb-capture-matrix.md`. Add the integration suites under `electron/ipc/recording/ios/` and native tests. Add `verify:ios-capture-fixture` to `package.json`. + +**Consumes:** implemented capture/finalisation/editor contracts. **Produces:** reproducible software tests and a physical-device acceptance report for A01–A16. + +The new verification command has this explicit interface: + +```bash +npm run verify:ios-capture-fixture -- \ + --session-dir /absolute/path/to/synthetic-or-test-session \ + --expected /absolute/path/to/expected.json \ + --report /absolute/path/to/report.json +``` + +The verifier parses named arguments without a shell, rejects unknown/missing arguments and exits nonzero for failed assertions. It uses the native media inspector and the resolved FFmpeg binary. It is an engineering utility operating on explicitly supplied test files, not a renderer-accessible path API. + +- [ ] Define `expected.json` fields: fixture version/name, expected display geometry/transform, video mode, duration/tolerance, known audio event times per stream, permitted sample loss, expected source colour metadata, and expected terminal status. Keep created test media synthetic and reproducible. Include a declared licence/provenance for any externally sourced fixture. +- [ ] Generate a short UI-like test clip with moving fine text, single-pixel edge markers, labelled colour patches, timed flashes and corresponding audio clicks. Generate portrait, landscape, odd-dimension, delayed-microphone, negative-offset, internal-gap and silent/static variants. Use a 30-minute timing variant for the hardware gate and a shorter version in routine CI. +- [ ] Unit-test the verifier against a deliberately truncated movie, wrong rotation, 250 ms audio displacement, missing soundtrack and incorrect duration. Expected: it fails the corresponding checks instead of producing an unconditional “verified” report. +- [ ] Run deterministic software integration through native writers → journal → finalisation → session reload. Compare decoded geometry/timing and eligible compressed video payloads; compare payloads after demuxing, not entire MOV file hashes, because container metadata changes during remux. +- [ ] Validate source and final-export colour with measured metadata and same-source frame comparisons. The physical on-device/QuickTime comparison is a separate visual record; do not claim pixel identity between independent recordings of a moving screen. +- [ ] Execute A01–A16 on the advertised physical matrix: at least a USB-C iPhone and a Lightning iPhone where support is claimed, an iPad before advertising iPad support, and Apple Silicon/Intel Macs before claiming both. Record exact OS versions, device family, cable/connection type, tested build SHA, result and evidence location. An untested row says **Not tested**, not Pass. +- [ ] Measure the 30-minute sync at both ends against the ≤80 ms target; five-minute static/dark duration; preview-stall isolation; 20 recording cycles; interrupted recovery; and real installed-app permissions. Record memory/counter trends rather than an unqualified “fast” label. +- [ ] Re-run existing macOS desktop, Windows WGC/browser and Linux portal regression suites. Existing hardware-dependent platform smoke checks remain required in their respective environments; an iOS-only Mac test cannot certify them. +- [ ] Commit: `test: verify ios capture fidelity recovery and desktop regressions`. + +### Task 17 — Ship signed helpers and automate noninteractive checks + +**Files:** Modify `scripts/build-ios-device-helper.mjs`, `scripts/build-native-helpers.mjs`, `scripts/smoke-packaged-binaries.mjs`, `scripts/verify-macos-distribution.mjs`, `electron-builder.json5`, signing entitlement files and `package.json`. Create `.github/workflows/ios-capture.yml`. + +**Consumes:** working helper, self-test/inspection modes and the repo's existing build/signing pipeline. **Produces:** correctly staged, signed and linked helpers plus reproducible CI checks. Installation testing is part of this task, not a documentation-only follow-up. + +- [ ] Write failing packaging tests for missing executable, incorrect executable bit, wrong architecture, absent embedded usage strings, missing required signing entitlement and mismatched protocol version. Reject a build that would compile the helper on the user's computer. +- [ ] Build/stage both target architectures under `electron/native/bin/darwin-arm64/recordly-ios-device-helper` and `darwin-x64/recordly-ios-device-helper`. Keep macOS 14 as the deployment target. Ensure the package's Info.plist is actually embedded in the executable, rather than merely present in the source directory. +- [ ] Preserve existing native helpers and build scripts. On non-macOS hosts, skip the new native compile intentionally without falsely marking Mac native tests as passed. Declare `test:ios-native` and `verify:ios-capture-fixture` scripts explicitly; do not assume they existed on the baseline. +- [ ] Update the app's camera usage text to cover connected-device screen capture. Sign the helper with the narrow entitlements established by G1; verify the parent and helper separately. Leave unchanged unrelated permissions, architectures and signing policy. No speculative blanket entitlements. +- [ ] Extend packaged smoke discovery to require the new helper only in a build that includes the feature. Run its `--self-test` without an iPhone or TCC prompt. Inspect Mach-O architecture/deployment, dynamic linkage, embedded privacy metadata and signatures for each staged architecture. Cross-compiling is not physical compatibility testing. +- [ ] Add a macOS workflow for dependency installation, native tests, TypeScript/Vitest, helper build and noninteractive self-test. Reuse existing release signing/notarisation integration rather than adding a duplicate secret-dependent release system. Use the repository's supported runner/Xcode versions; document any SDK requirement separately from the macOS deployment floor. +- [ ] Build the normal application and execute its existing smoke/distribution checks with their inspected argument contract. In an environment configured for the repository's normal Mac builds, the standard build entry point is: + +```bash +npm run build:mac +npm run smoke:packaged-binaries +``` + +The inspected distribution verifier requires `--arch` (`arm64` or `x64`) and `--team-id`; its release directory defaults to `release`. Run it against the actual signed artifacts for both architectures, with the configured Apple Team ID: + +```bash +: "${APPLE_TEAM_ID:?Set APPLE_TEAM_ID to the team that signed these artifacts}" +for arch in arm64 x64; do + npm run verify:macos-distribution -- \ + --arch "$arch" \ + --team-id "$APPLE_TEAM_ID" \ + --release-dir release \ + --report "release/macos-distribution-report-$arch.json" +done +``` + +The variable is a required release-environment input, not a Team ID supplied by this document. Preserve the existing signature/notarisation checks; an unsigned local build cannot satisfy them. + +- [ ] Install a signed/notarised artifact on a clean test account without the repository, Xcode or command-line tools. Test first permission prompt, denial, later grant/relaunch, discovery, recording, audio, export and recovery. Inspect that the prompt identifies the intended application, not Terminal or a temporary helper. +- [ ] Mark A15 passed only for the installed architectures tested. A Mac CI compile and a mock capture run are not sufficient. Commit: `build: package sign and verify ios device capture helpers`. + +### Task 18 — Complete documentation, release gates and controlled enablement + +**Files:** Create/update `docs/ios-usb-capture.md`, `docs/testing/ios-usb-capture-feasibility.md`, `docs/testing/ios-usb-capture-matrix.md`, feature policy tests and user-facing locale/release copy. Update the repository README only with the tested supported feature set. + +**Consumes:** all implementation/test results, G1–G4 evidence and the actual support matrix. **Produces:** a releasable feature whose user-facing promises match verified behaviour. + +- [ ] Document the supported cable/trust flow, source selection, device audio versus Mac narration, fixed orientation, no pause/webcam in v1, interruption recovery and permission-denied recovery. Explain that notifications and protected content are not automatically fixed or redacted. +- [ ] Document modes as received-stream preservation versus a validated high-quality encode. Do not use “lossless iPhone recording,” guaranteed native frame rate, guaranteed HDR/P3, or guaranteed zero-loss crash recovery. Preserve the distinction between source recording quality and a later edited export. +- [ ] Populate the acceptance matrix with evidence or an explicit failure/not-tested status. Cross-check every F and A requirement against the traceability table below. Remove unsupported device/OS claims from copy; do not silently weaken an acceptance criterion after a failing test. +- [ ] Keep the internal default off until G1–G3 pass. Enable a labelled beta through the normal build policy, not a remote configuration service. General enablement requires G4 and the complete applicable acceptance matrix. Narration cannot remain broken while calling this full v1 complete. +- [ ] Test rollback/disablement: the source category disappears, no new device capture starts, and previously committed videos/projects plus recovery candidates remain accessible. Enabling/disabling new capture must not corrupt existing data. +- [ ] Run the final quality commands in section 5 and record their actual results. List baseline failures separately, without claiming they are passing. Get a review of native timing/format logic and a separate review of IPC/storage/security boundaries. +- [ ] Commit: `docs: publish tested ios capture workflow and release evidence`. Change the compiled default only in the release commit that includes the successful gate evidence. + +## 4. Dependencies and reviewable delivery slices + +Use this order when implementing sequentially: **01 → 02 → 03 → 04 → 05 → 06 → 07 → 08 → 09 → 10 → 11 → 12 → 13 → 14 → 15 → 16 → 17 → 18**. + +The feasibility probe comes before polishing the source picker. Tasks 05–08 can be developed in partially parallel branches after the shared contracts/package are stable, but they share `CaptureEngine.swift`; assign one integration owner rather than allowing overlapping rewrites. Controller tests can use the fake helper while native work proceeds. Real integration still waits for the actual native contracts and media evidence. + +A practical review sequence is: + +| Review slice | Tasks | Independently reviewable result | +|---|---|---| +| Feasibility and contracts | 01–04 | Documented platform evidence, protocol, testable helper skeleton and staging. | +| Native media | 05–08 | Correct source selection, native capture/audio/timing and independent preview. | +| Main lifecycle and storage | 09–11 | Process control, safe IPC, exclusive ownership and atomic media commit. | +| Product integration | 12–14 | Launcher/HUD experience and durable editor/session behaviour. | +| Reliability | 15–16 | Recovery, fault injection and media/regression evidence. | +| Distribution and release | 17–18 | Installed-app verification, documented support and controlled enablement. | + +Each slice stays behind the disabled feature flag until its dependencies are ready. Do not merge a half-wired source category that accidentally records the desktop. Native APIs and file formats are reviewed before downstream UI assumptions are locked in. + +## 5. Verification commands and evidence rules + +These baseline scripts were observed in the inspected package. Run them from the implementation checkout; this plan has not executed them: + +```bash +npm test +npx tsc --noEmit +npm run lint +npm run format:check +npm run i18n:check +``` + +The implementation adds these commands; they are not present in the untouched baseline: + +```bash +# Requires macOS and the development Swift toolchain. +npm run test:ios-native + +# Creates/stages the new helper without building the entire app. +node scripts/build-ios-device-helper.mjs + +# Verifies a generated/test recording against explicit expectations. +npm run verify:ios-capture-fixture -- \ + --session-dir /absolute/path/to/session \ + --expected /absolute/path/to/expected.json \ + --report /absolute/path/to/report.json +``` + +The three paths in the fixture command are test-run inputs to substitute with actual generated file locations. They are not assumed files shipped with this documentation. The generator/verifier task defines the necessary file format and rejects missing inputs. + +Use the repository's normal packaging workflow for release artifacts, followed by the extended packaged smoke and distribution verification. Run native self-tests in CI without devices or permission prompts. Run physical capture tests separately on the installed app. + +A release evidence record contains test ID, build SHA, OS/architecture, device family, test setup, expected result, observed result and evidence location. Record no raw phone serial/UDID or personal device name. A test result is **Pass**, **Fail**, **Not tested** or **Not applicable with reason**; “probably works” is not a status. + +A nonzero test command is not hidden by `|| true`, excluding relevant tests, deleting a gate, or changing a target until it matches the observed output. Distinguish pre-existing repository failures from newly introduced failures, and disclose both at handoff. + +## 6. Requirements-to-task traceability + +| Spec requirement | Implemented/tested by | +|---|---| +| F01 macOS source/discovery/help | 02, 05, 11, 13, 17 | +| F02 positive screen-source classification | 02, 05, 16 | +| F03 one device, preview and actual format | 05, 06, 08, 11, 13 | +| F04 native video dimensions/timing | 02, 06, 07, 10, 16 | +| F05 device audio and native narration | 05, 07, 10, 12, 13, 16 | +| F06 countdown/Record/Stop/Discard | 09, 11, 12, 13, 15 | +| F07 editor handoff/defaults | 09, 10, 14, 16 | +| F08 interruptions and storage/process failures | 06, 07, 09, 10, 15, 16 | +| F09 raw session/project provenance | 10, 14, 16 | +| F10 prebuilt signed helpers | 04, 17 | +| F11 redacted diagnostics/recovery | 07, 10, 15, 18 | +| F12 desktop/platform regressions | 11, 12, 14, 16, 17 | + +| Acceptance criterion | Evidence tasks | +|---|---| +| A01 discovery/identity | 02, 05, 13, 16 | +| A02 permissions | 02, 05, 11, 12, 17 | +| A03 readiness/start cancellation | 06, 09, 12, 16 | +| A04 geometry | 06, 08, 14, 16 | +| A05 compressed passthrough | 06, 10, 16 | +| A06 validated H.264 encoding | 06, 16 | +| A07 colour | 02, 06, 14, 16 | +| A08 audio timing | 07, 10, 16 | +| A09 static/dark duration | 06, 07, 10, 16 | +| A10 interruptions | 09, 10, 15, 16 | +| A11 session/project persistence | 10, 14, 16 | +| A12 editor/export agreement | 14, 16 | +| A13 performance/backpressure | 06, 08, 09, 16 | +| A14 repetition/cleanup/idempotency | 05, 09, 12, 15, 16 | +| A15 installed distribution | 04, 17 | +| A16 desktop regressions | 11, 12, 14, 16, 17 | + +**G1:** Tasks 02, 05 and 17 establish source classification and signed permission behaviour. **G2:** Tasks 02, 06, 10, 14 and 16 establish media fidelity and decoder compatibility. **G3:** Tasks 07, 10 and 16 establish actual clock/audio synchronisation. **G4:** Tasks 14–18 establish recovery, persistence, distribution and release readiness. + +## 7. Final implementation handoff checklist + +- [ ] Specification and plan are checked into their documented repository locations; no requirement was silently dropped. +- [ ] Shared contract names and version fields match in Swift, main, preload, renderer and tests. +- [ ] There is no device-to-desktop fallback, unrelated camera selection or hidden desktop permission prerequisite. +- [ ] No source or accepted audio is silently discarded; known unsupported paths are reported before recording. +- [ ] Session metadata survives without a webcam; existing projects preserve their behaviour and new projects preserve user edits. +- [ ] Recovery preserves native timing and media after parent failure and is available with capture disabled. +- [ ] Desktop, native and packaged checks have actual recorded results; hardware gates have physical evidence. +- [ ] Installed architecture/OS claims match the support matrix; source/export quality wording matches measurements. +- [ ] The release commit explicitly records feature-flag status, remaining known limitations and the tested artifact. + +**Implementation status of this document:** planned, not implemented. Source inspection establishes integration points; it does not establish working device capture, a successful macOS build or passing application tests. The first execution task creates that baseline evidence, and the later gates establish runtime correctness. diff --git a/docs/superpowers/specs/2026-09-08-ios-usb-capture.md b/docs/superpowers/specs/2026-09-08-ios-usb-capture.md new file mode 100644 index 000000000..df7b0db0c --- /dev/null +++ b/docs/superpowers/specs/2026-09-08-ios-usb-capture.md @@ -0,0 +1,470 @@ +# Recordly — USB iPhone / iPad Screen Capture +## Feature specification v1.0 + +Date: 8 September 2026 +Repository: `webadderallorg/Recordly` +Inspected baseline: `4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1` (`chore: release v1.4.0`). Its parent is `68bca43f6a8c1842ab9acfab75cfad817f737cb4`, used in the earlier feasibility assessment. +Companion document: `02-implementation-plan.md`. + +This is a proposed product and engineering specification, not a claim that the feature has been implemented or tested. Repository integration points and documented API behaviour were inspected. A connected-device test, a macOS build, and signed-distribution permission tests have not been performed for this specification. Numerical thresholds below are proposed acceptance targets, not observed performance. + +## 1. Product decision + +Add a first-class **iPhone / iPad** recording source to Recordly on macOS. A user connects a device with a data-capable USB cable, unlocks it and grants the normal device trust permission, selects its screen, records, and lands in Recordly's existing editor. + +Use a bundled Swift helper with CoreMediaIO and AVFoundation. Do not capture an iPhone Mirroring or QuickTime window. Do not implement an iOS app, network transport, screen-control service, or a replacement editor. + +The delivered video should be as faithful to the **stream macOS receives from the device** as practical. This is deliberately not a promise of lossless access to the physical display framebuffer, 4K, 60/120 fps, HDR, Display P3, or parity with an on-device recording. Input capabilities must be measured rather than inferred from the phone model. + +The primary path preserves a compatible compressed device stream without another video encode. An explicitly identified H.264 encode path handles supported uncompressed SDR inputs. Both feed the existing editor with a normal local media file. Normal capture finalisation must not re-encode the video a second time. + +### 1.1 Success from the user's perspective + +“I plugged in my phone, selected it in Recordly, recorded my app with its sound and optional narration, and immediately edited a clean portrait or landscape video. There was no desktop window border, stray Mac cursor, or unexplained quality drop.” + +### 1.2 Existing code that matters + +At the inspected baseline: + +- `electron/ipc/types.ts` and launcher source types describe screen/window sources, not physical iOS capture sources. +- `SourcePopover.tsx` fetches Electron desktop sources; `launchPopoverTypes.ts` defaults an unrecognised source to a screen. A phone must bypass that default. +- `useScreenRecorder.ts` orchestrates recording and selects native desktop capture using screen/window identifiers. Routing must happen before desktop permissions, cursor capture and browser fallback. +- `electron/ipc/project/session.ts` writes session-manifest version 2 and removes the manifest when no webcam is attached. Mobile metadata cannot survive through that path unchanged. +- `projectPersistence.ts` has project version 2, a `videoPath`, and editor settings. Mobile provenance requires an optional, validated addition without resetting user edits. +- Native helpers are already packaged for `darwin-arm64` and `darwin-x64` with a macOS 14 deployment target. Native binaries are unpacked from ASAR; users must not need Xcode to run a packaged app. + +These observations justify a new capture backend plus targeted source, session and launcher changes—not a rewrite of the recording architecture. See the source register at the end. + +## 2. Scope and explicit exclusions + +### 2.1 Required for v1 + +| ID | Requirement | +|---|---| +| F01 | macOS-only source category with lazy device discovery and connect/unlock/trust guidance. | +| F02 | Distinguish eligible device-screen sources from built-in cameras, webcams, virtual cameras and ordinary Continuity Camera lens feeds. | +| F03 | Select one device; show a low-bandwidth live preview and actual observed format information. | +| F04 | Record the device screen through a native pipeline at its delivered dimensions and timing. | +| F05 | Record device audio when exposed and enabled; optionally record a selected Mac microphone for narration. | +| F06 | Use the existing countdown, Record, Stop and Discard flow; accurately report starting, recording and finalising states. | +| F07 | Open a validated recording in the existing editor with mobile-safe initial defaults. | +| F08 | Handle device removal, permission denial, format changes, storage failures and process failures without reporting false success or deleting recovery candidates. | +| F09 | Persist capture metadata through raw-session reopen and project save/load. | +| F10 | Ship prebuilt, signed and verified native helpers for the architectures advertised as supported. | +| F11 | Provide opt-in, redacted diagnostics and an explicit recovery flow. | +| F12 | Preserve desktop capture behaviour on macOS, Windows and Linux. | + +Support iPhone and iPad through the same backend. Advertise iPad support only after the iPad acceptance cases pass. There is no hardcoded hardware-model list in the UI. The support matrix lists tested device/OS combinations; device discovery is not itself a compatibility guarantee. + +### 2.2 Deliberately outside v1 + +Wireless/AirPlay capture; Windows/Linux physical iPhone capture; Android capture; an iOS companion app; Mac-controlled touch/keyboard input; touch coordinates, automatic touch-driven zooms or gesture overlays; simultaneous multi-device recording; simultaneous desktop and phone capture; webcam overlays while recording this source; pause/resume; automatic reconnection into the same take; seamless orientation changes within a take; HDR/wide-gamut guarantees; ProRes/HEVC quality selectors; device bezels, fake status bars or notch reconstruction; independent device/narration mixing controls in the editor; cloud upload or telemetry services. + +Existing manual zoom, backgrounds, annotation, trimming and export features remain available. This feature does not introduce another subscription, login, or service dependency. + +### 2.3 Why not the alternatives? + +A mirrored-window integration depends on another application, visible window geometry and its interaction/permission behaviour. Importing a QuickTime recording is a useful fallback but is not integrated capture. A companion iOS application adds installation, distribution and transport concerns without being necessary for the selected wired workflow. Apple documents recording a connected device directly in QuickTime; CoreMediaIO exposes an opt-in for screen-capture devices. [A1, A2] + +## 3. UX and interaction contract + +### 3.1 Source picker + +Add an **iPhone / iPad** tab or section to the existing source picker on macOS only. Reuse Recordly's existing popovers, buttons, typography, spacing, semantic colours and Phosphor icons. Do not add a SwiftUI settings window or a second visual design system. + +Do not enumerate desktop sources merely to populate the device tab. In particular, the device flow must not require Mac Screen Recording or Accessibility permission when its native capture path does not need them. Source-specific permission preparation must also cover launcher startup behaviour, not only the Record button. + +Empty-state copy: + +> Connect your iPhone or iPad with a USB cable. Unlock it and tap Trust if asked. + +Secondary help: “Use a cable that supports data. Close QuickTime or other apps using the device.” Actions: **Refresh**, **Connection help**. Do not claim the device is untrusted or locked just because discovery returned an empty array. The public capture API may not distinguish those states. + +A discovered row shows its human-readable name, a device icon and availability. Duplicate names must remain distinguishable, using a session-local ordinal or other non-sensitive distinction. Identification and routing use opaque IDs, never names. + +No automatic recording, no automatic selection of a different device, and no global camera permission prompt merely from opening Recordly. Permissions are initiated by the device-source interaction. + +### 3.2 Preview and preparation + +Selecting a row obtains an exclusive prepared session and displays a correctly oriented, aspect-fit preview. The same native session continues into recording; opening/closing the popover must not recreate the physical capture session. + +Preview is deliberately lightweight: longest image edge at most 480 pixels and at most 5 updates/second. It is a framing/status preview, not a measure of capture quality. Show **“Preview quality only — recording uses the device stream.”** Do not persist preview frames. + +After media arrives, show **“1170 × 2532 · SDR”** or equivalent *observed* values. Show an observed frame-rate range only once sufficient samples exist; mark it as observed. Never fill it from a marketing model table. A source that has been discovered but has not produced frames is not Ready. + +When preview is hidden, suspend its JPEG conversion and delivery, not capture. A frozen/blocked renderer must not obstruct media writing. + +### 3.3 Recording controls + +For this source, replace the meaning of the desktop System Audio control with a separate, clearly labelled **Device audio** setting. Do not silently repurpose the stored desktop preference. Default device audio on when available, Mac narration off, and use a separate mobile preference namespace. + +Narration uses the native helper's microphone list. Chromium device IDs are not assumed to equal AVFoundation IDs. Do not resolve microphones by label when names collide. Exclude the selected phone's audio endpoint from narration choices to avoid recording it twice. + +Hide or disable webcam and pause/resume controls with an explanation. The corresponding keyboard shortcuts and IPC endpoints must reject unsupported operations too. Preserve desktop settings while displaying effective mobile settings. + +Record begins only from Ready, following the existing countdown. During countdown, Cancel returns to Ready and keeps no media. Revalidate source identity and availability before arming. Show **Starting recording…** until the helper has accepted the first decodable video sample after the recording boundary. Only then show the recording indicator and timer. + +Stop changes the UI to **Saving recording…**. Disable further starts until the recording is committed or a recoverable failure is displayed. Keep the native recording status authoritative across all launcher/HUD windows. + +Discard before media starts cancels immediately. Discard after capture starts asks **“Discard this recording?”**, with **Keep recording** and **Discard**. Deletion is limited to the current session directory and must never follow arbitrary renderer paths. + +### 3.4 During recording + +Show the selected device, elapsed time, active audio sources and Stop. The phone preview is view-only: clicking it does not interact with iOS. The user continues operating the physical phone. + +A quiet source or a dark picture is not an error. Do not identify phone lock, DRM, or a stalled stream by black-pixel heuristics. No-frame warnings describe the observation, not a guessed cause. Do not stop a sparse/static stream solely because its pixels or frame timestamps have not changed; this must be covered by the media-duration tests. + +On an observed incompatible format change, stop before appending incompatible samples and preserve the preceding take. Show: + +> The device's video format changed. Your recording up to that point was saved. Keep one orientation during a take and start a new recording. + +This is not a promise to detect every physical rotation: some device streams change their content without an identifiable format/transform change. Tell the user to use a fixed orientation. Recordly does not lock the phone orientation remotely. + +### 3.5 Accessibility and polish + +All selection and recording controls are keyboard-operable, have visible focus, and expose names and states to accessibility APIs. Announce Ready, Recording, Saving and terminal errors, not every progress tick. Preserve focus after discovery updates. Never rely on colour alone. + +Use existing motion tokens; status crossfades should be approximately 120–180 ms, respect reduced motion, and never animate live-preview layout continuously. Put detailed troubleshooting in a disclosure, not a mandatory setup wizard. All new strings use the existing localisation system; untranslated keys follow the repository's established fallback policy and pass `i18n:check`. + +## 4. Architecture and ownership + +### 4.1 Components + +| Component | Owns | Must not own | +|---|---|---| +| React source UI / recorder adapter | User intent, preview rendering, accessible status presentation | Capture process lifetime, destination paths, raw full-resolution media | +| Electron `IOSCaptureController` | Exclusive recording lease, helper lifecycle, validated IPC, progress/status snapshot, storage, finalisation and editor handoff | Native camera frames or UI-specific React state | +| Swift helper | Discovery, AVFoundation sessions, native identities, sample timing, native media writing, preview conversion | Project editor settings, arbitrary renderer instructions, network services | +| Existing editor integration | Local media import, persisted presentation defaults, save/load and export | Keeping the phone connected or driving its capture session | + +Full-resolution samples flow directly from AVFoundation to native writers. Only low-resolution JPEG previews cross into JavaScript. Use a separate Swift package for a testable capture-core library and a small executable. Do not turn `ScreenCaptureKitRecorder.swift` into a desktop/mobile switchboard. + +### 4.2 Source identity + +Introduce a discriminated `CaptureSource` union containing the existing desktop source and a new `IOSDeviceSource`. The new member has `sourceType: "ios-device"`, `id: "ios-device:"`, `deviceToken`, `displayName` and observed capabilities. Desktop-only functions retain desktop-specific argument types. + +The token maps to AVFoundation's identity inside the helper, is stable through discovery refreshes within that helper lifetime, and is invalidated when that identity disappears or the helper restarts. Never persist it as a reconnect guarantee. v1 does not auto-select a remembered phone on the next application launch. + +Capabilities include video availability, device-audio availability (`unknown | available | unavailable`), native microphone options, recording mode, format descriptor, and explicit `supportsPause: false`, `supportsWebcam: false`, `supportsTouchTelemetry: false`. + +Do not pass a mobile source through `mapRawSource`, `desktopCapturer`, display/window parsing, window highlighting, cursor tracking, or browser desktop fallback. An iOS capture failure remains an iOS capture failure; recording the Mac desktop instead is unacceptable. + +### 4.3 Lifetime and exclusivity + +There is one main-process controller and at most one mobile helper at a time. Start it lazily for discovery. Retain the selected native session for preview and capture. Close an unselected discovery-only helper once no device UI client remains; stop preview when hidden. Release a prepared device on deselection or explicit close. + +Integrate a small shared recording lease with existing desktop start paths so no window can start desktop recording while mobile capture is arming, recording or finalising, and vice versa. This is a narrowly scoped concurrency guard, not a universal backend framework rewrite. + +Main-process state is authoritative. Renderer subscriptions receive a complete snapshot immediately and ordered updates subsequently. Unmounting a popover cannot stop a take. A renderer crash while recording should trigger safe stop/finalisation in the main process and preserve the result for reopen. + +## 5. Native discovery and permission contract + +### 5.1 Discovery + +Before enumeration, set `kCMIOHardwarePropertyAllowScreenCaptureDevices` through CoreMediaIO and check its `OSStatus`. Do not enable the wireless-screen-capture property. Keep this opt-in scoped to the dedicated helper process. [A2] + +Use an availability-aware AVFoundation discovery session. Probe muxed external devices. Allow an additional discovery category only when the supported OS exposes a positively identified iOS screen source there. A recent idb change proposes including Continuity-type discovery for macOS 26 and filtering muxed devices with the model identifier `iOS Device`; this is implementation evidence, not an Apple guarantee. The discovery classifier must therefore be a small, separately tested compatibility policy. [P1] + +Classifier rule, corrected after the 2026-09-08 USB discovery probe: a candidate must advertise muxed media and match the observed `iOS Device` model signature. The physical screen source advertises muxed media without a standalone video media type; actual video samples are validated during preparation before readiness. Any extra model signature requires recorded hardware evidence and a fixture. A device's user-assigned name, its portrait dimensions, or `.external` alone is never sufficient. Ordinary Continuity Camera lens feeds remain excluded. Never use a sole arbitrary camera as fallback. See [discovery evidence](../../testing/ios-usb-capture-implementation.md). + +Subscribe to connect/disconnect and discovery changes. Coalesce changes for 250 ms. While the device UI or a prepared source is active, a 2-second reconciliation poll may repair missed events. Initial enumeration may settle asynchronously; retry within a 10-second discovery window, without blocking the UI. After that, display the empty state while continuing event-driven discovery. + +### 5.2 Permissions + +Camera-style media access may be required even though the feature records a device screen. Explain this before prompting: **“macOS uses camera access to receive your connected device's screen.”** Request audio access only for enabled audio capture that requires it. + +Use the app's existing Electron permission entry point where appropriate; the native helper independently verifies actual authorisation before opening inputs. Do not assume a parent app grant automatically authorises every unsigned helper. The release gate exercises the signed installed application and helper, not only Terminal or Xcode. + +Update the parent usage strings to include connected-device screen capture. The helper has an embedded Info.plist with appropriate identifiers and usage strings, and is signed with the required hardened-runtime camera/audio-input entitlements. Preserve the existing ContinuityCamera declaration where needed. An Info.plist boolean named like an entitlement is not a substitute for an actual signing entitlement. [A9, R7] + +Do not use private APIs, root privileges, developer mode, jailbreaks, WebDriverAgent, AppleScript UI automation, or custom USB drivers. Device trust is performed by the user through Apple's system UI. If a supported configuration cannot work with the public capture path and standard distribution, fail its compatibility gate instead of adding an undocumented workaround. + +## 6. Video capture and quality policy + +### 6.1 Prepare and inspect + +Configure `AVCaptureSession` on a serial queue; `startRunning()` is blocking and must not run on a UI queue. Attach one selected input and video/audio outputs as needed. Observe runtime errors, input removal and delivered format descriptions. [A3] + +Request device-native video samples using `AVCaptureVideoDataOutput.videoSettings = [:]`. The empty dictionary and `nil` are not interchangeable: Apple documents the former as device-native and the latter as a default uncompressed format. Inspect actual samples to identify compressed versus pixel-buffer input. Prevent automatic preview-size downscaling where the SDK supports that setting; validate the dimensions that actually arrive. [A4, A5] + +Do not force desktop 3840 × 2160 or minimum-60-fps constants onto the device. Prefer the delivered/default screen format. Any format negotiation must use advertised formats and explicit capability checks, not assumptions about iPhone screen dimensions. Preserve variable-rate presentation timing. + +### 6.2 Mode A: compressed passthrough + +For a compatible H.264 SDR stream, use `AVAssetWriter` with `.mov`, an appropriate source format hint, and an `AVAssetWriterInput` whose `outputSettings` is `nil`. Apple documents this as passing samples through without re-encoding. This does not undo compression performed before the Mac receives them. [A6] + +A recording begins on a decodable sync sample at or after the requested start boundary. Do not include preview/pre-countdown footage just to get an earlier keyframe. Preserve decode order, composition offsets, durations, codec configuration and colour metadata. B-frames mean presentation timestamps need not be monotonically increasing in decode order: do not incorrectly reject them. + +The prepare gate must verify that both Recordly's preview and export decode this media correctly. Do not accept a codec merely because AVFoundation can write it. Do not assume MOV implies H.264 or safe metadata. + +### 6.3 Mode B: H.264 encode + +For supported uncompressed SDR input, use a native H.264 writer path with source dimensions and observed timing. Derive compatible settings from AVFoundation after configuring the session, then apply the explicit quality policy. Validate `canApply` / `canAdd`; fail preparation if the encoder cannot handle the source. [A7] + +Proposed initial bitrate target: `clamp(width × height × observedFPS × 0.12, 12,000,000, 60,000,000)` bits/second. This is a starting tuning parameter, not an image-quality guarantee. Aim for a keyframe interval no longer than 2 seconds in the encode path. Do not request duplicate 60-fps frames to make a 30-fps source look higher-spec. + +Preserve visible pixel dimensions. If the encoder requires even coded dimensions, pad to the next valid dimension and represent the original display aperture correctly; do not truncate a row/column or stretch the screen. Include odd-dimension fixtures. + +Keep the source's known colour interpretation and perform a real colour conversion when needed. Do not “fix” colour by relabelling unconverted pixels as Rec.709. v1 certifies tested SDR input/output paths. Untested HDR/wide-gamut combinations must be rejected or explicitly marked unsupported at preparation, not silently tone-mapped under a native-quality claim. Unknown metadata requires a validated compatibility policy and a diagnostic flag rather than invented certainty. If native compressed delivery cannot satisfy the passthrough gates, preparation may explicitly renegotiate a supported uncompressed output for this encode mode, revalidate geometry/colour and select that mode before Record. If no validated raw-output route exists, report Unsupported format; do not pretend the compressed packets can be appended to a raw encoder unchanged. + +Mode selection happens before recording. A mode switch during a take is not allowed. Show the chosen mode in recording details as **Original device stream** or **High-quality H.264**; this is diagnostic information, not a complicated quality selector. + +### 6.4 Writing, preview and backpressure + +Write video into a session-owned `source-video.mov`. Enable movie fragments before writing, initially targeting a 1-second first fragment and 10-second subsequent fragments, consistent with Apple's documented recovery/performance guidance. Evaluate a shorter steady interval only with evidence. Recovery still depends on what was durably written; do not promise an exact maximum loss interval. [A8] + +Use bounded work queues. A proposed media-queue ceiling is the first of 6 pending video buffers, 64 MiB or 200 ms of queued media; tune only with recorded evidence. Never let preview buffers hold the capture pool indefinitely. Preview gets a latest-frame-only, independent, disposable queue and is dropped before recording work. + +Do not drop arbitrary compressed inter-frame packets and keep recording as though the result is intact. When compressed sample continuity is lost, stop safely and mark the result incomplete. For a validated raw-input path, late frame drops may be counted and surfaced; sustained overload must stop rather than consume unbounded memory. Record delivered, accepted and dropped counts separately. + +Check every append and writer status. A file existing on disk does not imply successful recording. Stop only reports a complete take once writers have finished and the media is validated. + +## 7. Time and audio contract + +### 7.1 Common clock + +Use media timestamps, not `Date.now()`, IPC arrival time or browser MediaRecorder chunk time, as the basis of A/V alignment. Convert samples from each capture session's synchronisation clock into a common host-clock domain. Use the current `synchronizationClock` API where available and a deployment-compatible `masterClock` adapter where needed. CoreMedia provides conversion and relative-rate APIs. [A10, A3] + +Define `T0` as the host-clock presentation time of the first accepted decodable video sample. Store native time values as integer ticks plus timescale; use strings for 64-bit values crossing JSON. JSON floating-point milliseconds are for display only. + +For stream j, map sample time into the common domain: `relativeTime = convertToHost(sampleTime, clock_j) - T0`. Preserve video decode/presentation relationships. Mapping an entire stream by subtracting its *own* first timestamp without retaining its offset is forbidden: that falsely aligns delayed microphones to zero. + +A capture-session clock that is absent or cannot be mapped reliably is a preparation failure for the affected combination, not permission to estimate alignment from callback receipt. The hardware gate validates this requirement. + +### 7.2 Separate native audio sidecars + +Keep phone audio and narration as independent native sidecars during capture. Use audio-only fragmented MOV files (`device-audio.mov`, `microphone.mov`) with uncompressed PCM where available so audio processing is postponed to finalisation. If a device provides compressed audio, decode through the documented native audio path; do not assume byte payloads are PCM. + +Each sidecar can start when its first sample arrives, independently of video. This avoids delaying the video writer until a quiet phone produces audio. The sidecar begins at its own local zero, while the journal records its common-clock offset, format and timebase mapping. Preserve internal discontinuities with explicit sample timestamps/PCM silence or documented segment records; normalising just the first offset is not sufficient. + +Narration is captured natively from a separately selected Mac input, not by reusing a browser microphone recorder with unrelated timing. A microphone disappearing does not switch to the default microphone. Stop the take safely and retain the preceding audio/video with an interruption warning. + +For device audio with no observed samples, show an unavailable/not-received warning, preserve video, and do not declare microphone denial. A silent valid audio stream is not the same as no audio stream. If required audio could not be opened before start, require the user to disable it or fix it; do not silently change their choice. + +### 7.3 Final audio assembly + +The main process uses the existing bundled FFmpeg path resolver to create `recording.mov` from source video and available audio sidecars. Copy the video stream. Align audio using the stored common-clock offsets, trim audio that precedes T0, insert leading and internal silence where required, and resample to a common 48 kHz output timeline. + +Do not repair drift by blindly stretching every audio file to video duration. Use measured timing/clock mapping; test correction on fixtures with known offsets and rates. Resampling must preserve intended timing, not hide a stopped microphone. Trim/pad the mixed audio to the validated recording boundary without shortening the video to the shortest audio stream. + +With one enabled source, use unity gain. With two, mix each at 0.5 linear gain (approximately -6 dB) with automatic loudness normalisation disabled. Encode the resulting single soundtrack as AAC, initially 192 kbps stereo, with a mono-only narration configuration permitted at 128 kbps. Record the actual output configuration. Retain original sidecars until the user discards/deletes the take under the normal media-retention policy. + +v1 exposes one mixed soundtrack in the existing editor; independently editable narration and phone-audio tracks are not promised. A failed mux/mix never destroys the source video. Offer retry or **Open without audio** as an explicit user action. + +A sidecar's timestamps and timing metadata must agree about gaps. Do not insert a gap twice when it is already represented in the media timeline. The finaliser applies one mapping from each sidecar's local timeline to the common video timeline and tests both encoded silence and missing-sample gaps. + +### 7.4 Duration + +Take duration is bounded by the actual start/stop media timeline. Tests must cover sparse/static delivery and quiet audio: a valid five-minute static-screen take must not become a one-frame short clip, and late/absent audio must not clip video duration. Represent a held final frame through the stop boundary using valid sample/container timing; do not fabricate motion or report synthetic frames as captured frames. If a source's timing cannot satisfy this contract, preparation must use a validated compatible path or reject that configuration. + +## 8. Helper and Electron protocol + +### 8.1 Transport + +Spawn the bundled helper directly with `shell: false`. Use stdin/stdout for UTF-8 newline-delimited JSON. Reserve stdout for protocol messages and stderr for bounded, redacted diagnostics. Drain both continuously. Do not use a localhost web server, TCP port or executable path supplied by a renderer. + +Every command carries `protocolVersion: 1`, `requestId`, `command`, and relevant `sessionId` / generation. Responses distinguish command acceptance from state completion. Events carry a monotonic sequence and session/generation so late events from a previous device cannot mutate the current UI. Device-source generations validate inventory selection; preparation/preview generations are a separate counter incremented on each preparation and helper restart. Active-session media is not invalidated merely because another device changes the inventory. + +Reject malformed JSON, unsupported versions, unknown commands, overlong IDs and lines larger than 64 KiB. Cap outstanding commands at 8. Duplicate state-changing requests with the same ID and session return the same recorded result; they must not start another writer. Retain a bounded recent-request cache. + +### 8.2 Commands and events + +| Command | Purpose / completion condition | +|---|---| +| `hello` | Report helper build, protocol version and supported backend capabilities. | +| `discover` | Start/refresh device and microphone inventory; returns a snapshot, then inventory events. | +| `prepare` | Open selected source and optional microphone; complete only after valid video samples and a selected recording mode. | +| `setPreviewEnabled` | Enable/disable preview work for the current prepared generation. | +| `start` | Arm files; emit `recordingStarted` only after the first eligible sample is accepted. | +| `stop` | Idempotently finish writers and return native source paths plus timing metadata. | +| `cancel` | Cancel a not-yet-started take or finish a take marked for explicit discard. Main owns deletion. | +| `release` | Release a prepared session; reject during active recording unless it first stops. | +| `inspectMedia` | Inspect a main-approved session file without starting capture or requesting camera access. | +| `shutdown` | Safe stop/release on application exit or stdin EOF. | + +Events: `inventoryChanged`, `prepared`, `recordingStarted`, `progress`, `warning`, `nativeFinalized`, `error`. `progress` is at most once per second and reports actual counters and timeline data, not synthetic percentage-complete values. Native finalisation and final editor commit are distinct events. + +### 8.3 Preview pipe + +Use an additional binary pipe (child fd 3), not base64 in JSON control messages. Frame records have a fixed 24-byte, big-endian header: magic `RLIP` (4 bytes), version (u16), flags (u16), generation (u32), sequence (u32), JPEG length (u32), reserved zero (u32), followed by JPEG bytes. Maximum JPEG payload: 128 KiB. Invalid headers close the preview stream and surface a warning without corrupting the recording control parser. + +The helper writes previews on an independent non-blocking/async queue. At most one queued preview is retained at each boundary. Main forwards the latest JPEG only to the authorised requesting window; the renderer replaces/revokes object URLs. Closing a preview consumer must never deadlock the native writer. + +### 8.4 Renderer API + +Expose a small namespaced API, for example `electronAPI.iosCapture`, with `getSnapshot`, `discover`, `prepare`, `start`, `stop`, `cancel`, `release`, `onState`, and `onPreview`. Each subscription returns an unsubscribe callback. `prepare` accepts settings and a source token; `start` accepts the main-issued prepared session ID. Neither accepts a destination file path or command line. Main revalidates that prepared session before arming. + +Main validates sender frame/window identity, argument schemas, ownership and current session state on every call. Map helper result paths back to the main-allocated session directory; reject escapes, symlinks and unexpected basenames. Renderer clients cannot choose a helper executable or invoke arbitrary native commands. + +## 9. State machine and failure semantics + +State sequence: + +`unavailable → idle/discovering → preparing → ready → countdown → starting → recording → stopping → finalising → completed` + +Alternative terminals: `cancelled`, `failed`, `interrupted`, `recoveryAvailable`. This sequence includes UI presentation: countdown is a UI substate of Ready, while main remains Ready until the start request. Cancelling the countdown returns to Ready without creating media. Discovery and preview visibility are orthogonal to the active recording state. + +| Situation | Required action | +|---|---| +| Start while another backend holds the lease | Reject `RECORDING_BUSY`; no native/session changes. | +| Device removed during countdown/preparation | Cancel start, release resources, show connection help. | +| Start succeeds as a command but no decodable video arrives within 10 s | `NO_VIDEO_SAMPLES`; no red recording state and no successful empty take. | +| Stop requested while starting | Cancel the arm; finalise only if actual media was accepted. | +| Double Stop or racing Stop/disconnect | One native finalisation and one final editor commit. | +| Confirmed device removal/runtime capture failure | Stop, finalise valid preceding media, mark interrupted. | +| Incompatible width/height/codec/transform/colour format change | Reject new samples, safely stop the preceding take. | +| No new frames / black frames | Describe observed lack of frames if needed; do not infer lock or stop solely from a static/dark screen. | +| Permission denied/restricted | Specific explanatory state and settings action; no retry prompt loop. | +| Device busy | Explain another app may be using it; explicit Retry. No process killing. | +| Audio absent or interrupted | Follow section 7; no silent device substitution or undetected soundtrack loss. | +| Storage reserve reached | Graceful stop and warning; preserve recovery data. | +| Helper exits/crashes | Release lease after cleanup, reject pending calls, offer validated recovery candidates. | +| Preview backpressure | Drop preview only. | +| Writer backpressure / append failure | Stop or fail according to section 6; never unbounded buffering. | +| Mux/finalisation error | Keep native files and journal; Retry or explicit video-only open. | +| Renderer crash | Main safely finalises, retains result and restores state after renderer restart. | +| System sleep / application quit | Request safe stop immediately; native stdin EOF also initiates finalisation. No automatic recording on wake. | +| Packaged helper missing / incompatible protocol | Mark feature unavailable with reinstall/update guidance. No runtime compilation. | + +Use stable machine-readable error codes and separate localised user messages. Proposed codes include `UNSUPPORTED_PLATFORM`, `HELPER_UNAVAILABLE`, `PROTOCOL_MISMATCH`, `PERMISSION_DENIED`, `DEVICE_NOT_FOUND`, `DEVICE_BUSY`, `UNSUPPORTED_FORMAT`, `NO_VIDEO_SAMPLES`, `CLOCK_MAPPING_UNAVAILABLE`, `RECORDING_BUSY`, `FORMAT_CHANGED`, `DEVICE_DISCONNECTED`, `AUDIO_INTERRUPTED`, `DISK_SPACE_LOW`, `WRITER_FAILED`, `FINALIZATION_FAILED`, `HELPER_EXITED`, `INVALID_REQUEST`, `UNSUPPORTED_OPERATION`. + +A 10-second native-stop grace period begins graceful termination escalation, not deletion. If an encoder or FFmpeg remains alive, preserve files before termination; FFmpeg receives a progress/stall watchdog rather than a tiny fixed total duration. A long take can legitimately require longer audio processing. UI elapsed timers never substitute for native acknowledgements. + +## 10. Storage, recovery and project persistence + +### 10.1 Session layout + +Allocate a unique directory under Recordly's approved recording storage, not beside an arbitrary renderer path: + +``` +/ios-/ + capture-journal.json + native-timing.json # helper-owned, survives loss of the parent + source-video.mov + device-audio.mov # only when samples exist + microphone.mov # only when samples exist + recording.pending.mov # finalisation output, never opened as complete + recording.mov # validated, committed editor source + + diagnostics.json # redacted and bounded +``` + +Video-only sessions may commit `source-video.mov` directly as their editor source rather than create a duplicate. A/V finalisation writes a separate destination. For that case preflight finalisation capacity for another video-sized file; a mux operation is not free in disk space just because it copies video packets. + +Before recording, require a writable local destination and at least 1 GiB available. During capture check periodically; the reserve must cover approximately 60 seconds of measured incoming data plus space for the projected final copy and a 256 MiB minimum safety buffer. Compute with byte rates, not resolution alone. Gracefully stop before exhausting that reserve. Report storage estimates as estimates, not remaining-time promises. + +The journal is created before writers and atomically replaced for lifecycle changes. Main and helper have non-overlapping ownership: main owns lifecycle/files/commit, helper owns native timing counters and finalisation results; main merges those into the journal. Do not have both processes independently rewrite the same file. The helper atomically maintains a separate `native-timing.json` containing stream offsets, timing/format changes and its terminal result; update it at first samples, relevant changes, periodic checkpoints and stop. Recovery uses that file if the parent crashed before consuming stdout. Loss of timing metadata permits an explicitly labelled video-only recovery, not a guessed A/V mix. + +Record source kind, session state, source-format fingerprint, recording mode, actual media geometry and colour metadata, timebase/offset/rate information, enabled and observed audio sources, stop reason, created time, expected relative files and validation result. Do not store the phone's raw unique ID, serial number or personal name. + +### 10.2 Commit boundary + +Native finalisation completes first. Main checks file size, presence of a decodable video track, nonzero valid duration, expected dimensions and selected audio results. Use a new native `AVURLAsset`-based inspector and the application's actual decode smoke test where needed; do not assume `ffprobe-static` is available in packaged macOS builds—the inspected packaging excludes its Darwin binaries. + +After optional mix/remux, validate the final file, atomically rename it, write the recording-session manifest, mark the journal committed, and hand it to the editor exactly once. A recovered/interrupted take is visibly distinguished from a normal completed take. + +### 10.3 Session manifest + +Introduce version 3 for mobile recording-session manifests. Keep readers for versions 1 and 2. Add optional `captureMetadata` and persist `hideOverlayCursorByDefault`; remove the current “no webcam means delete manifest” condition when meaningful capture metadata exists. Existing desktop/webcam manifests can remain version 2 unless the new fields are used. + +Read relative paths only after validating they resolve inside the session directory. Missing optional audio/diagnostic sidecars do not make an otherwise valid committed video unopenable. Malformed provenance falls back safely to ordinary media presentation, never executable behaviour. + +### 10.4 Saved projects and presentation + +Extend `EditorProjectData` with an optional validated `captureMetadata` field without unnecessarily changing the main project version. Preserve it in save/load, snapshots, Save As and reopen. File-path-bearing recovery data stays in the capture journal, not exported project provenance. User-visible editor settings are already explicit and must remain authoritative. + +For a fresh mobile take: no cursor overlay, no cursor telemetry loading, no automatic mouse-driven zoom generation, no crop, and a source-native canvas/aspect so no pixels are cut off. Use existing wallpaper/padding/shadow defaults that do not crop the source; set initial corner radius to zero. The user may then choose 9:16, 16:9, 1:1 or other existing presentation options. An iPhone screen is not assumed to be exactly 9:16. + +Apply these defaults once on fresh import, not on every open. Saving and reopening must preserve manual zooms, annotations, aspect/crop and user-changed cursor settings. Existing desktop projects must not acquire mobile defaults. + +### 10.5 Recovery + +At launch, inspect only session directories created by this feature that contain an incomplete journal. Validate candidate media before offering **Recover recording**, **Open folder**, or **Discard**. Never label an unrecoverable file as recovered. Never automatically delete incomplete sessions by age. Respect the user's normal recording-retention/deletion actions. + +Fragmentation improves the chance of retaining already written media; the last fragment or an extremely short recording may still be lost. Power loss, filesystem failure and an encoder crash are not all equally recoverable. Record the limits honestly. [A8] + +## 11. Privacy, security and platform constraints + +All capture, preview, audio and diagnostics remain local. No analytics SDK, pairing server, local web listener or network permission is introduced. Device content is never logged. Display names appear only where needed in the current UI; diagnostics use session tokens and generic device family. + +Do not bypass protected/DRM content or attempt to decode capture-protected frames. Do not infer a protected app from black pixels. Remind users that notifications and sensitive on-screen information may appear in recordings; do not promise to suppress notifications or redact content automatically. + +Validate IPC request sizes/types, sequence numbers, allowed senders, relative media paths and symlink boundaries. Escape device labels as text. Bind device/microphone selection to current inventory generations. Do not accept arbitrary FFmpeg filters or shell strings from the renderer. Execute bundled binaries by exact resolved paths, with argument arrays and bounded output logs. + +Use an appropriate sleep-prevention assertion during an active take, released on every terminal path. This does not prevent forced sleep or guarantee recording through lid closure. No background recording begins without explicit user action. + +## 12. Build and distribution + +Keep the current macOS 14 minimum target unless a separately approved repository change raises it. Build the new helper for both existing macOS architectures, but advertise native recording on an architecture only after its hardware/runtime acceptance gates pass. Do not treat Rosetta execution as a substitute for testing an Intel support claim. + +The new Swift package compiles into `recordly-ios-device-helper`, staged under `electron/native/bin/darwin-arm64/` and `darwin-x64/`. Extend `scripts/build-native-helpers.mjs` to build/stage it without altering existing helpers. It may compile from source in development; installed builds must use the bundled binary and fail clearly when missing. + +Include the helper in package contents, ASAR-unpack checks, signature/notarisation verification and packaged-binary smoke checks. Validate its embedded privacy metadata and actual entitlements in each Mach-O slice. Smoke mode `--self-test` must run without connecting a phone or triggering TCC permission dialogs. It validates protocol, file layout, clock-independent policies and linkage—not hardware capture. + +Add macOS-only native tests; Windows/Linux CI must not attempt Swift device capture. Keep their existing tests and build checks. Update user docs with cable/trust steps, audio semantics, fixed-orientation limitation, quality wording and recovery behaviour. + +## 13. Acceptance criteria and release evidence + +| ID | Required evidence | +|---|---| +| A01 Discovery | Distinct tokens for two same-name devices; unrelated webcams and physical Continuity Camera feeds excluded; connect/unplug refresh works. | +| A02 Permissions | Clean signed installation: allow, deny, change in Settings and relaunch; no requirement to open QuickTime first; no desktop-screen permission gate for video-only phone capture. | +| A03 Readiness | Discovered-only device cannot record; 10-second no-sample failure is understandable; cancel/stop during start produces no false-success take. | +| A04 Native fidelity | Delivered geometry, visible aperture and orientation preserved; preview resizing cannot change recording geometry. | +| A05 Passthrough | For eligible deterministic fixtures, captured compressed payloads remain unchanged through normal video-copy finalisation; decode starts cleanly including keyframe/B-frame cases. | +| A06 Encode | Moving UI/text test at delivered resolution passes visual review; no forced scaling, stretched aspect or persistent overload. | +| A07 Colour | On-device reference, QuickTime USB, native source and final Recordly export compared on the same test UI; no unexplained gamma/range/colour shift relative to the received source. Wider-gamut differences documented, not concealed. | +| A08 Audio sync | Phone-only, narration-only and both; known positive/negative start offsets; start and end alignment within 80 ms on a 30-minute flash/click fixture; delayed start and internal gaps handled. | +| A09 Static duration | A five-minute static/dark UI and quiet audio retain the intended timeline; no false lock detection or one-frame short take. | +| A10 Interruptions | Unplug, format change, microphone removal, disk pressure, helper exit, renderer crash and quit produce honest completion/interruption states and retain valid preceding media. | +| A11 Persistence | Fresh recording, raw media reopen, project save/reopen, Save As and missing optional sidecars preserve intended settings and do not alter desktop projects. | +| A12 Export | Portrait and landscape source; native aspect, 9:16 and 16:9 compositions; manual zoom and annotation; existing preview and export agree on geometry, duration and sound. | +| A13 Performance | Preview backpressure cannot stall recording; no monotonic memory growth in a 30-minute take; writing queues remain bounded. For tested encode configurations, unexplained video loss below 1%; passthrough packet loss is not silently accepted. | +| A14 Repeatability | 20 consecutive start/stop cycles without leaked helper/session/input; double-stop and stale callbacks do not produce duplicate editor opens. | +| A15 Packaging | Installed signed/notarised app works without Xcode or repo files; both advertised architectures have smoke and physical-device evidence. | +| A16 Regression | Existing macOS desktop/audio capture, Windows WGC/browser paths, Linux portal paths, webcam sessions and old project formats retain baseline behaviour. | + +Minimum physical matrix: Apple Silicon with a current USB-C iPhone; a Lightning iPhone/cable if advertised; at least one iPad before advertising iPad support; a real Intel Mac if Intel capture is advertised; oldest supported macOS and the latest stable macOS used for release. Record exact macOS, iOS/iPadOS, device family, cable/hub, app build and recording mode. Include at least one direct-cable and one hub test. Do not market OS beta compatibility based only on compilation. + +A proposed already-trusted-device readiness target is within 5 seconds under normal test conditions; the UI remains correct when slower. This is a measured UX target, not a guarantee of Apple's enumeration latency. + +## 14. Gates, rollout and definition of done + +**G1 — API/permission feasibility:** prove positive device-screen classification, first frames and signed-helper permission identity on the target systems. Failure blocks feature enablement on that combination. + +**G2 — media fidelity:** prove input format, native/passthrough viability, editor compatibility, colour and sparse-stream duration. Failure selects a validated encode path or marks the configuration unsupported. It does not justify a desktop-window fallback. + +**G3 — audio:** prove common-clock mapping, sidecar timing and 30-minute sync. Until it passes, narration stays developer-only and the feature is not called v1 complete. + +**G4 — reliability and distribution:** interrupted-media recovery, session/project compatibility, resource cleanup and signed installed-app testing pass before general availability. + +Start behind an internal build-time flag, default off. Keep source UI absent when disabled. Progress to a labelled beta after G1–G3, then enable generally after G4 and the acceptance matrix. No server-side rollout service is necessary. Already recorded videos must remain openable when capture is disabled later. + +Done means all F01–F12 requirements, A01–A16 acceptance cases and G1–G4 gates have evidence, the shipped binary includes the helper, and no success/quality claims exceed the actual support matrix. A working preview alone is not done. + +## 15. Source register + +All repository references below use the inspected baseline. API behaviour is documented at the linked sources; device-specific behaviour still requires the gates above. URLs were consulted on 8 September 2026. + +- R1 — Repository baseline: https://github.com/webadderallorg/Recordly/commit/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1 +- R2 — Package/scripts: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/package.json +- R3 — Source popover: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/src/components/launch/popovers/SourcePopover.tsx +- R4 — Launcher source mapper: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/src/components/launch/popovers/launchPopoverTypes.ts +- R5 — Recorder hook: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/src/hooks/useScreenRecorder.ts +- R6 — Session persistence: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/electron/ipc/project/session.ts +- R7 — Packaging: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/electron-builder.json5 +- R8 — Native builds: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/scripts/build-native-helpers.mjs +- R9 — Project persistence: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/src/components/video-editor/projectPersistence.ts +- R10 — Packaged-binary smoke checks: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/scripts/smoke-packaged-binaries.mjs +- R11 — Native helper paths: https://github.com/webadderallorg/Recordly/blob/4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1/electron/ipc/paths/binaries.ts +- A1 — Apple connected-device QuickTime recording: https://support.apple.com/en-au/guide/quicktime-player/qtp356b55534/mac +- A2 — CoreMediaIO screen-device opt-in: https://developer.apple.com/documentation/coremediaio/kcmiohardwarepropertyallowscreencapturedevices +- A3 — AVCaptureSession, configuration and synchronisation clocks: https://developer.apple.com/documentation/avfoundation/avcapturesession +- A4 — Native versus uncompressed video settings: https://developer.apple.com/documentation/avfoundation/avcapturevideodataoutput/videosettings +- A5 — Automatic preview-size buffer configuration: https://developer.apple.com/documentation/avfoundation/avcapturevideodataoutput/automaticallyconfiguresoutputbufferdimensions +- A6 — AVAssetWriterInput passthrough semantics: https://developer.apple.com/documentation/avfoundation/avassetwriterinput/outputsettings +- A7 — Recommended settings after configuring the session: https://developer.apple.com/documentation/avfoundation/avcapturevideodataoutput/recommendedvideosettings(forvideocodectype:assetwriteroutputfiletype:) +- A8 — Fragment recovery: https://developer.apple.com/documentation/avfoundation/avassetwriter/moviefragmentinterval ; https://developer.apple.com/documentation/avfoundation/avassetwriter/initialmoviefragmentinterval +- A9 — Electron media permission API and usage descriptions: https://www.electronjs.org/docs/latest/api/system-preferences +- A10 — CoreMedia clock conversion/rate APIs: https://developer.apple.com/documentation/coremedia/cmsyncgetrelativerate(_:relativeto:) +- P1 — Primary implementation evidence, not a platform guarantee: https://github.com/facebook/idb/pull/938 (discovery/authorisation changes for macOS 26; do not copy code without reviewing its licence and suitability). diff --git a/docs/testing/ios-usb-capture-feasibility.md b/docs/testing/ios-usb-capture-feasibility.md new file mode 100644 index 000000000..d5642fc9d --- /dev/null +++ b/docs/testing/ios-usb-capture-feasibility.md @@ -0,0 +1,93 @@ +# iOS USB capture feasibility evidence + +Implementation baseline: `4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1`. +Development host: macOS 26.2 (25C5048a), arm64, Xcode 26.6 (17F113), +Apple Swift 6.3.3. This beta host does not certify stable macOS compatibility. + +## Baseline — 8 September 2026 + +| Check | Observed result | +| --- | --- | +| `npm ci --cache /private/tmp/recordly-npm-cache` | Dependencies installed; postinstall failed rebuilding uiohook because Homebrew Python 3.14 could not load an expat symbol. | +| `npm_config_python=/usr/bin/python3 npm run rebuild:native` | Pass. No source/dependency changes needed. | +| `npm test` using initially selected Node | Could not start: app-bundled Node refused Rollup's native library because of signing Team ID mismatch. | +| `PATH=/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin npm test` | Pass: 120 files, 1,083 tests. Homebrew Node 26.5.0. | +| `npx tsc --noEmit` | Pass. | +| `npm run lint` | Pass: 574 files. | +| `npm run format:check` | Pass: 573 files. | +| `npm run i18n:check` | Pass: locale structures consistent. | + +Local baseline logs are `/private/tmp/recordly-baseline-*.log`; these are development +artifacts, not hardware recordings or release evidence. No baseline application +test failures remain after selecting working runtimes. Native platform-helper and +packaged application builds are separate checks. + +## Release gates + +| Gate | Status | Required evidence | +| --- | --- | --- | +| G1 API and permissions | Partial | One USB iPhone's discovery signature, actual samples and Ready/live preview observed; one recording/editor handoff user-confirmed. Permission allow/deny/relaunch and signed installed-app identity remain untested. | +| G2 media fidelity | Partial | Physical 1206 × 2622 H.264 High → `420v`, Rec.709/sRGB/Rec.709 metadata observed; physical color, sparse timing and editor/export comparison remain untested. | +| G3 audio | Not tested | Real clock mapping and 30-minute device/narration sync within 80 ms. | +| G4 reliability and distribution | Not tested | Physical interruption tests and clean signed installed-app matrix. | + +Software fixtures do not certify these gates. The feature must remain default-off. +No device family, signing team or installed release has been certified by this +record. The probes recorded in [implementation evidence](ios-usb-capture-implementation.md) +observed the `iOS Device` signature without a standalone video media type, followed +by actual frame metadata. One successful development session does not certify the +remaining devices, permissions, media-fidelity or release matrix. + +## Implemented software evidence — 8 September 2026 + +The native helper and disabled-by-default application integration are now present. +The latest capture-fix native run passed 46 XCTest tests and built both arm64 and x86_64 helper +artifacts with a macOS 14 deployment target and embedded privacy metadata. Cross-build +results do not establish Intel runtime or installed permission behavior. + +The final full JavaScript/TypeScript integration run passed 147 files +and 1,235 tests, with one explicitly opt-in native-media suite skipped in that run. +That suite was run separately and passed using synthetic media through the staged +native inspector, production finalizer, bundled FFmpeg, manifest reopen and verifier. +TypeScript, full lint, formatting and localization checks also passed. + +A development `build:mac` produced arm64/x64 DMG and ZIP artifacts signed with an +Apple Development identity. Notarization was skipped because release credentials +were unavailable. That build preceded final review edits. Packaged smoke then found +an x64 `otool` output-unit parsing defect; the parser and regression test were fixed. +A final-source arm64 preview bundle was rebuilt with an ad-hoc signature, and packaged +smoke passed for that bundle and both helper slices. Packaging does not satisfy any +release gate; G1 and G2 have partial physical evidence and G3–G4 remain **Not tested**. + +Synthetic media evidence includes seven passing positive variants and five expected +negative cases. Packet preservation is checked after demuxing rather than by hashing +whole MOV containers. A synthetic 300-second static timeline and rational clock/audio +tests pass. The latest native suite also round-trips 1206 × 2622 `420v` media with +exact geometry and Rec.709 / `IEC_sRGB` / Rec.709 tags, then checks decoded flat-patch +values. The policy preserves sRGB transfer on macOS 15+; it does not relabel the +source or enable unverified macOS 14 sRGB encoding. These synthetic results do not +establish physical color/export fidelity, frame rate, 30-minute sync or interruption recovery. + +## Physical capture follow-up — 8 September 2026 + +A metadata-only probe observed a 1206 × 2622 H.264 High-profile stream, full range, +with Rec.709 primaries/matrix and `IEC_sRGB` transfer. Negotiating `420v` yielded +video-range raw samples with the same geometry and color triplet. The original +transfer restriction caused the observed rejection. The probe saved no screen or +audio media. Earlier no-frame timeouts did not show that the phone was disconnected: +this macOS 26 host uses `SPUSBHostDataType`, and the USB registry confirmed the phone. + +After the transfer-policy fix, Ready and a live preview were directly observed in +the isolated development test app. The user subsequently confirmed that recording +completed and the editor opened, and that 9:41 with full status icons appeared on +the phone. A later direct accessibility inspection of the editor showed **Device +recording saved**, **Device audio track recorded**, and approximately **20.2 seconds** +of clip duration. This verifies the displayed outcome and metadata; audible playback, +audio sync, decoded-media quality and export were not checked. No long-run result +was recorded. The phone OS, cable/hub topology and exact running artifact checksum +remain unspecified. These results advance G1/G2 to partial evidence without passing +either gate; G3/G4 remain untested. + +See [implementation evidence](ios-usb-capture-implementation.md) for task status and +commands, and [the acceptance matrix](ios-usb-capture-matrix.md) for the still-open +physical evidence. diff --git a/docs/testing/ios-usb-capture-implementation.md b/docs/testing/ios-usb-capture-implementation.md new file mode 100644 index 000000000..223db9e90 --- /dev/null +++ b/docs/testing/ios-usb-capture-implementation.md @@ -0,0 +1,156 @@ +# iOS USB capture implementation evidence + +Date: 8 September 2026. Baseline: `4b20a1a76ed3564bd70aee7a0cc7167ea2e5e9c1`. + +This record separates implemented software behavior from physical and release proof. Tasks 03–15, deterministic portions of Task 16, packaging automation from Task 17 and documentation/policy work from Task 18 are implemented. Task 02 now has partial physical discovery, sample-format and readiness evidence, plus user-confirmed recording/editor handoff; Task 16's wider physical matrix remains pending. The release feature remains disabled by default. + +## Implementation by task + +| Task | Implemented software | Remaining evidence | +| --- | --- | --- | +| 01 | Baseline results and evidence templates recorded; final broad software gates passed. | None for the software baseline. | +| 02 | Public-API probes confirmed the USB iPhone's muxed-only screen signature and actual sample metadata; Ready/live preview were observed, and recording/editor handoff were user-confirmed. | Broader devices/OS versions, TCC flows, physical color comparison, rotation, unplug, sparse delivery and signed installed app. | +| 03 | Versioned shared source, snapshot, protocol, native-time, metadata and error-code validators. Mobile and desktop types narrow at dispatch boundaries. | Hardware-derived support policy remains unverified. | +| 04 | macOS 14 Swift package, bounded NDJSON loop, permission-free self-test, embedded privacy metadata, development/packaged resolution and dual-architecture staging. | Runtime/signing proof for each advertised architecture. | +| 05 | Checked CMIO opt-in, classifier with a physical muxed-only discovery regression, opaque tokens, coalesced discovery/reconciliation, native microphones and permission policy. | Broader physical device classification, signed TCC and repeated physical selection. | +| 06 | Baseline H.264 passthrough, supported 8-bit 4:2:0 encode with Rec.709 primaries/matrix and Rec.709 or macOS 15+ sRGB transfer, odd-size padding, fragmented MOV and format/boundary checks. The observed 1206 × 2622 source has a synthetic geometry/tag/decoded-patch regression. | Broader physical formats, physical color/export comparison, overload and sparse/static capture. | +| 07 | Host-clock rational timing, signed PCM sidecars, represented gaps and atomic native timing checkpoints. | Physical clock mapping and 30-minute device/narration sync. | +| 08 | Bounded JPEG/RLIP preview parser and nonblocking latest-frame delivery, with orientation/aperture handling. | Sustained physical preview-stall and resource measurements. | +| 09 | Helper process client, authoritative controller state, operation epochs, timeouts, idempotent terminal path and retained exactly-once completion. | Physical helper/device failure behavior. | +| 10 | Safe session allocation, journals, native inspection, FFmpeg audio alignment, video-copy commit, progress watchdog and retryable finalization. The user confirmed one physical recording completed and reached the editor. | Independent inspection of that take, requested audio outcomes and installed bundled-runtime exercise. | +| 11 | Sender-validated namespaced IPC, source narrowing, default-off policy and exclusive desktop/mobile recording lease. | Hardware/platform desktop regressions. | +| 12 | Mobile recorder routing before desktop acquisition, countdown, HUD, unsupported-operation enforcement and native-authoritative status. | Mounted multiwindow and physical interaction run. | +| 13 | Device picker, factual help, native microphone selection, preview disposal, warning presentation and locale structure. | Manual keyboard, screen-reader, reduced-motion and physical preview review. | +| 14 | Manifest v3, v1/v2 compatibility, project provenance, Save As, native-aspect fresh defaults and editor interruption notice. | Physical portrait/landscape editor and export comparison. | +| 15 | Validated recovery registry, explicit video-only recovery, lifecycle cleanup, diagnostics redaction and abortable finalization. | Real fragmented interruption, unplug, sleep, quit and power-loss recovery. | +| 16 | Reproducible synthetic generator/verifier, positive and negative media matrix, and native-inspector-to-finalizer-to-manifest integration. | A01–A16 physical/cross-platform matrix, 30-minute and repetition runs. | +| 17 | Helper build/staging, packaged binary checks, architecture/deployment/plist/entitlement checks, CI workflow and distribution verification integration. | Signed/notarized clean-account installation; real Intel runtime. | +| 18 | User workflow, evidence documents and immutable default-off policy pending release gates. | G1–G4 and support claims required before enabling or README marketing. | + +## Software verification snapshot + +| Check | Result | Scope and limit | +| --- | --- | --- | +| Native XCTest | Pass: 46 tests, zero failures | Synthetic AVFoundation/CoreMedia media, discovery lifecycle, format transition and exact-size sRGB round trip; no device or TCC prompt in the suite. | +| Native helper build | Pass: arm64 and x86_64 staged, macOS 14 target, embedded plist | Cross-build is not Intel runtime or signing evidence. | +| Helper CLI smoke | Pass | Self-test, duplicate request replay, synthetic inspection and EOF finalization. | +| Full JavaScript/TypeScript tests | Pass: 147 files, 1,235 tests; one explicit opt-in native suite skipped | The skipped native integration test was run separately and passed before this discovery/UI update. | +| Opt-in native finalization integration | Pass: one test | Synthetic raw video/PCM → native inspector → production finalizer/FFmpeg → manifest reopen → verifier. | +| TypeScript, localization, full lint and formatting checks | Pass on the reviewed source | No hardware or interactive UI claim. | +| Synthetic media matrix | Pass: 7 positive variants; expected rejection: 5 negative variants | Physical-device evidence is explicitly false in reports. | +| Development macOS package build | Pass for both architecture artifacts: DMG and ZIP, Apple Development signed; notarization skipped | Preceded final review edits; distribution artifacts require a fresh release build. Does not satisfy G4. | +| Final-source host bundle | Pass: normal build pipeline with `--dir --arm64 --publish never`, automatic identity discovery disabled | Refreshed arm64 app has an ad-hoc signature. No release was published; not an installed distribution test. | +| Packaged binary smoke | Pass after final-source bundle refresh, including both helper slices | An x64 `otool` byte-format mismatch was fixed with a regression test; this does not validate notarization or device permission identity. | + +The positive synthetic variants are portrait, landscape, odd dimension, delayed microphone, negative offset, internal gap represented as encoded silence, and silent/static. Negative cases verify rejection of wrong rotation, 250 ms displaced audio, missing audio, a two-second truncation and a byte-truncated MOV. Video preservation is compared by hashing demuxed compressed packet payloads rather than whole MOV containers. + +The native suite also proves a synthetic 300-second static baseline passthrough and raw single-frame timeline. This is container timing evidence, not a physical five-minute phone test. `observedFrameRate` remains `null`; no estimator or frame-rate guarantee is claimed. + +## USB discovery correction — 8 September 2026 + +On macOS 26.2 (25C5048a), arm64, an unlocked and trusted iPhone connected by USB was missing from the development test app built from `3dd5bd26215860ab781b6ff814adf1b8635bebbb`. The user confirmed the cable, unlock and trust state; the USB registry independently confirmed an iPhone was present. Cable/hub topology and the phone OS version were not recorded. + +A noncapturing AVFoundation probe enabled the public CMIO screen-device discovery property successfully. Both modern and legacy muxed discovery returned a connected external source with `modelID: "iOS Device"`, `hasMuxed: true`, `hasVideo: false`, and `hasAudio: false` after approximately two seconds. A separate iPhone camera source advertised `modelID: "iPhone18,1"`, video and no muxed media. No device names, serials or native identifiers were retained in this evidence. + +The original classifier rejected the screen source because it required a standalone video media type. The corrected rule requires the exact `iOS Device` model plus muxed media. Preparation still requires actual video samples before readiness; audio availability remains unknown until samples establish it. The observed camera source and generic muxed webcams remain excluded. + +The new regression failed against the original classifier, then passed with all 31 native tests after the fix. Both helper architectures were rebuilt. The corrected arm64 helper's real `discover` command returned one device with unknown audio availability and no errors, without preparation or recording. Development logs are `/private/tmp/recordly-discovery-red.log`, `/private/tmp/recordly-discovery-native-tests.log`, and `/private/tmp/recordly-discovery-helper-build.log`; the sanitized discovery result is `/private/tmp/recordly-discovery-fixed-check.json`, and the noncapturing probe source is `/private/tmp/recordly-discovery-probe.swift`. This is partial G1/A01 discovery evidence only: no recording, first frame, audio, duplicate-label/removal test or permission prompt was exercised. + +Subsequent fresh helper launches exposed a second startup issue. Retaining the discovery session alone was insufficient. With the app closed, sequential eight-second probes found the screen after approximately two seconds when CMIO and AVFoundation discovery initialized on the main thread with a running main run loop. Main-thread initialization with `dispatchMain()`, and background initialization with a main run loop, both stayed empty. The helper now initializes discovery on the main thread, retains the discovery session until discovery stops, and services the main run loop while its serial engine queue continues to own inventory and capture state. The probe source and sanitized results are `/private/tmp/recordly-discovery-lifecycle-probe.swift` and `/private/tmp/recordly-discovery-main-thread-results.log`. + +After this correction, two consecutive fresh launches of the exact development-signed test-app helper returned one iPhone, unknown audio availability, and no errors, without preparation or recording. Reports are `/private/tmp/recordly-discovery-context-packaged-1.json` and `/private/tmp/recordly-discovery-context-packaged-2.json`. All 31 native tests passed again, both architectures rebuilt, and the updated test bundle passed deep signature verification. The native tests and the physical discovery probes are separate evidence; neither establishes first-frame, audio or installed-release permission behavior. + +The rebuilt `Recordly iOS Test` app (1.4.0-ios-test.2, arm64, isolated test identity) was reopened and its iPhone/iPad picker visibly listed the connected phone with the ready-to-select discovery message. The phone was not selected and no capture permission or recording was started. A separate delayed-hello process check confirmed the helper stays alive before commands arrive and shuts down cleanly on stdin EOF, without discovery. + +## Picker, refresh and preparation follow-up — 8 September 2026 + +The launcher now keeps permission-free discovery active while mounted, so closing the source picker does not shut down the helper and repeat USB startup. Releasing a prepared source, including through another window or desktop selection, restarts discovery. The recovery section was removed from the picker and More menu; saved media and the internal recovery/storage validation remain intact. + +Idle Refresh rebuilds the main-thread CMIO discovery observation. Prepared sources reuse their existing observation. Device-list KVO schedules reconciliation without waiting for the two-second fallback poll, and generation checks ignore callbacks from replaced or stopped observations. The controller waits through immediate empty inventory for up to five seconds, accepts later device arrivals, coalesces repeated requests, and rejects refresh during capture transitions. Refresh shows progress, and successful shared inventory clears obsolete discovery errors. + +Preparation had a separate format-transition defect: after requesting raw output, queued compressed samples were immediately rejected. It now requests advertised `420v` or `420f` once and waits within the existing ten-second deadline. Geometry, color and first-sample validation remain required. Encoder recommendations are queried only for advertised H.264 encoding, avoiding an unsupported AVFoundation codec query during passthrough. + +Thirteen added controller tests, six native discovery-lifecycle tests and seven native format-negotiation/capability tests cover the discovery update. Regression runs reproduced stale discovery, late helper failures and premature compressed-frame rejection before their fixes. The capture follow-up verification snapshot is 1,221 JavaScript/TypeScript tests and 46 native tests, including the two subsequent color-policy/round-trip tests described below. Type, lint, formatting and locale checks also pass. A browser fixture mounting the actual picker verified warm discovery across opening/closing, busy refresh feedback, automatic rediscovery after another window releases a prepared source, clearing obsolete errors after shared recovery, and no recovery-list requests. + +A subsequent metadata-only probe identified the physical rejection: the connected iPhone supplies 1206 × 2622 H.264 High-profile video with Rec.709 primaries/matrix and an `IEC_sRGB` transfer function. Advertised `420v` renegotiation succeeds, preserving the same color triplet, but the original Rec.709-only transfer check rejects it. The initial compressed sample is full range and the negotiated `420v` sample is video range; coded dimensions, clean aperture and presentation dimensions all match. No screen or audio media was saved by the probe. Earlier no-frame timeouts did not establish disconnection: `SPUSBDataType` is obsolete on this macOS 26 host; `SPUSBHostDataType` and the USB registry confirm the phone is connected. The sanitized sample/rejection record is `/private/tmp/recordly-format-probe-srgb-rejection.jsonl`. + +The corrected policy accepts this SDR transfer on macOS 15 and later and preserves its actual `IEC_sRGB` metadata through encoding instead of relabeling it Rec.709. Rec.709 primaries/matrix, supported pixel formats, geometry and first-frame checks remain required; unsupported HDR, P3, matrix and transfer combinations remain rejected. macOS 14 sRGB encoding is not enabled or certified by this change. + +The synthetic native regression encodes and decodes a 1206 × 2622 `420v` frame, verifies exact coded/display geometry and the Rec.709 / `IEC_sRGB` / Rec.709 tags, and checks five flat-patch centers within three code values of their inputs. The observed decoded luma values were `[16, 64, 128, 192, 235]`. This is controlled codec evidence, not a physical display or final-export color comparison. The focused red/green logs are `/private/tmp/recordly-srgb-red.log` and `/private/tmp/recordly-srgb-green.log`; `/private/tmp/recordly-srgb-native-tests.log` records all 46 native tests passing. + +After rebuilding the isolated `Recordly iOS Test` app with this fix, **Ready and a live phone preview were directly observed** on the same Apple Silicon development host. The user then **confirmed that a physical recording completed and opened in the editor**, and that the phone displayed **9:41 with full status icons**. A subsequent direct accessibility inspection of the editor showed **Device recording saved**, **Device audio track recorded**, and a clip duration of approximately **20.2 seconds**. These are observed UI and metadata outcomes; no audible playback, audio-sync measurement, decoded-media comparison, final export or long-run test was performed. Status-bar appearance remains user-confirmed. The phone OS, cable/hub topology and exact running artifact checksum were not recorded in this evidence. G1 and G2 are partial; no release gate has passed. + +The disposable test build was then cleaned without replacing the running app. Its 12 `electron/native/ios-device-capture/Sources` files were synchronized exactly with the root sources, removing `FormatProbe.swift` and its temporary instrumentation. Resources and all 13 test files, including fixtures, were hashed unchanged. A clean helper build exited successfully and staged arm64 and x86_64 binaries; `lipo` and `vtool` confirmed both architectures and macOS 14.0 deployment targets. Probe identifiers were absent from the sources and both binaries' strings/symbols. The staged helper SHA-256 values are `d12e7f3ada6a4241b46b6f87beefa7eba64dc2aea0df92e3294f0bfb521dc665` for `.tmp/ios-test-build/electron/native/bin/darwin-arm64/recordly-ios-device-helper` and `bf7b73988c132e7ed022af9fb64bd665c8691ab7efcf3b1d3d66d00d9360c7e8` for the corresponding `darwin-x64` path. This cleanup did not install, sign, restart or repackage the app, or create a ZIP. + +## Review findings resolved + +The editor's Scene → Frame section now offers None, Black and White iPhone 16 Pro +frames using bundled licensed PNG artwork. Preview, modern MP4, legacy MP4/GIF +and project thumbnails use the same physical aperture. Source aspect and original +orientation are preserved, including letterboxing for other crops. Frame images +load before export; missing images fail export explicitly. A preview image failure +keeps the editor and its frame controls available. Texture tests cover stale loads, +style changes, teardown and failure, and persistence tests cover saved projects, +preferences and presets. The final frame suite passes 1,235 JavaScript/TypeScript +tests across 147 files, with the one opt-in native integration suite skipped in +this ordinary run; native capture remains 46 tests. Browser checks used the actual +preview and both export renderers with synthetic screen content to verify the +photo frame in portrait and landscape, including zero padding. This is renderer +evidence, not physical recording/export color comparison. + +Independent integration and media reviews found defects at real ownership boundaries. Corrections include: + +- preparation/session epochs now prevent an asynchronous cancelled prepare from reopening inputs without a lease; +- the standalone picker remains alive through mobile preparation; +- editor handoff is reserved and memoized before asynchronous work, preventing concurrent duplicate opens; +- last-client cleanup releases prepared resources while a disposable preview close does not end a take; +- legacy source-selection IPC applies the same sender policy before changing mobile state; +- desktop native exit releases the recording lease independently of renderer cleanup; +- malformed preview framing closes that stream and surfaces a warning without corrupting capture control; +- capture warnings and committed interruption/audio outcomes are visible in the picker, HUD and editor; +- disk reserve uses actual media-file growth and reserves a second video-sized output only when audio assembly needs it; +- clock conversion maps durations through converted endpoints and permits one PCM sample of rounding tolerance; +- active sessions are excluded from recovery scans, and recovery cannot manufacture missing timing or counters; +- aborting finalization terminates the child process with bounded escalation while retaining partial output. + +Focused regression tests cover these fixes. They remain software evidence and do not replace physical acceptance. + +## Running the software checks + +```bash +npm test +npx tsc --noEmit +npm run lint +npm run format:check +npm run i18n:check +npm run test:ios-native +npm run build:ios-helper +``` + +Generate and verify a synthetic session with explicit paths: + +```bash +node scripts/fixtures/ios-capture/generate.mjs \ + --output-dir /absolute/new-directory \ + --variant portrait + +npm run verify:ios-capture-fixture -- \ + --session-dir /absolute/session \ + --expected /absolute/expected.json \ + --report /absolute/report.json +``` + +The verifier rejects unknown, missing and nonabsolute arguments. It uses the native media inspector on macOS and the bundled FFmpeg resolver. Some media checks require normal macOS codec/IOSurface access and can fail under a restrictive sandbox. + +For local application packaging, this host required system Python for the native Electron rebuild: + +```bash +npm_config_python=/usr/bin/python3 npm run build:mac +npm run smoke:packaged-binaries +``` + +Release distribution verification additionally requires the repository's signing/notarization environment and explicit `--arch` and `--team-id` inputs. A local Apple Development signature is not release proof. + +## Release boundary + +G1 API/permissions and G2 physical media fidelity have **Partial** evidence from one phone: discovery, sample metadata and Ready/live preview were observed, and recording/editor handoff were user-confirmed. G3 physical long-duration audio and G4 reliability/distribution remain **Not tested**. No release gate has passed. Keep the release feature disabled; the user-requested isolated development test build is not a release. Do not advertise iPhone/iPad or Intel support or add a README product claim from these results. Physical results belong in [the acceptance matrix](ios-usb-capture-matrix.md), with exact build, OS, generic device family, cable/setup and artifact location. diff --git a/docs/testing/ios-usb-capture-matrix.md b/docs/testing/ios-usb-capture-matrix.md new file mode 100644 index 000000000..71285b412 --- /dev/null +++ b/docs/testing/ios-usb-capture-matrix.md @@ -0,0 +1,76 @@ +# iOS USB capture acceptance matrix + +Every physical row starts **Not tested**. Record the exact build, OS, architecture, +generic device family, cable/hub, actions, expected/observed results and evidence +location when running it. Exclude serials, raw device IDs and personal device names. + +| ID | Acceptance case | Status | Evidence | +| --- | --- | --- | --- | +| A01 | Positive discovery, duplicate labels, removal | Partial | One trusted USB iPhone's muxed-only screen signature observed on the development host; duplicates and removal untested. See implementation evidence. | +| A02 | Installed permission allow/deny/relaunch, no desktop permission prerequisite | Not tested | Signed installation required | +| A03 | Readiness and start cancellation | Partial | Ready/live preview directly observed; recording completion user-confirmed. Start cancellation and physical no-sample behavior remain untested. | +| A04 | Delivered geometry/aperture/orientation | Partial | Physical coded, clean-aperture and presentation dimensions all 1206 × 2622 with zero aperture origin. Other orientations/transitions remain untested. | +| A05 | Passthrough packets and clean first decode | Not tested | Observed phone uses High-profile H.264 → raw encode, not baseline passthrough. Synthetic baseline packet/decode checks are separate. | +| A06 | Encode quality and overload | Partial | Physical raw-encode preparation/live preview observed and a completed take user-confirmed; independent quality and overload measurements remain untested. | +| A07 | Colour comparison | Partial | Physical Rec.709 / `IEC_sRGB` / Rec.709 tags observed. Synthetic encode/decode preserves tags and flat patches; physical device/QuickTime/source/export comparison remains untested. | +| A08 | Audio sync | Not tested | 30-minute flash/click, each audio combination | +| A09 | Static/dark duration | Not tested | Five-minute sparse source | +| A10 | Interruptions | Not tested | Unplug, rotation, microphone loss, disk/crashes/quit | +| A11 | Persistence | Not tested | Raw session, saved project, Save As | +| A12 | Editor and export | Partial | User confirmed one physical recording opened in the editor; direct editor inspection showed saved recording, recorded device-audio metadata and approximately 20.2 seconds. Export and portrait/landscape/native/9:16/16:9 comparisons remain untested. | +| A13 | Performance and preview isolation | Partial | Live preview directly observed. Sustained preview isolation, 30-minute memory and counter measurements remain untested. | +| A14 | Repeated sessions | Not tested | 20 cycles, duplicate stop/stale event checks | +| A15 | Installed distribution | Not tested | Signed notarised app, clean account, each advertised architecture | +| A16 | Desktop regression | Partial | Baseline 1,083 tests and capture-fix integrated suite of 1,221 tests pass; platform hardware tests not run | + +### Observed development session — 8 September 2026 + +Host: macOS 26.2 (25C5048a), Apple Silicon, isolated `Recordly iOS Test` build with +the sRGB transfer fix. Generic device: one USB iPhone. The phone OS, cable/hub topology +and exact running artifact checksum were not recorded; this is not a clean-account +installed-release test. + +The metadata-only probe recorded H.264 High-profile, 1206 × 2622, full range, followed +by negotiated `420v` video-range samples with the same geometry and Rec.709 primaries, +`IEC_sRGB` transfer and Rec.709 matrix. Evidence: +`/private/tmp/recordly-format-probe-srgb-rejection.jsonl`. Ready and a live preview +were directly observed after the policy fix. Recording completion/editor handoff and +9:41/full status icons were subsequently confirmed by the user in the development +session. A later direct accessibility inspection of the editor showed **Device recording +saved**, **Device audio track recorded**, and approximately **20.2 seconds** of clip +duration. Audible playback, audio sync, decoded-media quality, export and long-duration +behavior were not checked. These observations only partially satisfy the +rows above. + +## Physical combinations + +| Combination | Status | +| --- | --- | +| Apple Silicon + USB iPhone, cable/hub topology not recorded | Partial: metadata and Ready/live preview observed; recording/editor handoff user-confirmed | +| Apple Silicon + USB-C iPhone + direct cable | Not tested | +| Apple Silicon + USB-C iPhone + hub | Not tested | +| Lightning iPhone, if advertised | Not tested | +| iPad, before advertising support | Not tested | +| Real Intel Mac, before advertising Intel capture | Not tested | +| macOS 14 minimum and current stable release | Not tested | + +## Software evidence + +Software-only evidence does not change any physical row above: + +| Evidence set | Status | Result | +| --- | --- | --- | +| Native XCTest | Pass | 46 native tests, zero failures; includes discovery, raw negotiation and the observed geometry/color-triplet regression | +| Native helper cross-build | Pass | arm64 and x86_64 staged for macOS 14; no Intel runtime claim | +| Integrated JS/TS checks | Pass | 147 files, 1,235 tests; one opt-in native suite skipped and passed separately | +| Native finalization integration | Pass | One synthetic native-inspector → finalizer → manifest → verifier test | +| Positive synthetic media variants | Pass | 7: portrait, landscape, odd dimension, delayed microphone, negative offset, internal gap, silent/static | +| Negative synthetic media cases | Pass by rejection | 5: wrong rotation, displaced audio, missing audio, duration truncation, byte-truncated MOV | +| Synthetic sRGB round trip | Pass | 1206 × 2622 coded/display geometry and Rec.709 / `IEC_sRGB` / Rec.709 tags preserved; decoded flat-patch luma `[16, 64, 128, 192, 235]`, tolerance three code values. macOS 15+ policy, no physical color/export guarantee. | +| Development macOS packaging | Partial | Both architecture artifacts built and Apple Development signed; final-source arm64 bundle refreshed with ad-hoc signing. Release build and notarization remain pending. | +| Packaged smoke | Pass | Final-source host bundle and both helper slices checked; x64 `otool` byte parsing fixed with a regression test | + +See [feasibility and baseline](ios-usb-capture-feasibility.md) and +[implementation evidence](ios-usb-capture-implementation.md). G1 and G2 have **Partial** +physical evidence; G3–G4 remain **Not tested**. No release gate has passed, and no +software fixture is a substitute for the remaining physical and installed-app tests. diff --git a/electron-builder.json5 b/electron-builder.json5 index 73d57fb8b..c00554efd 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -71,7 +71,7 @@ "artifactName": "${productName}-${arch}.${ext}", "extendInfo": { "NSAudioCaptureUsageDescription": "Recordly needs audio capture permission to record system audio.", - "NSCameraUsageDescription": "Recordly needs camera access to record webcam video.", + "NSCameraUsageDescription": "Recordly needs camera access to record webcam video or receive a connected iPhone or iPad screen.", "NSMicrophoneUsageDescription": "Recordly needs microphone access to record voice audio.", "NSCameraUseContinuityCameraDeviceType": true, "com.apple.security.device.audio-input": true, diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index c00ef3226..6c583f171 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -197,6 +197,7 @@ interface RendererNativeExportCapabilities { interface Window { electronAPI: { + iosCapture: import("../src/shared/iosCaptureAPI").IOSCaptureAPI; hudOverlaySetIgnoreMouse: (ignore: boolean) => void; hudOverlaySetSourceSelectionActive: (active: boolean) => void; hudOverlayDrag: (phase: "start" | "move" | "end", screenX: number, screenY: number) => void; @@ -216,11 +217,11 @@ interface Window { getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; openSourceSelector: () => Promise; - selectSource: (source: ProcessedDesktopSource) => Promise; + selectSource: (source: SelectedCaptureSource) => Promise; showSourceHighlight: (source: ProcessedDesktopSource) => Promise<{ success: boolean }>; - getSelectedSource: () => Promise; + getSelectedSource: () => Promise; onSelectedSourceChanged: ( - callback: (source: ProcessedDesktopSource | null) => void, + callback: (source: SelectedCaptureSource | null) => void, ) => () => void; startNativeScreenRecording: ( source: ProcessedDesktopSource, @@ -708,6 +709,7 @@ interface Window { ) => Promise<{ success: boolean; webcamPath: string | null }>; setCurrentRecordingSession: ( session: { + captureMetadata?: import("../src/shared/iosCapture").CaptureMetadata; videoPath: string; webcamPath?: string | null; timeOffsetMs?: number; @@ -718,6 +720,7 @@ interface Window { getCurrentRecordingSession: () => Promise<{ success: boolean; session?: { + captureMetadata?: import("../src/shared/iosCapture").CaptureMetadata; videoPath: string; webcamPath?: string | null; timeOffsetMs?: number; @@ -907,6 +910,9 @@ interface Window { }; } +type SelectedCaptureSource = + import("../src/shared/iosCapture").CaptureSource; + interface ProcessedDesktopSource { id: string; name: string; diff --git a/electron/ipc/cursor/bounds.ts b/electron/ipc/cursor/bounds.ts index 1b8b8eaa8..a7d488d0e 100644 --- a/electron/ipc/cursor/bounds.ts +++ b/electron/ipc/cursor/bounds.ts @@ -172,7 +172,7 @@ export function stopWindowBoundsCapture() { } async function refreshSelectedWindowBounds() { - if (!selectedSource?.id?.startsWith("window:")) { + if (!selectedSource?.id?.startsWith("window:") || selectedSource.sourceType === "ios-device") { setSelectedWindowBounds(null); return; } diff --git a/electron/ipc/cursor/telemetry.ts b/electron/ipc/cursor/telemetry.ts index 73f62714e..264783d16 100644 --- a/electron/ipc/cursor/telemetry.ts +++ b/electron/ipc/cursor/telemetry.ts @@ -194,7 +194,9 @@ export function getNormalizedCursorPoint() { }; } - const sourceDisplayId = Number(selectedSource?.display_id); + const sourceDisplayId = Number( + selectedSource?.sourceType === "ios-device" ? undefined : selectedSource?.display_id, + ); const sourceDisplay = Number.isFinite(sourceDisplayId) ? (getScreen() .getAllDisplays() diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 2a0f998eb..da742de64 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -6,6 +6,7 @@ import { registerExportHandlers } from "./register/export"; import { registerPermissionHandlers } from "./register/permissions"; import { registerProjectHandlers } from "./register/project"; import { registerRecordingHandlers } from "./register/recording"; +import { registerIOSCaptureHandlers } from "./register/iosCapture"; import { registerSettingsHandlers } from "./register/settings"; import { registerSourceHandlers } from "./register/sources"; import { @@ -54,10 +55,16 @@ export function killWindowsCaptureProcess() { export function registerIpcHandlers( createEditorWindow: () => void, createSourceSelectorWindow: () => BrowserWindow, - _getMainWindow: () => BrowserWindow | null, + getMainWindow: () => BrowserWindow | null, getSourceSelectorWindow: () => BrowserWindow | null, onRecordingStateChange?: (recording: boolean, sourceName: string) => void, ) { + registerIOSCaptureHandlers({ + getMainWindow, + getSourceSelectorWindow, + createEditorWindow, + onRecordingStateChange, + }); registerSourceHandlers({ createEditorWindow, createSourceSelectorWindow, diff --git a/electron/ipc/paths/binaries.ts b/electron/ipc/paths/binaries.ts index 77c8dd35f..c080193b8 100644 --- a/electron/ipc/paths/binaries.ts +++ b/electron/ipc/paths/binaries.ts @@ -93,6 +93,32 @@ export function getNativeCaptureHelperBinaryPath(): string { return path.join(app.getPath("userData"), "native-tools", "recordly-screencapturekit-helper"); } +export function getIOSDeviceCaptureHelperBinaryPath(): string { + return getPrebundledNativeHelperPath("recordly-ios-device-helper"); +} + +export async function ensureIOSDeviceCaptureHelperBinary(): Promise { + if (process.platform !== "darwin") throw new Error("UNSUPPORTED_PLATFORM"); + const binary = getIOSDeviceCaptureHelperBinaryPath(); + if (!existsSync(binary)) { + if (app.isPackaged) throw new Error("HELPER_UNAVAILABLE"); + await execFileAsync( + process.execPath, + [path.join(app.getAppPath(), "scripts", "build-ios-device-helper.mjs")], + { + env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, + maxBuffer: 1024 * 1024, + }, + ); + } + try { + await fs.access(binary, fsConstants.X_OK); + } catch { + throw new Error("HELPER_UNAVAILABLE"); + } + return binary; +} + export function getSystemCursorHelperSourcePath(): string { return resolveUnpackedAppPath("electron", "native", "SystemCursorAssets.swift"); } diff --git a/electron/ipc/project/session.test.ts b/electron/ipc/project/session.test.ts new file mode 100644 index 000000000..5d6b06321 --- /dev/null +++ b/electron/ipc/project/session.test.ts @@ -0,0 +1,94 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { + getRecordingSessionManifestPath, + persistRecordingSessionManifest, + resolveRecordingSessionManifest, +} from "./session"; +vi.mock("electron", () => ({ + app: { getPath: () => "/private/tmp", getName: () => "Recordly", isPackaged: false }, +})); +const roots: string[] = []; +const metadata = { + version: 1 as const, + sourceKind: "ios-device" as const, + mode: "passthrough" as const, + format: { + codedWidth: 1920, + codedHeight: 1080, + displayWidth: 1080, + displayHeight: 1920, + codec: "h264", + colorPrimaries: null, + transferFunction: null, + ycbcrMatrix: null, + fullRange: null, + transform: [0, 1, -1, 0, 1080, 0] as const, + observedFrameRate: 30, + fingerprint: "format1", + }, + deviceAudioRecorded: false, + narrationRecorded: false, + stopReason: "user-stop", + interrupted: false, +}; +async function fixture() { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ios-session-")); + roots.push(dir); + const videoPath = path.join(dir, "source-video.mov"); + await fs.writeFile(videoPath, "fixture"); + return { + videoPath, + webcamPath: null, + captureMetadata: metadata, + hideOverlayCursorByDefault: true, + }; +} +afterEach(async () => { + await Promise.all(roots.splice(0).map((p) => fs.rm(p, { recursive: true, force: true }))); +}); +it("keeps device metadata without a webcam", async () => { + const session = await fixture(); + await persistRecordingSessionManifest(session); + const result = await resolveRecordingSessionManifest(session.videoPath); + expect(result?.captureMetadata).toEqual(metadata); + expect(result?.hideOverlayCursorByDefault).toBe(true); + expect(result?.webcamPath).toBeNull(); +}); +it("ignores malformed optional provenance and rejects linked traversal or symlinks", async () => { + const session = await fixture(); + const manifest = getRecordingSessionManifestPath(session.videoPath); + await fs.writeFile( + manifest, + JSON.stringify({ + version: 3, + videoFileName: "source-video.mov", + webcamFileName: "../outside.mov", + captureMetadata: { sourceKind: "ios-device", secret: "x" }, + }), + ); + const result = await resolveRecordingSessionManifest(session.videoPath); + expect(result?.webcamPath).toBeNull(); + expect(result?.captureMetadata).toBeUndefined(); + const outside = await fixture(); + await fs.symlink(outside.videoPath, path.join(path.dirname(session.videoPath), "webcam.mov")); + await fs.writeFile( + manifest, + JSON.stringify({ + version: 2, + videoFileName: "source-video.mov", + webcamFileName: "webcam.mov", + }), + ); + expect((await resolveRecordingSessionManifest(session.videoPath))?.webcamPath).toBeNull(); +}); +it.each([1, 2])("reads legacy manifest v%s", async (version) => { + const s = await fixture(); + await fs.writeFile( + getRecordingSessionManifestPath(s.videoPath), + JSON.stringify({ version, videoFileName: "source-video.mov", timeOffsetMs: 12 }), + ); + expect((await resolveRecordingSessionManifest(s.videoPath))?.timeOffsetMs).toBe(12); +}); diff --git a/electron/ipc/project/session.ts b/electron/ipc/project/session.ts index 7b2131aee..884a25dc3 100644 --- a/electron/ipc/project/session.ts +++ b/electron/ipc/project/session.ts @@ -1,3 +1,5 @@ +import { parseCaptureMetadata } from "../../../src/shared/iosCapture"; +import { randomUUID } from "node:crypto"; import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; @@ -26,19 +28,37 @@ export async function persistRecordingSessionManifest( const normalizedWebcamPath = normalizeVideoSourcePath(session.webcamPath ?? null); const manifestPath = getRecordingSessionManifestPath(normalizedVideoPath); - if (!normalizedWebcamPath) { + const captureMetadata = normalizeCaptureMetadata(session.captureMetadata); + if (!normalizedWebcamPath && !captureMetadata && !session.hideOverlayCursorByDefault) { await fs.rm(manifestPath, { force: true }); return; } const manifest: RecordingSessionManifest = { - version: 2, + version: captureMetadata || session.hideOverlayCursorByDefault ? 3 : 2, + ...(captureMetadata ? { captureMetadata } : {}), + hideOverlayCursorByDefault: session.hideOverlayCursorByDefault, videoFileName: path.basename(normalizedVideoPath), - webcamFileName: path.basename(normalizedWebcamPath), + webcamFileName: + normalizedWebcamPath && + path.resolve(path.dirname(normalizedWebcamPath)) === + path.resolve(path.dirname(normalizedVideoPath)) + ? path.basename(normalizedWebcamPath) + : null, timeOffsetMs: normalizeRecordingTimeOffsetMs(session.timeOffsetMs), }; - await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf-8"); + const temporaryPath = `${manifestPath}.${randomUUID()}.tmp`; + try { + await fs.writeFile(temporaryPath, JSON.stringify(manifest, null, 2), { + encoding: "utf-8", + flag: "wx", + mode: 0o600, + }); + await fs.rename(temporaryPath, manifestPath); + } finally { + await fs.rm(temporaryPath, { force: true }); + } } export async function resolveRecordingSessionManifest( @@ -52,12 +72,19 @@ export async function resolveRecordingSessionManifest( const manifestPath = getRecordingSessionManifestPath(normalizedVideoPath); try { + const manifestStat = await fs.lstat(manifestPath); + if (!manifestStat.isFile() || manifestStat.isSymbolicLink()) return null; const content = await fs.readFile(manifestPath, "utf-8"); const parsed = parseJsonWithByteOrderMark>(content); - if (parsed.version !== 1 && parsed.version !== 2) { + if (parsed.version !== 1 && parsed.version !== 2 && parsed.version !== 3) { return null; } + if (parsed.videoFileName !== path.basename(normalizedVideoPath)) return null; + const provenance = { + captureMetadata: normalizeCaptureMetadata(parsed.captureMetadata), + hideOverlayCursorByDefault: parsed.hideOverlayCursorByDefault === true, + }; const webcamFileName = typeof parsed.webcamFileName === "string" && parsed.webcamFileName.trim() ? parsed.webcamFileName.trim() @@ -65,20 +92,18 @@ export async function resolveRecordingSessionManifest( if (!webcamFileName) { return { videoPath: normalizedVideoPath, + ...provenance, webcamPath: null, timeOffsetMs: normalizeRecordingTimeOffsetMs(parsed.timeOffsetMs), }; } - const webcamPath = path.join(path.dirname(normalizedVideoPath), webcamFileName); - const webcamExists = await fs - .access(webcamPath, fsConstants.F_OK) - .then(() => true) - .catch(() => false); + const webcamPath = await resolveSessionLinkedFile(normalizedVideoPath, webcamFileName); return { videoPath: normalizedVideoPath, - webcamPath: webcamExists ? webcamPath : null, + webcamPath, + ...provenance, timeOffsetMs: normalizeRecordingTimeOffsetMs(parsed.timeOffsetMs), }; } catch { @@ -138,3 +163,32 @@ export async function resolveRecordingSession( webcamPath: linkedWebcamPath, }; } + +async function resolveSessionLinkedFile(videoPath: string, name: string): Promise { + if ( + !name || + name !== path.basename(name) || + name.includes("\\") || + name === "." || + name === ".." + ) + return null; + try { + const directory = await fs.realpath(path.dirname(videoPath)); + const linked = path.join(directory, name); + const stat = await fs.lstat(linked); + return stat.isFile() && !stat.isSymbolicLink() && (await fs.realpath(linked)) === linked + ? linked + : null; + } catch { + return null; + } +} + +function normalizeCaptureMetadata(value: unknown) { + try { + return parseCaptureMetadata(value); + } catch { + return undefined; + } +} diff --git a/electron/ipc/recording/ios/__fixtures__/fake-ios-helper.mjs b/electron/ipc/recording/ios/__fixtures__/fake-ios-helper.mjs new file mode 100644 index 000000000..3c1a4713d --- /dev/null +++ b/electron/ipc/recording/ios/__fixtures__/fake-ios-helper.mjs @@ -0,0 +1,46 @@ +import readline from "node:readline"; +import fs from "node:fs"; + +const scenario = process.argv[2]; +let sequence = 0; +const input = readline.createInterface({ input: process.stdin }); +input.on("line", (line) => { + const command = JSON.parse(line); + if (command.command === "shutdown") { + input.close(); + process.exit(0); + } + if (scenario === "crash") process.exit(2); + if (scenario === "silent") return; + if (scenario === "malformed") { + process.stdout.write("broken\n"); + return; + } + if (scenario === "bad-preview") fs.writeSync(3, Buffer.alloc(24)); + const event = + JSON.stringify({ + protocolVersion: 1, + event: "accepted", + requestId: command.requestId, + sequence: ++sequence, + payload: { + build: "test", + protocolVersion: 1, + capabilities: { + supportsPause: false, + supportsWebcam: false, + supportsTouchTelemetry: false, + previewMaxLongestEdge: 480, + previewMaxFramesPerSecond: 5, + previewMaxJpegBytes: 131072, + protocolVersion: 1, + }, + }, + }) + "\n"; + if (scenario === "split") { + process.stderr.write("x".repeat(32768)); + process.stdout.write(event.slice(0, 20)); + setTimeout(() => process.stdout.write(event.slice(20)), 5); + } else process.stdout.write(event); +}); +input.on("close", () => process.exit(0)); diff --git a/electron/ipc/recording/ios/controller.lifecycle.integration.test.ts b/electron/ipc/recording/ios/controller.lifecycle.integration.test.ts new file mode 100644 index 000000000..b4d199834 --- /dev/null +++ b/electron/ipc/recording/ios/controller.lifecycle.integration.test.ts @@ -0,0 +1,131 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { IOSCaptureEvent } from "../../../../src/shared/iosCapture"; +import { getRecordingLease } from "../recordingLease"; +import { IOSCaptureController, type IOSControllerDependencies } from "./controller"; +import type { IOSHelperTransport } from "./helperProcess"; +import type { IOSSessionStorage } from "./storage"; + +const source = { + sourceType: "ios-device", + id: "ios-device:phone", + deviceToken: "phone", + displayName: "Phone", + generation: 1, + deviceAudio: "unknown", +} as const; +const controllers: IOSCaptureController[] = []; +function transport() { + const listeners = new Set<(event: IOSCaptureEvent) => void>(); + let sequence = 0; + const helper: IOSHelperTransport = { + request: vi.fn(async (input) => { + if (input.command === "discover") + queueMicrotask(() => { + const event: IOSCaptureEvent = { + protocolVersion: 1, + event: "inventoryChanged", + sequence: ++sequence, + payload: { devices: [source], microphones: [], inventoryGeneration: 1 }, + }; + for (const listener of listeners) listener(event); + }); + return { + protocolVersion: 1, + event: "accepted", + requestId: "test", + sequence: ++sequence, + payload: {}, + }; + }), + onEvent: (listener) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + onExit: () => () => undefined, + onPreview: () => () => undefined, + shutdown: vi.fn(async () => undefined), + }; + return helper; +} +function controller(overrides: Partial) { + const instance = new IOSCaptureController({ + enabled: true, + createHelper: async () => transport(), + allocate: async () => { + throw new Error("allocation unexpected"); + }, + permissions: async () => undefined, + finalize: async () => { + throw new Error("finalization unexpected"); + }, + discard: async () => undefined, + onCommitted: () => undefined, + ...overrides, + }); + controllers.push(instance); + return instance; +} +afterEach(async () => { + for (const instance of controllers.splice(0)) await instance.shutdown(); +}); + +it("shutdown waits for a pending helper creation and closes it before any hello or discover command", async () => { + let created!: (helper: IOSHelperTransport) => void; + const creation = new Promise((resolve) => { + created = resolve; + }); + const instance = controller({ createHelper: () => creation }); + const discovery = instance.discover(); + const rejected = expect(discovery).rejects.toThrow("HELPER_UNAVAILABLE"); + let stopped = false; + const shutdown = instance.shutdown().then(() => { + stopped = true; + }); + await Promise.resolve(); + expect(stopped).toBe(false); + const helper = transport(); + created(helper); + await Promise.all([rejected, shutdown]); + expect(helper.request).not.toHaveBeenCalled(); + expect(helper.shutdown).toHaveBeenCalledOnce(); + expect(getRecordingLease()).toBeNull(); +}); + +it("cancelling while allocation is pending deletes only the newly allocated empty session and cannot reopen inputs", async () => { + let allocated!: (storage: IOSSessionStorage) => void; + const allocation = new Promise((resolve) => { + allocated = resolve; + }); + const helper = transport(); + const discard = vi.fn(async () => undefined); + const instance = controller({ + createHelper: async () => helper, + allocate: () => allocation, + discard, + }); + await instance.discover(); + const preparation = instance.prepare({ + deviceToken: "phone", + generation: 1, + options: { deviceAudio: false, microphoneToken: null }, + }); + const rejected = expect(preparation).rejects.toThrow("INVALID_REQUEST"); + await Promise.resolve(); + const sessionId = instance.getSnapshot().sessionId!; + await instance.release(sessionId); + const storage = { + sessionId, + directory: `/tmp/ios-${sessionId}`, + journalPath: `/tmp/ios-${sessionId}/capture-journal.json`, + }; + allocated(storage); + await rejected; + expect(discard).toHaveBeenCalledExactlyOnceWith(storage); + expect( + vi.mocked(helper.request).mock.calls.some(([request]) => request.command === "prepare"), + ).toBe(false); + expect(instance.getSnapshot()).toMatchObject({ phase: "idle", sessionId: null }); + expect(getRecordingLease()).toBeNull(); +}); diff --git a/electron/ipc/recording/ios/controller.test.ts b/electron/ipc/recording/ios/controller.test.ts new file mode 100644 index 000000000..07a5224fe --- /dev/null +++ b/electron/ipc/recording/ios/controller.test.ts @@ -0,0 +1,569 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { + IOSCaptureEvent, + IOSCaptureSnapshot, + IOSDeviceSource, +} from "../../../../src/shared/iosCapture"; +import { IOSCaptureController, type IOSControllerDependencies } from "./controller"; +import type { IOSHelperRequest, IOSHelperTransport } from "./helperProcess"; +import { getRecordingLease } from "../recordingLease"; + +const source = { + sourceType: "ios-device", + id: "ios-device:phone", + deviceToken: "phone", + displayName: "Phone", + generation: 1, + deviceAudio: "unknown", +} as const; +const options = { deviceAudio: false, microphoneToken: null }; +const format = { + codedWidth: 640, + codedHeight: 480, + displayWidth: 640, + displayHeight: 480, + codec: "avc1", + colorPrimaries: "ITU_R_709_2", + transferFunction: "ITU_R_709_2", + ycbcrMatrix: "ITU_R_709_2", + fullRange: false, + transform: [1, 0, 0, 1, 0, 0], + observedFrameRate: 30, + fingerprint: "test-format", +} as const; +const controllers: IOSCaptureController[] = []; + +function harness( + overrides: Partial = {}, + initialDevices: readonly IOSDeviceSource[] | null = [source], +) { + let discoveryDevices = initialDevices; + let listener: (event: IOSCaptureEvent) => void = () => {}; + let exit: (error: Error) => void = () => {}; + let sequence = 0; + const requests: IOSHelperRequest[] = []; + const committed = vi.fn(); + const finalize = vi.fn(async (storage) => ({ + sessionId: storage.sessionId, + videoPath: `${storage.directory}/source-video.mov`, + hideOverlayCursorByDefault: true, + captureMetadata: { + version: 1, + sourceKind: "ios-device", + mode: "passthrough", + format, + deviceAudioRecorded: false, + narrationRecorded: false, + stopReason: "stopped", + interrupted: false, + }, + })); + const controller = new IOSCaptureController({ + enabled: true, + createHelper: async () => ({ + request: async (input) => { + requests.push(input); + const devices = discoveryDevices; + if (input.command === "discover" && devices !== null) + queueMicrotask(() => + emit( + "inventoryChanged", + { devices, microphones: [], inventoryGeneration: 1 }, + false, + ), + ); + if (input.command === "prepare") + queueMicrotask(() => + emit("prepared", { source, options, format, mode: "passthrough" }), + ); + return { + protocolVersion: 1, + event: "accepted", + sequence: ++sequence, + requestId: "r", + payload: {}, + } as IOSCaptureEvent; + }, + onEvent: (callback) => { + listener = callback; + return () => {}; + }, + onExit: (callback) => { + exit = callback; + return () => {}; + }, + onPreview: () => () => {}, + shutdown: async () => {}, + }), + allocate: async (sessionId) => ({ + sessionId, + directory: `/tmp/ios-${sessionId}`, + journalPath: `/tmp/ios-${sessionId}/capture-journal.json`, + }), + permissions: async () => {}, + finalize, + discard: async () => {}, + onCommitted: committed, + ...overrides, + }); + function emit(event: string, payload: unknown, scoped = true) { + const s = controller.getSnapshot(); + listener({ + protocolVersion: 1, + event, + sequence: ++sequence, + ...(scoped ? { sessionId: s.sessionId, generation: s.generation } : {}), + payload, + } as IOSCaptureEvent); + } + async function ready() { + await controller.discover(); + await controller.prepare({ deviceToken: source.deviceToken, generation: 1, options }); + return controller.getSnapshot().sessionId!; + } + function nativeFinished() { + const sessionId = controller.getSnapshot().sessionId!; + emit("nativeFinalized", { + result: { + sessionId, + stopReason: "stopped", + mode: "passthrough", + format, + timingFile: "native-timing.json", + video: { + relativeName: "source-video.mov", + mediaKind: "video", + firstHostTime: { value: "100", timescale: 1 }, + duration: { value: "1", timescale: 1 }, + sampleCount: 1, + videoFormat: format, + }, + }, + }); + } + controllers.push(controller); + return { + controller, + requests, + ready, + emit, + exit: () => exit(new Error("HELPER_EXITED")), + nativeFinished, + finalize, + committed, + setDiscoveryDevices: (devices: readonly IOSDeviceSource[] | null) => { + discoveryDevices = devices; + }, + }; +} +afterEach(async () => { + for (const controller of controllers.splice(0)) await controller.shutdown(); + vi.useRealTimers(); +}); + +function helperWithDelayedRequest(command?: "hello" | "discover") { + const listeners = new Set<(event: IOSCaptureEvent) => void>(); + let sequence = 0; + let rejectRequest!: (error: Error) => void; + const delayed = new Promise((_resolve, reject) => { + rejectRequest = reject; + }); + const shutdown = vi.fn(async () => { + listeners.clear(); + }); + const helper: IOSHelperTransport = { + request: async (input) => { + if (input.command === command) return delayed; + if (input.command === "discover") + queueMicrotask(() => { + const event: IOSCaptureEvent = { + protocolVersion: 1, + event: "inventoryChanged", + sequence: ++sequence, + payload: { devices: [source], microphones: [], inventoryGeneration: 1 }, + }; + for (const listener of listeners) listener(event); + }); + return { + protocolVersion: 1, + event: "accepted", + requestId: "test", + sequence: ++sequence, + payload: {}, + }; + }, + onEvent: (listener) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + onExit: () => () => {}, + onPreview: () => () => {}, + shutdown, + }; + return { helper, rejectRequest, shutdown }; +} + +it.each([ + "hello", + "discover", +] as const)("replaces a helper whose %s fails after the discovery observation window", async (command) => { + vi.useFakeTimers(); + const delayed = helperWithDelayedRequest(command); + const fresh = helperWithDelayedRequest(); + const createHelper = vi + .fn() + .mockResolvedValueOnce(delayed.helper) + .mockResolvedValueOnce(fresh.helper); + const h = harness({ createHelper }); + const first = h.controller.discover(); + await vi.advanceTimersByTimeAsync(5_000); + expect(await first).toMatchObject({ phase: "idle", devices: [] }); + delayed.rejectRequest(new Error("HELPER_UNAVAILABLE")); + await vi.advanceTimersByTimeAsync(0); + expect(h.controller.getSnapshot()).toMatchObject({ + phase: "failed", + error: { code: "HELPER_UNAVAILABLE" }, + }); + expect(delayed.shutdown).toHaveBeenCalledOnce(); + const refreshed = h.controller.discover(); + await vi.advanceTimersByTimeAsync(0); + expect(await refreshed).toMatchObject({ phase: "idle", devices: [source], error: null }); + expect(createHelper).toHaveBeenCalledTimes(2); +}); + +it("ignores a delayed hello failure from a helper replaced after shutdown", async () => { + vi.useFakeTimers(); + const delayed = helperWithDelayedRequest("hello"); + const fresh = helperWithDelayedRequest(); + const createHelper = vi + .fn() + .mockResolvedValueOnce(delayed.helper) + .mockResolvedValueOnce(fresh.helper); + const h = harness({ createHelper }); + const rejected = expect(h.controller.discover()).rejects.toThrow("HELPER_UNAVAILABLE"); + await vi.advanceTimersByTimeAsync(0); + await h.controller.shutdown(); + await rejected; + const refreshed = h.controller.discover(); + await vi.advanceTimersByTimeAsync(0); + await refreshed; + delayed.rejectRequest(new Error("HELPER_UNAVAILABLE")); + await vi.advanceTimersByTimeAsync(0); + expect(h.controller.getSnapshot()).toMatchObject({ + phase: "idle", + devices: [source], + error: null, + }); + expect(fresh.shutdown).not.toHaveBeenCalled(); +}); + +it("keeps an empty initial inventory discovering until a device arrives", async () => { + vi.useFakeTimers(); + const h = harness({}, []); + let result: IOSCaptureSnapshot | undefined; + const discovery = h.controller.discover().then((snapshot) => { + result = snapshot; + }); + await vi.advanceTimersByTimeAsync(0); + expect(h.controller.getSnapshot()).toMatchObject({ phase: "discovering", devices: [] }); + expect(result).toBeUndefined(); + h.emit( + "inventoryChanged", + { devices: [source], microphones: [], inventoryGeneration: 1 }, + false, + ); + await discovery; + expect(result).toMatchObject({ phase: "idle", devices: [source] }); +}); + +it("shares a pending discovery across repeated refresh requests", async () => { + vi.useFakeTimers(); + const h = harness({}, []); + const first = h.controller.discover(); + await vi.advanceTimersByTimeAsync(0); + const second = h.controller.discover(); + await vi.advanceTimersByTimeAsync(0); + expect(h.requests.filter((request) => request.command === "discover")).toHaveLength(1); + h.emit( + "inventoryChanged", + { devices: [source], microphones: [], inventoryGeneration: 1 }, + false, + ); + expect(await first).toEqual(await second); + expect((await first).devices).toEqual([source]); +}); + +it("settles empty discovery after five seconds and still accepts later device arrivals", async () => { + vi.useFakeTimers(); + const h = harness({}, []); + let result: IOSCaptureSnapshot | undefined; + const discovery = h.controller.discover().then((snapshot) => { + result = snapshot; + }); + await vi.advanceTimersByTimeAsync(4_999); + expect(result).toBeUndefined(); + expect(h.controller.getSnapshot().phase).toBe("discovering"); + await vi.advanceTimersByTimeAsync(1); + await discovery; + expect(result).toMatchObject({ phase: "idle", devices: [] }); + h.emit( + "inventoryChanged", + { devices: [source], microphones: [], inventoryGeneration: 1 }, + false, + ); + expect(h.controller.getSnapshot()).toMatchObject({ phase: "idle", devices: [source] }); + expect(h.requests.filter((request) => request.command === "discover")).toHaveLength(1); +}); + +it("retains a prepared source and ready phase while refresh waits for inventory", async () => { + vi.useFakeTimers(); + const h = harness(); + const id = await h.ready(); + h.setDiscoveryDevices([]); + let result: IOSCaptureSnapshot | undefined; + const refresh = h.controller.discover().then((snapshot) => { + result = snapshot; + }); + await vi.advanceTimersByTimeAsync(0); + expect(h.controller.getSnapshot()).toMatchObject({ + phase: "ready", + source, + sessionId: id, + format, + }); + expect(result).toBeUndefined(); + h.emit( + "inventoryChanged", + { devices: [source], microphones: [], inventoryGeneration: 1 }, + false, + ); + await refresh; + expect(result).toMatchObject({ phase: "ready", source, sessionId: id, format }); + expect(h.requests.filter((request) => request.command === "prepare")).toHaveLength(1); +}); + +it("shutdown rejects pending discovery and cancels its wait without a timeout", async () => { + vi.useFakeTimers(); + const h = harness({}, null); + let outcome: unknown; + const discovery = h.controller.discover().then( + (snapshot) => { + outcome = snapshot; + }, + (error) => { + outcome = error; + }, + ); + await vi.advanceTimersByTimeAsync(0); + await h.controller.shutdown(); + await vi.advanceTimersByTimeAsync(0); + expect(outcome).toEqual(new Error("HELPER_UNAVAILABLE")); + expect(h.controller.getSnapshot().phase).toBe("idle"); + expect(vi.getTimerCount()).toBe(0); + await discovery; +}); + +it.each([ + "preparing", + "starting", + "recording", + "stopping", + "finalising", +] as const)("rejects discovery during %s without replacing recording inputs", async (phase) => { + let allowPermission!: () => void; + const permissions = new Promise((resolve) => { + allowPermission = resolve; + }); + const h = harness( + phase === "preparing" + ? { permissions: () => permissions } + : phase === "finalising" + ? { finalize: () => new Promise(() => {}) } + : {}, + ); + const pending: Promise[] = []; + try { + await h.controller.discover(); + if (phase === "preparing") { + pending.push( + h.controller + .prepare({ deviceToken: "phone", generation: 1, options }) + .catch(() => {}), + ); + } else { + await h.controller.prepare({ deviceToken: "phone", generation: 1, options }); + const id = h.controller.getSnapshot().sessionId!; + const start = h.controller.start(id); + pending.push(start.catch(() => {})); + if (phase !== "starting") { + h.emit("recordingStarted", { + firstHostTime: { value: "100", timescale: 1 }, + acceptedVideoSamples: 1, + }); + await start; + } + if (phase === "stopping" || phase === "finalising") + pending.push(h.controller.stop(id).catch(() => {})); + if (phase === "finalising") h.nativeFinished(); + } + expect(h.controller.getSnapshot().phase).toBe(phase); + await expect(h.controller.discover()).rejects.toThrow("RECORDING_BUSY"); + expect(h.controller.getSnapshot().phase).toBe(phase); + expect(h.requests.filter((request) => request.command === "discover")).toHaveLength(1); + } finally { + h.exit(); + allowPermission(); + await Promise.all(pending); + } +}); + +it("does not mark recording on command acceptance alone", async () => { + const h = harness(); + const id = await h.ready(); + const started = h.controller.start(id); + await Promise.resolve(); + expect(h.controller.getSnapshot().phase).toBe("starting"); + h.emit("recordingStarted", { + firstHostTime: { value: "100", timescale: 1 }, + acceptedVideoSamples: 1, + }); + await started; + expect(h.controller.getSnapshot().phase).toBe("recording"); + const stop1 = h.controller.stop(id); + const stop2 = h.controller.stop(id); + h.nativeFinished(); + expect(await stop1).toEqual(await stop2); + expect(h.finalize).toHaveBeenCalledTimes(1); + expect(h.committed).toHaveBeenCalledTimes(1); + expect(h.requests.filter((r) => r.command === "stop")).toHaveLength(1); +}); +it("commits spontaneous interruption without a surviving renderer stop promise", async () => { + const h = harness(); + const id = await h.ready(); + const start = h.controller.start(id); + h.emit("recordingStarted", { + firstHostTime: { value: "100", timescale: 1 }, + acceptedVideoSamples: 1, + }); + await start; + h.nativeFinished(); + await vi.waitFor(() => expect(h.committed).toHaveBeenCalledTimes(1)); + expect(h.controller.getSnapshot().phase).toBe("completed"); +}); +it("rejects stale tokens and source generations before creating a prepared session", async () => { + const h = harness(); + await h.controller.discover(); + await expect( + h.controller.prepare({ deviceToken: "other", generation: 1, options }), + ).rejects.toThrow("DEVICE_NOT_FOUND"); + await expect( + h.controller.prepare({ deviceToken: "phone", generation: 0, options }), + ).rejects.toThrow("DEVICE_NOT_FOUND"); + expect(h.requests.some((r) => r.command === "prepare")).toBe(false); +}); +it("helper exit rejects pending start and never reports an empty take complete", async () => { + const h = harness(); + const id = await h.ready(); + const start = h.controller.start(id); + const failure = expect(start).rejects.toThrow("HELPER_EXITED"); + h.exit(); + await failure; + expect(h.controller.getSnapshot().phase).toBe("failed"); + expect(h.committed).not.toHaveBeenCalled(); +}); + +it("ignores native finalization outside an armed take and request-scoped failures", async () => { + const h = harness(); + await h.ready(); + h.nativeFinished(); + await Promise.resolve(); + expect(h.finalize).not.toHaveBeenCalled(); + expect(h.controller.getSnapshot().phase).toBe("ready"); +}); +it("reprepares on the same helper so inventory tokens remain valid", async () => { + const h = harness(); + await h.ready(); + await h.controller.prepare({ deviceToken: "phone", generation: 1, options }); + expect(h.requests.filter((r) => r.command === "hello")).toHaveLength(1); + expect(h.requests.filter((r) => r.command === "prepare")).toHaveLength(2); +}); + +it("does not resume a cancelled preparation after permission resolves", async () => { + let allow!: () => void; + const permissions = new Promise((resolve) => { + allow = resolve; + }); + const allocate = vi.fn(async (sessionId: string) => ({ + sessionId, + directory: `/tmp/ios-${sessionId}`, + journalPath: `/tmp/ios-${sessionId}/capture-journal.json`, + })); + const h = harness({ permissions: () => permissions, allocate }); + await h.controller.discover(); + const preparing = h.controller.prepare({ deviceToken: "phone", generation: 1, options }); + const failed = expect(preparing).rejects.toThrow("INVALID_REQUEST"); + await h.controller.release(h.controller.getSnapshot().sessionId!); + allow(); + await failed; + expect(allocate).not.toHaveBeenCalled(); + expect(h.requests.filter((r) => r.command === "prepare")).toHaveLength(0); + expect(h.controller.getSnapshot()).toMatchObject({ phase: "idle", sessionId: null }); + expect(getRecordingLease()).toBeNull(); +}); + +it("keeps terminal work exclusive until editor handoff has finished", async () => { + let finishHandoff!: () => void; + const handoff = new Promise((resolve) => { + finishHandoff = resolve; + }); + const onCommitted = vi.fn(() => handoff); + const h = harness({ onCommitted }); + const id = await h.ready(); + const started = h.controller.start(id); + h.emit("recordingStarted", { + firstHostTime: { value: "100", timescale: 1 }, + acceptedVideoSamples: 1, + }); + await started; + const stopped = h.controller.stop(id); + h.nativeFinished(); + await vi.waitFor(() => expect(onCommitted).toHaveBeenCalledOnce()); + expect(h.controller.getSnapshot().phase).toBe("finalising"); + expect(getRecordingLease()?.owner).toBe("ios-device"); + await expect( + h.controller.prepare({ deviceToken: "phone", generation: 1, options }), + ).rejects.toThrow("RECORDING_BUSY"); + finishHandoff(); + await stopped; + expect(h.controller.getSnapshot().phase).toBe("completed"); + expect(getRecordingLease()).toBeNull(); +}); +it("bounds quit while a finalizer is stalled and retains the take without handoff", async () => { + let abortSignal: AbortSignal | undefined; + const h = harness({ + shutdownGraceMs: 20, + finalize: (_storage, _result, signal) => { + abortSignal = signal; + return new Promise((_resolve, reject) => + signal?.addEventListener("abort", () => reject(new Error("FINALIZATION_FAILED")), { + once: true, + }), + ); + }, + }); + const id = await h.ready(); + const started = h.controller.start(id); + h.emit("recordingStarted", { + firstHostTime: { value: "100", timescale: 1 }, + acceptedVideoSamples: 1, + }); + await started; + h.nativeFinished(); + await h.controller.shutdown(); + expect(abortSignal?.aborted).toBe(true); + expect(h.controller.getSnapshot().phase).toBe("recoveryAvailable"); + expect(h.committed).not.toHaveBeenCalled(); + expect(getRecordingLease()).toBeNull(); +}); diff --git a/electron/ipc/recording/ios/controller.ts b/electron/ipc/recording/ios/controller.ts new file mode 100644 index 000000000..8d6633b0d --- /dev/null +++ b/electron/ipc/recording/ios/controller.ts @@ -0,0 +1,666 @@ +import { randomUUID } from "node:crypto"; +import { + IOS_CAPTURE_ERROR_CODES, + type CommittedIOSRecording, + type IOSCaptureEvent, + type IOSCaptureSnapshot, + type IOSRecordingOptions, + type NativeCaptureResult, +} from "../../../../src/shared/iosCapture"; +import { + acquireRecordingLease, + releaseRecordingLease, + type RecordingLease, +} from "../recordingLease"; +import type { IOSHelperTransport } from "./helperProcess"; +import type { IOSPreviewFrame } from "./preview"; +import type { IOSCaptureJournal, IOSSessionStorage } from "./storage"; + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; + reject: (error: Error) => void; +} +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + void promise.catch(() => { + /* Failure is retained by the authoritative controller. */ + }); + return { promise, resolve, reject }; +} + +export interface IOSControllerDependencies { + enabled: boolean; + createHelper: () => Promise; + allocate: (sessionId: string) => Promise; + permissions: (options: IOSRecordingOptions) => Promise; + finalize: ( + storage: IOSSessionStorage, + result: NativeCaptureResult, + signal?: AbortSignal, + ) => Promise; + discard: (storage: IOSSessionStorage) => Promise; + onCommitted: (recording: CommittedIOSRecording) => void | Promise; + journal?: (storage: IOSSessionStorage, patch: Partial) => Promise; + checkSpace?: (storage: IOSSessionStorage, elapsedMs: number) => Promise; + setCapturing?: (capturing: boolean) => void; + shutdownGraceMs?: number; +} + +export function initialIOSCaptureSnapshot(enabled: boolean): IOSCaptureSnapshot { + return { + sequence: 0, + generation: 0, + sessionId: null, + phase: enabled ? "idle" : "unavailable", + devices: [], + microphones: [], + source: null, + options: null, + format: null, + mode: null, + elapsedMs: 0, + acceptedVideoSamples: 0, + warningCodes: [], + error: null, + }; +} + +/** The sole owner of capture state, terminal work, and renderer-independent handoff. */ +export class IOSCaptureController { + private state: IOSCaptureSnapshot; + private listeners = new Set<(state: IOSCaptureSnapshot) => void>(); + private previewListeners = new Set<(frame: IOSPreviewFrame) => void>(); + private helper?: IOSHelperTransport; + private helperPromise?: Promise; + private helperEpoch = 0; + private preparationRevision = 0; + private closingHelper?: Promise; + private cleanupHelper: Array<() => void> = []; + private lastEvent = -1; + private lease?: RecordingLease; + private storage?: IOSSessionStorage; + private inventoryWait?: Deferred; + private inventoryTimer?: ReturnType; + private prepareWait?: Deferred; + private startWait?: Deferred; + private terminal?: Deferred; + private finalization?: Promise; + private finalizationAbort?: AbortController; + private timer?: ReturnType; + private stopSent = false; + private stopReason = "user-stop"; + private discardRequested = false; + private previewSequence = -1; + private checkingSpace = false; + private committed?: CommittedIOSRecording; + + constructor(private readonly deps: IOSControllerDependencies) { + this.state = initialIOSCaptureSnapshot(deps.enabled); + } + getSnapshot(): IOSCaptureSnapshot { + return structuredClone(this.state); + } + getCommitted(): CommittedIOSRecording | undefined { + return this.committed; + } + subscribe(listener: (state: IOSCaptureSnapshot) => void) { + this.listeners.add(listener); + listener(this.getSnapshot()); + return () => { + this.listeners.delete(listener); + }; + } + onPreview(listener: (frame: IOSPreviewFrame) => void) { + this.previewListeners.add(listener); + return () => { + this.previewListeners.delete(listener); + }; + } + private update(patch: Partial) { + this.state = { ...this.state, ...patch, sequence: this.state.sequence + 1 }; + for (const listener of this.listeners) listener(this.getSnapshot()); + } + private assertEnabled() { + if (!this.deps.enabled) throw new Error("UNSUPPORTED_PLATFORM"); + } + private assertSession(id: string) { + if (!id || id !== this.state.sessionId) throw new Error("INVALID_REQUEST"); + } + private clearTimer() { + if (this.timer) clearTimeout(this.timer); + this.timer = undefined; + } + private settleInventory(error?: Error) { + if (this.inventoryTimer) clearTimeout(this.inventoryTimer); + this.inventoryTimer = undefined; + const waiting = this.inventoryWait; + this.inventoryWait = undefined; + if (error) waiting?.reject(error); + else waiting?.resolve(this.getSnapshot()); + } + private assertCanDiscover() { + if ( + ["preparing", "starting", "recording", "stopping", "finalising"].includes( + this.state.phase, + ) + ) + throw new Error("RECORDING_BUSY"); + } + private releaseLease() { + if (!this.lease) return; + releaseRecordingLease(this.lease); + this.lease = undefined; + this.deps.setCapturing?.(false); + } + private armTimeout(code: string) { + this.clearTimer(); + this.timer = setTimeout(() => { + this.fail(new Error(code)); + void this.closeHelper(); + }, 10_000); + } + private fail(error: Error) { + this.preparationRevision++; + this.clearTimer(); + this.prepareWait?.reject(error); + this.prepareWait = undefined; + this.startWait?.reject(error); + this.startWait = undefined; + this.settleInventory(error); + this.terminal?.reject(error); + const code = + IOS_CAPTURE_ERROR_CODES.find((candidate) => candidate === error.message) ?? + "HELPER_EXITED"; + this.update({ + phase: this.state.acceptedVideoSamples ? "recoveryAvailable" : "failed", + error: { code, recoverable: !!this.storage }, + }); + void this.closeHelper().finally(() => this.releaseLease()); + if (this.storage) + void this.deps.journal?.(this.storage, { state: "interrupted" }).catch(() => { + /* Failure is retained by the authoritative controller. */ + }); + } + private async ensureHelper(): Promise { + if (this.closingHelper) await this.closingHelper; + if (this.helperPromise) return this.helperPromise; + const epoch = this.helperEpoch; + this.helperPromise = (async () => { + const helper = await this.deps.createHelper(); + if (epoch !== this.helperEpoch) { + await helper.shutdown(); + throw new Error("HELPER_UNAVAILABLE"); + } + this.helper = helper; + this.lastEvent = -1; + this.cleanupHelper = [ + helper.onEvent((event) => this.receive(event)), + helper.onExit((error) => { + this.fail(error); + void this.closeHelper(); + }), + helper.onPreview((frame) => { + if ( + frame.generation !== this.state.generation || + frame.sequence <= this.previewSequence + ) + return; + this.previewSequence = frame.sequence; + for (const listener of this.previewListeners) listener(frame); + }), + helper.onWarning?.((code) => + this.update({ + warningCodes: [...new Set([...this.state.warningCodes, code])].slice(-32), + }), + ) ?? (() => undefined), + ]; + await helper.request({ command: "hello" }); + return helper; + })(); + return this.helperPromise; + } + private async closeHelper() { + this.settleInventory(new Error("HELPER_UNAVAILABLE")); + if (this.closingHelper) return this.closingHelper; + this.helperEpoch++; + const helper = this.helper; + const pending = this.helperPromise; + this.helper = undefined; + this.helperPromise = undefined; + for (const cleanup of this.cleanupHelper.splice(0)) cleanup(); + this.update({ + devices: [], + microphones: [], + phase: this.state.phase === "discovering" ? "idle" : this.state.phase, + }); + this.closingHelper = helper + ? helper.shutdown() + : pending + ? pending.then( + () => undefined, + () => undefined, + ) + : Promise.resolve(); + try { + await this.closingHelper; + } finally { + this.closingHelper = undefined; + } + } + async discover(): Promise { + this.assertEnabled(); + this.assertCanDiscover(); + if (this.inventoryWait) return this.inventoryWait.promise; + this.inventoryWait = deferred(); + const waiting = this.inventoryWait; + if (["idle", "failed", "unavailable", "completed", "cancelled"].includes(this.state.phase)) + this.update({ phase: "discovering", error: null }); + // Bound the observation window without stopping native discovery or its later updates. + this.inventoryTimer = setTimeout(() => { + if (this.inventoryWait !== waiting) return; + this.update({ phase: this.state.phase === "discovering" ? "idle" : this.state.phase }); + this.settleInventory(); + }, 5_000); + const helperEpoch = this.helperEpoch; + void (async () => { + try { + const helper = await this.ensureHelper(); + if (helper !== this.helper) return; + // A recording may have started while helper startup or its hello was pending. + this.assertCanDiscover(); + await helper.request({ command: "discover" }); + } catch (error) { + // Observation may have timed out while this same helper was still starting. + if (helperEpoch !== this.helperEpoch) return; + if ((error as Error).message === "RECORDING_BUSY") { + if (this.inventoryWait === waiting) this.settleInventory(error as Error); + } else this.fail(error as Error); + } + })(); + return waiting.promise; + } + async prepare(input: { + deviceToken: string; + generation: number; + options: IOSRecordingOptions; + }): Promise { + this.assertEnabled(); + if ( + this.closingHelper || + ["preparing", "starting", "recording", "stopping", "finalising"].includes( + this.state.phase, + ) + ) + throw new Error("RECORDING_BUSY"); + const source = this.state.devices.find( + (device) => + device.deviceToken === input.deviceToken && device.generation === input.generation, + ); + if (!source) throw new Error("DEVICE_NOT_FOUND"); + if ( + input.options.microphoneToken && + !this.state.microphones.some((mic) => mic.token === input.options.microphoneToken) + ) + throw new Error("DEVICE_NOT_FOUND"); + const previousSessionId = this.state.phase === "ready" ? this.state.sessionId : null; + if (!previousSessionId) this.lease = acquireRecordingLease("ios-device"); + const revision = ++this.preparationRevision; + const assertPreparing = () => { + if (revision !== this.preparationRevision || this.state.phase !== "preparing") + throw new Error("INVALID_REQUEST"); + }; + this.stopSent = false; + this.stopReason = "user-stop"; + this.discardRequested = false; + this.finalization = undefined; + this.committed = undefined; + this.terminal = deferred(); + this.prepareWait = deferred(); + const waiting = this.prepareWait; + this.previewSequence = -1; + const sessionId = randomUUID(); + this.storage = undefined; + this.update({ + phase: "preparing", + generation: this.state.generation + 1, + sessionId, + source, + options: input.options, + format: null, + mode: null, + acceptedVideoSamples: 0, + elapsedMs: 0, + warningCodes: [], + error: null, + }); + try { + if (previousSessionId) { + // Retain the helper and lease across input release so inventory tokens stay valid. + await this.helper?.request({ command: "release", sessionId: previousSessionId }); + assertPreparing(); + } + await this.deps.permissions(input.options); + assertPreparing(); + const storage = await this.deps.allocate(sessionId); + if (revision !== this.preparationRevision) { + await this.deps.discard(storage); + throw new Error("INVALID_REQUEST"); + } + assertPreparing(); + this.storage = storage; + const helper = await this.ensureHelper(); + assertPreparing(); + this.armTimeout("NO_VIDEO_SAMPLES"); + await helper.request({ + command: "prepare", + sessionId, + generation: this.state.generation, + payload: { + deviceToken: input.deviceToken, + inventoryGeneration: input.generation, + options: input.options, + }, + storage: { + sessionRoot: this.storage.directory, + allowedRelativeNames: [ + "source-video.mov", + "device-audio.mov", + "microphone.mov", + "native-timing.json", + ], + }, + }); + return await waiting.promise; + } catch (error) { + if (revision === this.preparationRevision) { + this.fail(error as Error); + await this.closeHelper(); + } + throw error; + } + } + async setPreviewEnabled(enabled: boolean): Promise { + if (!this.state.sessionId || !this.helper || this.state.phase === "preparing") return; + await this.helper.request({ + command: "setPreviewEnabled", + sessionId: this.state.sessionId, + generation: this.state.generation, + payload: { enabled }, + }); + } + start(sessionId: string): Promise { + this.assertSession(sessionId); + if (this.state.phase !== "ready" || !this.helper || !this.storage) + return Promise.reject(new Error("INVALID_REQUEST")); + this.startWait = deferred(); + const waiting = this.startWait; + this.update({ phase: "starting", acceptedVideoSamples: 0 }); + this.armTimeout("NO_VIDEO_SAMPLES"); + void (async () => { + try { + await this.deps.journal?.(this.storage!, { state: "starting" }); + if (this.state.phase !== "starting") return; + await this.helper!.request({ command: "start", sessionId }); + } catch (error) { + this.fail(error as Error); + await this.closeHelper(); + } + })(); + return waiting.promise; + } + stop(sessionId: string, reason = "user-stop"): Promise { + this.assertSession(sessionId); + if (this.stopReason === "user-stop") this.stopReason = reason; + if (this.committed) return Promise.resolve(this.committed); + if ( + !this.terminal || + !["starting", "recording", "stopping", "finalising"].includes(this.state.phase) + ) + return Promise.reject(new Error("INVALID_REQUEST")); + if (!this.stopSent && !this.finalization) { + this.stopSent = true; + this.clearTimer(); + this.startWait?.reject(new Error("NO_VIDEO_SAMPLES")); + this.startWait = undefined; + this.update({ phase: "stopping" }); + this.armTimeout("FINALIZATION_FAILED"); + void this.helper + ?.request({ command: "stop", sessionId }) + .catch((error) => this.fail(error)); + } + return this.terminal.promise; + } + async cancel(sessionId: string, discardAcceptedMedia: boolean): Promise { + this.assertSession(sessionId); + if (["starting", "recording", "stopping", "finalising"].includes(this.state.phase)) { + if (!discardAcceptedMedia && this.state.acceptedVideoSamples > 0) + throw new Error("INVALID_REQUEST"); + this.discardRequested = discardAcceptedMedia; + try { + await this.stop(sessionId); + } catch (error) { + if ((error as Error).message !== "NO_VIDEO_SAMPLES") throw error; + } + return; + } + await this.release(sessionId); + if (discardAcceptedMedia && this.storage) await this.deps.discard(this.storage); + this.update({ phase: "cancelled" }); + } + async release(sessionId: string): Promise { + this.assertSession(sessionId); + if (["starting", "recording", "stopping", "finalising"].includes(this.state.phase)) + throw new Error("RECORDING_BUSY"); + this.preparationRevision++; + this.prepareWait?.reject(new Error("INVALID_REQUEST")); + this.prepareWait = undefined; + this.clearTimer(); + try { + if (this.helper) await this.helper.request({ command: "release", sessionId }); + } finally { + await this.closeHelper(); + this.releaseLease(); + this.update({ + phase: "idle", + sessionId: null, + source: null, + options: null, + format: null, + mode: null, + devices: [], + microphones: [], + }); + } + } + private receive(event: IOSCaptureEvent) { + if (event.sequence <= this.lastEvent) return; + this.lastEvent = event.sequence; + if (event.event === "inventoryChanged") { + const hasDevices = event.payload.devices.length > 0; + this.update({ + devices: event.payload.devices, + microphones: event.payload.microphones, + phase: this.state.phase === "discovering" && hasDevices ? "idle" : this.state.phase, + }); + if (hasDevices) this.settleInventory(); + return; + } + if (event.event === "accepted" || (event.event === "error" && event.requestId)) return; + if ( + (event.sessionId && event.sessionId !== this.state.sessionId) || + (event.generation !== undefined && event.generation !== this.state.generation) + ) + return; + switch (event.event) { + case "prepared": + if ( + this.state.phase !== "preparing" || + event.payload.source.deviceToken !== this.state.source?.deviceToken + ) + return; + this.clearTimer(); + this.update({ + phase: "ready", + format: event.payload.format, + mode: event.payload.mode, + }); + if (this.storage) + void this.deps + .journal?.(this.storage, { + format: event.payload.format, + mode: event.payload.mode, + }) + .catch((error) => this.fail(error)); + this.prepareWait?.resolve(this.getSnapshot()); + this.prepareWait = undefined; + break; + case "recordingStarted": + if (!["starting", "stopping"].includes(this.state.phase)) return; + if (!this.stopSent) this.clearTimer(); + this.update({ + phase: this.stopSent ? "stopping" : "recording", + acceptedVideoSamples: event.payload.acceptedVideoSamples, + }); + this.deps.setCapturing?.(true); + this.startWait?.resolve(this.getSnapshot()); + this.startWait = undefined; + if (this.storage) + void this.deps + .journal?.(this.storage, { state: "recording" }) + .catch((error) => this.fail(error)); + break; + case "progress": + if (!["recording", "stopping"].includes(this.state.phase)) return; + this.update({ + elapsedMs: event.payload.elapsedMs, + acceptedVideoSamples: event.payload.acceptedVideoSamples, + }); + if (this.storage && this.deps.checkSpace && !this.checkingSpace) { + this.checkingSpace = true; + const sessionId = this.state.sessionId; + void this.deps + .checkSpace(this.storage, this.state.elapsedMs) + .then((enough) => { + if ( + !enough && + sessionId === this.state.sessionId && + this.state.phase === "recording" + ) { + this.update({ + warningCodes: [...this.state.warningCodes, "DISK_SPACE_LOW"], + }); + void this.stop(sessionId!, "DISK_SPACE_LOW").catch(() => { + /* Failure is retained by the authoritative controller. */ + }); + } + }) + .catch(() => { + if (sessionId && sessionId === this.state.sessionId) + void this.stop(sessionId).catch(() => { + /* Failure is retained by the authoritative controller. */ + }); + }) + .finally(() => { + this.checkingSpace = false; + }); + } + break; + case "warning": + this.update({ + warningCodes: [ + ...new Set([...this.state.warningCodes, event.payload.code]), + ].slice(-32), + }); + break; + case "nativeFinalized": + this.finish(event.payload.result); + break; + case "error": + this.fail(new Error(event.payload.code)); + break; + } + } + private finish(result: NativeCaptureResult) { + if ( + this.finalization || + !this.storage || + result.sessionId !== this.state.sessionId || + !["starting", "recording", "stopping"].includes(this.state.phase) || + result.video.sampleCount < 1 + ) + return; + this.clearTimer(); + if (["user-stop", "stopped"].includes(result.stopReason)) { + const reason = + this.stopReason !== "user-stop" + ? this.stopReason + : this.state.warningCodes.includes("AUDIO_INTERRUPTED") + ? "AUDIO_INTERRUPTED" + : result.stopReason; + result = { ...result, stopReason: reason }; + } + const storage = this.storage; + const terminal = this.terminal; + const abort = new AbortController(); + this.finalizationAbort = abort; + this.finalization = (async () => { + try { + this.update({ phase: "finalising" }); + if (this.discardRequested) { + await this.closeHelper(); + await this.deps.discard(storage); + this.releaseLease(); + this.update({ phase: "cancelled" }); + terminal?.reject(new Error("NO_VIDEO_SAMPLES")); + return; + } + const recording = await this.deps.finalize(storage, result, abort.signal); + if (abort.signal.aborted) throw new Error("FINALIZATION_FAILED"); + await this.closeHelper(); + await this.deps.onCommitted(recording); + this.committed = recording; + this.releaseLease(); + this.update({ + phase: recording.captureMetadata.interrupted ? "interrupted" : "completed", + }); + terminal?.resolve(recording); + } catch (error) { + this.fail(error as Error); + } + })(); + } + async shutdown(): Promise { + this.preparationRevision++; + this.prepareWait?.reject(new Error("INVALID_REQUEST")); + this.prepareWait = undefined; + if ( + this.state.sessionId && + ["starting", "recording", "stopping", "finalising"].includes(this.state.phase) + ) { + let timeout: ReturnType | undefined; + const finished = await Promise.race([ + this.stop(this.state.sessionId, "app-quit").then( + () => true, + () => true, + ), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), this.deps.shutdownGraceMs ?? 10_000); + }), + ]); + if (timeout) clearTimeout(timeout); + if (!finished) { + this.finalizationAbort?.abort(); + this.fail(new Error("FINALIZATION_FAILED")); + } + } + this.clearTimer(); + await this.closeHelper(); + this.releaseLease(); + } +} diff --git a/electron/ipc/recording/ios/featurePolicy.test.ts b/electron/ipc/recording/ios/featurePolicy.test.ts new file mode 100644 index 000000000..389dc6b1f --- /dev/null +++ b/electron/ipc/recording/ios/featurePolicy.test.ts @@ -0,0 +1,10 @@ +import { expect, it } from "vitest"; +import { IOS_CAPTURE_ENABLED_BY_DEFAULT, isIOSCaptureEnabled } from "./featurePolicy"; +it("is default off on every platform and packaged builds ignore development overrides", () => { + expect(IOS_CAPTURE_ENABLED_BY_DEFAULT).toBe(false); + expect(isIOSCaptureEnabled("darwin", false, {})).toBe(false); + expect(isIOSCaptureEnabled("darwin", true, { RECORDLY_ENABLE_IOS_CAPTURE: "1" })).toBe(false); + expect(isIOSCaptureEnabled("linux", false, { RECORDLY_ENABLE_IOS_CAPTURE: "1" })).toBe(false); + expect(isIOSCaptureEnabled("win32", false, { RECORDLY_ENABLE_IOS_CAPTURE: "1" })).toBe(false); + expect(isIOSCaptureEnabled("darwin", false, { RECORDLY_ENABLE_IOS_CAPTURE: "1" })).toBe(true); +}); diff --git a/electron/ipc/recording/ios/featurePolicy.ts b/electron/ipc/recording/ios/featurePolicy.ts new file mode 100644 index 000000000..d886c3350 --- /dev/null +++ b/electron/ipc/recording/ios/featurePolicy.ts @@ -0,0 +1,14 @@ +/** G1–G4 require recorded device and signed-install evidence before changing this. */ +export const IOS_CAPTURE_ENABLED_BY_DEFAULT = false; + +export function isIOSCaptureEnabled( + platform: string, + packaged: boolean, + environment: Readonly>, +): boolean { + return ( + platform === "darwin" && + (IOS_CAPTURE_ENABLED_BY_DEFAULT || + (!packaged && environment.RECORDLY_ENABLE_IOS_CAPTURE === "1")) + ); +} diff --git a/electron/ipc/recording/ios/finalize.test.ts b/electron/ipc/recording/ios/finalize.test.ts new file mode 100644 index 000000000..2775193dd --- /dev/null +++ b/electron/ipc/recording/ios/finalize.test.ts @@ -0,0 +1,422 @@ +import { existsSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it, vi } from "vitest"; +import type { NativeCaptureResult } from "../../../../src/shared/iosCapture"; +import { allocateIOSSessionStorage, readIOSJournal } from "./storage"; +import { + buildAudioAlignment, + finalizeIOSRecording, + nativeTimeDifferenceMs, + buildIOSFFmpegArguments, +} from "./finalize"; +vi.mock("electron", () => ({ app: { getPath: () => "/private/tmp", isPackaged: false } })); +export const id = "3d594650-3436-4a5a-b6a7-5ff45ecf73d0"; +export const format = { + codedWidth: 1920, + codedHeight: 1080, + displayWidth: 1080, + displayHeight: 1920, + codec: "h264", + colorPrimaries: null, + transferFunction: null, + ycbcrMatrix: null, + fullRange: null, + transform: [0, 1, -1, 0, 1080, 0] as const, + observedFrameRate: 30, + fingerprint: "format1", +}; +export const result: NativeCaptureResult = { + sessionId: id, + stopReason: "user-stop", + mode: "passthrough", + format, + video: { + relativeName: "source-video.mov", + mediaKind: "video", + firstHostTime: { value: "9007199254740993000", timescale: 1000 }, + duration: { value: "5000", timescale: 1000 }, + sampleCount: 10, + mediaFormat: { codec: "h264", width: 1920, height: 1080 }, + }, + timingFile: "native-timing.json", +}; +const roots: string[] = []; +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "ios-finalize-"))); + roots.push(root); + const storage = await allocateIOSSessionStorage(root, id); + await fs.writeFile(path.join(storage.directory, "source-video.mov"), "video-fixture"); + return storage; +} +afterEach(async () => { + await Promise.all(roots.splice(0).map((p) => fs.rm(p, { recursive: true, force: true }))); +}); +it("aligns signed offsets and subtracts rational clocks before conversion", () => { + expect(buildAudioAlignment(250)).toEqual({ trimStartMs: 0, delayMs: 250 }); + expect(buildAudioAlignment(-120)).toEqual({ trimStartMs: 120, delayMs: 0 }); + expect( + nativeTimeDifferenceMs( + { value: "9007199254740993250", timescale: 1000 }, + result.video.firstHostTime, + ), + ).toBe(250); +}); +it("commits inspected video without copying it and retry retains the source", async () => { + const storage = await fixture(); + const inspectMedia = vi.fn(async () => ({ + decodable: true, + duration: result.video.duration, + video: format, + })); + const runFFmpeg = vi.fn(); + const committed = await finalizeIOSRecording( + { storage, nativeResult: result }, + { inspectMedia, runFFmpeg }, + ); + expect(committed.videoPath).toBe(path.join(storage.directory, "source-video.mov")); + expect(runFFmpeg).not.toHaveBeenCalled(); + expect((await readIOSJournal(storage)).state).toBe("committed"); + expect( + await finalizeIOSRecording({ storage, nativeResult: result }, { inspectMedia, runFFmpeg }), + ).toEqual(committed); +}); +it("does not commit nondecodable, wrong geometry or zero duration files", async () => { + const storage = await fixture(); + for (const inspection of [ + { decodable: false, duration: result.video.duration, video: format }, + { decodable: true, duration: { value: "0", timescale: 1000 }, video: format }, + { + decodable: true, + duration: result.video.duration, + video: { ...format, displayWidth: 99 }, + }, + ]) { + await expect( + finalizeIOSRecording( + { storage, nativeResult: result }, + { inspectMedia: async () => inspection }, + ), + ).rejects.toThrow(); + } + expect((await readIOSJournal(storage)).state).not.toBe("committed"); +}); +it("constructs video copy and one unity soundtrack with silence to video duration, never shortest", () => { + const args = buildIOSFFmpegArguments({ + videoPath: "/video.mov", + outputPath: "/pending.mov", + durationMs: 5000, + audio: [{ path: "/mic.mov", offsetMs: 250, channels: 1 }], + }); + expect(args).toContain("copy"); + expect(args).not.toContain("-shortest"); + expect(args.join(" ")).toContain("volume=1"); + expect(args.join(" ")).toContain("adelay=250"); + expect(args).toContain("128k"); + const two = buildIOSFFmpegArguments({ + videoPath: "/v", + outputPath: "/o", + durationMs: 5000, + audio: [ + { path: "/a", offsetMs: 0, channels: 2 }, + { path: "/b", offsetMs: -120, channels: 1 }, + ], + }); + expect(two.join(" ")).toContain("normalize=0"); + expect(two.join(" ")).toContain("volume=0.5"); +}); + +async function audioFixture() { + const storage = await fixture(); + await fs.writeFile(path.join(storage.directory, "microphone.mov"), "audio-fixture"); + const microphone: NativeCaptureResult["microphone"] = { + relativeName: "microphone.mov", + mediaKind: "microphone", + firstHostTime: { value: "9007199254740993250", timescale: 1000 }, + duration: { value: "4750", timescale: 1000 }, + sampleCount: 200, + mediaFormat: { codec: "aac", sampleRate: 48000, channels: 1 }, + }; + const timing = { + version: 1 as const, + timeline: "host-mapped" as const, + gapsRepresentedInMedia: true as const, + streams: [ + { + mediaKind: "microphone" as const, + firstHostTime: microphone.firstHostTime, + duration: microphone.duration, + rate: { numerator: "10001", denominator: "10000" }, + clockAnchor: { + hostTime: microphone.firstHostTime, + mediaTime: { value: "0", timescale: 1 }, + }, + gaps: [ + { + start: { value: "1000", timescale: 1000 }, + duration: { value: "250", timescale: 1000 }, + }, + ], + }, + ], + }; + await fs.writeFile(path.join(storage.directory, "native-timing.json"), JSON.stringify(timing)); + const nativeResult = { ...result, microphone, timing }; + const inspectMedia = async (p: string) => + p.endsWith("microphone.mov") + ? { + decodable: true, + duration: microphone.duration, + audio: { codec: "aac", sampleRate: 48000, channels: 1 }, + } + : { + decodable: true, + duration: result.video.duration, + video: format, + ...(p.endsWith("recording.mov") || p.endsWith("recording.pending.mov") + ? { audio: { codec: "aac", sampleRate: 48000, channels: 1 } } + : {}), + }; + const runFFmpeg = vi.fn(async (args: readonly string[]) => { + await fs.writeFile(args.at(-1)!, "assembled"); + }); + return { storage, nativeResult, inspectMedia, runFFmpeg }; +} +it("retries after rename without assembling again and preserves all original sidecars", async () => { + const f = await audioFixture(); + let fail = true; + const persistManifest = vi.fn(async () => { + if (fail) throw new Error("manifest failure"); + }); + await expect(finalizeIOSRecording(f, { ...f, persistManifest })).rejects.toThrow( + "manifest failure", + ); + expect((await readIOSJournal(f.storage)).state).toBe("renamed"); + fail = false; + await finalizeIOSRecording(f, { ...f, persistManifest }); + expect(f.runFFmpeg).toHaveBeenCalledTimes(1); + expect(await fs.readFile(path.join(f.storage.directory, "source-video.mov"), "utf8")).toBe( + "video-fixture", + ); + expect(await fs.readFile(path.join(f.storage.directory, "microphone.mov"), "utf8")).toBe( + "audio-fixture", + ); + const filter = f.runFFmpeg.mock.calls[0][0].join(" "); + expect(filter).not.toContain("atempo"); + expect(filter.match(/adelay=/g)).toHaveLength(1); +}); +it("preserves recoverable journal on mux failure and rejects symlink outputs", async () => { + const f = await audioFixture(); + await expect( + finalizeIOSRecording(f, { + ...f, + runFFmpeg: async () => { + throw new Error("mux failed"); + }, + }), + ).rejects.toThrow("mux failed"); + expect((await readIOSJournal(f.storage)).nativeResult).toEqual(f.nativeResult); + await fs.symlink( + path.join(f.storage.directory, "source-video.mov"), + path.join(f.storage.directory, "recording.mov"), + ); + await expect(finalizeIOSRecording(f, f)).rejects.toThrow("UNSAFE_ARTIFACT"); +}); +it("preflights duplicate-video capacity before audio assembly", async () => { + const f = await audioFixture(); + await expect(finalizeIOSRecording(f, { ...f, availableBytes: async () => 10 })).rejects.toThrow( + "DISK_SPACE_LOW", + ); + expect(f.runFFmpeg).not.toHaveBeenCalled(); +}); +it("can finish a validated renamed movie when optional native audio or timing is missing", async () => { + const f = await audioFixture(); + await expect( + finalizeIOSRecording(f, { + ...f, + persistManifest: async () => { + throw new Error("crash"); + }, + }), + ).rejects.toThrow(); + await fs.rm(path.join(f.storage.directory, "microphone.mov")); + await fs.rm(path.join(f.storage.directory, "native-timing.json")); + const committed = await finalizeIOSRecording(f, f); + expect(committed.captureMetadata.narrationRecorded).toBe(true); + expect(f.runFFmpeg).toHaveBeenCalledTimes(1); +}); +it.skipIf( + !existsSync( + path.resolve( + "node_modules/ffmpeg-static", + process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg", + ), + ), +)( + "runs bundled FFmpeg video-copy assembly and preserves delayed mono audio and full video duration", + async () => { + const { execFileSync } = await import("node:child_process"); + const { runIOSFFmpeg } = await import("./finalize"); + const binary = path.resolve( + "node_modules/ffmpeg-static", + process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg", + ); + const storage = await fixture(); + const videoPath = path.join(storage.directory, "fixture.mov"); + const audioPath = path.join(storage.directory, "fixture-audio.mov"); + const outputPath = path.join(storage.directory, "fixture-output.mov"); + execFileSync(binary, [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=red:s=64x96:r=10:d=2", + "-an", + "-c:v", + "libx264", + videoPath, + ]); + execFileSync(binary, [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=0.5", + "-c:a", + "pcm_s16le", + audioPath, + ]); + await runIOSFFmpeg( + buildIOSFFmpegArguments({ + videoPath, + outputPath, + durationMs: 2000, + audio: [{ path: audioPath, offsetMs: 250, channels: 1 }], + }), + ); + const samples = execFileSync(binary, [ + "-hide_banner", + "-loglevel", + "error", + "-i", + outputPath, + "-map", + "0:a:0", + "-f", + "s16le", + "-ac", + "1", + "-ar", + "48000", + "pipe:1", + ]); + function energy(start: number, end: number) { + let sum = 0; + for (let i = Math.round(start * 48000); i < Math.round(end * 48000); i++) + sum += Math.abs(samples.readInt16LE(i * 2)); + return sum / ((end - start) * 48000); + } + expect(samples.length / 2 / 48000).toBeGreaterThanOrEqual(1.99); + expect(energy(0, 0.2)).toBeLessThan(1); + expect(energy(0.3, 0.6)).toBeGreaterThan(100); + expect(energy(1, 1.8)).toBeLessThan(1); + const sourcePackets = execFileSync( + binary, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + videoPath, + "-map", + "0:v:0", + "-c:v", + "copy", + "-f", + "hash", + "-hash", + "sha256", + "pipe:1", + ], + { encoding: "utf8" }, + ); + const finalPackets = execFileSync( + binary, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + outputPath, + "-map", + "0:v:0", + "-c:v", + "copy", + "-f", + "hash", + "-hash", + "sha256", + "pipe:1", + ], + { encoding: "utf8" }, + ); + expect(finalPackets).toBe(sourcePackets); + }, + 20000, +); +it("uses advancing progress rather than a total deadline and kills a stalled process", async () => { + const { runIOSFFmpeg } = await import("./finalize"); + await runIOSFFmpeg( + [ + "-e", + "let n=0;const timer=setInterval(()=>{process.stdout.write(`out_time_us=${++n}\n`);if(n===30){clearInterval(timer);}},30)", + ], + { binary: process.execPath, stallTimeoutMs: 500 }, + ); + await expect( + runIOSFFmpeg(["-e", 'setInterval(()=>process.stdout.write("out_time_us=1\\n"),10)'], { + binary: process.execPath, + stallTimeoutMs: 500, + }), + ).rejects.toThrow("FFMPEG_STALLED"); +}, 5000); +it.skipIf(process.platform === "win32")( + "aborts finalization, escalates an ignored SIGTERM, preserves partial output and removes its listener", + async () => { + const { runIOSFFmpeg } = await import("./finalize"); + const storage = await fixture(); + const partial = path.join(storage.directory, "recording.pending.mov"); + const ready = path.join(storage.directory, "abort-ready.json"); + const terminated = path.join(storage.directory, "terminated.txt"); + const controller = new AbortController(); + const removeListener = vi.spyOn(controller.signal, "removeEventListener"); + const script = `const fs=require('node:fs');process.on('SIGTERM',()=>fs.writeFileSync(${JSON.stringify(terminated)},'term'));fs.writeFileSync(${JSON.stringify(partial)},'partial media');fs.writeFileSync(${JSON.stringify(ready)},JSON.stringify({pid:process.pid}));let n=0;setInterval(()=>{process.stdout.write('out_time_us='+ ++n+'\\n');process.stderr.write('drained diagnostics\\n');},20);setTimeout(()=>process.exit(0),6500);`; + const running = runIOSFFmpeg(["-e", script], { + binary: process.execPath, + signal: controller.signal, + }); + const rejection = expect(running).rejects.toThrow("FFMPEG_ABORTED"); + await vi.waitFor(() => expect(existsSync(ready)).toBe(true)); + const { pid } = JSON.parse(await fs.readFile(ready, "utf8")); + controller.abort(); + await rejection; + expect(await fs.readFile(terminated, "utf8")).toBe("term"); + expect(await fs.readFile(partial, "utf8")).toBe("partial media"); + expect(() => process.kill(pid, 0)).toThrow(); + expect(removeListener).toHaveBeenCalledWith("abort", expect.any(Function)); + }, + 10000, +); +it("rejects an already-aborted signal before resolving or spawning an executable", async () => { + const { runIOSFFmpeg } = await import("./finalize"); + const controller = new AbortController(); + controller.abort(); + await expect( + runIOSFFmpeg([], { binary: "/nonexistent-ffmpeg", signal: controller.signal }), + ).rejects.toThrow("FFMPEG_ABORTED"); +}); diff --git a/electron/ipc/recording/ios/finalize.ts b/electron/ipc/recording/ios/finalize.ts new file mode 100644 index 000000000..479950f1a --- /dev/null +++ b/electron/ipc/recording/ios/finalize.ts @@ -0,0 +1,465 @@ +import fs from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { + parseNativeCaptureResult, + parseCaptureMetadata, + parseNativeTiming, + validateNativeTime, + type CaptureMetadata, + type CommittedIOSRecording, + type IOSAudioFormat, + type IOSMediaInspection, + type IOSCaptureMode, + type IOSVideoFormat, + type NativeCaptureResult, + type NativeTime, + type NativeTiming, +} from "../../../../src/shared/iosCapture"; +import { getFfmpegBinaryPath } from "../../ffmpeg/binary"; +import { persistRecordingSessionManifest } from "../../project/session"; +import { + availableIOSStorageBytes, + readIOSJournal, + resolveIOSArtifact, + updateIOSJournal, + type IOSSessionStorage, +} from "./storage"; +export type { IOSMediaInspection } from "../../../../src/shared/iosCapture"; +export interface IOSFinalizeDependencies { + inspectMedia: (filePath: string) => Promise; + runFFmpeg?: (args: readonly string[]) => Promise; + availableBytes?: (directory: string) => Promise; + persistManifest?: typeof persistRecordingSessionManifest; + checkpoint?: ( + phase: "native-validated" | "output-renamed" | "before-manifest", + ) => Promise; +} +const zero: NativeTime = { value: "0", timescale: 1 }; +export function nativeTimeDifferenceMs(a: NativeTime, b: NativeTime): number { + if (!validateNativeTime(a) || !validateNativeTime(b)) throw new Error("INVALID_NATIVE_TIME"); + const numerator = + (BigInt(a.value) * BigInt(b.timescale) - BigInt(b.value) * BigInt(a.timescale)) * 1000n; + const denominator = BigInt(a.timescale) * BigInt(b.timescale); + const quotient = numerator / denominator; + // Limit duration arithmetic after exact common-clock subtraction (seven days). + if (quotient > 604800000n || quotient < -604800000n) throw new Error("INVALID_MEDIA_DURATION"); + return Number(quotient) + Number(numerator % denominator) / Number(denominator); +} +export function buildAudioAlignment(offsetMs: number): { trimStartMs: number; delayMs: number } { + if (!Number.isFinite(offsetMs) || Math.abs(offsetMs) > 604800000) + throw new Error("INVALID_AUDIO_OFFSET"); + return { trimStartMs: Math.max(0, -offsetMs), delayMs: Math.max(0, offsetMs) }; +} +export function buildIOSFFmpegArguments(input: { + videoPath: string; + outputPath: string; + durationMs: number; + audio: readonly { + path: string; + offsetMs: number; + channels: number; + kind?: "device-audio" | "microphone"; + }[]; +}): string[] { + if ( + !Number.isFinite(input.durationMs) || + input.durationMs <= 0 || + input.durationMs > 604800000 || + input.audio.length < 1 || + input.audio.length > 2 + ) + throw new Error("INVALID_AUDIO_PLAN"); + const args = ["-hide_banner", "-nostdin", "-n", "-i", input.videoPath]; + const filters: string[] = []; + const layout = + input.audio.length === 1 && + input.audio[0].channels === 1 && + input.audio[0].kind !== "device-audio" + ? "mono" + : "stereo"; + input.audio.forEach((audio, index) => { + if (![1, 2].includes(audio.channels)) throw new Error("UNSUPPORTED_CHANNEL_LAYOUT"); + args.push("-i", audio.path); + const alignment = buildAudioAlignment(audio.offsetMs); + // Native sidecars already map rate and gaps into host time. Resampling materializes + // timestamp holes once; never concatenate another silence segment from the journal. + filters.push( + `[${index + 1}:a:0]asetpts=PTS-STARTPTS,aresample=48000:async=1:first_pts=0,atrim=start=${alignment.trimStartMs / 1000},asetpts=PTS-STARTPTS,adelay=${alignment.delayMs}:all=1,aformat=sample_rates=48000:channel_layouts=${layout},volume=${input.audio.length === 1 ? 1 : 0.5},apad,atrim=duration=${input.durationMs / 1000}[a${index}]`, + ); + }); + filters.push( + input.audio.length === 1 + ? "[a0]anull[outa]" + : `[a0][a1]amix=inputs=2:duration=longest:normalize=0,atrim=duration=${input.durationMs / 1000}[outa]`, + ); + args.push( + "-filter_complex", + filters.join(";"), + "-map", + "0:v:0", + "-map", + "[outa]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + layout === "mono" ? "128k" : "192k", + "-ar", + "48000", + "-ac", + layout === "mono" ? "1" : "2", + "-t", + String(input.durationMs / 1000), + "-movflags", + "+faststart", + "-progress", + "pipe:1", + input.outputPath, + ); + return args; +} +export async function runIOSFFmpeg( + args: readonly string[], + options: { stallTimeoutMs?: number; binary?: string; signal?: AbortSignal } = {}, +): Promise { + if (options.signal?.aborted) throw new Error("FFMPEG_ABORTED"); + const binary = options.binary ?? getFfmpegBinaryPath(); + await new Promise((resolve, reject) => { + const child = spawn(binary, [...args], { shell: false, stdio: ["ignore", "pipe", "pipe"] }); + let lastActivity = Date.now(); + let settled = false; + let terminationError: Error | undefined; + let progressBuffer = ""; + const progressValues = new Map(); + let killTimer: ReturnType | undefined; + const terminate = (error: Error) => { + if (settled || terminationError) return; + terminationError = error; + child.kill("SIGTERM"); + killTimer = setTimeout(() => child.kill("SIGKILL"), 5000); + }; + const onAbort = () => terminate(new Error("FFMPEG_ABORTED")); + const watchdog = setInterval( + () => { + if (Date.now() - lastActivity > (options.stallTimeoutMs ?? 120000)) { + terminate(new Error("FFMPEG_STALLED")); + } + }, + Math.min(1000, options.stallTimeoutMs ?? 120000), + ); + const finish = (error?: Error) => { + if (settled) return; + settled = true; + clearInterval(watchdog); + options.signal?.removeEventListener("abort", onAbort); + if (killTimer) clearTimeout(killTimer); + if (error) reject(error); + else resolve(); + }; + child.stdout.on("data", (chunk: Buffer) => { + progressBuffer = (progressBuffer + chunk.toString("utf8")).slice(-65536); + while (true) { + const newline = progressBuffer.indexOf("\n"); + if (newline < 0) break; + const line = progressBuffer.slice(0, newline).trim(); + progressBuffer = progressBuffer.slice(newline + 1); + const match = /^(out_time_us|frame|total_size)=(\d+)$/.exec(line); + if (match) { + const value = BigInt(match[2]); + if (value > (progressValues.get(match[1]) ?? -1n)) { + progressValues.set(match[1], value); + lastActivity = Date.now(); + } + } + } + }); + // Always drain diagnostics, but repeated warnings are not progress. + child.stderr.on("data", () => undefined); + child.once("error", finish); + child.once("close", (code) => + finish(terminationError ?? (code === 0 ? undefined : new Error("FFMPEG_FAILED"))), + ); + options.signal?.addEventListener("abort", onAbort, { once: true }); + // Cover an abort between the initial check and listener registration. + if (options.signal?.aborted) onAbort(); + }); +} +export function validateIOSVideoInspection( + inspection: IOSMediaInspection, + format?: IOSVideoFormat, + expectedDurationMs?: number, +): number { + const durationMs = nativeTimeDifferenceMs(inspection.duration, zero); + if (!inspection.decodable || !inspection.video || durationMs <= 0) + throw new Error("INVALID_VIDEO"); + const v = inspection.video; + if (v.codedWidth <= 0 || v.codedHeight <= 0 || v.displayWidth <= 0 || v.displayHeight <= 0) + throw new Error("INVALID_VIDEO_GEOMETRY"); + if ( + format && + (v.codedWidth !== format.codedWidth || + v.codedHeight !== format.codedHeight || + v.displayWidth !== format.displayWidth || + v.displayHeight !== format.displayHeight || + JSON.stringify(v.transform) !== JSON.stringify(format.transform)) + ) + throw new Error("VIDEO_GEOMETRY_MISMATCH"); + if ( + expectedDurationMs !== undefined && + Math.abs(durationMs - expectedDurationMs) > + Math.max(100, 1000 / (format?.observedFrameRate ?? 30)) + ) + throw new Error("VIDEO_DURATION_MISMATCH"); + return durationMs; +} +export async function readIOSNativeTiming( + storage: IOSSessionStorage, + result: NativeCaptureResult, +): Promise { + // Read the helper-owned checkpoint even if stdout carried timing, to retain crash semantics. + const parsed = JSON.parse( + await fs.readFile(await resolveIOSArtifact(storage, "native-timing.json"), "utf8"), + ); + const timing = parseNativeTiming(parsed.timing ?? parsed); + if (result.timing && JSON.stringify(result.timing) !== JSON.stringify(timing)) + throw new Error("TIMING_MISMATCH"); + return timing; +} +const finalizations = new Map>(); +export function finalizeIOSRecording( + input: { storage: IOSSessionStorage; nativeResult: NativeCaptureResult }, + deps: IOSFinalizeDependencies, +): Promise { + const existing = finalizations.get(input.storage.directory); + if (existing) return existing; + const work = finalize(input, deps); + finalizations.set(input.storage.directory, work); + void work + .finally(() => { + if (finalizations.get(input.storage.directory) === work) + finalizations.delete(input.storage.directory); + }) + .catch(() => undefined); + return work; +} +async function finalize( + { storage, nativeResult }: { storage: IOSSessionStorage; nativeResult: NativeCaptureResult }, + deps: IOSFinalizeDependencies, +): Promise { + const result = parseNativeCaptureResult(nativeResult); + if (result.sessionId !== storage.sessionId || result.video.relativeName !== "source-video.mov") + throw new Error("INVALID_NATIVE_RESULT"); + const journal = await readIOSJournal(storage); + // A crash after rename must resume from the inspected movie even when optional + // native sidecars are unavailable; no second encode or alignment pass is needed. + if (journal.nativeResult && (result.deviceAudio || result.microphone)) { + const existingOutput = await resolveIOSArtifact(storage, "recording.mov", true); + const present = await fs + .stat(existingOutput) + .then(() => true) + .catch(() => false); + if (present) { + const inspected = await deps.inspectMedia(existingOutput); + validateIOSVideoInspection( + inspected, + result.format, + nativeTimeDifferenceMs(result.video.duration, zero), + ); + if ( + !inspected.audio || + inspected.audio.sampleRate !== 48000 || + inspected.audio.channels !== + (result.microphone && + !result.deviceAudio && + (result.microphone.mediaFormat as IOSAudioFormat).channels === 1 + ? 1 + : 2) + ) + throw new Error("INVALID_FINAL_AUDIO"); + return commitIOSResult( + storage, + result, + existingOutput, + "recording.mov", + journal.state === "committed", + deps, + ); + } + } + const source = await resolveIOSArtifact(storage, "source-video.mov"); + const video = await deps.inspectMedia(source); + if (video.audio) throw new Error("INVALID_NATIVE_VIDEO_TRACKS"); + const durationMs = validateIOSVideoInspection( + video, + result.format, + nativeTimeDifferenceMs(result.video.duration, zero), + ); + await deps.checkpoint?.("native-validated"); + const artifacts = [result.deviceAudio, result.microphone].filter( + (a): a is NonNullable => Boolean(a), + ); + const audio: { + path: string; + offsetMs: number; + channels: number; + kind?: "device-audio" | "microphone"; + }[] = []; + if (artifacts.length) { + const timing = await readIOSNativeTiming(storage, result); + const kinds = new Set(timing.streams.map((s) => s.mediaKind)); + if (kinds.size !== timing.streams.length) throw new Error("INVALID_TIMING"); + for (const a of artifacts) { + const expected = a.mediaKind === "device-audio" ? "device-audio.mov" : "microphone.mov"; + if (a.relativeName !== expected) throw new Error("INVALID_NATIVE_RESULT"); + const stream = timing.streams.find((s) => s.mediaKind === a.mediaKind); + if (!stream || nativeTimeDifferenceMs(stream.firstHostTime, a.firstHostTime) !== 0) + throw new Error("CLOCK_MAPPING_UNAVAILABLE"); + const p = await resolveIOSArtifact(storage, expected); + const inspection = await deps.inspectMedia(p); + const f = a.mediaFormat as IOSAudioFormat; + if ( + !inspection.decodable || + !inspection.audio || + inspection.audio.channels !== f.channels || + inspection.audio.sampleRate !== f.sampleRate || + nativeTimeDifferenceMs(inspection.duration, zero) <= 0 + ) + throw new Error("INVALID_AUDIO"); + audio.push({ + path: p, + offsetMs: nativeTimeDifferenceMs(stream.firstHostTime, result.video.firstHostTime), + channels: f.channels, + kind: a.mediaKind as "device-audio" | "microphone", + }); + } + } + const finalName = audio.length ? "recording.mov" : "source-video.mov"; + if (journal.state === "committed" && journal.committedFile !== finalName) + throw new Error("ALREADY_COMMITTED"); + if (journal.state !== "committed") + await updateIOSJournal(storage, { state: "finalising", nativeResult: result }); + const output = await resolveIOSArtifact(storage, finalName, true); + if (audio.length) { + let exists = true; + try { + await fs.access(output); + } catch { + exists = false; + } + if (!exists) { + const bytes = (await fs.stat(source)).size; + if ( + (await (deps.availableBytes ?? availableIOSStorageBytes)(storage.directory)) < + bytes + 256 * 1024 ** 2 + ) + throw new Error("DISK_SPACE_LOW"); + const pending = await resolveIOSArtifact(storage, "recording.pending.mov", true); + await fs.rm(pending, { force: true }); + await (deps.runFFmpeg ?? runIOSFFmpeg)( + buildIOSFFmpegArguments({ + videoPath: source, + outputPath: pending, + durationMs, + audio, + }), + ); + const inspected = await deps.inspectMedia( + await resolveIOSArtifact(storage, "recording.pending.mov"), + ); + validateIOSVideoInspection(inspected, result.format, durationMs); + if ( + !inspected.audio || + inspected.audio.sampleRate !== 48000 || + inspected.audio.channels !== + (audio.length === 1 && audio[0].channels === 1 && audio[0].kind === "microphone" + ? 1 + : 2) + ) + throw new Error("INVALID_FINAL_AUDIO"); + await fs.rename(pending, output); + await updateIOSJournal(storage, { state: "renamed", committedFile: finalName }); + await deps.checkpoint?.("output-renamed"); + } + const inspected = await deps.inspectMedia(await resolveIOSArtifact(storage, finalName)); + validateIOSVideoInspection(inspected, result.format, durationMs); + if (!inspected.audio) throw new Error("INVALID_FINAL_AUDIO"); + } + return commitIOSResult(storage, result, output, finalName, journal.state === "committed", deps); +} +async function commitIOSResult( + storage: IOSSessionStorage, + result: Pick< + NativeCaptureResult, + "mode" | "format" | "stopReason" | "deviceAudio" | "microphone" + >, + output: string, + finalName: "recording.mov" | "source-video.mov", + alreadyCommitted: boolean, + deps: IOSFinalizeDependencies, +): Promise { + const captureMetadata: CaptureMetadata = parseCaptureMetadata({ + version: 1, + sourceKind: "ios-device", + mode: result.mode, + format: result.format, + deviceAudioRecorded: Boolean(result.deviceAudio), + narrationRecorded: Boolean(result.microphone), + stopReason: result.stopReason, + interrupted: !["user-stop", "userStop", "stopped", "stop", "completed"].includes( + result.stopReason, + ), + }); + const committed: CommittedIOSRecording = { + sessionId: storage.sessionId, + videoPath: output, + hideOverlayCursorByDefault: true, + captureMetadata, + }; + if (!alreadyCommitted) { + await deps.checkpoint?.("before-manifest"); + await (deps.persistManifest ?? persistRecordingSessionManifest)({ + ...committed, + webcamPath: null, + }); + await updateIOSJournal(storage, { + state: "committed", + committedFile: finalName, + ...(result.deviceAudio || result.microphone + ? { + outputAudio: { + codec: "aac", + sampleRate: 48000, + channels: + result.microphone && + !result.deviceAudio && + (result.microphone.mediaFormat as IOSAudioFormat).channels === 1 + ? 1 + : 2, + }, + } + : {}), + }); + } + return committed; +} + +export async function finalizeRecoveredIOSVideo( + input: { storage: IOSSessionStorage; mode: IOSCaptureMode; format: IOSVideoFormat }, + deps: IOSFinalizeDependencies, +): Promise { + const { storage, mode, format } = input; + const journal = await readIOSJournal(storage); + if (journal.state === "committed" && journal.committedFile !== "source-video.mov") + throw new Error("ALREADY_COMMITTED"); + const source = await resolveIOSArtifact(storage, "source-video.mov"); + validateIOSVideoInspection(await deps.inspectMedia(source), format); + return commitIOSResult( + storage, + { mode, format, stopReason: "recovered-interruption" }, + source, + "source-video.mov", + journal.state === "committed", + deps, + ); +} diff --git a/electron/ipc/recording/ios/fixtureVerifier.test.ts b/electron/ipc/recording/ios/fixtureVerifier.test.ts new file mode 100644 index 000000000..38de0fb6f --- /dev/null +++ b/electron/ipc/recording/ios/fixtureVerifier.test.ts @@ -0,0 +1,145 @@ +import { expect, it } from "vitest"; +const verifierPath = new URL("../../../../scripts/verify-ios-capture-fixture.mjs", import.meta.url) + .href; +const fixture = { + fixtureVersion: 1, + name: "test", + sessionId: "3d594650-3436-4a5a-b6a7-5ff45ecf73d0", + videoFile: "recording.mov", + videoMode: "passthrough", + durationMs: 4000, + durationToleranceMs: 50, + displayGeometry: { width: 240, height: 426, transform: [1, 0, 0, 1, 0, 0] }, + audio: { + required: true, + channels: 1, + eventToleranceMs: 80, + streams: [{ kind: "microphone", eventTimesMs: [500, 2000] }], + }, + expectedVideoSamples: 40, + permittedSampleLoss: 0, + sourceColour: { + colorPrimaries: null, + transferFunction: null, + ycbcrMatrix: null, + fullRange: null, + }, + terminalStatus: "completed", + provenance: "Synthetic generated fixture; CC0-1.0", + compareSourceVideoPayload: true, +}; +const observation = { + inspection: { + decodable: true, + duration: { value: "4000", timescale: 1000 }, + video: { + displayWidth: 240, + displayHeight: 426, + transform: [1, 0, 0, 1, 0, 0], + colorPrimaries: null, + transferFunction: null, + ycbcrMatrix: null, + fullRange: null, + }, + audio: { channels: 1, sampleRate: 48000 }, + }, + decodeSucceeded: true, + decodedVideoSamples: 40, + audioEventTimesMs: [500, 2000], + videoMode: "passthrough", + terminalStatus: "completed", + sourcePayloadHash: "same", + finalPayloadHash: "same", +}; +it("rejects unknown/missing CLI args and malformed expectations", async () => { + const { parseArguments, validateExpected } = await import(verifierPath); + expect(() => parseArguments(["--session-dir", "/tmp"])).toThrow(); + expect(() => parseArguments(["--unknown", "x"])).toThrow(); + expect(() => validateExpected({ ...fixture, durationMs: -1 })).toThrow(); +}); +it("reports honest failed checks for rotation, displaced/missing audio, wrong duration and truncation", async () => { + const { evaluateFixture } = await import(verifierPath); + expect(evaluateFixture(fixture, observation).passed).toBe(true); + for (const changed of [ + { + inspection: { + ...observation.inspection, + video: { ...observation.inspection.video, transform: [0, 1, -1, 0, 426, 0] }, + }, + }, + { audioEventTimesMs: [750, 2250] }, + { inspection: { ...observation.inspection, audio: undefined } }, + { inspection: { ...observation.inspection, duration: { value: "2000", timescale: 1000 } } }, + { decodeSucceeded: false }, + { decodedVideoSamples: 39 }, + { finalPayloadHash: "different" }, + ]) { + const report = evaluateFixture(fixture, { ...observation, ...changed }); + expect(report.passed).toBe(false); + expect(report.checks.some((check: { passed: boolean }) => !check.passed)).toBe(true); + } +}); + +it("measures displaced soundtrack events in the video timeline with real FFmpeg", async (context) => { + const fs = await import("node:fs/promises"); + const { existsSync } = await import("node:fs"); + const path = await import("node:path"); + const os = await import("node:os"); + const { execFileSync } = await import("node:child_process"); + const binary = path.resolve( + "node_modules/ffmpeg-static", + process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg", + ); + if (!existsSync(binary)) { + context.skip(); + return; + } + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "ios-fixture-audio-")); + try { + const source = path.join(directory, "source.mov"); + const shifted = path.join(directory, "shifted.mov"); + execFileSync(binary, [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=red:s=64x96:r=10:d=2", + "-f", + "lavfi", + "-i", + "aevalsrc=0.8*sin(2*PI*1000*t)*between(t\\,0.5\\,0.51):s=48000:d=2", + "-c:v", + "libx264", + "-c:a", + "aac", + source, + ]); + execFileSync(binary, [ + "-hide_banner", + "-loglevel", + "error", + "-i", + source, + "-itsoffset", + "0.25", + "-i", + source, + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c", + "copy", + shifted, + ]); + const { decodeFixtureAudio } = await import(verifierPath); + const normal = await decodeFixtureAudio(binary, source), + displaced = await decodeFixtureAudio(binary, shifted); + expect(normal[0]).toBeCloseTo(500, 0); + expect(Math.abs(displaced[0] - normal[0] - 250)).toBeLessThan(2); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } +}); diff --git a/electron/ipc/recording/ios/handoff.test.ts b/electron/ipc/recording/ios/handoff.test.ts new file mode 100644 index 000000000..4bbca2b8d --- /dev/null +++ b/electron/ipc/recording/ios/handoff.test.ts @@ -0,0 +1,37 @@ +import { expect, it, vi } from "vitest"; +import type { CommittedIOSRecording } from "../../../../src/shared/iosCapture"; +import { IOSRecordingHandoff } from "./handoff"; + +it("delivers one editor when concurrent recovery and completion callers await the same session", async () => { + let finish!: () => void; + const deliver = vi.fn( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const handoff = new IOSRecordingHandoff(deliver); + const recording = { sessionId: "one" } as CommittedIOSRecording; + const first = handoff.run(recording); + const second = handoff.run(recording); + expect(second).toBe(first); + await Promise.resolve(); + expect(deliver).toHaveBeenCalledOnce(); + finish(); + await first; + await handoff.run(recording); + expect(deliver).toHaveBeenCalledOnce(); +}); + +it("allows a failed delivery to be retried without an extra successful delivery", async () => { + const deliver = vi + .fn() + .mockRejectedValueOnce(new Error("editor unavailable")) + .mockResolvedValue(undefined); + const handoff = new IOSRecordingHandoff(deliver); + const recording = { sessionId: "one" } as CommittedIOSRecording; + await expect(handoff.run(recording)).rejects.toThrow("editor unavailable"); + await handoff.run(recording); + await handoff.run(recording); + expect(deliver).toHaveBeenCalledTimes(2); +}); diff --git a/electron/ipc/recording/ios/handoff.ts b/electron/ipc/recording/ios/handoff.ts new file mode 100644 index 000000000..5b96f5d21 --- /dev/null +++ b/electron/ipc/recording/ios/handoff.ts @@ -0,0 +1,23 @@ +import type { CommittedIOSRecording } from "../../../../src/shared/iosCapture"; + +/** Concurrent recovery callers and native completion share one editor delivery. */ +export class IOSRecordingHandoff { + private readonly deliveries = new Map>(); + constructor(private readonly deliver: (recording: CommittedIOSRecording) => Promise) {} + run(recording: CommittedIOSRecording): Promise { + const existing = this.deliveries.get(recording.sessionId); + if (existing) return existing; + const work = Promise.resolve().then(() => this.deliver(recording)); + this.deliveries.set(recording.sessionId, work); + void work.then( + () => { + if (this.deliveries.size > 64) + this.deliveries.delete(this.deliveries.keys().next().value!); + }, + () => { + this.deliveries.delete(recording.sessionId); + }, + ); + return work; + } +} diff --git a/electron/ipc/recording/ios/helperProcess.test.ts b/electron/ipc/recording/ios/helperProcess.test.ts new file mode 100644 index 000000000..c235d8e63 --- /dev/null +++ b/electron/ipc/recording/ios/helperProcess.test.ts @@ -0,0 +1,48 @@ +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { IOSHelperProcess } from "./helperProcess"; + +const helpers: IOSHelperProcess[] = []; +function helper(scenario = "normal") { + const result = new IOSHelperProcess({ + binaryPath: process.execPath, + args: [ + path.resolve("electron/ipc/recording/ios/__fixtures__/fake-ios-helper.mjs"), + scenario, + ], + requestTimeoutMs: 200, + }); + helpers.push(result); + return result; +} +afterEach(async () => { + await Promise.all(helpers.splice(0).map((h) => h.shutdown())); +}); + +it("handshakes with an actual child, including split output and stderr flood", async () => { + const client = helper("split"); + const hello = await client.request({ command: "hello" }); + expect(hello.event).toBe("accepted"); + expect(client.diagnosticBytes).toBeLessThanOrEqual(8192); +}); +it("rejects malformed protocol and pending commands on child crash", async () => { + await expect(helper("malformed").request({ command: "hello" })).rejects.toThrow(); + await expect(helper("crash").request({ command: "hello" })).rejects.toThrow("HELPER_EXITED"); +}); +it("bounds pending requests and times out an unresponsive helper", async () => { + const client = helper("silent"); + const pending = Array.from({ length: 8 }, () => + client.request({ command: "hello" }).catch((e: Error) => e.message), + ); + await expect(client.request({ command: "hello" })).rejects.toThrow("RECORDING_BUSY"); + expect(await Promise.all(pending)).toEqual(Array(8).fill("HELPER_UNAVAILABLE")); +}); +it("warns once for an invalid preview while continuing the control protocol", async () => { + const client = helper("bad-preview"); + const warning = vi.fn(); + client.onWarning(warning); + await client.request({ command: "hello" }); + await vi.waitFor(() => expect(warning).toHaveBeenCalledWith("PREVIEW_UNAVAILABLE")); + expect((await client.request({ command: "hello" })).event).toBe("accepted"); + expect(warning).toHaveBeenCalledOnce(); +}); diff --git a/electron/ipc/recording/ios/helperProcess.ts b/electron/ipc/recording/ios/helperProcess.ts new file mode 100644 index 000000000..39dad11cc --- /dev/null +++ b/electron/ipc/recording/ios/helperProcess.ts @@ -0,0 +1,207 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import type { Readable } from "node:stream"; +import { + parseIOSCaptureEvent, + parseNativeIOSCaptureCommand, + type IOSCaptureEvent, +} from "../../../../src/shared/iosCapture"; +import { PreviewFrameDecoder, type IOSPreviewFrame } from "./preview"; +import { NDJSONDecoder } from "./protocol"; + +export interface IOSHelperRequest { + command: string; + sessionId?: string; + generation?: number; + payload?: Record; + storage?: { sessionRoot: string; allowedRelativeNames: readonly string[] }; +} + +export interface IOSHelperTransport { + request(input: IOSHelperRequest): Promise; + onEvent(listener: (event: IOSCaptureEvent) => void): () => void; + onExit(listener: (error: Error) => void): () => void; + onPreview(listener: (frame: IOSPreviewFrame) => void): () => void; + onWarning?(listener: (code: string) => void): () => void; + shutdown(): Promise; +} + +export class IOSHelperProcess implements IOSHelperTransport { + private child: ChildProcess; + private events = new Set<(event: IOSCaptureEvent) => void>(); + private exits = new Set<(error: Error) => void>(); + private previews = new Set<(frame: IOSPreviewFrame) => void>(); + private warnings = new Set<(code: string) => void>(); + private previewFailed = false; + private pending = new Map< + string, + { + resolve: (event: IOSCaptureEvent) => void; + reject: (error: Error) => void; + timer: ReturnType; + } + >(); + private closed = false; + private intentionalExit = false; + private lastSequence = -1; + private shutdownPromise?: Promise; + private exitPromise: Promise; + private failure?: Error; + private latestPreview?: IOSPreviewFrame; + private previewTick?: ReturnType; + /** Only the bounded byte count is retained; native diagnostics may contain private labels. */ + diagnosticBytes = 0; + + constructor( + private readonly options: { + binaryPath: string; + args?: readonly string[]; + requestTimeoutMs?: number; + }, + ) { + this.child = spawn(options.binaryPath, [...(options.args ?? [])], { + shell: false, + stdio: ["pipe", "pipe", "pipe", "pipe"], + windowsHide: true, + }); + const decoder = new NDJSONDecoder((line) => parseIOSCaptureEvent(JSON.parse(line))); + const preview = new PreviewFrameDecoder(); + this.child.stdout?.on("data", (bytes: Buffer) => { + try { + for (const event of decoder.push(bytes)) this.receive(event); + } catch { + this.fail(new Error("PROTOCOL_MISMATCH")); + this.child.kill(); + } + }); + this.child.stderr?.on("data", (bytes: Buffer) => { + this.diagnosticBytes = Math.min(8192, this.diagnosticBytes + bytes.length); + }); + (this.child.stdio[3] as Readable | null)?.on("data", (bytes: Buffer) => { + try { + for (const frame of preview.push(bytes)) this.latestPreview = frame; + if (!this.previewTick && this.latestPreview) { + this.previewTick = setImmediate(() => { + this.previewTick = undefined; + const frame = this.latestPreview; + this.latestPreview = undefined; + if (frame) for (const listener of this.previews) listener(frame); + }); + } + } catch { + // Invalid previews are disposable. Continue draining fd3 and control/media. + this.latestPreview = undefined; + if (!this.previewFailed) { + this.previewFailed = true; + for (const listener of this.warnings) listener("PREVIEW_UNAVAILABLE"); + } + } + }); + this.child.stdin?.on("error", () => this.fail(new Error("HELPER_EXITED"))); + this.child.on("error", () => this.fail(new Error("HELPER_UNAVAILABLE"))); + this.exitPromise = new Promise((resolve) => { + this.child.once("close", () => { + this.closed = true; + if (this.previewTick) clearImmediate(this.previewTick); + this.latestPreview = undefined; + try { + decoder.finish(); + } catch { + this.fail(new Error("PROTOCOL_MISMATCH")); + } + if (!this.failure) this.fail(new Error("HELPER_EXITED")); + resolve(); + }); + }); + } + + private fail(error: Error) { + for (const command of this.pending.values()) { + clearTimeout(command.timer); + command.reject(error); + } + this.pending.clear(); + if (!this.failure) { + this.failure = error; + if (!this.intentionalExit) for (const listener of this.exits) listener(error); + } + } + + private receive(event: IOSCaptureEvent) { + if (event.sequence <= this.lastSequence) return; + this.lastSequence = event.sequence; + if (event.requestId && (event.event === "accepted" || event.event === "error")) { + const command = this.pending.get(event.requestId); + if (command) { + clearTimeout(command.timer); + this.pending.delete(event.requestId); + if (event.event === "error") command.reject(new Error(event.payload.code)); + else command.resolve(event); + } + } + for (const listener of this.events) listener(event); + } + + async request(input: IOSHelperRequest): Promise { + if (this.closed || this.failure) throw this.failure ?? new Error("HELPER_EXITED"); + if (this.pending.size >= 8) throw new Error("RECORDING_BUSY"); + const requestId = randomUUID(); + const command = parseNativeIOSCaptureCommand({ protocolVersion: 1, requestId, ...input }); + const encoded = JSON.stringify(command) + "\n"; + if (Buffer.byteLength(encoded) > 64 * 1024) throw new Error("INVALID_REQUEST"); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + reject(new Error("HELPER_UNAVAILABLE")); + }, this.options.requestTimeoutMs ?? 10_000); + this.pending.set(requestId, { resolve, reject, timer }); + this.child.stdin?.write(encoded, (error) => { + if (error) this.fail(new Error("HELPER_EXITED")); + }); + }); + } + + onEvent(listener: (event: IOSCaptureEvent) => void) { + this.events.add(listener); + return () => { + this.events.delete(listener); + }; + } + onExit(listener: (error: Error) => void) { + this.exits.add(listener); + return () => { + this.exits.delete(listener); + }; + } + onPreview(listener: (frame: IOSPreviewFrame) => void) { + this.previews.add(listener); + return () => { + this.previews.delete(listener); + }; + } + onWarning(listener: (code: string) => void) { + this.warnings.add(listener); + return () => { + this.warnings.delete(listener); + }; + } + + shutdown(): Promise { + if (this.shutdownPromise) return this.shutdownPromise; + this.intentionalExit = true; + this.shutdownPromise = (async () => { + if (this.closed) return; + // EOF is a native safe-finalisation request even when control output has failed. + this.child.stdin?.end(); + const graceful = setTimeout(() => this.child.kill("SIGTERM"), 10_000); + const forced = setTimeout(() => this.child.kill("SIGKILL"), 12_000); + try { + await this.exitPromise; + } finally { + clearTimeout(graceful); + clearTimeout(forced); + } + })(); + return this.shutdownPromise; + } +} diff --git a/electron/ipc/recording/ios/ipcPolicy.test.ts b/electron/ipc/recording/ios/ipcPolicy.test.ts new file mode 100644 index 000000000..2ca1a1b05 --- /dev/null +++ b/electron/ipc/recording/ios/ipcPolicy.test.ts @@ -0,0 +1,34 @@ +import { expect, it } from "vitest"; +import { assertIOSCaptureSender, parseIOSPrepareInput } from "./ipcPolicy"; +it("accepts only the known application's top-level frame at its trusted URL", () => { + const allowed = { + knownWindow: true, + mainFrame: true, + url: "http://localhost:5173/?windowType=hud-overlay", + trustedUrls: ["http://localhost:5173/"], + }; + expect(() => assertIOSCaptureSender(allowed)).not.toThrow(); + for (const patch of [ + { knownWindow: false }, + { mainFrame: false }, + { url: "https://evil.example/" }, + { url: "http://localhost:5173/remote" }, + ]) { + expect(() => assertIOSCaptureSender({ ...allowed, ...patch })).toThrow("INVALID_REQUEST"); + } +}); +it("rejects renderer paths, shell arguments and unknown prepare fields", () => { + const input = { + deviceToken: "phone", + generation: 1, + options: { deviceAudio: false, microphoneToken: null }, + }; + expect(parseIOSPrepareInput(input)).toEqual(input); + for (const payload of [ + { ...input, outputPath: "/tmp/evil.mov" }, + { ...input, args: ["-vf", "crop"] }, + { ...input, options: { ...input.options, filter: "evil" } }, + ]) { + expect(() => parseIOSPrepareInput(payload)).toThrow(); + } +}); diff --git a/electron/ipc/recording/ios/ipcPolicy.ts b/electron/ipc/recording/ios/ipcPolicy.ts new file mode 100644 index 000000000..2d5a24771 --- /dev/null +++ b/electron/ipc/recording/ios/ipcPolicy.ts @@ -0,0 +1,48 @@ +import { + parseIOSCaptureCommand, + type IOSRecordingOptions, +} from "../../../../src/shared/iosCapture"; + +export function assertIOSCaptureSender(input: { + knownWindow: boolean; + mainFrame: boolean; + url: string; + trustedUrls: readonly string[]; +}): void { + let trusted = false; + try { + const actual = new URL(input.url); + trusted = input.trustedUrls.some((base) => { + const expected = new URL(base); + return ( + actual.protocol === expected.protocol && + actual.host === expected.host && + actual.pathname === expected.pathname + ); + }); + } catch { + /* Malformed and foreign URLs are untrusted. */ + } + if (!input.knownWindow || !input.mainFrame || !trusted) throw new Error("INVALID_REQUEST"); +} + +export function parseIOSPrepareInput(value: unknown): { + deviceToken: string; + generation: number; + options: IOSRecordingOptions; +} { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("INVALID_REQUEST"); + const input = value as Record; + if (Object.keys(input).some((key) => !["deviceToken", "generation", "options"].includes(key))) + throw new Error("INVALID_REQUEST"); + const command = parseIOSCaptureCommand({ + protocolVersion: 1, + requestId: "ipc", + command: "prepare", + generation: input.generation, + payload: { deviceToken: input.deviceToken, options: input.options }, + }); + if (command.command !== "prepare") throw new Error("INVALID_REQUEST"); + return { generation: command.generation, ...command.payload }; +} diff --git a/electron/ipc/recording/ios/nativeBridge.test.ts b/electron/ipc/recording/ios/nativeBridge.test.ts new file mode 100644 index 000000000..838850fd4 --- /dev/null +++ b/electron/ipc/recording/ios/nativeBridge.test.ts @@ -0,0 +1,25 @@ +import { existsSync } from "node:fs"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { IOSHelperProcess } from "./helperProcess"; + +const binary = path.resolve( + `electron/native/bin/darwin-${process.arch === "arm64" ? "arm64" : "x64"}/recordly-ios-device-helper`, +); +it.skipIf(process.platform !== "darwin" || !existsSync(binary))( + "decodes the actual Swift helper hello through the production TypeScript transport without TCC", + async () => { + const helper = new IOSHelperProcess({ binaryPath: binary }); + try { + const event = await helper.request({ command: "hello" }); + expect(event.event).toBe("accepted"); + if (event.event === "accepted") + expect(event.payload).toMatchObject({ + protocolVersion: 1, + capabilities: { supportsPause: false }, + }); + } finally { + await helper.shutdown(); + } + }, +); diff --git a/electron/ipc/recording/ios/nativeFinalization.test.ts b/electron/ipc/recording/ios/nativeFinalization.test.ts new file mode 100644 index 000000000..ac8f0aedf --- /dev/null +++ b/electron/ipc/recording/ios/nativeFinalization.test.ts @@ -0,0 +1,61 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { expect, it, vi } from "vitest"; +import { finalizeIOSRecording } from "./finalize"; +import { resolveRecordingSessionManifest } from "../../project/session"; +vi.mock("electron", () => ({ app: { getPath: () => "/private/tmp", isPackaged: false } })); +const generatorUrl = new URL( + "../../../../scripts/fixtures/ios-capture/generate.mjs", + import.meta.url, +).href; +const verifierUrl = new URL("../../../../scripts/verify-ios-capture-fixture.mjs", import.meta.url) + .href; +it.skipIf(process.platform !== "darwin" || process.env.RECORDLY_RUN_NATIVE_MEDIA_TESTS !== "1")( + "finalizes synthetic native-sidecar fixtures through the actual inspector and reopens provenance", + async () => { + const root = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "ios-native-finalization-")), + ); + try { + const { generateFixture } = await import(generatorUrl); + const generated = await generateFixture( + path.join(root, "generated"), + "delayed-microphone", + ); + const expected = JSON.parse(await fs.readFile(generated.expectedPath, "utf8")); + const directory = path.join(root, `ios-${expected.sessionId}`); + await fs.rename(generated.directory, directory); + const journalPath = path.join(directory, "capture-journal.json"); + const journal = JSON.parse(await fs.readFile(journalPath, "utf8")); + journal.state = "finalising"; + delete journal.committedFile; + await fs.writeFile(journalPath, JSON.stringify(journal)); + await fs.rm(path.join(directory, "recording.mov")); + const { inspectFixtureMedia, verifyFixture } = await import(verifierUrl); + const committed = await finalizeIOSRecording( + { + storage: { sessionId: expected.sessionId, directory, journalPath }, + nativeResult: journal.nativeResult, + }, + { + inspectMedia: (file) => + inspectFixtureMedia(directory, path.basename(file), expected.sessionId), + }, + ); + const reopened = await resolveRecordingSessionManifest(committed.videoPath); + expect(reopened?.captureMetadata?.narrationRecorded).toBe(true); + expect(reopened?.webcamPath).toBeNull(); + expect(reopened?.hideOverlayCursorByDefault).toBe(true); + const report = await verifyFixture({ + sessionDir: directory, + expectedPath: path.join(directory, "expected.json"), + reportPath: path.join(directory, "report.json"), + }); + expect(report.passed).toBe(true); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }, + 30000, +); diff --git a/electron/ipc/recording/ios/packaging.test.ts b/electron/ipc/recording/ios/packaging.test.ts new file mode 100644 index 000000000..262c5ddd8 --- /dev/null +++ b/electron/ipc/recording/ios/packaging.test.ts @@ -0,0 +1,44 @@ +import { expect, it } from "vitest"; +import { + decodeMachOPlistSection, + validateIOSHelperFacts, +} from "../../../../scripts/ios-helper-policy.mjs"; + +const valid = { + executable: true, + architecture: "arm64", + minimumOS: "14.0", + protocolVersion: 1, + privacy: { + NSCameraUsageDescription: "Connected device screen", + NSMicrophoneUsageDescription: "Narration", + }, + entitlements: { + "com.apple.security.device.camera": true, + "com.apple.security.device.audio-input": true, + }, +}; +it("rejects missing executable, wrong slice/floor/protocol/privacy and unsigned capture entitlements", () => { + expect(validateIOSHelperFacts(valid, { architecture: "arm64", signed: true })).toEqual([]); + for (const patch of [ + { executable: false }, + { architecture: "x86_64" }, + { minimumOS: "26.0" }, + { protocolVersion: 2 }, + { privacy: {} }, + { entitlements: {} }, + ]) { + expect( + validateIOSHelperFacts({ ...valid, ...patch }, { architecture: "arm64", signed: true }) + .length, + ).toBeGreaterThan(0); + } +}); + +it("reads both byte-oriented Intel and word-oriented arm64 otool sections", () => { + const expected = "\n"; + const intel = "0000000100044700\t3c 70 6c 69 73 74 3e 3c 2f 70 6c 69 73 74 3e 0a"; + const arm = "000000010003e790\t696c703c 3c3e7473 696c702f 0a3e7473"; + expect(decodeMachOPlistSection(intel).toString()).toBe(expected); + expect(decodeMachOPlistSection(arm).toString()).toBe(expected); +}); diff --git a/electron/ipc/recording/ios/permissions.test.ts b/electron/ipc/recording/ios/permissions.test.ts new file mode 100644 index 000000000..1f668490a --- /dev/null +++ b/electron/ipc/recording/ios/permissions.test.ts @@ -0,0 +1,27 @@ +import { expect, it, vi } from "vitest"; +import { prepareIOSPermissions } from "./permissions"; + +it("does not prompt again after denial and never asks for audio when disabled", async () => { + const request = vi.fn(async () => true); + await expect( + prepareIOSPermissions( + { deviceAudio: false, microphoneToken: null }, + { + status: () => "denied", + request, + }, + ), + ).rejects.toThrow("PERMISSION_DENIED"); + expect(request).not.toHaveBeenCalled(); + const status = vi.fn(() => "granted"); + await prepareIOSPermissions({ deviceAudio: false, microphoneToken: null }, { status, request }); + expect(status).toHaveBeenCalledExactlyOnceWith("camera"); +}); +it("requests only undetermined camera and selected audio access", async () => { + const request = vi.fn(async () => true); + await prepareIOSPermissions( + { deviceAudio: true, microphoneToken: null }, + { status: () => "not-determined", request }, + ); + expect(request.mock.calls).toEqual([["camera"], ["microphone"]]); +}); diff --git a/electron/ipc/recording/ios/permissions.ts b/electron/ipc/recording/ios/permissions.ts new file mode 100644 index 000000000..d53c501d4 --- /dev/null +++ b/electron/ipc/recording/ios/permissions.ts @@ -0,0 +1,18 @@ +import type { IOSRecordingOptions } from "../../../../src/shared/iosCapture"; + +export async function prepareIOSPermissions( + options: IOSRecordingOptions, + permissions: { + status: (media: "camera" | "microphone") => string; + request: (media: "camera" | "microphone") => Promise; + }, +): Promise { + const media: Array<"camera" | "microphone"> = ["camera"]; + if (options.deviceAudio || options.microphoneToken) media.push("microphone"); + for (const kind of media) { + const status = permissions.status(kind); + if (status === "granted") continue; + if (status !== "not-determined" || !(await permissions.request(kind))) + throw new Error("PERMISSION_DENIED"); + } +} diff --git a/electron/ipc/recording/ios/preview.test.ts b/electron/ipc/recording/ios/preview.test.ts new file mode 100644 index 000000000..495588e45 --- /dev/null +++ b/electron/ipc/recording/ios/preview.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { PreviewFrameDecoder } from "./preview"; + +function frame(sequence = 1, generation = 2, jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xd9])) { + const header = Buffer.alloc(24); + header.write("RLIP"); + header.writeUInt16BE(1, 4); + header.writeUInt32BE(generation, 8); + header.writeUInt32BE(sequence, 12); + header.writeUInt32BE(jpeg.length, 16); + return Buffer.concat([header, jpeg]); +} + +describe("RLIP preview boundary", () => { + it("decodes split headers and payloads", () => { + const decoder = new PreviewFrameDecoder(); + const wire = frame(); + expect(decoder.push(wire.subarray(0, 11))).toEqual([]); + expect(decoder.push(wire.subarray(11, 25))).toEqual([]); + expect(decoder.push(wire.subarray(25))).toEqual([ + { generation: 2, sequence: 1, jpeg: wire.subarray(24) }, + ]); + }); + it("retains only the latest frame from a blocked consumer's batch", () => { + const decoder = new PreviewFrameDecoder(); + expect(decoder.push(Buffer.concat([frame(1), frame(2)])).map((f) => f.sequence)).toEqual([ + 2, + ]); + }); + it.each([0, 4, 6, 20])("permanently closes malformed headers at byte %i", (offset) => { + const decoder = new PreviewFrameDecoder(); + const wire = frame(); + wire[offset] = 99; + expect(() => decoder.push(wire)).toThrow("INVALID_PREVIEW"); + expect(decoder.push(frame())).toEqual([]); + }); + it("rejects oversized payloads before buffering their body", () => { + const wire = frame(); + wire.writeUInt32BE(128 * 1024 + 1, 16); + expect(() => new PreviewFrameDecoder().push(wire)).toThrow("INVALID_PREVIEW"); + }); +}); diff --git a/electron/ipc/recording/ios/preview.ts b/electron/ipc/recording/ios/preview.ts new file mode 100644 index 000000000..726197c21 --- /dev/null +++ b/electron/ipc/recording/ios/preview.ts @@ -0,0 +1,53 @@ +export interface IOSPreviewFrame { + generation: number; + sequence: number; + jpeg: Uint8Array; +} + +const HEADER_BYTES = 24; +const MAX_JPEG_BYTES = 128 * 1024; + +/** A damaged preview closes only this disposable stream, never the control pipe. */ +export class PreviewFrameDecoder { + private partial = Buffer.alloc(0); + private expected = HEADER_BYTES; + private closed = false; + + push(chunk: Uint8Array): IOSPreviewFrame[] { + if (this.closed) return []; + let latest: IOSPreviewFrame | undefined; + let offset = 0; + while (offset < chunk.byteLength) { + const amount = Math.min(this.expected - this.partial.length, chunk.byteLength - offset); + this.partial = Buffer.concat([this.partial, chunk.subarray(offset, offset + amount)]); + offset += amount; + if (this.partial.length !== this.expected) continue; + if (this.expected === HEADER_BYTES) { + const header = this.partial; + const size = header.readUInt32BE(16); + if ( + header.toString("ascii", 0, 4) !== "RLIP" || + header.readUInt16BE(4) !== 1 || + header.readUInt16BE(6) !== 0 || + header.readUInt32BE(20) !== 0 || + size === 0 || + size > MAX_JPEG_BYTES + ) { + this.closed = true; + this.partial = Buffer.alloc(0); + throw new Error("INVALID_PREVIEW"); + } + this.expected += size; + } else { + latest = { + generation: this.partial.readUInt32BE(8), + sequence: this.partial.readUInt32BE(12), + jpeg: this.partial.subarray(HEADER_BYTES), + }; + this.partial = Buffer.alloc(0); + this.expected = HEADER_BYTES; + } + } + return latest ? [latest] : []; + } +} diff --git a/electron/ipc/recording/ios/protocol.test.ts b/electron/ipc/recording/ios/protocol.test.ts new file mode 100644 index 000000000..0cf8b4fa2 --- /dev/null +++ b/electron/ipc/recording/ios/protocol.test.ts @@ -0,0 +1,18 @@ +import { expect, it } from "vitest"; +import { NDJSONDecoder } from "./protocol"; + +it("reassembles split multibyte UTF-8 without corrupting control messages", () => { + const wire = Buffer.from('{"name":"📱"}\n{"ok":true}\n'); + const decoder = new NDJSONDecoder(JSON.parse); + expect(decoder.push(wire.subarray(0, 11))).toEqual([]); + expect(decoder.push(wire.subarray(11))).toEqual([{ name: "📱" }, { ok: true }]); + decoder.finish(); +}); +it("rejects oversized, malformed and incomplete control messages", () => { + expect(() => new NDJSONDecoder(JSON.parse).push(Buffer.alloc(65 * 1024, 97))).toThrow(); + expect(() => new NDJSONDecoder(JSON.parse).push(Buffer.from([0xff, 10]))).toThrow(); + expect(() => new NDJSONDecoder(JSON.parse).push(Buffer.from("oops\n"))).toThrow(); + const split = new NDJSONDecoder(JSON.parse); + split.push(Buffer.from('{"ok":')); + expect(() => split.finish()).toThrow("INCOMPLETE_PROTOCOL"); +}); diff --git a/electron/ipc/recording/ios/protocol.ts b/electron/ipc/recording/ios/protocol.ts new file mode 100644 index 000000000..a1eddb9aa --- /dev/null +++ b/electron/ipc/recording/ios/protocol.ts @@ -0,0 +1,29 @@ +/** Byte-bounded framing with fatal UTF-8 validation, independent of event schema. */ +export class NDJSONDecoder { + private partial = Buffer.alloc(0); + private readonly utf8 = new TextDecoder("utf-8", { fatal: true }); + + constructor(private readonly parse: (line: string) => T) {} + + push(chunk: Uint8Array): T[] { + const bytes = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + const output: T[] = []; + let offset = 0; + while (offset < bytes.length) { + const end = bytes.indexOf(10, offset); + const stop = end === -1 ? bytes.length : end; + if (this.partial.length + stop - offset > 64 * 1024) throw new Error("INVALID_REQUEST"); + this.partial = Buffer.concat([this.partial, bytes.subarray(offset, stop)]); + if (end === -1) break; + if (this.partial.length === 0) throw new Error("INVALID_REQUEST"); + output.push(this.parse(this.utf8.decode(this.partial))); + this.partial = Buffer.alloc(0); + offset = end + 1; + } + return output; + } + + finish(): void { + if (this.partial.length) throw new Error("INCOMPLETE_PROTOCOL"); + } +} diff --git a/electron/ipc/recording/ios/recovery.test.ts b/electron/ipc/recording/ios/recovery.test.ts new file mode 100644 index 000000000..4bdfdb41c --- /dev/null +++ b/electron/ipc/recording/ios/recovery.test.ts @@ -0,0 +1,128 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it, vi } from "vitest"; +import { allocateIOSSessionStorage, updateIOSJournal } from "./storage"; +import { IOSRecoveryRegistry } from "./recovery"; +vi.mock("electron", () => ({ app: { getPath: () => "/private/tmp", isPackaged: false } })); +const id = "3d594650-3436-4a5a-b6a7-5ff45ecf73d0"; +const format = { + codedWidth: 100, + codedHeight: 200, + displayWidth: 100, + displayHeight: 200, + codec: "h264", + colorPrimaries: null, + transferFunction: null, + ycbcrMatrix: null, + fullRange: null, + transform: [1, 0, 0, 1, 0, 0] as const, + observedFrameRate: 30, + fingerprint: "format", +}; +const roots: string[] = []; +async function fixture() { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "ios-recovery-"))); + roots.push(root); + const storage = await allocateIOSSessionStorage(root, id); + await fs.writeFile(path.join(storage.directory, "source-video.mov"), "video"); + await updateIOSJournal(storage, { state: "recording", format, mode: "passthrough" }); + return { root, storage }; +} +afterEach(async () => { + await Promise.all(roots.splice(0).map((p) => fs.rm(p, { recursive: true, force: true }))); +}); +const inspectMedia = async () => ({ + decodable: true, + duration: { value: "5000", timescale: 1000 }, + video: format, +}); +it("does not inspect any session while disabled by default", async () => { + const { root } = await fixture(); + const inspect = vi.fn(inspectMedia); + expect(await new IOSRecoveryRegistry({ inspectMedia: inspect }).scan(root)).toEqual([]); + expect(inspect).not.toHaveBeenCalled(); +}); +it("offers inspected video-only recovery when timing is lost and resolves only scanned UUIDs", async () => { + const { root, storage } = await fixture(); + const registry = new IOSRecoveryRegistry({ inspectMedia }, true); + const candidates = await registry.scan(root); + expect(candidates).toEqual([ + { + sessionId: id, + status: "recoverable-video", + durationMs: 5000, + reasonCode: "TIMING_UNAVAILABLE", + }, + ]); + expect(JSON.stringify(candidates)).not.toContain(root); + await expect(registry.recover({ sessionId: "../evil", mode: "video-only" })).rejects.toThrow(); + await expect(registry.recover({ sessionId: id, mode: "with-audio" })).rejects.toThrow(); + const committed = await registry.recover({ sessionId: id, mode: "video-only" }); + expect(committed.captureMetadata.interrupted).toBe(true); + expect(committed.videoPath).toBe(path.join(storage.directory, "source-video.mov")); +}); +it("rejects invalid video, partial journal and directory symlinks; discard is explicit", async () => { + const { root, storage } = await fixture(); + const registry = new IOSRecoveryRegistry( + { + inspectMedia: async () => ({ + decodable: false, + duration: { value: "0", timescale: 1 }, + }), + }, + true, + ); + expect((await registry.scan(root))[0]?.status).toBe("unrecoverable"); + await registry.discard(id); + await expect(fs.access(storage.directory)).rejects.toThrow(); + const fixture2 = await fixture(); + await fs.writeFile(fixture2.storage.journalPath, "{"); + expect(await registry.scan(fixture2.root)).toEqual([]); +}); +it("uses real helper checkpoint evidence to recover a partially written main journal", async () => { + const { root, storage } = await fixture(); + const nativeResult = { + sessionId: id, + mode: "passthrough", + format, + stopReason: "helper-exited", + video: { + relativeName: "source-video.mov", + mediaKind: "video", + firstHostTime: { value: "100", timescale: 1 }, + duration: { value: "5000", timescale: 1000 }, + sampleCount: 150, + mediaFormat: { codec: "h264", width: 100, height: 200 }, + }, + timingFile: "native-timing.json", + }; + await fs.writeFile(storage.journalPath, '{"state":'); + await fs.writeFile( + path.join(storage.directory, "native-timing.json"), + JSON.stringify({ result: nativeResult }), + ); + const registry = new IOSRecoveryRegistry({ inspectMedia }, true); + expect((await registry.scan(root))[0]?.status).toBe("recoverable-video"); + await registry.recover({ sessionId: id, mode: "video-only" }); + const journal = JSON.parse(await fs.readFile(storage.journalPath, "utf8")); + expect(journal.nativeResult.video.sampleCount).toBe(150); + expect(journal.state).toBe("committed"); +}); +it("never manufactures native host time or accepted counters for missing-timing recovery", async () => { + const { root, storage } = await fixture(); + const registry = new IOSRecoveryRegistry({ inspectMedia }, true); + await registry.scan(root); + await registry.recover({ sessionId: id, mode: "video-only" }); + const journal = JSON.parse(await fs.readFile(storage.journalPath, "utf8")); + expect(journal.nativeResult).toBeUndefined(); +}); +it("never inspects or registers an excluded active recording", async () => { + const { root } = await fixture(); + const inspect = vi.fn(inspectMedia); + const registry = new IOSRecoveryRegistry({ inspectMedia: inspect }, true); + expect(await registry.scan(root, new Set([id]))).toEqual([]); + expect(inspect).not.toHaveBeenCalled(); + await expect(registry.resolveDirectory(id)).rejects.toThrow("INVALID_RECOVERY_REQUEST"); + await expect(registry.discard(id)).rejects.toThrow("INVALID_RECOVERY_REQUEST"); +}); diff --git a/electron/ipc/recording/ios/recovery.ts b/electron/ipc/recording/ios/recovery.ts new file mode 100644 index 000000000..6a11da92e --- /dev/null +++ b/electron/ipc/recording/ios/recovery.ts @@ -0,0 +1,262 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { + parseNativeCaptureResult, + type CommittedIOSRecording, + type IOSSessionId, + type NativeCaptureResult, + type IOSCaptureMode, + type IOSVideoFormat, +} from "../../../../src/shared/iosCapture"; +import { + finalizeIOSRecording, + finalizeRecoveredIOSVideo, + readIOSNativeTiming, + validateIOSVideoInspection, + type IOSFinalizeDependencies, +} from "./finalize"; +import { + IOS_SESSION_UUID, + readIOSJournal, + repairIOSJournal, + resolveIOSArtifact, + validateIOSStorage, + type IOSSessionStorage, + type IOSCaptureJournal, +} from "./storage"; +export interface IOSRecoveryCandidate { + sessionId: IOSSessionId; + status: "recoverable-av" | "recoverable-video" | "unrecoverable"; + durationMs: number | null; + reasonCode: string; +} +interface Registered { + storage: IOSSessionStorage; + candidate: IOSRecoveryCandidate; + result: NativeCaptureResult | null; + needsJournalRepair?: boolean; + recoveryMetadata?: { mode: IOSCaptureMode; format: IOSVideoFormat }; +} +export class IOSRecoveryRegistry { + private readonly sessions = new Map(); + private readonly pending = new Map>(); + constructor( + private readonly deps: IOSFinalizeDependencies, + private readonly platformSupported = false, + ) {} + async scan( + recordingsRoot: string, + excludedSessionIds: ReadonlySet = new Set(), + ): Promise { + this.sessions.clear(); + if (!this.platformSupported) return []; + const root = await fs.realpath(recordingsRoot); + if ((await fs.lstat(recordingsRoot)).isSymbolicLink()) + throw new Error("UNSAFE_RECORDINGS_ROOT"); + for (const entry of await fs.readdir(root, { withFileTypes: true })) { + const sessionId = entry.name.slice(4); + if ( + !entry.isDirectory() || + !entry.name.startsWith("ios-") || + !IOS_SESSION_UUID.test(sessionId) || + excludedSessionIds.has(sessionId) + ) + continue; + const directory = path.join(root, entry.name); + const storage = { + sessionId, + directory, + journalPath: path.join(directory, "capture-journal.json"), + }; + let journal: IOSCaptureJournal; + let needsJournalRepair = false; + try { + journal = await readIOSJournal(storage); + } catch { + try { + // Only a real validated helper terminal checkpoint can reconstruct a lost main journal. + await resolveIOSArtifact(storage, "capture-journal.json"); + const checkpoint = JSON.parse( + await fs.readFile( + await resolveIOSArtifact(storage, "native-timing.json"), + "utf8", + ), + ); + const nativeResult = parseNativeCaptureResult( + checkpoint.nativeResult ?? checkpoint.result, + ); + if (nativeResult.sessionId !== sessionId) continue; + journal = { + version: 1, + sessionId, + createdAt: "", + state: "interrupted", + nativeResult, + format: nativeResult.format, + mode: nativeResult.mode, + }; + needsJournalRepair = true; + } catch { + continue; + } + } + if (journal.state === "committed") continue; + let result: NativeCaptureResult | null = null; + try { + result = parseNativeCaptureResult(journal.nativeResult); + } catch { + /* Missing optional evidence must not acquire inferred values. */ + } + let durationMs: number | null = null; + let status: IOSRecoveryCandidate["status"] = "unrecoverable"; + let reasonCode = "INVALID_VIDEO"; + try { + const source = await resolveIOSArtifact(storage, "source-video.mov"); + const inspection = await this.deps.inspectMedia(source); + durationMs = validateIOSVideoInspection(inspection); + try { + result = parseNativeCaptureResult(journal.nativeResult); + } catch { + try { + const checkpoint = JSON.parse( + await fs.readFile( + await resolveIOSArtifact(storage, "native-timing.json"), + "utf8", + ), + ); + result = parseNativeCaptureResult( + checkpoint.nativeResult ?? checkpoint.result, + ); + } catch { + /* Missing optional evidence must not acquire inferred values. */ + } + } + if (result && result.sessionId !== sessionId) throw new Error("INVALID_SESSION"); + + // Unknown original encoder mode is not guessed from the codec. + if (!result && (!journal.mode || !journal.format)) { + reasonCode = "PROVENANCE_UNAVAILABLE"; + } else { + status = "recoverable-video"; + reasonCode = "TIMING_UNAVAILABLE"; + try { + if (!result) throw new Error("TIMING_UNAVAILABLE"); + await readIOSNativeTiming(storage, result); + const artifacts = [result.deviceAudio, result.microphone].filter((a) => + Boolean(a), + ); + if (artifacts.length) { + for (const artifact of artifacts) { + const p = await resolveIOSArtifact(storage, artifact!.relativeName); + const inspected = await this.deps.inspectMedia(p); + if (!inspected.decodable || !inspected.audio) + throw new Error("INVALID_AUDIO"); + } + status = "recoverable-av"; + reasonCode = "INTERRUPTED"; + } + } catch { + /* Missing optional evidence must not acquire inferred values. */ + } + } + } catch { + /* Missing optional evidence must not acquire inferred values. */ + } + if (result && (result.deviceAudio || result.microphone)) { + try { + const finalPath = await resolveIOSArtifact(storage, "recording.mov"); + const finalInspection = await this.deps.inspectMedia(finalPath); + durationMs = validateIOSVideoInspection(finalInspection, result.format); + if (finalInspection.audio) { + status = "recoverable-av"; + reasonCode = "FINALIZED_MOVIE_AVAILABLE"; + } + } catch { + /* Missing optional evidence must not acquire inferred values. */ + } + } + const candidate = { sessionId, status, durationMs, reasonCode }; + this.sessions.set(sessionId, { + storage, + candidate, + result, + needsJournalRepair, + recoveryMetadata: + journal.mode && journal.format + ? { mode: journal.mode, format: journal.format } + : undefined, + }); + } + return [...this.sessions.values()].map((s) => s.candidate); + } + recover(input: { + sessionId: IOSSessionId; + mode: "with-audio" | "video-only"; + }): Promise { + const registered = this.sessions.get(input.sessionId); + if ( + !this.platformSupported || + !registered || + (!registered.result && !registered.recoveryMetadata) || + registered.candidate.status === "unrecoverable" || + !["with-audio", "video-only"].includes(input.mode) + ) + return Promise.reject(new Error("INVALID_RECOVERY_REQUEST")); + if (input.mode === "with-audio" && registered.candidate.status !== "recoverable-av") + return Promise.reject(new Error("CLOCK_MAPPING_UNAVAILABLE")); + const pending = this.pending.get(input.sessionId); + if (pending) return pending; + const result = registered.result; + const work = (async () => { + if (registered.needsJournalRepair && result) + await repairIOSJournal(registered.storage, result); + return input.mode === "video-only" + ? finalizeRecoveredIOSVideo( + { + storage: registered.storage, + ...(result + ? { mode: result.mode, format: result.format } + : registered.recoveryMetadata!), + }, + this.deps, + ) + : finalizeIOSRecording( + { + storage: registered.storage, + nativeResult: { ...result!, stopReason: "recovered-interruption" }, + }, + this.deps, + ); + })(); + this.pending.set(input.sessionId, work); + void work + .then(() => this.sessions.delete(input.sessionId)) + .finally(() => this.pending.delete(input.sessionId)) + .catch(() => undefined); + return work; + } + async resolveDirectory(sessionId: IOSSessionId): Promise { + const registered = this.sessions.get(sessionId); + if (!this.platformSupported || !registered) throw new Error("INVALID_RECOVERY_REQUEST"); + await validateIOSStorage(registered.storage); + return registered.storage.directory; + } + async discard(sessionId: IOSSessionId): Promise { + if (this.pending.has(sessionId)) throw new Error("RECOVERY_BUSY"); + const directory = await this.resolveDirectory(sessionId); + await fs.rm(directory, { recursive: true, force: false }); + this.sessions.delete(sessionId); + } +} +export function scanIOSRecoveryCandidates( + recordingsRoot: string, + registry: IOSRecoveryRegistry, +): Promise { + return registry.scan(recordingsRoot); +} +export function recoverIOSRecording( + input: { sessionId: IOSSessionId; mode: "with-audio" | "video-only" }, + registry: IOSRecoveryRegistry, +): Promise { + return registry.recover(input); +} diff --git a/electron/ipc/recording/ios/storage.test.ts b/electron/ipc/recording/ios/storage.test.ts new file mode 100644 index 000000000..ed0329a7e --- /dev/null +++ b/electron/ipc/recording/ios/storage.test.ts @@ -0,0 +1,71 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + allocateIOSSessionStorage, + checkIOSStorageCapacity, + readIOSJournal, + resolveIOSArtifact, + updateIOSJournal, +} from "./storage"; +const id = "3d594650-3436-4a5a-b6a7-5ff45ecf73d0"; +const roots: string[] = []; +async function root() { + const p = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "ios-storage-"))); + roots.push(p); + return p; +} +afterEach(async () => { + await Promise.all(roots.splice(0).map((p) => fs.rm(p, { recursive: true, force: true }))); +}); +describe("iOS storage", () => { + it("allocates unique UUID directories and refuses collisions and traversal", async () => { + const p = await root(); + const s = await allocateIOSSessionStorage(p, id); + expect(s.directory).toBe(path.join(p, `ios-${id}`)); + expect((await readIOSJournal(s)).state).toBe("allocated"); + await expect(allocateIOSSessionStorage(p, id)).rejects.toThrow(); + await expect(allocateIOSSessionStorage(p, "../escape")).rejects.toThrow(); + }); + it("rejects symlink roots and artifacts", async () => { + const p = await root(); + const other = await root(); + await fs.symlink(other, path.join(p, "link")); + await expect(allocateIOSSessionStorage(path.join(p, "link"), id)).rejects.toThrow(); + const s = await allocateIOSSessionStorage(p, id); + await fs.symlink(other, path.join(s.directory, "source-video.mov")); + await expect(resolveIOSArtifact(s, "source-video.mov")).rejects.toThrow(); + await expect(resolveIOSArtifact(s, "../source-video.mov")).rejects.toThrow(); + }); + it("replaces journals atomically and cannot downgrade a committed source", async () => { + const s = await allocateIOSSessionStorage(await root(), id); + await updateIOSJournal(s, { state: "committed", committedFile: "source-video.mov" }); + await expect(updateIOSJournal(s, { state: "recording" })).rejects.toThrow(); + expect((await readIOSJournal(s)).state).toBe("committed"); + expect((await fs.readdir(s.directory)).filter((n) => n.endsWith(".tmp"))).toEqual([]); + }); + it("reserves incoming bytes and second video capacity only when mixing", async () => { + expect( + checkIOSStorageCapacity({ + availableBytes: 1024 ** 3, + videoBytes: 800 * 1024 ** 2, + observedBytes: 0, + elapsedMs: 0, + mixing: false, + }), + ).toBe(true); + expect( + checkIOSStorageCapacity({ + availableBytes: 1024 ** 3, + videoBytes: 800 * 1024 ** 2, + observedBytes: 0, + elapsedMs: 0, + mixing: true, + }), + ).toBe(false); + await expect( + allocateIOSSessionStorage(await root(), id, { availableBytes: async () => 100 }), + ).rejects.toThrow("DISK_SPACE_LOW"); + }); +}); diff --git a/electron/ipc/recording/ios/storage.ts b/electron/ipc/recording/ios/storage.ts new file mode 100644 index 000000000..55c266423 --- /dev/null +++ b/electron/ipc/recording/ios/storage.ts @@ -0,0 +1,241 @@ +import { constants } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { parseNativeCaptureResult } from "../../../../src/shared/iosCapture"; +import type { + IOSAudioFormat, + IOSSessionId, + NativeCaptureResult, + IOSVideoFormat, + IOSCaptureMode, +} from "../../../../src/shared/iosCapture"; +export interface IOSSessionStorage { + sessionId: IOSSessionId; + directory: string; + journalPath: string; +} +export interface IOSCaptureJournal { + version: 1; + sessionId: IOSSessionId; + createdAt: string; + state: + | "allocated" + | "starting" + | "recording" + | "stopping" + | "interrupted" + | "finalising" + | "renamed" + | "committed" + | "failed"; + format?: IOSVideoFormat; + mode?: IOSCaptureMode; + nativeResult?: NativeCaptureResult; + outputAudio?: IOSAudioFormat; + committedFile?: "recording.mov" | "source-video.mov"; +} +export const IOS_SESSION_UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +export const IOS_ARTIFACT_NAMES = [ + "capture-journal.json", + "native-timing.json", + "source-video.mov", + "device-audio.mov", + "microphone.mov", + "recording.pending.mov", + "recording.mov", + "diagnostics.json", +] as const; +export async function availableIOSStorageBytes(directory: string): Promise { + const s = await fs.statfs(directory, { bigint: true }); + const bytes = s.bavail * s.bsize; + return Number( + bytes > BigInt(Number.MAX_SAFE_INTEGER) ? BigInt(Number.MAX_SAFE_INTEGER) : bytes, + ); +} +export function checkIOSStorageCapacity(input: { + availableBytes: number; + videoBytes: number; + observedBytes: number; + elapsedMs: number; + mixing: boolean; +}): boolean { + const { availableBytes, videoBytes, observedBytes, elapsedMs, mixing } = input; + if ( + ![availableBytes, videoBytes, observedBytes, elapsedMs].every( + (n) => Number.isFinite(n) && n >= 0, + ) + ) + return false; + const rate = elapsedMs > 0 ? observedBytes / (elapsedMs / 1000) : 0; + return availableBytes >= 256 * 1024 ** 2 + rate * 60 + (mixing ? videoBytes : 0); +} +export async function validateIOSStorage(storage: IOSSessionStorage): Promise { + if ( + !IOS_SESSION_UUID.test(storage.sessionId) || + path.basename(storage.directory) !== `ios-${storage.sessionId}` || + storage.journalPath !== path.join(storage.directory, "capture-journal.json") + ) + throw new Error("INVALID_SESSION_STORAGE"); + const st = await fs.lstat(storage.directory); + if ( + !st.isDirectory() || + st.isSymbolicLink() || + (await fs.realpath(storage.directory)) !== path.resolve(storage.directory) + ) + throw new Error("UNSAFE_SESSION_PATH"); +} +export async function resolveIOSArtifact( + storage: IOSSessionStorage, + name: string, + allowMissing = false, +): Promise { + await validateIOSStorage(storage); + if (!(IOS_ARTIFACT_NAMES as readonly string[]).includes(name)) + throw new Error("INVALID_ARTIFACT_NAME"); + const p = path.join(storage.directory, name); + try { + const st = await fs.lstat(p); + if (!st.isFile() || st.isSymbolicLink() || (await fs.realpath(p)) !== p) + throw new Error("UNSAFE_ARTIFACT"); + } catch (e) { + if (!(allowMissing && (e as NodeJS.ErrnoException).code === "ENOENT")) throw e; + } + return p; +} +export async function allocateIOSSessionStorage( + root: string, + sessionId: IOSSessionId, + deps: { availableBytes?: (directory: string) => Promise } = {}, +): Promise { + if (!IOS_SESSION_UUID.test(sessionId) || !path.isAbsolute(root)) + throw new Error("INVALID_SESSION_STORAGE"); + const st = await fs.lstat(root); + if ( + !st.isDirectory() || + st.isSymbolicLink() || + (await fs.realpath(root)) !== path.resolve(root) + ) + throw new Error("UNSAFE_RECORDINGS_ROOT"); + await fs.access(root, constants.W_OK); + if ((await (deps.availableBytes ?? availableIOSStorageBytes)(root)) < 1024 ** 3) + throw new Error("DISK_SPACE_LOW"); + const directory = path.join(root, `ios-${sessionId}`); + await fs.mkdir(directory, { mode: 0o700 }); + const storage = { + sessionId, + directory, + journalPath: path.join(directory, "capture-journal.json"), + }; + await writeIOSJournal(storage, { + version: 1, + sessionId, + createdAt: new Date().toISOString(), + state: "allocated", + }); + return storage; +} +async function writeIOSJournal( + storage: IOSSessionStorage, + journal: IOSCaptureJournal, +): Promise { + await resolveIOSArtifact(storage, "capture-journal.json", true); + const temporary = path.join(storage.directory, `.journal-${randomUUID()}.tmp`); + try { + const file = await fs.open(temporary, "wx", 0o600); + try { + await file.writeFile(JSON.stringify(journal)); + await file.sync(); + } finally { + await file.close(); + } + await fs.rename(temporary, storage.journalPath); + const directory = await fs.open(storage.directory, "r"); + try { + await directory.sync(); + } finally { + await directory.close(); + } + } finally { + await fs.rm(temporary, { force: true }); + } +} +export async function readIOSJournal(storage: IOSSessionStorage): Promise { + const journal = JSON.parse( + await fs.readFile(await resolveIOSArtifact(storage, "capture-journal.json"), "utf8"), + ) as IOSCaptureJournal; + const states = [ + "allocated", + "starting", + "recording", + "stopping", + "interrupted", + "finalising", + "renamed", + "committed", + "failed", + ]; + if ( + journal.version !== 1 || + journal.sessionId !== storage.sessionId || + !states.includes(journal.state) || + typeof journal.createdAt !== "string" + ) + throw new Error("INVALID_JOURNAL"); + if ( + journal.committedFile !== undefined && + !["recording.mov", "source-video.mov"].includes(journal.committedFile) + ) + throw new Error("INVALID_JOURNAL"); + return journal; +} +const journalWrites = new Map>(); +export async function updateIOSJournal( + storage: IOSSessionStorage, + patch: Partial>, +): Promise { + const previous = journalWrites.get(storage.directory) ?? Promise.resolve(); + const work = previous + .catch(() => undefined) + .then(async () => { + const current = await readIOSJournal(storage); + if ( + current.state === "committed" && + ((patch.state && patch.state !== "committed") || + (patch.committedFile && patch.committedFile !== current.committedFile)) + ) + throw new Error("ALREADY_COMMITTED"); + await writeIOSJournal(storage, { ...current, ...patch }); + }); + journalWrites.set(storage.directory, work); + try { + await work; + } finally { + if (journalWrites.get(storage.directory) === work) journalWrites.delete(storage.directory); + } +} + +export async function repairIOSJournal( + storage: IOSSessionStorage, + result: NativeCaptureResult, +): Promise { + const nativeResult = parseNativeCaptureResult(result); + if (nativeResult.sessionId !== storage.sessionId) throw new Error("INVALID_NATIVE_RESULT"); + try { + const journal = await readIOSJournal(storage); + if (journal.state === "committed") throw new Error("ALREADY_COMMITTED"); + return; + } catch (error) { + if ((error as Error).message === "ALREADY_COMMITTED") throw error; + } + await writeIOSJournal(storage, { + version: 1, + sessionId: storage.sessionId, + createdAt: new Date().toISOString(), + state: "interrupted", + nativeResult, + format: nativeResult.format, + mode: nativeResult.mode, + }); +} diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 57a9e6881..f2a81cda8 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -1,6 +1,7 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process"; import fs from "node:fs/promises"; import { BrowserWindow } from "electron"; +import { endDesktopRecording, getRecordingLease } from "./recordingLease"; import { persistPendingCursorTelemetry, snapshotCursorTelemetryForPersistence, @@ -191,6 +192,7 @@ export async function muxNativeMacRecordingWithAudio( } export function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStreams) { + const lease = getRecordingLease(); process.once("close", () => { const wasActive = nativeScreenRecordingActive; setNativeCaptureProcess(null); @@ -200,13 +202,17 @@ export function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStr } setNativeScreenRecordingActive(false); + endDesktopRecording(lease); console.log("[mac-finalize] Optimization active: skipping safety-net muxing."); setNativeCaptureTargetPath(null); setNativeCaptureStopRequested(false); setNativeCaptureSystemAudioPath(null); setNativeCaptureMicrophonePath(null); - const sourceName = selectedSource?.name ?? "Screen"; + const sourceName = + selectedSource?.sourceType === "ios-device" + ? selectedSource.displayName + : (selectedSource?.name ?? "Screen"); BrowserWindow.getAllWindows().forEach((window) => { if (!window.isDestroyed()) { window.webContents.send("recording-state-changed", { diff --git a/electron/ipc/recording/recordingLease.test.ts b/electron/ipc/recording/recordingLease.test.ts new file mode 100644 index 000000000..575ed5f70 --- /dev/null +++ b/electron/ipc/recording/recordingLease.test.ts @@ -0,0 +1,36 @@ +import { afterEach, expect, it } from "vitest"; +import { + acquireRecordingLease, + beginDesktopRecording, + endDesktopRecording, + getRecordingLease, + releaseRecordingLease, +} from "./recordingLease"; + +afterEach(() => { + const current = getRecordingLease(); + if (current) releaseRecordingLease(current); +}); +it("rejects competing mobile and desktop capture", () => { + const mobile = acquireRecordingLease("ios-device"); + expect(() => acquireRecordingLease("desktop")).toThrow("RECORDING_BUSY"); + expect(getRecordingLease()).toEqual(mobile); +}); +it("duplicate or stale release cannot release a later owner's lease", () => { + const old = acquireRecordingLease("desktop"); + releaseRecordingLease(old); + releaseRecordingLease(old); + const current = acquireRecordingLease("ios-device"); + releaseRecordingLease(old); + expect(getRecordingLease()).toEqual(current); + expect(() => acquireRecordingLease("ios-device")).toThrow("RECORDING_BUSY"); +}); +it("an old desktop helper close cannot release a new desktop recording", () => { + const old = beginDesktopRecording(); + endDesktopRecording(old); + const current = beginDesktopRecording(); + endDesktopRecording(old); + expect(getRecordingLease()).toEqual(current); + endDesktopRecording(current); + expect(getRecordingLease()).toBeNull(); +}); diff --git a/electron/ipc/recording/recordingLease.ts b/electron/ipc/recording/recordingLease.ts new file mode 100644 index 000000000..120acb4a3 --- /dev/null +++ b/electron/ipc/recording/recordingLease.ts @@ -0,0 +1,33 @@ +import { randomUUID } from "node:crypto"; + +export interface RecordingLease { + owner: "desktop" | "ios-device"; + token: string; +} + +let current: RecordingLease | null = null; +let desktopLease: RecordingLease | null = null; + +export function acquireRecordingLease(owner: RecordingLease["owner"]): RecordingLease { + if (current) throw new Error("RECORDING_BUSY"); + current = Object.freeze({ owner, token: randomUUID() }); + return current; +} + +export function releaseRecordingLease(lease: RecordingLease): void { + if (current?.owner === lease.owner && current.token === lease.token) current = null; +} + +export function getRecordingLease(): RecordingLease | null { + return current; +} + +export function beginDesktopRecording(): RecordingLease { + desktopLease = acquireRecordingLease("desktop"); + return desktopLease; +} +export function endDesktopRecording(expected?: RecordingLease | null): void { + if (expected !== undefined && desktopLease?.token !== expected?.token) return; + if (desktopLease) releaseRecordingLease(desktopLease); + desktopLease = null; +} diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index 650cdca98..8d1ea6445 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -2,6 +2,7 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process"; import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import { BrowserWindow } from "electron"; +import { endDesktopRecording, getRecordingLease } from "./recordingLease"; import { getWindowsCaptureExePath } from "../paths/binaries"; import { selectedSource, @@ -171,6 +172,7 @@ export function waitForWindowsCaptureStop( } export function attachWindowsCaptureLifecycle(proc: ChildProcessWithoutNullStreams) { + const lease = getRecordingLease(); proc.once("close", () => { const wasActive = windowsNativeCaptureActive; setWindowsCaptureProcess(null); @@ -180,9 +182,13 @@ export function attachWindowsCaptureLifecycle(proc: ChildProcessWithoutNullStrea } setWindowsNativeCaptureActive(false); + endDesktopRecording(lease); setWindowsCaptureStopRequested(false); - const sourceName = selectedSource?.name ?? "Screen"; + const sourceName = + selectedSource?.sourceType === "ios-device" + ? selectedSource.displayName + : (selectedSource?.name ?? "Screen"); BrowserWindow.getAllWindows().forEach((window) => { if (!window.isDestroyed()) { window.webContents.send("recording-state-changed", { diff --git a/electron/ipc/register/iosCapture.integration.test.ts b/electron/ipc/register/iosCapture.integration.test.ts new file mode 100644 index 000000000..6b47997c9 --- /dev/null +++ b/electron/ipc/register/iosCapture.integration.test.ts @@ -0,0 +1,529 @@ +import { EventEmitter } from "node:events"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { IpcMainInvokeEvent } from "electron"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import type { + CommittedIOSRecording, + IOSCaptureEvent, + IOSCaptureSnapshot, +} from "../../../src/shared/iosCapture"; +import type { IOSHelperRequest } from "../recording/ios/helperProcess"; + +interface TestWindow { + webContents: EventEmitter & { + mainFrame: { url: string }; + send: ReturnType; + isDestroyed: () => boolean; + }; + isDestroyed: () => boolean; + close: ReturnType; +} +interface TestHelper { + requests: IOSHelperRequest[]; + shutdown: ReturnType; +} +const mocks = vi.hoisted(() => ({ + handlers: new Map unknown>(), + windows: [] as TestWindow[], + helpers: [] as TestHelper[], + enabled: false, + root: "", + hud: undefined as TestWindow | undefined, + selectedSource: undefined as unknown, + scan: vi.fn(), + recover: vi.fn(), + resolveDirectory: vi.fn(), + discard: vi.fn(), + approveRead: vi.fn(), + showMessageBox: vi.fn(), + ensureBinary: vi.fn(), + device: { + sourceType: "ios-device", + id: "ios-device:phone", + deviceToken: "phone", + displayName: "Phone", + generation: 1, + deviceAudio: "unknown", + } as const, + format: { + codedWidth: 640, + codedHeight: 480, + displayWidth: 640, + displayHeight: 480, + codec: "avc1", + colorPrimaries: "ITU_R_709_2", + transferFunction: "ITU_R_709_2", + ycbcrMatrix: "ITU_R_709_2", + fullRange: false, + transform: [1, 0, 0, 1, 0, 0], + observedFrameRate: 30, + fingerprint: "test", + } as const, +})); +vi.mock("electron", () => ({ + app: { getAppPath: () => "/recordly", isPackaged: false, getVersion: () => "test" }, + BrowserWindow: { + fromWebContents: (sender: unknown) => + mocks.windows.find((win) => win.webContents === sender), + getAllWindows: () => mocks.windows, + }, + ipcMain: { + handle: ( + channel: string, + handler: (event: IpcMainInvokeEvent, value?: unknown) => unknown, + ) => mocks.handlers.set(channel, handler), + on: vi.fn(), + }, + dialog: { showMessageBox: mocks.showMessageBox, showSaveDialog: vi.fn() }, + powerSaveBlocker: { start: vi.fn(() => 1), stop: vi.fn() }, + systemPreferences: { getMediaAccessStatus: () => "granted", askForMediaAccess: vi.fn() }, + shell: { openPath: vi.fn() }, + desktopCapturer: {}, +})); +vi.mock("../../rendererServer", () => ({ getPackagedRendererBaseUrl: () => undefined })); +vi.mock("../../windows", () => ({ + getHudOverlayWindow: () => mocks.hud, + reassertHudOverlayMousePassthrough: vi.fn(), +})); +vi.mock("../paths/binaries", () => ({ ensureIOSDeviceCaptureHelperBinary: mocks.ensureBinary })); +vi.mock("../project/manager", () => ({ rememberApprovedLocalReadPath: mocks.approveRead })); +vi.mock("../recording/ios/featurePolicy", () => ({ isIOSCaptureEnabled: () => mocks.enabled })); +vi.mock("../recording/ios/finalize", () => ({ + finalizeIOSRecording: vi.fn(async ({ storage, nativeResult }) => + recording(storage.sessionId, nativeResult.stopReason), + ), +})); +vi.mock("../recording/ios/recovery", () => ({ + IOSRecoveryRegistry: class { + scan = mocks.scan; + recover = mocks.recover; + resolveDirectory = mocks.resolveDirectory; + discard = mocks.discard; + }, +})); +vi.mock("../recording/ios/storage", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + allocateIOSSessionStorage: async (root: string, sessionId: string) => ({ + sessionId, + directory: path.join(root, `ios-${sessionId}`), + journalPath: path.join(root, `ios-${sessionId}`, "capture-journal.json"), + }), + updateIOSJournal: vi.fn(async () => undefined), + validateIOSStorage: vi.fn(async () => undefined), + }; +}); +vi.mock("../state", () => ({ + get selectedSource() { + return mocks.selectedSource; + }, + setSelectedSource: (source: unknown) => { + mocks.selectedSource = source; + }, + setCurrentProjectPath: vi.fn(), + setCurrentRecordingSession: vi.fn(), + setCurrentVideoPath: vi.fn(), +})); +vi.mock("../utils", () => ({ + getRecordingsDir: async () => mocks.root, + getScreen: vi.fn(), + parseWindowId: () => null, +})); +vi.mock("../constants", () => ({ ALLOW_RECORDLY_WINDOW_CAPTURE: false })); +vi.mock("../cursor/bounds", () => ({ + getNativeMacWindowSources: vi.fn(), + resolveLinuxWindowBounds: vi.fn(), + resolveMacWindowBounds: vi.fn(), + stopWindowBoundsCapture: vi.fn(), +})); +vi.mock("../recording/ffmpeg", () => ({ + getDisplayBoundsForSource: vi.fn(), + getDisplayWorkAreaForSource: vi.fn(), +})); +vi.mock("../windowsWindowControl", () => ({ + bringWindowsWindowForward: vi.fn(), + resolveWindowsWindowBounds: vi.fn(), +})); +vi.mock("../recording/ios/helperProcess", () => ({ + IOSHelperProcess: class { + requests: IOSHelperRequest[] = []; + private listeners = new Set<(event: IOSCaptureEvent) => void>(); + private sequence = 0; + private sessionId?: string; + private generation = 0; + shutdown = vi.fn(async () => undefined); + constructor() { + mocks.helpers.push(this); + } + private emit(event: string, payload: unknown, scoped = true) { + const message = { + protocolVersion: 1, + sequence: ++this.sequence, + event, + payload, + ...(scoped ? { sessionId: this.sessionId, generation: this.generation } : {}), + } as IOSCaptureEvent; + for (const listener of this.listeners) listener(message); + } + async request(input: IOSHelperRequest) { + this.requests.push(input); + if (input.command === "discover") + queueMicrotask(() => + this.emit( + "inventoryChanged", + { devices: [mocks.device], microphones: [], inventoryGeneration: 1 }, + false, + ), + ); + if (input.command === "prepare") { + this.sessionId = input.sessionId; + this.generation = input.generation!; + queueMicrotask(() => + this.emit("prepared", { + source: mocks.device, + options: input.payload!.options, + format: mocks.format, + mode: "passthrough", + }), + ); + } + if (input.command === "start") + queueMicrotask(() => + this.emit("recordingStarted", { + firstHostTime: { value: "1", timescale: 1 }, + acceptedVideoSamples: 1, + }), + ); + if (input.command === "stop") + queueMicrotask(() => + this.emit("nativeFinalized", { + result: { + sessionId: this.sessionId, + stopReason: "stopped", + mode: "passthrough", + format: mocks.format, + timingFile: "native-timing.json", + video: { + relativeName: "source-video.mov", + mediaKind: "video", + firstHostTime: { value: "1", timescale: 1 }, + duration: { value: "1", timescale: 1 }, + sampleCount: 1, + videoFormat: mocks.format, + }, + }, + }), + ); + return { + protocolVersion: 1, + event: "accepted", + sequence: ++this.sequence, + requestId: "test", + payload: {}, + }; + } + onEvent(listener: (event: IOSCaptureEvent) => void) { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + } + onExit() { + return () => undefined; + } + onPreview() { + return () => undefined; + } + }, +})); + +function recording(sessionId: string, stopReason = "stopped"): CommittedIOSRecording { + return { + sessionId, + videoPath: path.join(mocks.root, `ios-${sessionId}`, "source-video.mov"), + hideOverlayCursorByDefault: true, + captureMetadata: { + version: 1, + sourceKind: "ios-device", + mode: "passthrough", + format: mocks.format, + deviceAudioRecorded: false, + narrationRecorded: false, + stopReason, + interrupted: stopReason !== "stopped", + }, + }; +} +function createWindow(): TestWindow { + const webContents = Object.assign(new EventEmitter(), { + mainFrame: { url: "file:///recordly/dist/index.html?windowType=hud-overlay" }, + send: vi.fn(), + isDestroyed: () => false, + }); + const win = { webContents, isDestroyed: () => false, close: vi.fn() }; + mocks.windows.push(win); + return win; +} +function event(win: TestWindow, patch: Partial = {}): IpcMainInvokeEvent { + return { + sender: win.webContents, + senderFrame: win.webContents.mainFrame, + ...patch, + } as unknown as IpcMainInvokeEvent; +} +async function invoke(channel: string, sender: IpcMainInvokeEvent, value?: unknown) { + const handler = mocks.handlers.get(channel); + if (!handler) throw new Error(`Handler missing: ${channel}`); + return handler(sender, value); +} +let closeController: (() => Promise) | undefined; +beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); + vi.unstubAllEnvs(); + mocks.handlers.clear(); + mocks.windows.length = 0; + mocks.helpers.length = 0; + mocks.hud = undefined; + mocks.selectedSource = undefined; + mocks.enabled = false; + mocks.root = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-ios-ipc-test-")); + mocks.ensureBinary.mockResolvedValue("/recordly/helper"); + mocks.approveRead.mockResolvedValue(undefined); + mocks.scan.mockResolvedValue([]); + mocks.resolveDirectory.mockResolvedValue(mocks.root); + mocks.showMessageBox.mockResolvedValue({ response: 1 }); +}); +afterEach(async () => { + await closeController?.(); + closeController = undefined; + await fs.rm(mocks.root, { recursive: true, force: true }); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); +async function register(enabled = true) { + mocks.enabled = enabled; + const main = createWindow(); + const picker = createWindow(); + const editor = createWindow(); + mocks.hud = main; + const createEditor = vi.fn(); + const recordingChanged = vi.fn(); + const module = await import("./iosCapture"); + const controller = module.registerIOSCaptureHandlers({ + getMainWindow: () => main as never, + getSourceSelectorWindow: () => picker as never, + createEditorWindow: createEditor, + onRecordingStateChange: recordingChanged, + }); + closeController = () => controller.shutdown(); + const sources = await import("./sources"); + sources.registerSourceHandlers({ + createEditorWindow: createEditor, + createSourceSelectorWindow: () => picker as never, + getSourceSelectorWindow: () => picker as never, + }); + return { main, picker, editor, controller, createEditor, recordingChanged }; +} +async function prepare(main: TestWindow) { + await invoke("ios-capture:discover", event(main)); + return (await invoke("ios-capture:prepare", event(main), { + deviceToken: "phone", + generation: 1, + options: { deviceAudio: false, microphoneToken: null }, + })) as IOSCaptureSnapshot; +} + +it("default-off registration, subscription and rejected discovery never spawn a helper", async () => { + const h = await register(false); + expect(await invoke("ios-capture:capabilities", event(h.main))).toEqual({ enabled: false }); + await invoke("ios-capture:subscribe", event(h.main)); + expect(await invoke("ios-capture:snapshot", event(h.main))).toMatchObject({ + phase: "unavailable", + }); + await expect(invoke("ios-capture:discover", event(h.main))).rejects.toThrow( + "UNSUPPORTED_PLATFORM", + ); + expect(mocks.ensureBinary).not.toHaveBeenCalled(); + expect(mocks.helpers).toHaveLength(0); +}); +it("registered IPC rejects foreign windows, subframes and documents before preparing or releasing inputs", async () => { + const h = await register(); + const ready = await prepare(h.main); + h.picker.webContents.mainFrame.url = "https://untrusted.example/"; + for (const untrusted of [ + event(h.editor), + event(h.main, { senderFrame: { url: h.main.webContents.mainFrame.url } as never }), + event(h.picker), + ]) { + await expect( + invoke("ios-capture:prepare", untrusted, { + deviceToken: "phone", + generation: 1, + options: { deviceAudio: false, microphoneToken: null }, + }), + ).rejects.toThrow("INVALID_REQUEST"); + await expect(invoke("ios-capture:release", untrusted, ready.sessionId)).rejects.toThrow( + "INVALID_REQUEST", + ); + await expect( + invoke("select-source", untrusted, { + sourceType: "screen", + id: "screen:1", + name: "Screen", + }), + ).rejects.toThrow("INVALID_REQUEST"); + } + expect(h.controller.getSnapshot()).toMatchObject({ + phase: "ready", + sessionId: ready.sessionId, + }); + expect( + mocks.helpers[0].requests.filter((request) => request.command === "prepare"), + ).toHaveLength(1); + expect(mocks.helpers[0].requests.some((request) => request.command === "release")).toBe(false); +}); +it("selecting a native device keeps the standalone picker alive through preparation", async () => { + const h = await register(); + await invoke("ios-capture:discover", event(h.picker)); + await invoke("select-source", event(h.picker), mocks.device); + expect(h.picker.close).not.toHaveBeenCalled(); + const prepared = await invoke("ios-capture:prepare", event(h.picker), { + deviceToken: "phone", + generation: 1, + options: { deviceAudio: false, microphoneToken: null }, + }); + expect(prepared).toMatchObject({ phase: "ready", source: mocks.device }); +}); +it("blocks every active-session recovery action even when an earlier scan registered that UUID", async () => { + const h = await register(); + const ready = await prepare(h.main); + const id = ready.sessionId!; + mocks.scan.mockResolvedValue([{ sessionId: id, status: "recoverable-video" }]); + const state = h.controller.getSnapshot(); + const snapshot = vi + .spyOn(h.controller, "getSnapshot") + .mockReturnValue({ ...state, sessionId: null, phase: "idle" }); + await invoke("ios-capture:recovery-list", event(h.main)); + snapshot.mockRestore(); + await invoke("ios-capture:recovery-list", event(h.main)); + expect(mocks.scan.mock.calls.at(-1)?.[1]).toEqual(new Set([id])); + for (const [channel, input] of [ + ["recover", { sessionId: id, mode: "video-only" }], + ["recovery-discard", id], + ["recovery-folder", id], + ["diagnostics", id], + ] as const) { + await expect(invoke(`ios-capture:${channel}`, event(h.main), input)).rejects.toThrow( + "RECORDING_BUSY", + ); + } + expect(mocks.recover).not.toHaveBeenCalled(); + expect(mocks.resolveDirectory).not.toHaveBeenCalled(); + expect(mocks.discard).not.toHaveBeenCalled(); + expect(mocks.showMessageBox).not.toHaveBeenCalled(); +}); +it("rechecks active-session ownership after a recovery discard confirmation resolves", async () => { + const h = await register(); + const ready = await prepare(h.main); + const id = ready.sessionId!; + const state = h.controller.getSnapshot(); + const snapshot = vi + .spyOn(h.controller, "getSnapshot") + .mockReturnValue({ ...state, sessionId: null, phase: "idle" }); + let confirm!: (value: { response: number }) => void; + mocks.showMessageBox.mockImplementationOnce( + () => + new Promise((resolve) => { + confirm = resolve; + }), + ); + const discarding = invoke("ios-capture:recovery-discard", event(h.main), id); + const failed = expect(discarding).rejects.toThrow("RECORDING_BUSY"); + await vi.waitFor(() => expect(mocks.showMessageBox).toHaveBeenCalledOnce()); + snapshot.mockRestore(); + confirm({ response: 1 }); + await failed; + expect(mocks.discard).not.toHaveBeenCalled(); +}); +it("concurrent recovery IPC calls deliver one editor while file approval is pending", async () => { + const h = await register(); + const result = recording("12345678-1234-4123-8123-123456789abc", "recovered-interruption"); + mocks.recover.mockResolvedValue(result); + let approve!: () => void; + mocks.approveRead.mockImplementationOnce( + () => + new Promise((resolve) => { + approve = resolve; + }), + ); + const first = invoke("ios-capture:recover", event(h.main), { + sessionId: result.sessionId, + mode: "video-only", + }); + const second = invoke("ios-capture:recover", event(h.picker), { + sessionId: result.sessionId, + mode: "video-only", + }); + await vi.waitFor(() => expect(mocks.approveRead).toHaveBeenCalledOnce()); + expect(h.createEditor).not.toHaveBeenCalled(); + approve(); + await Promise.all([first, second]); + expect(h.createEditor).toHaveBeenCalledOnce(); + expect( + h.main.webContents.send.mock.calls.filter( + ([channel]) => channel === "recording-session-changed", + ), + ).toHaveLength(1); +}); +it("keeps prepared inputs when the picker closes with a subscribed launcher, then releases on last-client loss", async () => { + const h = await register(); + await invoke("ios-capture:subscribe", event(h.main)); + await invoke("ios-capture:subscribe", event(h.picker)); + const ready = await prepare(h.picker); + h.picker.webContents.emit("destroyed"); + await Promise.resolve(); + expect(h.controller.getSnapshot()).toMatchObject({ + phase: "ready", + sessionId: ready.sessionId, + }); + expect(mocks.helpers[0].shutdown).not.toHaveBeenCalled(); + h.main.webContents.emit("destroyed"); + await vi.waitFor(() => + expect(h.controller.getSnapshot()).toMatchObject({ phase: "idle", sessionId: null }), + ); + expect(mocks.helpers[0].shutdown).toHaveBeenCalledOnce(); +}); +it("renderer crash stops an active take even when another subscribed window survives", async () => { + const h = await register(); + await invoke("ios-capture:subscribe", event(h.main)); + await invoke("ios-capture:subscribe", event(h.picker)); + const ready = await prepare(h.main); + await invoke("ios-capture:start", event(h.main), ready.sessionId); + expect(h.controller.getSnapshot().phase).toBe("recording"); + h.main.webContents.emit("render-process-gone"); + await vi.waitFor(() => expect(h.controller.getSnapshot().phase).toBe("interrupted")); + expect(h.controller.getCommitted()?.captureMetadata.stopReason).toBe("renderer-loss"); + expect(mocks.helpers[0].requests.filter((request) => request.command === "stop")).toHaveLength( + 1, + ); + expect(h.createEditor).toHaveBeenCalledOnce(); +}); +it("closing discovery does not report an unrelated desktop recording as stopped", async () => { + const h = await register(); + const leases = await import("../recording/recordingLease"); + leases.beginDesktopRecording(); + try { + await invoke("ios-capture:discovery-active", event(h.picker), true); + await invoke("ios-capture:discover", event(h.picker)); + await invoke("ios-capture:discovery-active", event(h.picker), false); + expect(mocks.helpers[0].shutdown).toHaveBeenCalledOnce(); + expect(leases.getRecordingLease()?.owner).toBe("desktop"); + expect(h.recordingChanged).not.toHaveBeenCalled(); + } finally { + leases.endDesktopRecording(); + } +}); diff --git a/electron/ipc/register/iosCapture.ts b/electron/ipc/register/iosCapture.ts new file mode 100644 index 000000000..39828e63b --- /dev/null +++ b/electron/ipc/register/iosCapture.ts @@ -0,0 +1,447 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + app, + BrowserWindow, + dialog, + ipcMain, + powerSaveBlocker, + shell, + systemPreferences, + type IpcMainEvent, + type IpcMainInvokeEvent, + type WebContents, +} from "electron"; +import { + type CommittedIOSRecording, + type IOSMediaInspection, + parseIOSCaptureCommand, +} from "../../../src/shared/iosCapture"; +import { getPackagedRendererBaseUrl } from "../../rendererServer"; +import { getHudOverlayWindow } from "../../windows"; +import { ensureIOSDeviceCaptureHelperBinary } from "../paths/binaries"; +import { rememberApprovedLocalReadPath } from "../project/manager"; +import { IOSCaptureController } from "../recording/ios/controller"; +import { isIOSCaptureEnabled } from "../recording/ios/featurePolicy"; +import { finalizeIOSRecording, runIOSFFmpeg } from "../recording/ios/finalize"; +import { IOSHelperProcess } from "../recording/ios/helperProcess"; +import { IOSRecordingHandoff } from "../recording/ios/handoff"; +import { getRecordingLease } from "../recording/recordingLease"; +import { assertIOSCaptureSender, parseIOSPrepareInput } from "../recording/ios/ipcPolicy"; +import { prepareIOSPermissions } from "../recording/ios/permissions"; +import type { IOSPreviewFrame } from "../recording/ios/preview"; +import { IOSRecoveryRegistry } from "../recording/ios/recovery"; +import { + allocateIOSSessionStorage, + availableIOSStorageBytes, + checkIOSStorageCapacity, + IOS_ARTIFACT_NAMES, + IOS_SESSION_UUID, + readIOSJournal, + resolveIOSArtifact, + updateIOSJournal, + validateIOSStorage, +} from "../recording/ios/storage"; +import { setCurrentProjectPath, setCurrentRecordingSession, setCurrentVideoPath } from "../state"; +import { getRecordingsDir } from "../utils"; + +let activeController: IOSCaptureController | undefined; +let validateSelectionSender: ((event: IpcMainInvokeEvent) => void) | undefined; +export function assertIOSCaptureClient(event: IpcMainInvokeEvent) { + if (!validateSelectionSender) throw new Error("UNSUPPORTED_PLATFORM"); + validateSelectionSender(event); +} +export function validateIOSSourceSelection(event: IpcMainInvokeEvent, source: unknown) { + assertIOSCaptureClient(event); + if (!activeController || !validateSelectionSender) throw new Error("UNSUPPORTED_PLATFORM"); + const state = activeController.getSnapshot(); + if (state.phase === "unavailable" || !source || typeof source !== "object") + throw new Error("INVALID_REQUEST"); + const input = source as Record; + const found = state.devices.find( + (device) => + device.deviceToken === input.deviceToken && + device.id === input.id && + device.generation === input.generation, + ); + if (!found) throw new Error("DEVICE_NOT_FOUND"); + return found; +} +export function getIOSCaptureController() { + return activeController; +} +export async function shutdownIOSCapture() { + await activeController?.shutdown(); +} +export async function interruptIOSCapture(reason = "system-suspend") { + const state = activeController?.getSnapshot(); + if ( + state?.sessionId && + ["starting", "recording", "stopping", "finalising"].includes(state.phase) + ) { + try { + await activeController?.stop(state.sessionId, reason); + } catch { + /* Retained for recovery. */ + } + } +} + +export function registerIOSCaptureHandlers(input: { + getMainWindow: () => BrowserWindow | null; + getSourceSelectorWindow: () => BrowserWindow | null; + createEditorWindow: () => void; + onRecordingStateChange?: (recording: boolean, sourceName: string) => void; +}) { + if (activeController) throw new Error("iOS capture handlers already registered"); + const enabled = isIOSCaptureEnabled(process.platform, app.isPackaged, process.env); + const root = async () => fs.realpath(await getRecordingsDir()); + const createHelper = async () => + new IOSHelperProcess({ binaryPath: await ensureIOSDeviceCaptureHelperBinary() }); + const subscribers = new Map(); + const discoveryClients = new Set(); + const previews = new Map< + WebContents, + { pending?: { generation: number; sequence: number }; latest?: IOSPreviewFrame } + >(); + const watched = new Set(); + let powerId: number | undefined; + const editorHandoff = new IOSRecordingHandoff(async (recording) => { + await rememberApprovedLocalReadPath(recording.videoPath); + setCurrentProjectPath(null); + setCurrentVideoPath(recording.videoPath); + setCurrentRecordingSession(recording); + for (const win of BrowserWindow.getAllWindows()) { + if (!win.isDestroyed()) win.webContents.send("recording-session-changed", recording); + } + input.createEditorWindow(); + }); + const handoff = (recording: CommittedIOSRecording) => editorHandoff.run(recording); + const inspectMedia = async (filePath: string): Promise => { + const directory = path.dirname(filePath); + const sessionId = path.basename(directory).slice(4); + await validateIOSStorage({ + sessionId, + directory, + journalPath: path.join(directory, "capture-journal.json"), + }); + const relativeName = path.basename(filePath); + await resolveIOSArtifact( + { sessionId, directory, journalPath: path.join(directory, "capture-journal.json") }, + relativeName, + ); + const helper = await createHelper(); + try { + await helper.request({ command: "hello" }); + const response = await helper.request({ + command: "inspectMedia", + sessionId, + payload: { relativeName }, + storage: { sessionRoot: directory, allowedRelativeNames: [relativeName] }, + }); + if (response.event !== "accepted" || !("inspection" in response.payload)) + throw new Error("FINALIZATION_FAILED"); + return response.payload.inspection; + } finally { + await helper.shutdown(); + } + }; + const finalizeDeps = { inspectMedia }; + const recovery = new IOSRecoveryRegistry(finalizeDeps, process.platform === "darwin"); + const controller: IOSCaptureController = new IOSCaptureController({ + enabled, + createHelper, + allocate: async (id) => allocateIOSSessionStorage(await root(), id), + permissions: (options) => + prepareIOSPermissions(options, { + status: (media) => systemPreferences.getMediaAccessStatus(media), + request: (media) => systemPreferences.askForMediaAccess(media), + }), + journal: updateIOSJournal, + finalize: (storage, nativeResult, signal) => + finalizeIOSRecording( + { storage, nativeResult }, + { ...finalizeDeps, runFFmpeg: (args) => runIOSFFmpeg(args, { signal }) }, + ), + discard: async (storage) => { + await validateIOSStorage(storage); + await fs.rm(storage.directory, { recursive: true }); + }, + checkSpace: async (storage, elapsedMs) => { + const sizes = await Promise.all( + ["source-video.mov", "device-audio.mov", "microphone.mov"].map(async (name) => { + try { + return (await fs.stat(await resolveIOSArtifact(storage, name))).size; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return 0; + throw error; + } + }), + ); + const options = controller.getSnapshot().options; + return checkIOSStorageCapacity({ + availableBytes: await availableIOSStorageBytes(storage.directory), + videoBytes: sizes[0], + observedBytes: sizes.reduce((a, b) => a + b, 0), + elapsedMs, + mixing: !!(options?.deviceAudio || options?.microphoneToken), + }); + }, + onCommitted: handoff, + setCapturing: (capturing) => { + if (capturing && powerId === undefined) + powerId = powerSaveBlocker.start("prevent-app-suspension"); + if (!capturing && powerId !== undefined) { + powerSaveBlocker.stop(powerId); + powerId = undefined; + } + input.onRecordingStateChange?.( + capturing, + controller.getSnapshot().source?.displayName ?? "iPhone / iPad", + ); + }, + }); + activeController = controller; + function assertSender(event: IpcMainInvokeEvent | IpcMainEvent) { + const win = BrowserWindow.fromWebContents(event.sender); + const trustedUrls = [pathToFileURL(path.join(app.getAppPath(), "dist", "index.html")).href]; + if (process.env.VITE_DEV_SERVER_URL) trustedUrls.push(process.env.VITE_DEV_SERVER_URL); + const packagedUrl = getPackagedRendererBaseUrl(); + if (packagedUrl) trustedUrls.push(new URL("/", packagedUrl).href); + assertIOSCaptureSender({ + knownWindow: + !!win && + [ + input.getMainWindow(), + input.getSourceSelectorWindow(), + getHudOverlayWindow(), + ].includes(win), + mainFrame: event.senderFrame === event.sender.mainFrame, + url: event.senderFrame?.url ?? "", + trustedUrls, + }); + if (!watched.has(event.sender)) { + watched.add(event.sender); + const cleanup = (crashed: boolean) => { + const participated = + subscribers.has(event.sender) || discoveryClients.has(event.sender); + subscribers.delete(event.sender); + discoveryClients.delete(event.sender); + previews.delete(event.sender); + watched.delete(event.sender); + void controller.setPreviewEnabled(previews.size > 0).catch(() => { + /* Failure is retained by the authoritative controller. */ + }); + if (crashed && participated) void interruptIOSCapture("renderer-loss"); + if (!subscribers.size && !discoveryClients.size) { + const state = controller.getSnapshot(); + if (state.sessionId && ["ready", "preparing"].includes(state.phase)) + void controller.release(state.sessionId).catch(() => { + /* Closing retains any media for recovery. */ + }); + else void controller.shutdown(); + } + }; + event.sender.once("destroyed", () => cleanup(false)); + event.sender.once("render-process-gone", () => cleanup(true)); + } + } + function handle( + channel: string, + action: (event: IpcMainInvokeEvent, value: unknown) => unknown, + ) { + ipcMain.handle(`ios-capture:${channel}`, (event, value: unknown) => { + assertSender(event); + if (value !== undefined && Buffer.byteLength(JSON.stringify(value)) > 64 * 1024) + throw new Error("INVALID_REQUEST"); + return action(event, value); + }); + } + validateSelectionSender = assertSender; + function session(value: unknown): string { + if (typeof value !== "string" || !IOS_SESSION_UUID.test(value)) + throw new Error("INVALID_REQUEST"); + return value; + } + function sendPreview(client: WebContents) { + const preview = previews.get(client); + if (!preview || preview.pending || !preview.latest || client.isDestroyed()) return; + const frame = preview.latest; + preview.latest = undefined; + preview.pending = { generation: frame.generation, sequence: frame.sequence }; + client.send("ios-capture:preview", frame); + } + controller.subscribe((state) => { + for (const client of subscribers.keys()) + if (!client.isDestroyed()) client.send("ios-capture:state", state); + }); + controller.onPreview((frame) => { + for (const [client, preview] of previews) { + preview.latest = frame; + sendPreview(client); + } + }); + handle("capabilities", () => ({ enabled })); + handle("snapshot", () => controller.getSnapshot()); + handle("subscribe", (event) => { + subscribers.set(event.sender, (subscribers.get(event.sender) ?? 0) + 1); + event.sender.send("ios-capture:state", controller.getSnapshot()); + }); + handle("unsubscribe", (event) => { + const count = (subscribers.get(event.sender) ?? 1) - 1; + if (count) subscribers.set(event.sender, count); + else subscribers.delete(event.sender); + }); + handle("discover", () => controller.discover()); + handle("prepare", async (_event, value) => { + const state = await controller.prepare(parseIOSPrepareInput(value)); + await controller.setPreviewEnabled(previews.size > 0); + return state; + }); + for (const command of ["start", "stop", "release"] as const) + handle(command, (_event, value) => { + const sessionId = session(value); + parseIOSCaptureCommand({ protocolVersion: 1, requestId: "ipc", command, sessionId }); + return controller[command](sessionId); + }); + handle("cancel", async (_event, value) => { + if (!value || typeof value !== "object") throw new Error("INVALID_REQUEST"); + const v = value as Record; + if ( + Object.keys(v).some((key) => !["sessionId", "discardAcceptedMedia"].includes(key)) || + typeof v.discardAcceptedMedia !== "boolean" + ) + throw new Error("INVALID_REQUEST"); + const id = session(v.sessionId); + if (id !== controller.getSnapshot().sessionId) throw new Error("INVALID_REQUEST"); + if (v.discardAcceptedMedia) { + const { response } = await dialog.showMessageBox({ + type: "warning", + message: "Discard this recording?", + buttons: ["Keep recording", "Discard"], + defaultId: 0, + cancelId: 0, + }); + if (response !== 1) return; + if (id !== controller.getSnapshot().sessionId) throw new Error("INVALID_REQUEST"); + } + return controller.cancel(id, v.discardAcceptedMedia); + }); + handle("preview-enabled", async (event, value) => { + if (typeof value !== "boolean") throw new Error("INVALID_REQUEST"); + if (value) { + if (!previews.has(event.sender)) previews.set(event.sender, {}); + } else previews.delete(event.sender); + await controller.setPreviewEnabled(previews.size > 0); + }); + ipcMain.on("ios-capture:preview-ack", (event, value: unknown) => { + try { + assertSender(event); + if (!value || typeof value !== "object") return; + const v = value as Record; + const preview = previews.get(event.sender); + if ( + preview && + preview.pending?.generation === v.generation && + preview.pending?.sequence === v.sequence + ) { + preview.pending = undefined; + sendPreview(event.sender); + } + } catch { + /* Untrusted preview acknowledgements cannot affect capture. */ + } + }); + handle("discovery-active", async (event, value) => { + if (typeof value !== "boolean") throw new Error("INVALID_REQUEST"); + if (value) discoveryClients.add(event.sender); + else discoveryClients.delete(event.sender); + if (!discoveryClients.size && !controller.getSnapshot().sessionId) + await controller.shutdown(); + }); + handle("recovery-list", async () => { + const state = controller.getSnapshot(); + const excluded = + state.sessionId && !canRecoverSession(state.sessionId) + ? new Set([state.sessionId]) + : new Set(); + return recovery.scan(await root(), excluded); + }); + function canRecoverSession(id: string) { + const state = controller.getSnapshot(); + return ( + id !== state.sessionId || + (["failed", "recoveryAvailable"].includes(state.phase) && + getRecordingLease()?.owner !== "ios-device") + ); + } + function recoverySession(value: unknown) { + const id = session(value); + if (!canRecoverSession(id)) throw new Error("RECORDING_BUSY"); + return id; + } + handle("recover", async (_event, value) => { + if (!value || typeof value !== "object") throw new Error("INVALID_REQUEST"); + const v = value as Record; + if ( + Object.keys(v).some((key) => !["sessionId", "mode"].includes(key)) || + !["with-audio", "video-only"].includes(String(v.mode)) + ) + throw new Error("INVALID_REQUEST"); + const id = recoverySession(v.sessionId); + const recording = await recovery.recover({ + sessionId: id, + mode: v.mode as "with-audio" | "video-only", + }); + await handoff(recording); + return recording; + }); + handle("recovery-folder", async (_event, value) => { + await shell.openPath(await recovery.resolveDirectory(recoverySession(value))); + }); + handle("recovery-discard", async (_event, value) => { + const id = recoverySession(value); + await recovery.resolveDirectory(id); + const { response } = await dialog.showMessageBox({ + type: "warning", + message: "Discard this recording?", + buttons: ["Keep recording", "Discard"], + defaultId: 0, + cancelId: 0, + }); + if (response === 1) await recovery.discard(recoverySession(id)); + }); + handle("diagnostics", async (_event, value) => { + const id = recoverySession(value); + const directory = await recovery.resolveDirectory(id); + const journal = await readIOSJournal({ + sessionId: id, + directory, + journalPath: path.join(directory, "capture-journal.json"), + }); + const result = await dialog.showSaveDialog({ + title: "Export recording diagnostics", + defaultPath: "recording-diagnostics.json", + filters: [{ name: "JSON", extensions: ["json"] }], + }); + if (!result.canceled && result.filePath) + await fs.writeFile( + result.filePath, + JSON.stringify( + { + version: 1, + appVersion: app.getVersion(), + platform: process.platform, + architecture: process.arch, + state: journal.state, + mode: journal.mode, + expectedFiles: IOS_ARTIFACT_NAMES, + }, + null, + 2, + ), + ); + }); + return controller; +} diff --git a/electron/ipc/register/project.ts b/electron/ipc/register/project.ts index ff309a19a..5f6808430 100644 --- a/electron/ipc/register/project.ts +++ b/electron/ipc/register/project.ts @@ -1,3 +1,4 @@ +import { parseCaptureMetadata, type CaptureMetadata } from "../../../src/shared/iosCapture"; import { randomUUID } from "node:crypto"; import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; @@ -650,7 +651,11 @@ export function registerProjectHandlers() { resolvedSession.webcamPath, ]); - if (nextSession.webcamPath) { + if ( + nextSession.webcamPath || + nextSession.captureMetadata || + nextSession.hideOverlayCursorByDefault + ) { await persistRecordingSessionManifest(nextSession); } @@ -673,6 +678,7 @@ export function registerProjectHandlers() { async ( _, session: { + captureMetadata?: CaptureMetadata; videoPath: string; webcamPath?: string | null; timeOffsetMs?: number; @@ -688,6 +694,13 @@ export function registerProjectHandlers() { webcamPath: normalizeVideoSourcePath(session.webcamPath ?? null), timeOffsetMs: normalizeRecordingTimeOffsetMs(session.timeOffsetMs), hideOverlayCursorByDefault: normalizeBoolean(session.hideOverlayCursorByDefault), + captureMetadata: (() => { + try { + return parseCaptureMetadata(session.captureMetadata); + } catch { + return undefined; + } + })(), }); await rememberApprovedLocalReadPath(currentRecordingSession!.videoPath); await rememberApprovedLocalReadPath(currentRecordingSession!.webcamPath); diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 438b6eeee..3da931cc7 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -156,6 +156,11 @@ import { } from "../utils"; import { resolveWindowsCaptureTarget } from "../windowsCaptureSelection"; import { bringSelectedWindowForward } from "./sources"; +import { + beginDesktopRecording, + endDesktopRecording, + getRecordingLease, +} from "../recording/recordingLease"; const execFileAsync = promisify(execFile); @@ -401,7 +406,39 @@ async function resolveExistingPath(...candidates: Array void, ) { - ipcMain.handle( + let desktopStop: Promise | undefined; + const handleDesktopStart = (channel: string, listener: Parameters[1]) => + ipcMain.handle(channel, async (event, ...args) => { + if ( + selectedSource?.sourceType === "ios-device" || + (args[0] as { sourceType?: string } | undefined)?.sourceType === "ios-device" + ) + throw new Error("UNSUPPORTED_OPERATION"); + const lease = beginDesktopRecording(); + try { + const result = await listener(event, ...args); + if (!result?.success) endDesktopRecording(lease); + return result; + } catch (error) { + endDesktopRecording(lease); + throw error; + } + }); + const handleDesktopStop = (channel: string, listener: Parameters[1]) => + ipcMain.handle(channel, async (event, ...args) => { + if (getRecordingLease()?.owner === "ios-device") + throw new Error("UNSUPPORTED_OPERATION"); + if (desktopStop) return desktopStop; + const lease = getRecordingLease(); + desktopStop = Promise.resolve().then(() => listener(event, ...args)); + try { + return await desktopStop; + } finally { + endDesktopRecording(lease); + desktopStop = undefined; + } + }); + handleDesktopStart( "start-native-screen-recording", async (_, source: SelectedSource, options?: NativeMacRecordingOptions) => { // Capture starts before the renderer publishes its recording-state @@ -937,7 +974,7 @@ export function registerRecordingHandlers( }, ); - ipcMain.handle("stop-native-screen-recording", async () => { + handleDesktopStop("stop-native-screen-recording", async () => { const start = Date.now(); console.log("[PERF:MAIN] Handler: stop-native-screen-recording: STARTED"); try { @@ -1306,6 +1343,7 @@ export function registerRecordingHandlers( }); ipcMain.handle("pause-native-screen-recording", async () => { + if (getRecordingLease()?.owner === "ios-device") throw new Error("UNSUPPORTED_OPERATION"); if (process.platform === "win32") { if (!windowsNativeCaptureActive || !windowsCaptureProcess) { return { success: false, message: "No native Windows screen recording is active." }; @@ -1362,6 +1400,7 @@ export function registerRecordingHandlers( }); ipcMain.handle("resume-native-screen-recording", async () => { + if (getRecordingLease()?.owner === "ios-device") throw new Error("UNSUPPORTED_OPERATION"); if (process.platform === "win32") { if (!windowsNativeCaptureActive || !windowsCaptureProcess) { return { success: false, message: "No native Windows screen recording is active." }; @@ -1566,7 +1605,7 @@ export function registerRecordingHandlers( } }); - ipcMain.handle("start-ffmpeg-recording", async (_, source: SelectedSource) => { + handleDesktopStart("start-ffmpeg-recording", async (_, source: SelectedSource) => { if (ffmpegCaptureProcess) { return { success: false, message: "An FFmpeg recording is already active." }; } @@ -1608,7 +1647,7 @@ export function registerRecordingHandlers( } }); - ipcMain.handle("stop-ffmpeg-recording", async () => { + handleDesktopStop("stop-ffmpeg-recording", async () => { if (!ffmpegScreenRecordingActive) { return { success: false, message: "No FFmpeg recording is active." }; } @@ -1861,6 +1900,20 @@ export function registerRecordingHandlers( }); ipcMain.handle("set-recording-state", (_, recording: boolean) => { + if ( + getRecordingLease()?.owner === "ios-device" || + selectedSource?.sourceType === "ios-device" + ) + throw new Error("UNSUPPORTED_OPERATION"); + if (recording && !getRecordingLease()) beginDesktopRecording(); + if ( + !recording && + !desktopStop && + !nativeCaptureProcess && + !windowsCaptureProcess && + !ffmpegCaptureProcess + ) + endDesktopRecording(); if (recording) { stopCursorCapture(); stopInteractionCapture(); diff --git a/electron/ipc/register/sources.ts b/electron/ipc/register/sources.ts index ba40df16b..36966d530 100644 --- a/electron/ipc/register/sources.ts +++ b/electron/ipc/register/sources.ts @@ -11,7 +11,12 @@ import { } from "../cursor/bounds"; import { getDisplayBoundsForSource, getDisplayWorkAreaForSource } from "../recording/ffmpeg"; import { selectedSource, setSelectedSource } from "../state"; -import type { SelectedSource, WindowBounds } from "../types"; +import type { SelectedSource, SelectedCaptureSource, WindowBounds } from "../types"; +import { + assertIOSCaptureClient, + getIOSCaptureController, + validateIOSSourceSelection, +} from "./iosCapture"; import { getScreen, parseWindowId } from "../utils"; import { bringWindowsWindowForward, resolveWindowsWindowBounds } from "../windowsWindowControl"; import { getScreenSourceIdForDisplay } from "./sourceMapping"; @@ -381,7 +386,22 @@ export function registerSourceHandlers({ } }); - ipcMain.handle("select-source", async (_, source: SelectedSource) => { + ipcMain.handle("select-source", async (event, source: SelectedCaptureSource) => { + const controller = getIOSCaptureController(); + const state = controller?.getSnapshot(); + if (state && ["starting", "recording", "stopping", "finalising"].includes(state.phase)) + throw new Error("RECORDING_BUSY"); + if (source.sourceType === "ios-device") { + source = validateIOSSourceSelection(event, source); + setSelectedSource(source); + broadcastSelectedSourceChange(); + stopWindowBoundsCapture(); + return source; + } + if (state?.sessionId) { + assertIOSCaptureClient(event); + await controller?.release(state.sessionId); + } if (source.id?.startsWith("window:")) { await bringSelectedWindowForward(source); } @@ -397,6 +417,8 @@ export function registerSourceHandlers({ }); ipcMain.handle("show-source-highlight", async (_, source: SelectedSource) => { + if ((source as { sourceType?: string }).sourceType === "ios-device") + return { success: false }; try { const isWindow = source.id?.startsWith("window:"); diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744e..1c2141f83 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -5,13 +5,13 @@ import type { CursorVisualType, NativeCaptureDiagnostics, RecordingSessionData, - SelectedSource, + SelectedCaptureSource, SystemCursorAsset, WindowBounds, } from "./types"; // ── Source selection ────────────────────────────────────────────────────────── -export let selectedSource: SelectedSource | null = null; +export let selectedSource: SelectedCaptureSource | null = null; // ── Project / video state ───────────────────────────────────────────────────── export let currentProjectPath: string | null = null; @@ -109,7 +109,7 @@ export type { CursorInteractionType, CursorTelemetryPoint }; // TypeScript exported `let` can be reassigned by the owning module but importers // cannot assign to them directly. Provide simple setters for cross-module writes. -export function setSelectedSource(v: SelectedSource | null) { +export function setSelectedSource(v: SelectedCaptureSource | null) { selectedSource = v; } export function setCurrentProjectPath(v: string | null) { diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 58f5425bd..b2a687fe1 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -1,3 +1,4 @@ +import type { CaptureMetadata } from "../../src/shared/iosCapture"; export type SelectedSource = { id?: string; name: string; @@ -8,6 +9,9 @@ export type SelectedSource = { [key: string]: unknown; }; +export type SelectedCaptureSource = + import("../../src/shared/iosCapture").CaptureSource; + export type NativeMacRecordingOptions = { capturesSystemAudio?: boolean; capturesMicrophone?: boolean; @@ -44,6 +48,7 @@ export type NativeCaptureDiagnostics = { }; export type RecordingSessionData = { + captureMetadata?: CaptureMetadata; videoPath: string; webcamPath?: string | null; timeOffsetMs?: number; @@ -56,7 +61,9 @@ export type PauseSegment = { }; export type RecordingSessionManifest = { - version: 1 | 2; + version: 1 | 2 | 3; + captureMetadata?: CaptureMetadata; + hideOverlayCursorByDefault?: boolean; videoFileName: string; webcamFileName?: string | null; timeOffsetMs?: number; diff --git a/electron/main.ts b/electron/main.ts index 890726670..8bfb0cf29 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -8,6 +8,7 @@ import { dialog, webContents as electronWebContents, ipcMain, + powerMonitor, Menu, nativeImage, session, @@ -25,6 +26,16 @@ import { killWindowsCaptureProcess, registerIpcHandlers, } from "./ipc/handlers"; +import { + getIOSCaptureController, + interruptIOSCapture, + shutdownIOSCapture, +} from "./ipc/register/iosCapture"; +import { + beginDesktopRecording, + endDesktopRecording, + getRecordingLease, +} from "./ipc/recording/recordingLease"; import { ensureMediaServer } from "./mediaServer"; import { hardenWebContentsNavigation, shouldHardenWebContentsType } from "./navigationPolicy"; import { shouldGrantDisplayCapture, shouldGrantMediaPermission } from "./permissionPolicy"; @@ -863,7 +874,16 @@ function createSourceSelectorWindowWrapper() { // On macOS, applications and their menu bar stay active until the user quits // explicitly with Cmd + Q. -app.on("before-quit", () => { +let iosQuitReady = false; +app.on("before-quit", (event) => { + if (!iosQuitReady && getIOSCaptureController()) { + event.preventDefault(); + void shutdownIOSCapture().finally(() => { + iosQuitReady = true; + app.quit(); + }); + return; + } isAppQuitting = true; killWindowsCaptureProcess(); showCursor(); @@ -889,6 +909,9 @@ app.on("second-instance", () => { // Register all IPC handlers when app is ready app.whenReady().then(async () => { + powerMonitor.on("suspend", () => { + void interruptIOSCapture(); + }); if (process.platform === "win32") { app.setAppUserModelId("dev.recordly.app"); } @@ -1055,7 +1078,15 @@ app.whenReady().then(async () => { // propagation and causes cursor: 'never' from the renderer to be silently // ignored by the native capture pipeline. session.defaultSession.setDisplayMediaRequestHandler(async (request, callback) => { + let acquiredLease = false; try { + if ( + getSelectedSourceId()?.startsWith("ios-device:") || + getRecordingLease()?.owner === "ios-device" + ) { + callback({}); + return; + } const frame = request.frame; const isLiveFrame = Boolean(frame && !frame.isDestroyed()); const requestingWebContents = @@ -1085,6 +1116,8 @@ app.whenReady().then(async () => { // Browser and Linux portal capture starts as soon as this callback // resolves, before recording-state-changed is emitted. + beginDesktopRecording(); + acquiredLease = true; reassertHudOverlayCaptureProtection(); const sourceId = getSelectedSourceId(); @@ -1115,9 +1148,11 @@ app.whenReady().then(async () => { video: { id: source.id, name: source.name }, }); } else { + endDesktopRecording(); callback({}); } } catch (error) { + if (acquiredLease) endDesktopRecording(); console.error("setDisplayMediaRequestHandler error:", error); callback({}); } diff --git a/electron/native/bin/darwin-arm64/recordly-ios-device-helper b/electron/native/bin/darwin-arm64/recordly-ios-device-helper new file mode 100755 index 000000000..677f497cf Binary files /dev/null and b/electron/native/bin/darwin-arm64/recordly-ios-device-helper differ diff --git a/electron/native/bin/darwin-x64/recordly-ios-device-helper b/electron/native/bin/darwin-x64/recordly-ios-device-helper new file mode 100755 index 000000000..ebf30abbd Binary files /dev/null and b/electron/native/bin/darwin-x64/recordly-ios-device-helper differ diff --git a/electron/native/ios-device-capture/Package.swift b/electron/native/ios-device-capture/Package.swift new file mode 100644 index 000000000..db83e41ac --- /dev/null +++ b/electron/native/ios-device-capture/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription +let package = Package(name: "IOSDeviceCapture", platforms: [.macOS(.v14)], products: [ + .library(name: "IOSCaptureCore", targets: ["IOSCaptureCore"]), + .executable(name: "recordly-ios-device-helper", targets: ["IOSDeviceCaptureHelper"]) +], targets: [ + .target(name: "IOSCaptureCore"), + .executableTarget(name: "IOSDeviceCaptureHelper", dependencies: ["IOSCaptureCore"]), + .testTarget(name: "IOSCaptureCoreTests", dependencies: ["IOSCaptureCore"], resources: [.copy("Fixtures")]) +], swiftLanguageModes: [.v5]) diff --git a/electron/native/ios-device-capture/README.md b/electron/native/ios-device-capture/README.md new file mode 100644 index 000000000..74a703b16 --- /dev/null +++ b/electron/native/ios-device-capture/README.md @@ -0,0 +1,70 @@ +# Native device capture helper + +This package is an internal, default-off capture backend. Compilation and synthetic +media tests do not establish USB-device, privacy-prompt, signed-app, or release support. + +Build from the repository root with `node scripts/build-ios-device-helper.mjs`. +Run native tests with `node scripts/test-ios-device-helper.mjs` on macOS. The package +uses Swift tools 6.0, Swift 5 language mode for AVFoundation/GCD interoperability, +and the existing macOS 14 deployment floor. CaptureEngine state is confined to its +serial queue; preview work and nonblocking control output have separate queues. + +The installed executable accepts protocol-v1 NDJSON on stdin/stdout and writes +bounded RLIP previews on fd 3. `--self-test`, `hello`, and `inspectMedia` do not +perform device discovery or request camera/microphone permission. Main allocates +session paths and grants an explicit filename allowlist. The helper never deletes +accepted recording media. + +## Current compatibility policy + +* The sole candidate screen-device signature is muxed + model `iOS Device`. USB screen sources can advertise muxed media without a standalone video media type; preparation validates actual video samples before readiness. + A USB iPhone discovery probe confirmed this metadata; first-frame and permission behavior remain unverified. See [discovery evidence](../../../docs/testing/ios-usb-capture-implementation.md). +* CMIO and AVFoundation discovery initialize on the running main run loop. The screen discovery session stays alive across inventory polls and is released when discovery stops. +* Passthrough is limited to positively identified H.264 Baseline profile with + explicit Rec.709 metadata. Baseline excludes B slices, allowing a bounded final + sample to preserve a sparse timeline without rebuilding a reordered GOP. +* Other compressed profiles request supported native 420 video-range output during + preparation. That output must pass the same observed geometry/colour checks. + There is no mode change during a take and no desktop/camera fallback. +* Raw encoding accepts 8-bit bi-planar Rec.709 SDR. Odd dimensions are padded on the + right/bottom and their original display aperture is carried into the MOV. RGB, + unknown/wide colour, HDR, non-square pixels and unsupported apertures fail before + recording. No colour labels are invented and no unvalidated tone mapping occurs. +* Incoming capture orientation is the delivered pixel orientation, with identity + output transform. Preview crops the source clean aperture before downscaling. + The independent movie inspector returns its actual preferred transform. +* Enabled audio outputs request signed 16-bit interleaved PCM. Each sidecar retains + its host-clock start offset. AVAssetWriter flattens PCM timestamp holes, so gaps + are explicitly filled with PCM silence and marked as represented in media. + A gap over one second, more than 128 gaps, timestamp regression or unavailable + clock mapping interrupts safely instead of inventing alignment or buffering + unbounded data. Capture counts exclude inserted silence. +* Clock mapping requires a synchronizationClock with measured relative rate within + one part per million of the host clock. Every PTS/DTS is mapped natively; callback + receipt times never determine alignment. Non-unit mappings outside this tolerance + are unsupported, rather than silently stretched. Duration is converted from mapped + end minus mapped start; PCM comparisons allow one sample of clock-rounding error. +* Preview retains at most one conversion and one partial pipe record. Compressed + preview frames are decoded independently, JPEG conversion is capped at 5 Hz, + and dropped compressed work resynchronizes at a keyframe. A blocked fd 3 never + blocks capture or control output. + +The hardware/signing gates, long-duration physical sync, physical rotation/unplug, +preview-stall recording load, corruption/fragment recovery and Intel runtime tests +remain required. Observed frame-rate estimation is not implemented; the optional +field remains null. The H.264 bitrate uses the documented initial policy with a +30 fps tuning input when no observed rate is available; that is not an FPS claim. + +## Synthetic evidence + +The native XCTest suite covers parser bounds/idempotency, inventory identity, +rational offsets, host-clock identity mapping, no-preparation/no-frame rejection, +wide-colour rejection, actual H.264 encoding and inspection, odd display aperture, +300-second sparse timeline, baseline compressed packet identity and terminal decode, +PCM offsets/internal silence, symlink boundaries and durable timing checkpoints. +The committed synthetic fixture has its generation command and CC0 provenance in +`Tests/IOSCaptureCoreTests/Fixtures/PROVENANCE.md`. + +Capture storage reserve uses actual native file sizes for measured write rate. It +reserves an additional video-sized final output only when audio mixing is requested; +uncompressed pixel-buffer memory is never used as a disk-rate estimate. diff --git a/electron/native/ios-device-capture/Resources/Info.plist b/electron/native/ios-device-capture/Resources/Info.plist new file mode 100644 index 000000000..25e7d5706 --- /dev/null +++ b/electron/native/ios-device-capture/Resources/Info.plist @@ -0,0 +1,11 @@ + + + +CFBundleIdentifiercom.recordly.ios-device-capture-helper +CFBundleNameRecordly Device Capture +RecordlyCaptureProtocolVersion1 +CFBundleVersion1 +NSCameraUsageDescriptionRecordly receives your connected iPhone or iPad screen using macOS camera access. +NSMicrophoneUsageDescriptionRecordly records enabled device audio and your selected narration microphone. +NSCameraUseContinuityCameraDeviceType + diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/AudioWriter.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/AudioWriter.swift new file mode 100644 index 000000000..b51ac86a1 --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/AudioWriter.swift @@ -0,0 +1,57 @@ +import Foundation +import AVFoundation +import CoreMedia + +public final class AudioWriter { + private let writer: AVAssetWriter + private let input: AVAssetWriterInput + private let description: CMFormatDescription + public let sampleRate: Double + public let channels: Int + public private(set) var firstHostTime: CMTime + public private(set) var endHostTime: CMTime + public private(set) var sampleCount = 0 + public private(set) var gaps: [[String: Any]] = [] + public init(url: URL, firstSample: CMSampleBuffer) throws { + guard let description = CMSampleBufferGetFormatDescription(firstSample), let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(description)?.pointee, + asbd.mFormatID == kAudioFormatLinearPCM, asbd.mBitsPerChannel == 16, asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger != 0, asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved == 0, asbd.mSampleRate > 0, asbd.mChannelsPerFrame > 0, asbd.mChannelsPerFrame <= 2 else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + self.description = description; sampleRate = asbd.mSampleRate; channels = Int(asbd.mChannelsPerFrame) + firstHostTime = CMSampleBufferGetPresentationTimeStamp(firstSample); endHostTime = firstHostTime + writer = try AVAssetWriter(outputURL: url, fileType: .mov) + writer.movieFragmentInterval = CMTime(value: 10, timescale: 1); writer.initialMovieFragmentInterval = CMTime(value: 1, timescale: 1) + input = AVAssetWriterInput(mediaType: .audio, outputSettings: nil, sourceFormatHint: description) + input.expectsMediaDataInRealTime = true + guard writer.canAdd(input) else { throw CaptureFailure("UNSUPPORTED_FORMAT") }; writer.add(input) + guard writer.startWriting() else { throw CaptureFailure("WRITER_FAILED") } + writer.startSession(atSourceTime: firstHostTime) + } + public func append(_ sample: CMSampleBuffer) throws { + guard let format = CMSampleBufferGetFormatDescription(sample), CMFormatDescriptionEqual(description, otherFormatDescription: format) else { throw CaptureFailure("FORMAT_CHANGED") } + let pts = CMSampleBufferGetPresentationTimeStamp(sample) + let duration = CMSampleBufferGetDuration(sample) + guard pts.isNumeric, duration.isNumeric, duration > .zero, CMTimeSubtract(endHostTime, pts).seconds <= 1 / sampleRate else { throw CaptureFailure("AUDIO_INTERRUPTED") } + if CMTimeSubtract(pts, endHostTime).seconds > 1 / sampleRate { + guard gaps.count < 128 else { throw CaptureFailure("AUDIO_INTERRUPTED") } + try appendSilence(until: pts) + gaps.append(["start": try jsonObject(NativeTime(CMTimeSubtract(endHostTime, firstHostTime))), "duration": try jsonObject(NativeTime(CMTimeSubtract(pts, endHostTime))), "representedInMedia": true]) + } + guard writer.status == .writing, input.isReadyForMoreMediaData, input.append(sample) else { throw CaptureFailure("WRITER_FAILED") } + endHostTime = CMTimeAdd(pts, duration); sampleCount += CMSampleBufferGetNumSamples(sample) + } + private func appendSilence(until stop: CMTime) throws { + let frames = Int((CMTimeSubtract(stop, endHostTime).seconds * sampleRate).rounded()) + guard frames > 0, frames <= Int(sampleRate) else { throw CaptureFailure("AUDIO_INTERRUPTED") } + let bytes = frames * channels * 2 + var block: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil, blockLength: bytes, blockAllocator: kCFAllocatorDefault, customBlockSource: nil, offsetToData: 0, dataLength: bytes, flags: 0, blockBufferOut: &block) == noErr, let block, + CMBlockBufferFillDataBytes(with: 0, blockBuffer: block, offsetIntoDestination: 0, dataLength: bytes) == noErr else { throw CaptureFailure("WRITER_FAILED") } + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: Int32(sampleRate)), presentationTimeStamp: endHostTime, decodeTimeStamp: .invalid) + var size = channels * 2 + var silence: CMSampleBuffer? + guard CMSampleBufferCreateReady(allocator: kCFAllocatorDefault, dataBuffer: block, formatDescription: description, sampleCount: frames, sampleTimingEntryCount: 1, sampleTimingArray: &timing, sampleSizeEntryCount: 1, sampleSizeArray: &size, sampleBufferOut: &silence) == noErr, let silence, input.isReadyForMoreMediaData, input.append(silence) else { throw CaptureFailure("WRITER_FAILED") } + } + public func finish(completion: @escaping (Result) -> Void) { + input.markAsFinished() + writer.finishWriting { [writer] in completion(writer.status == .completed && self.sampleCount > 0 ? .success(()) : .failure(CaptureFailure("FINALIZATION_FAILED"))) } + } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/CaptureClock.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/CaptureClock.swift new file mode 100644 index 000000000..834feacbe --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/CaptureClock.swift @@ -0,0 +1,44 @@ +import AVFoundation +import CoreMedia +public struct CaptureClock { + private let source: CMClock + public init(session: AVCaptureSession) throws { + guard let clock = session.synchronizationClock else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + try self.init(source: clock) + } + public init(source: CMClock) throws { + let rate = CMSyncGetRelativeRate(source, relativeTo: CMClockGetHostTimeClock()) + guard rate.isFinite, abs(rate - 1) < 0.000001 else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + self.source = source + } + public func toHost(_ time: CMTime) throws -> CMTime { + let value = CMSyncConvertTime(time, from: source, to: CMClockGetHostTimeClock()) + guard value.isNumeric else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + return value + } + public func map(_ sample: CMSampleBuffer) throws -> CMSampleBuffer { + try Self.retime(sample, converting: toHost) + } + public static func retime(_ sample: CMSampleBuffer, converting convert: (CMTime) throws -> CMTime) throws -> CMSampleBuffer { + var count = 0 + guard CMSampleBufferGetSampleTimingInfoArray(sample, entryCount: 0, arrayToFill: nil, entriesNeededOut: &count) == noErr, count > 0, count <= 65536 else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + var timing = Array(repeating: CMSampleTimingInfo(), count: count) + guard CMSampleBufferGetSampleTimingInfoArray(sample, entryCount: count, arrayToFill: &timing, entriesNeededOut: &count) == noErr else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + for index in timing.indices { + let sourcePTS = timing[index].presentationTimeStamp + let hostPTS = try convert(sourcePTS) + if timing[index].duration.isNumeric { + let hostEnd = try convert(CMTimeAdd(sourcePTS, timing[index].duration)) + timing[index].duration = CMTimeSubtract(hostEnd, hostPTS) + guard timing[index].duration >= .zero else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + } + timing[index].presentationTimeStamp = hostPTS + if timing[index].decodeTimeStamp.isNumeric { timing[index].decodeTimeStamp = try convert(timing[index].decodeTimeStamp) } + } + var output: CMSampleBuffer? + guard CMSampleBufferCreateCopyWithNewTiming(allocator: kCFAllocatorDefault, sampleBuffer: sample, sampleTimingEntryCount: count, sampleTimingArray: &timing, sampleBufferOut: &output) == noErr, let output else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + return output + } + public static func offset(firstHostTime: CMTime, videoStartHostTime: CMTime) -> CMTime { CMTimeSubtract(firstHostTime, videoStartHostTime) } + public static var now: CMTime { CMClockGetTime(CMClockGetHostTimeClock()) } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/CaptureEngine.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/CaptureEngine.swift new file mode 100644 index 000000000..64f2272e8 --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/CaptureEngine.swift @@ -0,0 +1,324 @@ +import Foundation +import AVFoundation +import CoreMedia +import IOKit.pwr_mgt + +/// All methods and mutable state run on `queue`; AVFoundation delegates use this same queue. +public final class CaptureEngine: NSObject, @unchecked Sendable, AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate { + public let queue = DispatchQueue(label: "recordly.ios.capture", qos: .userInitiated) + public var emit: ((String, [String: Any], String?, String?, UInt32) -> Void)? + public var didShutdown: (() -> Void)? + private lazy var discovery = DeviceDiscovery(queue: queue) + private let preview = PreviewEncoder() + private var session: AVCaptureSession? + private var micSession: AVCaptureSession? + private var videoOutput: AVCaptureVideoDataOutput? + private var deviceAudioOutput: AVCaptureAudioDataOutput? + private var micOutput: AVCaptureAudioDataOutput? + private var clock: CaptureClock? + private var micClock: CaptureClock? + private var sessionId: String? + private var generation: UInt32 = 0 + private var source: [String: Any]? + private var token: String? + private var options: [String: Any] = [:] + private var storage: SessionStorage? + private var timing: NativeTimingStore? + private var format: IOSVideoFormat? + private var mode: String? + private var descriptionHint: CMFormatDescription? + private var video: VideoWriter? + private var deviceAudio: AudioWriter? + private var microphone: AudioWriter? + private var phase = "idle" + private var prepareRequest: String? + private var startRequest: String? + private var stopRequest: String? + private var deadline: DispatchWorkItem? + private var progressTimer: DispatchSourceTimer? + private var observers: [NSObjectProtocol] = [] + private var finalResult: [String: Any]? + private var shutdownPending = false + private var delivered = 0 + private var dropped = 0 + private var assertion: IOPMAssertionID = 0 + private var armHostTime: CMTime? + private var rawVideoNegotiation = RawVideoNegotiation() + private var acceptedRequests = Set() + private var acceptedOrder = [String]() + public override init() { super.init() } + static func recommendedVideoSettings(mode: String, output: AVCaptureVideoDataOutput?) -> [String: Any]? { + guard mode == "h264-encode", let output, + output.availableVideoCodecTypesForAssetWriter(writingTo: .mov).contains(.h264) else { return nil } + return output.recommendedVideoSettings(forVideoCodecType: .h264, assetWriterOutputFileType: .mov) + } + private func event(_ name: String, _ payload: [String: Any] = [:], request: String? = nil) { + if name == "accepted", let request { + if acceptedRequests.insert(request).inserted { acceptedOrder.append(request) } + if acceptedOrder.count > 128 { acceptedRequests.remove(acceptedOrder.removeFirst()) } + } + // Once accepted, failures belong to the session lifecycle, not a settled request. + let responseRequest = name == "error" && request.map(acceptedRequests.contains) == true ? nil : request + emit?(name, payload, responseRequest, sessionId, generation) + } + public func handle(_ command: ProtocolCommand) { + dispatchPrecondition(condition: .onQueue(queue)) + do { + if let id = command.sessionId, command.command != "prepare", command.command != "inspectMedia", id != sessionId { throw CaptureFailure("INVALID_REQUEST") } + switch command.command { + case "hello": emit?("accepted", ["build": "recordly-ios-device-helper-1", "protocolVersion": 1, "capabilities": ["supportsPause": false, "supportsWebcam": false, "supportsTouchTelemetry": false, "previewMaxLongestEdge": 480, "previewMaxFramesPerSecond": 5, "previewMaxJpegBytes": 131072, "protocolVersion": 1]], command.requestId, nil, generation) + case "discover": + discovery.changed = { [weak self] snapshot in + guard let self else { return }; self.event("inventoryChanged", snapshot) + if let token = self.token, !self.discovery.contains(token: token) { self.interrupt("DEVICE_DISCONNECTED") } + } + let refreshDiscovery = ["idle", "failed", "completed"].contains(phase) + try discovery.start(refresh: refreshDiscovery); event("accepted", request: command.requestId); event("inventoryChanged", discovery.snapshot(), request: command.requestId) + case "prepare": try prepare(command) + case "setPreviewEnabled": + guard command.generation == generation, let enabled = command.payload["enabled"] as? Bool else { throw CaptureFailure("INVALID_REQUEST") } + preview.setEnabled(enabled); event("accepted", request: command.requestId) + case "start": try start(command) + case "stop", "cancel": + event("accepted", request: command.requestId) + stopRequest = command.requestId + finish(reason: command.command == "cancel" ? "cancelled" : "user-stop") + case "release": + guard !["starting", "recording", "finalising"].contains(phase) else { throw CaptureFailure("RECORDING_BUSY") } + releaseInputs(); phase = "idle"; event("accepted", request: command.requestId) + case "inspectMedia": + guard let settings = command.storage, let name = command.payload["relativeName"] as? String else { throw CaptureFailure("INVALID_REQUEST") } + let location = try SessionStorage(settings).file(name) + let requestId = command.requestId, requestedSession = command.sessionId + Task { + do { let result = try await MediaInspector.inspect(url: location); self.queue.async { self.emit?("accepted", ["inspection": result], requestId, requestedSession, self.generation) } } + catch { self.queue.async { self.event("error", ["code": "FINALIZATION_FAILED", "recoverable": true], request: requestId) } } + } + case "shutdown": event("accepted", request: command.requestId); shutdown() + default: throw CaptureFailure("INVALID_REQUEST") + } + } catch { if command.command == "prepare" { releaseInputs(); phase = "failed" }; event("error", ["code": (error as? CaptureFailure)?.code ?? "WRITER_FAILED", "recoverable": video != nil], request: command.requestId) } + } + private func prepare(_ command: ProtocolCommand) throws { + guard !["starting", "recording", "finalising", "preparing"].contains(phase) else { throw CaptureFailure("RECORDING_BUSY") } + guard AVCaptureDevice.authorizationStatus(for: .video) == .authorized else { throw CaptureFailure("PERMISSION_DENIED") } + let settings = command.payload["options"] as! [String: Any] + let withAudio = settings["deviceAudio"] as? Bool == true + let micToken = settings["microphoneToken"] as? String + if withAudio || micToken != nil { guard AVCaptureDevice.authorizationStatus(for: .audio) == .authorized else { throw CaptureFailure("PERMISSION_DENIED") } } + let selectedToken = command.payload["deviceToken"] as! String + let selected = try discovery.device(token: selectedToken, generation: (command.payload["inventoryGeneration"] as! NSNumber).intValue) + let destination = try SessionStorage(command.storage!) + guard try destination.availableBytes() >= 1024 * 1024 * 1024 else { throw CaptureFailure("DISK_SPACE_LOW") } + releaseInputs() + sessionId = command.sessionId; generation = command.generation ?? (generation &+ 1); source = discovery.source(token: selectedToken); token = selectedToken; options = settings; storage = destination + rawVideoNegotiation = RawVideoNegotiation(); format = nil; mode = nil; video = nil; deviceAudio = nil; microphone = nil; finalResult = nil; delivered = 0; dropped = 0 + let capture = AVCaptureSession(); capture.beginConfiguration() + let input = try AVCaptureDeviceInput(device: selected) + guard capture.canAddInput(input) else { throw CaptureFailure("DEVICE_BUSY") }; capture.addInput(input) + let output = AVCaptureVideoDataOutput(); output.videoSettings = [:]; output.alwaysDiscardsLateVideoFrames = true + output.setSampleBufferDelegate(self, queue: queue) + guard capture.canAddOutput(output) else { throw CaptureFailure("UNSUPPORTED_FORMAT") }; capture.addOutput(output) + if withAudio { + let audio = AVCaptureAudioDataOutput(); audio.audioSettings = [AVFormatIDKey: kAudioFormatLinearPCM, AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false, AVLinearPCMIsNonInterleaved: false] + audio.setSampleBufferDelegate(self, queue: queue) + guard capture.canAddOutput(audio) else { throw CaptureFailure("UNSUPPORTED_FORMAT") }; capture.addOutput(audio); deviceAudioOutput = audio + } + capture.commitConfiguration(); session = capture; videoOutput = output + if let micToken { + let device = try discovery.microphone(token: micToken) + guard device.uniqueID != selected.uniqueID else { throw CaptureFailure("INVALID_REQUEST") } + let mic = AVCaptureSession(); mic.beginConfiguration() + let micInput = try AVCaptureDeviceInput(device: device) + let audio = AVCaptureAudioDataOutput(); audio.audioSettings = [AVFormatIDKey: kAudioFormatLinearPCM, AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false, AVLinearPCMIsNonInterleaved: false]; audio.setSampleBufferDelegate(self, queue: queue) + guard mic.canAddInput(micInput), mic.canAddOutput(audio) else { throw CaptureFailure("DEVICE_BUSY") } + mic.addInput(micInput); mic.addOutput(audio); mic.commitConfiguration(); micSession = mic; micOutput = audio + } + for active in [capture, micSession].compactMap({ $0 }) { + observers.append(NotificationCenter.default.addObserver(forName: .AVCaptureSessionRuntimeError, object: active, queue: nil) { [weak self] _ in self?.queue.async { self?.interrupt("DEVICE_DISCONNECTED") } }) + observers.append(NotificationCenter.default.addObserver(forName: AVCaptureDevice.wasDisconnectedNotification, object: nil, queue: nil) { [weak self, weak active] notification in + guard let lost = notification.object as? AVCaptureDevice, active?.inputs.compactMap({ $0 as? AVCaptureDeviceInput }).contains(where: { $0.device.uniqueID == lost.uniqueID }) == true else { return } + self?.queue.async { self?.interrupt("DEVICE_DISCONNECTED") } + }) + } + phase = "preparing"; prepareRequest = command.requestId + event("accepted", request: command.requestId) + capture.startRunning(); micSession?.startRunning() + clock = try CaptureClock(session: capture) + if let micSession { micClock = try CaptureClock(session: micSession) } + armDeadline() + } + private func armDeadline() { + deadline?.cancel() + let work = DispatchWorkItem { [weak self] in guard let self, ["preparing", "starting"].contains(self.phase) else { return }; self.interrupt("NO_VIDEO_SAMPLES") } + deadline = work; queue.asyncAfter(deadline: .now() + 10, execute: work) + } + private func start(_ command: ProtocolCommand) throws { + guard phase == "ready", let storage, let format, let mode, let descriptionHint, let session, session.isRunning else { throw CaptureFailure("INVALID_REQUEST") } + guard try storage.availableBytes() >= 1024 * 1024 * 1024 else { throw CaptureFailure("DISK_SPACE_LOW") } + let boundary = CaptureClock.now + video = try VideoWriter(url: storage.file("source-video.mov", creating: true), format: format, mode: mode, description: descriptionHint, boundary: boundary, recommended: Self.recommendedVideoSettings(mode: mode, output: videoOutput)) + timing = try NativeTimingStore(storage: storage); armHostTime = boundary + phase = "starting"; startRequest = command.requestId + try checkpoint(reason: nil) + event("accepted", request: command.requestId); armDeadline() + let timer = DispatchSource.makeTimerSource(queue: queue); timer.schedule(deadline: .now() + 1, repeating: 1) + timer.setEventHandler { [weak self] in self?.tick() }; timer.resume(); progressTimer = timer + _ = IOPMAssertionCreateWithName(kIOPMAssertionTypePreventUserIdleSystemSleep as CFString, IOPMAssertionLevel(kIOPMAssertionLevelOn), "Device recording" as CFString, &assertion) + } + public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + autoreleasepool { + do { + guard CMSampleBufferGetNumSamples(sampleBuffer) > 0 else { return } + guard output === videoOutput || output === deviceAudioOutput || output === micOutput, let clock = output === micOutput ? micClock : clock else { return } + let sample = try VideoFormatPolicy.normalizeRawGeometry(clock.map(sampleBuffer)) + if output === videoOutput { + delivered += 1 + if phase == "preparing" { + let rawPixelFormat = CMSampleBufferGetImageBuffer(sample).map(CVPixelBufferGetPixelFormatType) + let supportsPassthrough = CMSampleBufferGetFormatDescription(sample).map(VideoFormatPolicy.supportsPassthrough) ?? false + switch try rawVideoNegotiation.observe(rawPixelFormat: rawPixelFormat, supportsPassthrough: supportsPassthrough, availablePixelFormats: videoOutput?.availableVideoPixelFormatTypes ?? []) { + case .requestRaw(let pixelFormat): + guard let videoOutput else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: pixelFormat] + return + case .waitForRaw: + // Keep the original preparation deadline; queued packets cannot extend it. + return + case .inspectSample: + break + } + + let (observed, selectedMode) = try VideoFormatPolicy.inspect(sample) + if selectedMode == "passthrough", !VideoFormatPolicy.isSync(sample) { return } + try MediaInspector.validateFirstSample(sample) + let preflightURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + _ = try VideoWriter(url: preflightURL, format: observed, mode: selectedMode, description: CMSampleBufferGetFormatDescription(sample)!, boundary: .zero, recommended: Self.recommendedVideoSettings(mode: selectedMode, output: videoOutput)) + try? FileManager.default.removeItem(at: preflightURL) + format = observed; mode = selectedMode; descriptionHint = CMSampleBufferGetFormatDescription(sample) + deadline?.cancel(); phase = "ready" + event("prepared", ["source": source!, "options": options, "format": try jsonObject(observed), "mode": selectedMode], request: prepareRequest) + } + if let format, ["ready", "starting", "recording"].contains(phase) { + let (observed, _) = try VideoFormatPolicy.inspect(sample) + guard observed.fingerprint == format.fingerprint else { throw CaptureFailure("FORMAT_CHANGED") } + preview.submit(sampleBuffer, generation: generation) + } + if ["starting", "recording"].contains(phase), let video, try video.append(sample), phase == "starting" { + deadline?.cancel(); phase = "recording"; try checkpoint(reason: nil) + event("recordingStarted", ["firstHostTime": try jsonObject(NativeTime(video.firstHostTime!)), "acceptedVideoSamples": video.sampleCount], request: startRequest) + } + } else if ["starting", "recording"].contains(phase), let storage, let armHostTime, CMSampleBufferGetPresentationTimeStamp(sample) >= armHostTime { + if output === deviceAudioOutput { + if deviceAudio == nil { deviceAudio = try AudioWriter(url: storage.file("device-audio.mov", creating: true), firstSample: sample) } + try deviceAudio?.append(sample) + } else if output === micOutput { + if microphone == nil { microphone = try AudioWriter(url: storage.file("microphone.mov", creating: true), firstSample: sample) } + try microphone?.append(sample) + } + if deviceAudio?.sampleCount == CMSampleBufferGetNumSamples(sample) || microphone?.sampleCount == CMSampleBufferGetNumSamples(sample) { try checkpoint(reason: nil) } + } + } catch { interrupt((error as? CaptureFailure)?.code ?? "WRITER_FAILED") } + } + } + public func captureOutput(_ output: AVCaptureOutput, didDrop sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + dropped += 1 + if ["starting", "recording"].contains(phase) { interrupt("WRITER_FAILED") } + } + private func tick() { + guard ["starting", "recording"].contains(phase) else { return } + do { + try checkpoint(reason: nil) + let elapsed = video?.firstHostTime.map { max(0, CMTimeSubtract(CaptureClock.now, $0).seconds) } ?? 0 + if let storage { + let bytes = try storage.writtenMediaBytes() + let mixAudio = options["deviceAudio"] as? Bool == true || options["microphoneToken"] is String + let reserve = StorageReservePolicy.requiredBytes(writtenBytes: bytes.total, videoBytes: bytes.video, elapsedSeconds: elapsed, mixAudio: mixAudio) + guard try storage.availableBytes() > reserve else { throw CaptureFailure("DISK_SPACE_LOW") } + } + event("progress", ["elapsedMs": elapsed * 1000, "acceptedVideoSamples": video?.sampleCount ?? 0]) + } catch { interrupt((error as? CaptureFailure)?.code ?? "WRITER_FAILED") } + } + private func interrupt(_ code: String) { + guard !["idle", "failed", "completed", "finalising"].contains(phase) else { return } + event("warning", ["code": code]) + if video != nil { finish(reason: code) } + else { event("error", ["code": code, "recoverable": false], request: prepareRequest); releaseInputs(); phase = "failed" } + } + private func artifact(name: String, kind: String, first: CMTime, end: CMTime, count: Int, metadata: [String: Any]) throws -> [String: Any] { + ["relativeName": name, "mediaKind": kind, "firstHostTime": try jsonObject(NativeTime(first)), "duration": try jsonObject(NativeTime(CMTimeSubtract(end, first))), "sampleCount": count, "mediaFormat": metadata] + } + private func artifacts() throws -> [[String: Any]] { + var items = [[String: Any]]() + if let video, let first = video.firstHostTime, let format { + items.append(try artifact(name: "source-video.mov", kind: "video", first: first, end: video.endHostTime ?? CaptureClock.now, count: video.sampleCount, metadata: ["codec": "h264", "width": format.codedWidth, "height": format.codedHeight])) + } + for (writer, name, kind) in [(deviceAudio, "device-audio.mov", "device-audio"), (microphone, "microphone.mov", "microphone")] { + if let writer, writer.sampleCount > 0 { items.append(try artifact(name: name, kind: kind, first: writer.firstHostTime, end: writer.endHostTime, count: writer.sampleCount, metadata: ["codec": "pcm", "sampleRate": writer.sampleRate, "channels": writer.channels])) } + } + return items + } + private func timingObject() throws -> [String: Any] { + let items = try artifacts() + let streams = items.map { item -> [String: Any] in + let kind = item["mediaKind"] as! String + let writer = kind == "device-audio" ? deviceAudio : (kind == "microphone" ? microphone : nil) + return ["mediaKind": kind, "firstHostTime": item["firstHostTime"]!, "duration": item["duration"]!, "rate": ["numerator": "1", "denominator": "1"], "clockAnchor": ["hostTime": item["firstHostTime"]!, "mediaTime": ["value": "0", "timescale": 1]], "gaps": writer?.gaps.map { ["start": $0["start"]!, "duration": $0["duration"]!] } ?? []] + } + return ["version": 1, "timeline": "host-mapped", "gapsRepresentedInMedia": true, "streams": streams] + } + private func checkpoint(reason: String?) throws { + guard let timing, let sessionId else { return } + var record: [String: Any] = ["version": 1, "sessionId": sessionId, "phase": phase, "timing": try timingObject(), "deliveredVideoSamples": delivered, "acceptedVideoSamples": video?.sampleCount ?? 0, "droppedVideoSamples": dropped] + if let reason { record["stopReason"] = reason }; if let finalResult { record["result"] = finalResult } + try timing.checkpoint(record) + } + private func finish(reason: String) { + if let finalResult { event("nativeFinalized", ["result": finalResult], request: stopRequest); return } + guard phase != "finalising" else { return } + deadline?.cancel(); progressTimer?.cancel(); progressTimer = nil + guard let video, video.sampleCount > 0 else { + try? checkpoint(reason: reason); releaseInputs(); phase = "cancelled" + event("error", ["code": "NO_VIDEO_SAMPLES", "recoverable": false], request: stopRequest ?? startRequest) + if shutdownPending { didShutdown?() }; return + } + let stop = CaptureClock.now; phase = "finalising" + session?.stopRunning(); micSession?.stopRunning(); preview.setEnabled(false) + let group = DispatchGroup(); var failure: Error? + group.enter(); video.finish(at: stop) { result in self.queue.async { if case .failure(let error) = result { failure = error }; group.leave() } } + for audio in [deviceAudio, microphone].compactMap({ $0 }) { group.enter(); audio.finish { result in self.queue.async { if case .failure(let error) = result { failure = error }; group.leave() } } } + group.notify(queue: queue) { + if let failure { self.completeFailure(failure, reason: reason); return } + guard let storage = self.storage else { self.completeFailure(CaptureFailure("FINALIZATION_FAILED"), reason: reason); return } + Task { + do { + let inspection = try await MediaInspector.inspect(url: storage.file("source-video.mov")) + guard inspection["decodable"] as? Bool == true else { throw CaptureFailure("FINALIZATION_FAILED") } + self.queue.async { self.completeSuccess(reason: reason) } + } catch { self.queue.async { self.completeFailure(error, reason: reason) } } + } + } + } + private func completeSuccess(reason: String) { + do { + let items = try artifacts() + guard let main = items.first(where: { $0["mediaKind"] as? String == "video" }), let format, let mode, let sessionId else { throw CaptureFailure("FINALIZATION_FAILED") } + var result: [String: Any] = ["sessionId": sessionId, "stopReason": reason, "format": try jsonObject(format), "mode": mode, "video": main, "timingFile": "native-timing.json", "timing": try timingObject()] + result["deviceAudio"] = items.first(where: { $0["mediaKind"] as? String == "device-audio" }); result["microphone"] = items.first(where: { $0["mediaKind"] as? String == "microphone" }) + if options["deviceAudio"] as? Bool == true && deviceAudio == nil { event("warning", ["code": "AUDIO_INTERRUPTED"]) } + finalResult = result; phase = "completed"; try checkpoint(reason: reason) + releaseInputs(); event("nativeFinalized", ["result": result], request: stopRequest) + if shutdownPending { didShutdown?() } + } catch { completeFailure(error, reason: reason) } + } + private func completeFailure(_ error: Error, reason: String) { phase = "failed"; try? checkpoint(reason: reason); releaseInputs(); event("error", ["code": (error as? CaptureFailure)?.code ?? "FINALIZATION_FAILED", "recoverable": true], request: stopRequest); if shutdownPending { didShutdown?() } } + private func releaseInputs() { + deadline?.cancel(); progressTimer?.cancel(); progressTimer = nil; preview.setEnabled(false) + session?.stopRunning(); micSession?.stopRunning() + videoOutput?.setSampleBufferDelegate(nil, queue: nil); deviceAudioOutput?.setSampleBufferDelegate(nil, queue: nil); micOutput?.setSampleBufferDelegate(nil, queue: nil) + observers.forEach(NotificationCenter.default.removeObserver); observers.removeAll() + session = nil; micSession = nil; videoOutput = nil; deviceAudioOutput = nil; micOutput = nil; clock = nil; micClock = nil + if assertion != 0 { IOPMAssertionRelease(assertion); assertion = 0 } + } + public func shutdown() { shutdownPending = true; discovery.stop(); if ["starting", "recording", "finalising"].contains(phase) { finish(reason: "parent-exit") } else { releaseInputs(); didShutdown?() } } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/DeviceClassifier.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/DeviceClassifier.swift new file mode 100644 index 000000000..1c4c50c50 --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/DeviceClassifier.swift @@ -0,0 +1,21 @@ +import Foundation +public struct DeviceFacts { + public let modelID: String + public let hasMuxed: Bool + public let hasVideo: Bool + public init(modelID: String, hasMuxed: Bool, hasVideo: Bool) { self.modelID = modelID; self.hasMuxed = hasMuxed; self.hasVideo = hasVideo } +} +public enum DeviceClassifier { + // USB screen sources advertise muxed media without necessarily advertising standalone video. + // Preparation validates actual video samples before the device can become ready. + public static func isEligible(_ facts: DeviceFacts) -> Bool { facts.modelID == "iOS Device" && facts.hasMuxed } +} +public struct TokenInventory { + private var tokens: [String: String] = [:] + public init() {} + public mutating func reconcile(_ identities: Set) -> [String: String] { + tokens = tokens.filter { identities.contains($0.key) } + for identity in identities where tokens[identity] == nil { tokens[identity] = UUID().uuidString.lowercased() } + return tokens + } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/DeviceDiscovery.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/DeviceDiscovery.swift new file mode 100644 index 000000000..de2131286 --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/DeviceDiscovery.swift @@ -0,0 +1,140 @@ +import Foundation +import AVFoundation +import CoreMediaIO + +/// Owns discovery observation and scheduling separately from the hardware inventory. +/// Every state transition and reconciliation runs on the engine queue. +final class DiscoveryLifecycle { + private let queue: DispatchQueue + private let pollInterval: TimeInterval + private let coalescingInterval: TimeInterval + private let observe: (@escaping () -> Void) throws -> (() -> Void) + private let reconcile: () -> Void + private var cancelObservation: (() -> Void)? + private var poll: DispatchSourceTimer? + private var coalesced: DispatchWorkItem? + private var generation = 0 + + init(queue: DispatchQueue, pollInterval: TimeInterval = 2, coalescingInterval: TimeInterval = 0.25, + observe: @escaping (@escaping () -> Void) throws -> (() -> Void), reconcile: @escaping () -> Void) { + self.queue = queue + self.pollInterval = pollInterval + self.coalescingInterval = coalescingInterval + self.observe = observe + self.reconcile = reconcile + } + + func start(refresh: Bool) throws { + dispatchPrecondition(condition: .onQueue(queue)) + if cancelObservation != nil && !refresh { reconcile(); return } + let nextGeneration = generation + 1 + // Create the replacement first: failed initialization leaves the existing + // observation and its fallback poll usable. + let cancel = try observe { [weak self] in + guard let self else { return } + self.queue.async { + guard self.generation == nextGeneration else { return } + self.coalesced?.cancel() + let work = DispatchWorkItem { [weak self] in + guard let self, self.generation == nextGeneration else { return } + self.reconcile() + } + self.coalesced = work + self.queue.asyncAfter(deadline: .now() + self.coalescingInterval, execute: work) + } + } + stop() + generation = nextGeneration + cancelObservation = cancel + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now() + pollInterval, repeating: pollInterval) + timer.setEventHandler { [weak self] in + guard let self, self.generation == nextGeneration else { return } + self.reconcile() + } + timer.resume() + poll = timer + reconcile() + } + + func stop() { + dispatchPrecondition(condition: .onQueue(queue)) + generation += 1 + poll?.cancel(); poll = nil + coalesced?.cancel(); coalesced = nil + cancelObservation?(); cancelObservation = nil + } +} + +/// Access is confined to the engine's serial queue. No discovery is performed by init. +public final class DeviceDiscovery { + private let queue: DispatchQueue + private var identities = TokenInventory() + private var microphones = TokenInventory() + private var videoDevices: [String: AVCaptureDevice] = [:] + private var audioDevices: [String: AVCaptureDevice] = [:] + private var screenDiscovery: AVCaptureDevice.DiscoverySession? + private lazy var lifecycle = DiscoveryLifecycle(queue: queue, observe: { [weak self] changed in + guard let self else { throw CaptureFailure("HELPER_UNAVAILABLE") } + return try self.observe(changed: changed) + }, reconcile: { [weak self] in self?.reconcile() }) + private var revision = 0 + private var signature = "" + public var changed: (([String: Any]) -> Void)? + public init(queue: DispatchQueue) { self.queue = queue } + /// Rebuild the CMIO discovery observation only when capture inputs are idle. + /// Prepared/active sources keep their observation, identities, and device handles. + public func start(refresh: Bool = false) throws { + try lifecycle.start(refresh: refresh) + } + private func observe(changed: @escaping () -> Void) throws -> (() -> Void) { + // CMIO installs run-loop sources on its initialization thread. The helper keeps + // the main run loop alive while this queue owns inventory and capture state. + let (discovery, observation) = try DispatchQueue.main.sync { + var address = CMIOObjectPropertyAddress(mSelector: CMIOObjectPropertySelector(kCMIOHardwarePropertyAllowScreenCaptureDevices), mScope: UInt32(kCMIOObjectPropertyScopeGlobal), mElement: UInt32(kCMIOObjectPropertyElementMain)) + var enabled: UInt32 = 1 + guard CMIOObjectSetPropertyData(CMIOObjectID(kCMIOObjectSystemObject), &address, 0, nil, UInt32(MemoryLayout.size), &enabled) == noErr else { throw CaptureFailure("HELPER_UNAVAILABLE") } + var categories: [AVCaptureDevice.DeviceType] = [.external] + if #available(macOS 14, *) { categories.append(.continuityCamera) } + // Device arrival is asynchronous. Observe the session itself so a newly + // vended screen source need not wait for the fallback poll. + let discovery = AVCaptureDevice.DiscoverySession(deviceTypes: categories, mediaType: .muxed, position: .unspecified) + let observation = discovery.observe(\.devices, options: []) { _, _ in changed() } + return (discovery, observation) + } + screenDiscovery = discovery + var observers: [NSObjectProtocol] = [] + for name in [AVCaptureDevice.wasConnectedNotification, AVCaptureDevice.wasDisconnectedNotification] { + observers.append(NotificationCenter.default.addObserver(forName: name, object: nil, queue: nil) { _ in + changed() + }) + } + return { + observation.invalidate() + observers.forEach(NotificationCenter.default.removeObserver) + } + } + private func reconcile() { + guard let screenDiscovery else { return } + let candidates = screenDiscovery.devices + .filter { DeviceClassifier.isEligible(.init(modelID: $0.modelID, hasMuxed: $0.hasMediaType(.muxed), hasVideo: $0.hasMediaType(.video))) }.prefix(32) + let mapping = identities.reconcile(Set(candidates.map(\.uniqueID))) + videoDevices = Dictionary(uniqueKeysWithValues: candidates.compactMap { device in mapping[device.uniqueID].map { ($0, device) } }) + let audio = AVCaptureDevice.DiscoverySession(deviceTypes: [.microphone, .external], mediaType: .audio, position: .unspecified).devices.filter { device in !candidates.contains(where: { $0.uniqueID == device.uniqueID }) && !device.hasMediaType(.muxed) }.prefix(32) + let audioMapping = microphones.reconcile(Set(audio.map(\.uniqueID))) + audioDevices = Dictionary(uniqueKeysWithValues: audio.compactMap { device in audioMapping[device.uniqueID].map { ($0, device) } }) + let next = (videoDevices.keys.sorted() + audioDevices.keys.sorted()).joined(separator: ":") + if next != signature || revision == 0 { signature = next; revision += 1; changed?(snapshot()) } + } + public func snapshot() -> [String: Any] { + ["inventoryGeneration": revision, "devices": videoDevices.keys.sorted().map { source(token: $0)! }, "microphones": audioDevices.keys.sorted().map { ["token": $0, "label": String(decoding: audioDevices[$0]!.localizedName.utf16.prefix(256), as: UTF16.self)] }] + } + public func source(token: String) -> [String: Any]? { + guard let device = videoDevices[token] else { return nil } + return ["sourceType": "ios-device", "id": "ios-device:\(token)", "deviceToken": token, "displayName": String(decoding: device.localizedName.utf16.prefix(256), as: UTF16.self), "generation": revision, "deviceAudio": device.hasMediaType(.audio) ? "available" : "unknown"] + } + public func device(token: String, generation: Int) throws -> AVCaptureDevice { guard generation == revision, let device = videoDevices[token] else { throw CaptureFailure("DEVICE_NOT_FOUND") }; return device } + public func microphone(token: String) throws -> AVCaptureDevice { guard let device = audioDevices[token] else { throw CaptureFailure("DEVICE_NOT_FOUND") }; return device } + public func contains(token: String) -> Bool { videoDevices[token] != nil } + public func stop() { lifecycle.stop(); screenDiscovery = nil; videoDevices.removeAll(); audioDevices.removeAll(); _ = identities.reconcile([]); _ = microphones.reconcile([]) } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/MediaInspector.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/MediaInspector.swift new file mode 100644 index 000000000..57eebac9f --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/MediaInspector.swift @@ -0,0 +1,65 @@ +import Foundation +import AVFoundation +import CoreMedia +import CryptoKit +public enum MediaInspector { + public static func inspect(url: URL) async throws -> [String: Any] { + let asset = AVURLAsset(url: url) + let duration = try await asset.load(.duration) + guard duration.isNumeric, duration > .zero else { throw CaptureFailure("FINALIZATION_FAILED") } + let videos = try await asset.loadTracks(withMediaType: .video) + let audios = try await asset.loadTracks(withMediaType: .audio) + var result: [String: Any] = ["duration": try jsonObject(NativeTime(duration)), "decodable": false] + if let video = videos.first { + let size = try await video.load(.naturalSize) + let transform = try await video.load(.preferredTransform) + let descriptions = try await video.load(.formatDescriptions) + guard size.width > 0, size.height > 0, let description = descriptions.first else { throw CaptureFailure("FINALIZATION_FAILED") } + let reader = try AVAssetReader(asset: asset) + let output = AVAssetReaderTrackOutput(track: video, outputSettings: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]) + guard reader.canAdd(output) else { throw CaptureFailure("FINALIZATION_FAILED") }; reader.add(output) + guard reader.startReading(), let first = output.copyNextSampleBuffer(), CMSampleBufferGetImageBuffer(first) != nil else { throw CaptureFailure("FINALIZATION_FAILED") } + reader.cancelReading() + result["decodable"] = true + let coded = CMVideoFormatDescriptionGetDimensions(description) + let display = CMVideoFormatDescriptionGetPresentationDimensions(description, usePixelAspectRatio: true, useCleanAperture: true) + let ext = (CMFormatDescriptionGetExtensions(description) ?? [:] as CFDictionary) as NSDictionary + let codec = CMFormatDescriptionGetMediaSubType(description) == kCMVideoCodecType_H264 ? "h264" : String(format: "%08x", CMFormatDescriptionGetMediaSubType(description)) + let hash = SHA256.hash(data: Data("\(coded.width):\(coded.height):\(codec)".utf8)).map { String(format: "%02x", $0) }.joined() + result["video"] = try jsonObject(IOSVideoFormat(codedWidth: Int(coded.width), codedHeight: Int(coded.height), displayWidth: display.width, displayHeight: display.height, codec: codec, colorPrimaries: ext[kCMFormatDescriptionExtension_ColorPrimaries] as? String, transferFunction: ext[kCMFormatDescriptionExtension_TransferFunction] as? String, ycbcrMatrix: ext[kCMFormatDescriptionExtension_YCbCrMatrix] as? String, fullRange: ext[kCMFormatDescriptionExtension_FullRangeVideo] as? Bool, transform: [transform.a, transform.b, transform.c, transform.d, transform.tx, transform.ty], observedFrameRate: nil, fingerprint: hash)) + + } + if let audio = audios.first, let description = try await audio.load(.formatDescriptions).first, let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(description)?.pointee { + result["audio"] = ["codec": asbd.mFormatID == kAudioFormatLinearPCM ? "pcm" : String(format: "%08x", asbd.mFormatID), "sampleRate": asbd.mSampleRate, "channels": Int(asbd.mChannelsPerFrame)] + if videos.isEmpty { + let reader = try AVAssetReader(asset: asset) + let output = AVAssetReaderTrackOutput(track: audio, outputSettings: [AVFormatIDKey: kAudioFormatLinearPCM]) + guard reader.canAdd(output) else { throw CaptureFailure("FINALIZATION_FAILED") }; reader.add(output) + guard reader.startReading(), output.copyNextSampleBuffer() != nil else { throw CaptureFailure("FINALIZATION_FAILED") } + reader.cancelReading(); result["decodable"] = true + } + } + return result + } +} + +import VideoToolbox +extension MediaInspector { + public static func validateFirstSample(_ sample: CMSampleBuffer) throws { + if CMSampleBufferGetImageBuffer(sample) != nil { return } + guard let description = CMSampleBufferGetFormatDescription(sample), VideoFormatPolicy.isSync(sample) else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + var decoder: VTDecompressionSession? + guard VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: description, decoderSpecification: nil, imageBufferAttributes: [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA] as CFDictionary, outputCallback: nil, decompressionSessionOut: &decoder) == noErr, let decoder else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + defer { VTDecompressionSessionInvalidate(decoder) } + let state = DecodeResult() + guard VTDecompressionSessionDecodeFrame(decoder, sampleBuffer: sample, flags: [], infoFlagsOut: nil, outputHandler: { status, _, image, _, _ in state.set(status == noErr && image != nil) }) == noErr else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + VTDecompressionSessionWaitForAsynchronousFrames(decoder) + guard state.value else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + } +} +private final class DecodeResult: @unchecked Sendable { + private let lock = NSLock() + private var valid = false + func set(_ next: Bool) { lock.lock(); valid = next; lock.unlock() } + var value: Bool { lock.lock(); defer { lock.unlock() }; return valid } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/NativeTimingStore.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/NativeTimingStore.swift new file mode 100644 index 000000000..9d1b36c5f --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/NativeTimingStore.swift @@ -0,0 +1,74 @@ +import Foundation +import Darwin +public struct SessionStorage { + public let root: URL + public let names: Set + public init(_ storage: [String: Any]) throws { + guard let path = storage["sessionRoot"] as? String, path.hasPrefix("/"), let names = storage["allowedRelativeNames"] as? [String] else { throw CaptureFailure("INVALID_REQUEST") } + let root = URL(fileURLWithPath: path, isDirectory: true).standardizedFileURL + guard root.resolvingSymlinksInPath().path == root.path else { throw CaptureFailure("INVALID_REQUEST") } + let values = try root.resourceValues(forKeys: [.isDirectoryKey, .isSymbolicLinkKey]) + guard values.isDirectory == true, values.isSymbolicLink != true else { throw CaptureFailure("INVALID_REQUEST") } + let allowed: Set = ["source-video.mov", "device-audio.mov", "microphone.mov", "native-timing.json", "recording.pending.mov", "recording.mov"] + guard Set(names).isSubset(of: allowed) else { throw CaptureFailure("INVALID_REQUEST") } + self.root = root; self.names = Set(names) + } + public func file(_ name: String, creating: Bool = false) throws -> URL { + guard names.contains(name), !name.contains("/"), root.resolvingSymlinksInPath().path == root.path else { throw CaptureFailure("INVALID_REQUEST") } + let url = root.appendingPathComponent(name) + guard url.resolvingSymlinksInPath().path == url.path else { throw CaptureFailure("INVALID_REQUEST") } + var info = stat() + if lstat(url.path, &info) == 0, (info.st_mode & S_IFMT) == S_IFLNK { throw CaptureFailure("INVALID_REQUEST") } + if creating && FileManager.default.fileExists(atPath: url.path) { throw CaptureFailure("INVALID_REQUEST") } + return url + } + public func availableBytes() throws -> Int64 { + var info = statfs() + guard statfs(root.path, &info) == 0 else { throw CaptureFailure("DISK_SPACE_LOW") } + return Int64(info.f_bavail) * Int64(info.f_bsize) + } +} +public final class NativeTimingStore { + private let url: URL + public init(storage: SessionStorage) throws { url = try storage.file("native-timing.json") } + public func checkpoint(_ object: [String: Any]) throws { + let data = try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) + guard data.count <= 1024 * 1024 else { throw CaptureFailure("WRITER_FAILED") } + try data.write(to: url, options: .atomic) + let fd = open(url.path, O_RDONLY | O_NOFOLLOW) + guard fd >= 0 else { throw CaptureFailure("WRITER_FAILED") } + defer { close(fd) } + guard fsync(fd) == 0 else { throw CaptureFailure("WRITER_FAILED") } + let directoryFD = open(url.deletingLastPathComponent().path, O_RDONLY) + if directoryFD >= 0 { _ = fsync(directoryFD); close(directoryFD) } + } +} + +public enum StorageReservePolicy { + /// Uses actual media-file bytes, never pixel-buffer size or a resolution estimate. + public static func requiredBytes(writtenBytes: Int64, videoBytes: Int64, elapsedSeconds: Double, mixAudio: Bool) -> Int64 { + let elapsed = elapsedSeconds.isFinite && elapsedSeconds > 1 ? elapsedSeconds : 1 + let rateReserve = Double(max(0, writtenBytes)) / elapsed * 60 + let copyReserve = mixAudio ? max(0, videoBytes) : 0 + let total = rateReserve + Double(copyReserve) + Double(256 * 1024 * 1024) + return Int64(min(Double(Int64.max / 2), total).rounded(.up)) + } +} +extension SessionStorage { + public func writtenMediaBytes() throws -> (video: Int64, total: Int64) { + var video: Int64 = 0, total: Int64 = 0 + for name in ["source-video.mov", "device-audio.mov", "microphone.mov"] where names.contains(name) { + let url = try file(name) + var info = stat() + if lstat(url.path, &info) != 0 { + if errno == ENOENT { continue } + throw CaptureFailure("WRITER_FAILED") + } + guard (info.st_mode & S_IFMT) == S_IFREG, info.st_size >= 0 else { throw CaptureFailure("INVALID_REQUEST") } + let bytes = Int64(info.st_size) + total += bytes + if name == "source-video.mov" { video = bytes } + } + return (video, total) + } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/PreviewEncoder.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/PreviewEncoder.swift new file mode 100644 index 000000000..0d428a89a --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/PreviewEncoder.swift @@ -0,0 +1,94 @@ +import Foundation +import AVFoundation +import CoreImage +import ImageIO +import VideoToolbox +import Darwin + +public enum PreviewFraming { + public static func record(jpeg: Data, generation: UInt32, sequence: UInt32) throws -> Data { + guard !jpeg.isEmpty, jpeg.count <= 128 * 1024 else { throw CaptureFailure("INVALID_REQUEST") } + var data = Data("RLIP".utf8) + for value: UInt16 in [1, 0] { var big = value.bigEndian; withUnsafeBytes(of: &big) { data.append(contentsOf: $0) } } + for value: UInt32 in [generation, sequence, UInt32(jpeg.count), 0] { var big = value.bigEndian; withUnsafeBytes(of: &big) { data.append(contentsOf: $0) } } + data.append(jpeg); return data + } +} +/// One queued conversion and one partial pipe record. Every descriptor write is nonblocking. +public final class PreviewEncoder { + private let queue = DispatchQueue(label: "recordly.ios.preview", qos: .utility) + private let lock = NSLock() + private var enabled = false + private var occupied = false + private var lastSubmission = 0.0 + private var lastEncoding = 0.0 + private var lostCompressedContinuity = true + private var sequence: UInt32 = 0 + private var packet: Data? + private var offset = 0 + private var closed = false + private let fd: Int32 + private let context = CIContext(options: [.cacheIntermediates: false]) + private var decoder: VTDecompressionSession? + public init(fd: Int32 = 3) { self.fd = fd; if fcntl(fd, F_GETFD) >= 0 { _ = fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK) } else { closed = true } } + public func setEnabled(_ value: Bool) { lock.lock(); enabled = value; if !value { lostCompressedContinuity = true }; lock.unlock() } + public func submit(_ sample: CMSampleBuffer, generation: UInt32) { + let now = ProcessInfo.processInfo.systemUptime + lock.lock() + let compressed = CMSampleBufferGetImageBuffer(sample) == nil + guard enabled, !occupied, compressed || now - lastSubmission >= 0.2 else { + if compressed { lostCompressedContinuity = true } + lock.unlock(); return + } + occupied = true; lastSubmission = now; lock.unlock() + queue.async { [self] in + defer { lock.lock(); occupied = false; lock.unlock() } + guard !closed else { return } + drain() + autoreleasepool { + let aperture = CMSampleBufferGetFormatDescription(sample).map { CMVideoFormatDescriptionGetCleanAperture($0, originIsAtTopLeft: false) } + if let pixels = CMSampleBufferGetImageBuffer(sample) { encode(pixels, generation: generation, aperture: aperture); return } + guard let description = CMSampleBufferGetFormatDescription(sample) else { return } + lock.lock(); let discontinuity = lostCompressedContinuity; lock.unlock() + if discontinuity { + guard VideoFormatPolicy.isSync(sample) else { return } + if let decoder { VTDecompressionSessionInvalidate(decoder) }; decoder = nil + lock.lock(); lostCompressedContinuity = false; lock.unlock() + } + if decoder == nil || !VTDecompressionSessionCanAcceptFormatDescription(decoder!, formatDescription: description) { + if let decoder { VTDecompressionSessionInvalidate(decoder) }; decoder = nil + guard VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: description, decoderSpecification: nil, imageBufferAttributes: [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA] as CFDictionary, outputCallback: nil, decompressionSessionOut: &decoder) == noErr else { return } + } + guard let decoder else { return } + _ = VTDecompressionSessionDecodeFrame(decoder, sampleBuffer: sample, flags: [], infoFlagsOut: nil) { [weak self] status, _, image, _, _ in + guard status == noErr, let image, let self else { return } + self.encode(image, generation: generation, aperture: aperture) + } + VTDecompressionSessionWaitForAsynchronousFrames(decoder) + } + } + } + private func encode(_ pixels: CVPixelBuffer, generation: UInt32, aperture: CGRect?) { + let now = ProcessInfo.processInfo.systemUptime + guard packet == nil, now - lastEncoding >= 0.2 else { return } + lastEncoding = now + var image = CIImage(cvPixelBuffer: pixels) + if let aperture, !aperture.isEmpty, image.extent.contains(aperture) { + image = image.cropped(to: aperture).transformed(by: CGAffineTransform(translationX: -aperture.minX, y: -aperture.minY)) + } + let scale = min(1, 480 / max(image.extent.width, image.extent.height)) + let resized = image.transformed(by: CGAffineTransform(scaleX: scale, y: scale)) + guard let colorSpace = CGColorSpace(name: CGColorSpace.sRGB), let jpeg = context.jpegRepresentation(of: resized, colorSpace: colorSpace, options: [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 0.55]), jpeg.count <= 128 * 1024 else { return } + sequence &+= 1; packet = try? PreviewFraming.record(jpeg: jpeg, generation: generation, sequence: sequence); offset = 0; drain() + } + private func drain() { + guard let packet else { return } + while offset < packet.count { + let written = packet.withUnsafeBytes { write(fd, $0.baseAddress!.advanced(by: offset), packet.count - offset) } + if written < 0 { if errno == EINTR { continue }; if errno == EAGAIN || errno == EWOULDBLOCK { return }; closed = true; self.packet = nil; return } + guard written > 0 else { return }; offset += written + } + self.packet = nil; offset = 0 + } + deinit { if let decoder { VTDecompressionSessionInvalidate(decoder) } } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/Protocol.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/Protocol.swift new file mode 100644 index 000000000..5616ac590 --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/Protocol.swift @@ -0,0 +1,111 @@ +import Foundation +import CoreMedia + +public struct CaptureFailure: Error, LocalizedError { + public let code: String + public init(_ code: String) { self.code = code } + public var errorDescription: String? { code } +} + +public struct NativeTime: Codable, Equatable, Sendable { + public let value: String + public let timescale: Int32 + public init(_ time: CMTime) throws { + guard time.isNumeric, time.timescale > 0, time.epoch == 0 else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + value = String(time.value); timescale = time.timescale + } + public var cmTime: CMTime { CMTime(value: Int64(value) ?? 0, timescale: timescale) } +} + +public struct ProtocolCommand { + public let requestId: String + public let command: String + public let sessionId: String? + public let generation: UInt32? + public let payload: [String: Any] + public let storage: [String: Any]? + public static func parse(_ data: Data) throws -> ProtocolCommand { + guard data.count <= 65536, String(data: data, encoding: .utf8) != nil, + let object = try JSONSerialization.jsonObject(with: data) as? [String: Any], + Set(object.keys).isSubset(of: ["protocolVersion", "requestId", "command", "sessionId", "generation", "payload", "storage"]), + (object["protocolVersion"] as? NSNumber)?.intValue == 1, + let requestId = object["requestId"] as? String, validToken(requestId), + let name = object["command"] as? String, + ["hello", "discover", "prepare", "setPreviewEnabled", "start", "stop", "cancel", "release", "inspectMedia", "shutdown"].contains(name) + else { throw CaptureFailure("INVALID_REQUEST") } + if let version = object["protocolVersion"] as? NSNumber, version.doubleValue != 1 || CFGetTypeID(version) == CFBooleanGetTypeID() { throw CaptureFailure("PROTOCOL_MISMATCH") } + let session = object["sessionId"] as? String + if object["sessionId"] != nil && (session == nil || UUID(uuidString: session!) == nil) { throw CaptureFailure("INVALID_REQUEST") } + var generation: UInt32? + if let raw = object["generation"] { + guard let number = raw as? NSNumber, CFGetTypeID(number) != CFBooleanGetTypeID(), number.doubleValue >= 0, number.doubleValue <= Double(UInt32.max), number.doubleValue.rounded() == number.doubleValue else { throw CaptureFailure("INVALID_REQUEST") } + generation = number.uint32Value + } + let payload = object["payload"] as? [String: Any] ?? [:] + if object["payload"] != nil && !(object["payload"] is [String: Any]) { throw CaptureFailure("INVALID_REQUEST") } + let allowed: Set + switch name { + case "prepare": allowed = ["deviceToken", "inventoryGeneration", "options"] + case "setPreviewEnabled": allowed = ["enabled"] + case "cancel": allowed = ["discardAcceptedMedia"] + case "inspectMedia": allowed = ["relativeName"] + default: allowed = [] + } + guard Set(payload.keys).isSubset(of: allowed) else { throw CaptureFailure("INVALID_REQUEST") } + if ["setPreviewEnabled", "cancel"].contains(name) { + let key = name == "cancel" ? "discardAcceptedMedia" : "enabled" + guard let value = payload[key] as? NSNumber, CFGetTypeID(value) == CFBooleanGetTypeID() else { throw CaptureFailure("INVALID_REQUEST") } + } + let storage = object["storage"] as? [String: Any] + if object["storage"] != nil { + guard ["prepare", "inspectMedia"].contains(name), let storage, + Set(storage.keys) == ["sessionRoot", "allowedRelativeNames"], storage["sessionRoot"] is String, storage["allowedRelativeNames"] is [String] else { throw CaptureFailure("INVALID_REQUEST") } + } + if ["prepare", "start", "stop", "cancel", "release"].contains(name), session == nil { throw CaptureFailure("INVALID_REQUEST") } + if name == "prepare" { + guard let token = payload["deviceToken"] as? String, validToken(token), + let inventory = payload["inventoryGeneration"] as? NSNumber, inventory.doubleValue >= 0, inventory.doubleValue.rounded() == inventory.doubleValue, + let options = payload["options"] as? [String: Any], Set(options.keys) == ["deviceAudio", "microphoneToken"], + let audio = options["deviceAudio"] as? NSNumber, CFGetTypeID(audio) == CFBooleanGetTypeID(), + options["microphoneToken"] is NSNull || (options["microphoneToken"] as? String).map(validToken) == true, + storage != nil, generation != nil else { throw CaptureFailure("INVALID_REQUEST") } + } + return ProtocolCommand(requestId: requestId, command: name, sessionId: session, generation: generation, payload: payload, storage: storage) + } +} +public func validToken(_ text: String) -> Bool { + !text.isEmpty && text.utf8.count <= 128 && text.utf8.allSatisfy { (48...57).contains($0) || (65...90).contains($0) || (97...122).contains($0) || $0 == 45 || $0 == 95 } +} +public struct BoundedLineReader { + private var buffer = Data() + public init() {} + public mutating func push(_ chunk: Data) throws -> [Data] { + var lines = [Data]() + for byte in chunk { + if byte == 10 { if !buffer.isEmpty { lines.append(buffer) }; buffer.removeAll(keepingCapacity: true) } + else { guard buffer.count < 65536 else { throw CaptureFailure("INVALID_REQUEST") }; buffer.append(byte) } + } + return lines + } + public func finish() throws { if !buffer.isEmpty { throw CaptureFailure("INVALID_REQUEST") } } +} +public struct RequestCache { + private var input: [String: Data] = [:] + private var results: [String: [Data]] = [:] + private var order: [String] = [] + public init() {} + public mutating func begin(id: String, input bytes: Data) throws { + if input[id] != nil { _ = try lookup(id: id, input: bytes); return } + guard order.filter({ results[$0] == nil }).count < 8 else { throw CaptureFailure("RECORDING_BUSY") } + if order.count >= 128, let index = order.firstIndex(where: { results[$0] != nil }) { let old = order.remove(at: index); input.removeValue(forKey: old); results.removeValue(forKey: old) } + input[id] = bytes; order.append(id) + } + public func lookup(id: String, input bytes: Data) throws -> [Data]? { + if let old = input[id], old != bytes { throw CaptureFailure("INVALID_REQUEST") } + return input[id] == nil ? nil : (results[id] ?? []) + } + public mutating func complete(id: String, events: [Data]) { results[id] = events } +} +public func jsonObject(_ value: T) throws -> [String: Any] { + try JSONSerialization.jsonObject(with: JSONEncoder().encode(value)) as! [String: Any] +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/RawVideoNegotiation.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/RawVideoNegotiation.swift new file mode 100644 index 000000000..facf7d285 --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/RawVideoNegotiation.swift @@ -0,0 +1,26 @@ +import CoreVideo + +/// Confined to the capture queue. The engine's existing preparation deadline bounds waiting. +struct RawVideoNegotiation { + enum Decision: Equatable { + case inspectSample + case requestRaw(OSType) + case waitForRaw + } + + private var requestedRaw = false + + mutating func observe(rawPixelFormat: OSType?, supportsPassthrough: Bool, availablePixelFormats: [OSType]) throws -> Decision { + if let rawPixelFormat { + guard [kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange].contains(rawPixelFormat) else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + return .inspectSample + } + // Changing output settings does not make already queued compressed samples raw. + if requestedRaw { return .waitForRaw } + if supportsPassthrough { return .inspectSample } + let supported = [kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] + guard let pixelFormat = supported.first(where: availablePixelFormats.contains) else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + requestedRaw = true + return .requestRaw(pixelFormat) + } +} diff --git a/electron/native/ios-device-capture/Sources/IOSCaptureCore/VideoWriter.swift b/electron/native/ios-device-capture/Sources/IOSCaptureCore/VideoWriter.swift new file mode 100644 index 000000000..554e9c9e0 --- /dev/null +++ b/electron/native/ios-device-capture/Sources/IOSCaptureCore/VideoWriter.swift @@ -0,0 +1,208 @@ +import Foundation +import AVFoundation +import CoreMedia +import CryptoKit + +public struct IOSVideoFormat: Codable { + public let codedWidth: Int + public let codedHeight: Int + public let displayWidth: Double + public let displayHeight: Double + public let codec: String + public let colorPrimaries: String? + public let transferFunction: String? + public let ycbcrMatrix: String? + public let fullRange: Bool? + public let transform: [Double] + public var observedFrameRate: Double? + public let fingerprint: String +} +public struct VideoBoundary { + public let boundary: CMTime + public private(set) var started = false + public init(boundary: CMTime) { self.boundary = boundary } + public mutating func accept(pts: CMTime, isSync: Bool) -> Bool { + if !started { guard pts >= boundary && isSync else { return false }; started = true } + return true + } +} +public enum VideoFormatPolicy { + public static func bitrate(width: Int, height: Int, fps: Double) -> Int { Int(min(60000000, max(12000000, Double(width * height) * fps * 0.12))) } + public static func validateGeometry(width: Int, height: Int) throws { + guard width > 0, height > 0, width <= 8192, height <= 8192, width % 2 == 0, height % 2 == 0 else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + } + public static func inspect(_ sample: CMSampleBuffer) throws -> (IOSVideoFormat, String) { + guard let description = CMSampleBufferGetFormatDescription(sample), CMFormatDescriptionGetMediaType(description) == kCMMediaType_Video else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + let mediaBytes = CMSampleBufferGetImageBuffer(sample).map(CVPixelBufferGetDataSize) ?? CMSampleBufferGetTotalSampleSize(sample) + guard mediaBytes <= 64 * 1024 * 1024 else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + let dimensions = CMVideoFormatDescriptionGetDimensions(description) + try validateGeometry(width: Int(dimensions.width), height: Int(dimensions.height)) + let aperture = CMVideoFormatDescriptionGetCleanAperture(description, originIsAtTopLeft: true) + guard aperture.origin == .zero, Int(dimensions.width) - Int(aperture.width) <= 1, Int(dimensions.height) - Int(aperture.height) <= 1, aperture.width > 0, aperture.height > 0 else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + let display = CMVideoFormatDescriptionGetPresentationDimensions(description, usePixelAspectRatio: true, useCleanAperture: true) + guard display.width == aperture.width, display.height == aperture.height else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + let extensions = (CMFormatDescriptionGetExtensions(description) ?? [:] as CFDictionary) as NSDictionary + let primaries = extensions[kCMFormatDescriptionExtension_ColorPrimaries] as? String + let transfer = extensions[kCMFormatDescriptionExtension_TransferFunction] as? String + let matrix = extensions[kCMFormatDescriptionExtension_YCbCrMatrix] as? String + let supportsSRGB: Bool + // USB screen capture can use sRGB transfer with Rec.709 primaries/matrix. + // Asset-writer sRGB tagging is supported from macOS 15; preserve its actual tag. + if #available(macOS 15, *) { supportsSRGB = transfer == kCMFormatDescriptionTransferFunction_sRGB as String } + else { supportsSRGB = false } + guard primaries == kCMFormatDescriptionColorPrimaries_ITU_R_709_2 as String, + transfer == kCMFormatDescriptionTransferFunction_ITU_R_709_2 as String || supportsSRGB, + matrix == kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2 as String else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + let subtype = CMFormatDescriptionGetMediaSubType(description) + let raw = CMSampleBufferGetImageBuffer(sample) != nil + guard raw ? [kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange].contains(subtype) : supportsPassthrough(description) else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + let codec = raw ? String(format: "%08x", subtype) : "h264" + let canonical: [String: Any] = ["width": dimensions.width, "height": dimensions.height, "subtype": subtype, "extensions": extensions] + // Property-list canonicalization includes codec configuration, aperture, range and colour. + let bytes = try JSONSerialization.data(withJSONObject: canonicalJSON(canonical), options: [.sortedKeys]) + let fingerprint = SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined() + let fullRange = raw ? subtype == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange : extensions[kCMFormatDescriptionExtension_FullRangeVideo] as? Bool + return (IOSVideoFormat(codedWidth: Int(dimensions.width), codedHeight: Int(dimensions.height), displayWidth: display.width, displayHeight: display.height, codec: codec, colorPrimaries: primaries, transferFunction: transfer, ycbcrMatrix: matrix, fullRange: fullRange, transform: [1,0,0,1,0,0], observedFrameRate: nil, fingerprint: fingerprint), raw ? "h264-encode" : "passthrough") + } + public static func supportsPassthrough(_ description: CMFormatDescription) -> Bool { + guard CMFormatDescriptionGetMediaSubType(description) == kCMVideoCodecType_H264 else { return false } + var bytes: UnsafePointer? + var size = 0 + guard CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, parameterSetIndex: 0, parameterSetPointerOut: &bytes, parameterSetSizeOut: &size, parameterSetCountOut: nil, nalUnitHeaderLengthOut: nil) == noErr, let bytes, size >= 4 else { return false } + // Baseline profile cannot contain B slices. Other profiles must renegotiate raw + // until their reordered sparse-tail behavior has a validated compatibility policy. + return bytes[1] == 66 + } + public static func isSync(_ sample: CMSampleBuffer) -> Bool { + guard let entries = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[CFString: Any]], let first = entries.first else { return true } + return !(first[kCMSampleAttachmentKey_NotSync] as? Bool ?? false) + } +} + +/// Confined to CaptureEngine.queue. One retained tail buffer; no unbounded media staging. +public final class VideoWriter { + private let writer: AVAssetWriter + private let input: AVAssetWriterInput + public let format: IOSVideoFormat + public let mode: String + private var boundary: VideoBoundary + private var pending: CMSampleBuffer? + private var maxPTS = CMTime.invalid + private var lastDTS = CMTime.invalid + public private(set) var firstHostTime: CMTime? + public private(set) var sampleCount = 0 + public private(set) var endHostTime: CMTime? + public init(url: URL, format: IOSVideoFormat, mode: String, description: CMFormatDescription, boundary: CMTime, recommended: [String: Any]?) throws { + self.format = format; self.mode = mode; self.boundary = VideoBoundary(boundary: boundary) + writer = try AVAssetWriter(outputURL: url, fileType: .mov) + writer.movieFragmentInterval = CMTime(value: 10, timescale: 1) + writer.initialMovieFragmentInterval = CMTime(value: 1, timescale: 1) + var settings: [String: Any]? + if mode == "h264-encode" { + guard var config = recommended else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + config[AVVideoCodecKey] = AVVideoCodecType.h264 + config[AVVideoWidthKey] = format.codedWidth; config[AVVideoHeightKey] = format.codedHeight + config[AVVideoCompressionPropertiesKey] = [AVVideoAverageBitRateKey: VideoFormatPolicy.bitrate(width: format.codedWidth, height: format.codedHeight, fps: format.observedFrameRate ?? 30), AVVideoMaxKeyFrameIntervalDurationKey: 2, AVVideoAllowFrameReorderingKey: false] + config[AVVideoCleanApertureKey] = [AVVideoCleanApertureWidthKey: format.displayWidth, AVVideoCleanApertureHeightKey: format.displayHeight, AVVideoCleanApertureHorizontalOffsetKey: (format.displayWidth - Double(format.codedWidth)) / 2, AVVideoCleanApertureVerticalOffsetKey: (format.displayHeight - Double(format.codedHeight)) / 2] + config[AVVideoColorPropertiesKey] = [AVVideoColorPrimariesKey: format.colorPrimaries!, AVVideoTransferFunctionKey: format.transferFunction!, AVVideoYCbCrMatrixKey: format.ycbcrMatrix!] + guard writer.canApply(outputSettings: config, forMediaType: .video) else { throw CaptureFailure("UNSUPPORTED_FORMAT") }; settings = config + } + input = AVAssetWriterInput(mediaType: .video, outputSettings: settings, sourceFormatHint: description) + input.expectsMediaDataInRealTime = true + guard writer.canAdd(input) else { throw CaptureFailure("UNSUPPORTED_FORMAT") }; writer.add(input) + } + public func append(_ sourceSample: CMSampleBuffer) throws -> Bool { + let sample = try VideoFormatPolicy.normalizeRawGeometry(sourceSample) + let (next, nextMode) = try VideoFormatPolicy.inspect(sample) + guard next.fingerprint == format.fingerprint, nextMode == mode else { throw CaptureFailure("FORMAT_CHANGED") } + let pts = CMSampleBufferGetPresentationTimeStamp(sample) + guard pts.isNumeric else { throw CaptureFailure("CLOCK_MAPPING_UNAVAILABLE") } + guard boundary.accept(pts: pts, isSync: mode == "h264-encode" || VideoFormatPolicy.isSync(sample)) else { return false } + let dts = CMSampleBufferGetDecodeTimeStamp(sample) + if dts.isNumeric, lastDTS.isNumeric, dts < lastDTS { throw CaptureFailure("WRITER_FAILED") } + if dts.isNumeric { lastDTS = dts } + if firstHostTime == nil { + guard writer.startWriting() else { throw CaptureFailure("WRITER_FAILED") } + writer.startSession(atSourceTime: pts); firstHostTime = pts + } + if let pending { try appendNow(pending) } + pending = sample; maxPTS = maxPTS.isNumeric ? CMTimeMaximum(maxPTS, pts) : pts + sampleCount += 1 + return true + } + private func appendNow(_ sample: CMSampleBuffer) throws { + guard writer.status == .writing, input.isReadyForMoreMediaData, input.append(sample) else { throw CaptureFailure("WRITER_FAILED") } + } + public func finish(at stop: CMTime, completion: @escaping (Result) -> Void) { + do { + guard let firstHostTime, let pending, sampleCount > 0 else { writer.cancelWriting(); throw CaptureFailure("NO_VIDEO_SAMPLES") } + let pts = CMSampleBufferGetPresentationTimeStamp(pending) + guard stop > firstHostTime, stop >= maxPTS else { throw CaptureFailure("FINALIZATION_FAILED") } + // A reordered final sample cannot be extended without retaining/rebuilding a GOP. + // Preserve existing fragments and fail honestly rather than inventing a sparse tail. + guard pts == maxPTS else { throw CaptureFailure("FINALIZATION_FAILED") } + var count = 0 + CMSampleBufferGetSampleTimingInfoArray(pending, entryCount: 0, arrayToFill: nil, entriesNeededOut: &count) + guard count == 1 else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + var info = CMSampleTimingInfo() + CMSampleBufferGetSampleTimingInfo(pending, at: 0, timingInfoOut: &info) + info.duration = CMTimeSubtract(stop, pts) + var extended: CMSampleBuffer? + guard CMSampleBufferCreateCopyWithNewTiming(allocator: kCFAllocatorDefault, sampleBuffer: pending, sampleTimingEntryCount: 1, sampleTimingArray: &info, sampleBufferOut: &extended) == noErr, let extended else { throw CaptureFailure("WRITER_FAILED") } + try appendNow(extended); self.pending = nil; endHostTime = stop + writer.endSession(atSourceTime: stop); input.markAsFinished() + writer.finishWriting { [writer] in completion(writer.status == .completed ? .success(()) : .failure(CaptureFailure("FINALIZATION_FAILED"))) } + } catch { + input.markAsFinished() + if writer.status == .writing { + writer.finishWriting { completion(.failure(error)) } + } else { completion(.failure(error)) } + } + } +} + +public func jsonObject(_ value: IOSVideoFormat) throws -> [String: Any] { + var object = try JSONSerialization.jsonObject(with: JSONEncoder().encode(value)) as! [String: Any] + for key in ["colorPrimaries", "transferFunction", "ycbcrMatrix", "fullRange", "observedFrameRate"] where object[key] == nil { object[key] = NSNull() } + return object +} + +private func canonicalJSON(_ value: Any) -> Any { + if let data = value as? Data { return data.base64EncodedString() } + if let dictionary = value as? [String: Any] { return dictionary.mapValues(canonicalJSON) } + if let array = value as? [Any] { return array.map(canonicalJSON) } + return value +} + +extension VideoFormatPolicy { + /// Copies planar SDR pixels exactly; the extra right/bottom edge is padding, never a crop. + public static func normalizeRawGeometry(_ sample: CMSampleBuffer) throws -> CMSampleBuffer { + guard let source = CMSampleBufferGetImageBuffer(sample) else { return sample } + let width = CVPixelBufferGetWidth(source), height = CVPixelBufferGetHeight(source) + guard width % 2 != 0 || height % 2 != 0 else { return sample } + let type = CVPixelBufferGetPixelFormatType(source) + guard [kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange].contains(type), width < 8192, height < 8192 else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + var output: CVPixelBuffer? + guard CVPixelBufferCreate(kCFAllocatorDefault, width + width % 2, height + height % 2, type, nil, &output) == kCVReturnSuccess, let output else { throw CaptureFailure("UNSUPPORTED_FORMAT") } + CVPixelBufferLockBaseAddress(source, .readOnly); CVPixelBufferLockBaseAddress(output, []) + defer { CVPixelBufferUnlockBaseAddress(source, .readOnly); CVPixelBufferUnlockBaseAddress(output, []) } + for plane in 0..<2 { + let rowBytes = CVPixelBufferGetBytesPerRowOfPlane(output, plane) + let destination = CVPixelBufferGetBaseAddressOfPlane(output, plane)! + memset(destination, plane == 0 ? (type == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ? 0 : 16) : 128, rowBytes * CVPixelBufferGetHeightOfPlane(output, plane)) + let input = CVPixelBufferGetBaseAddressOfPlane(source, plane)! + for row in 0.. 0 { pending.removeFirst(offset); offset = 0 } + pending.append(data); drain() + } } + private func drain() { + guard !closed else { return } + while offset < pending.count { + let count = pending.withUnsafeBytes { write(STDOUT_FILENO, $0.baseAddress!.advanced(by: offset), pending.count - offset) } + if count < 0 { if errno == EINTR { continue }; if errno == EAGAIN { return }; closed = true; pending.removeAll(); return } + if count == 0 { return }; offset += count + } + pending.removeAll(keepingCapacity: true); offset = 0 + } + func finish() { queue.sync { drain() } } +} +let engine = CaptureEngine() +let output = ControlOutput() +var cache = RequestCache() +var sequence: UInt64 = 0 +engine.emit = { event, payload, requestId, sessionId, generation in + sequence += 1 + var object: [String: Any] = ["protocolVersion": 1, "event": event, "sequence": sequence, "generation": generation, "payload": payload] + if let requestId, ["accepted", "error"].contains(event) { object["requestId"] = requestId }; if let sessionId, event != "inventoryChanged" { object["sessionId"] = sessionId } + guard var data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]), data.count < 65536 else { return } + data.append(10) + if let requestId { cache.complete(id: requestId, events: [data]) } + output.send(data) +} +engine.didShutdown = { output.finish(); exit(0) } +DispatchQueue.global(qos: .userInitiated).async { + var parser = BoundedLineReader() + var bytes = [UInt8](repeating: 0, count: 8192) + while true { + let count = read(STDIN_FILENO, &bytes, bytes.count) + if count <= 0 { if count < 0 && errno == EINTR { continue }; break } + do { + let lines = try parser.push(Data(bytes.prefix(count))) + for line in lines { + engine.queue.sync { + do { + let command = try ProtocolCommand.parse(line) + if let events = try cache.lookup(id: command.requestId, input: line) { + for saved in events { + if var object = try JSONSerialization.jsonObject(with: saved) as? [String: Any] { + sequence += 1; object["sequence"] = sequence + var reply = try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]); reply.append(10); output.send(reply) + } + } + return + } + try cache.begin(id: command.requestId, input: line) + engine.handle(command) + } catch { engine.emit?("error", ["code": (error as? CaptureFailure)?.code ?? "INVALID_REQUEST", "recoverable": false], nil, nil, 0) } + } + } + } catch { engine.queue.async { engine.emit?("error", ["code": "INVALID_REQUEST", "recoverable": false], nil, nil, 0); engine.shutdown() }; return } + } + let incomplete = (try? parser.finish()) == nil + engine.queue.async { if incomplete { engine.emit?("error", ["code": "INVALID_REQUEST", "recoverable": false], nil, nil, 0) }; engine.shutdown() } +} +// USB discovery needs CMIO's main-thread run-loop sources as well as dispatch work. +CFRunLoopRun() diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/AudioWriterTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/AudioWriterTests.swift new file mode 100644 index 000000000..bc171e2d7 --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/AudioWriterTests.swift @@ -0,0 +1,58 @@ +import XCTest +import AVFoundation +@testable import IOSCaptureCore +func syntheticAudioSample(at seconds: Double) throws -> CMSampleBuffer { + var asbd = AudioStreamBasicDescription(mSampleRate: 48000, mFormatID: kAudioFormatLinearPCM, mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, mBytesPerPacket: 2, mFramesPerPacket: 1, mBytesPerFrame: 2, mChannelsPerFrame: 1, mBitsPerChannel: 16, mReserved: 0) + var description: CMAudioFormatDescription? + XCTAssertEqual(CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &asbd, layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil, extensions: nil, formatDescriptionOut: &description), noErr) + var block: CMBlockBuffer? + XCTAssertEqual(CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil, blockLength: 960, blockAllocator: kCFAllocatorDefault, customBlockSource: nil, offsetToData: 0, dataLength: 960, flags: 0, blockBufferOut: &block), noErr) + XCTAssertEqual(CMBlockBufferFillDataBytes(with: 0, blockBuffer: block!, offsetIntoDestination: 0, dataLength: 960), noErr) + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: 48000), presentationTimeStamp: CMTime(seconds: seconds, preferredTimescale: 48000), decodeTimeStamp: .invalid) + var size = 2 + var sample: CMSampleBuffer? + XCTAssertEqual(CMSampleBufferCreateReady(allocator: kCFAllocatorDefault, dataBuffer: block, formatDescription: description, sampleCount: 480, sampleTimingEntryCount: 1, sampleTimingArray: &timing, sampleSizeEntryCount: 1, sampleSizeArray: &size, sampleBufferOut: &sample), noErr) + return try XCTUnwrap(sample) +} +final class AudioWriterTests: XCTestCase { + func testPCMOffsetSilenceAndInternalGapSurviveMovie() async throws { + let first = try syntheticAudioSample(at: 100.25) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try AudioWriter(url: url, firstSample: first) + try writer.append(first) + try writer.append(syntheticAudioSample(at: 100.29)) + XCTAssertEqual(writer.firstHostTime.seconds, 100.25, accuracy: 0.000001) + XCTAssertEqual(writer.gaps.count, 1) + XCTAssertEqual(writer.sampleCount, 960) + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in writer.finish { continuation.resume(with: $0) } } + let result = try await MediaInspector.inspect(url: url) + XCTAssertEqual(result["decodable"] as? Bool, true) + let duration = result["duration"] as! [String: Any] + XCTAssertEqual(Double(duration["value"] as! String)! / Double(duration["timescale"] as! Int32), 0.05, accuracy: 0.001) + } + func testBackwardAudioTimestampFailsInsteadOfFlattening() throws { + let first = try syntheticAudioSample(at: 100) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try AudioWriter(url: url, firstSample: first) + try writer.append(first) + XCTAssertThrowsError(try writer.append(syntheticAudioSample(at: 99))) + } +} + +extension AudioWriterTests { + func testAcceptedClockRateDoesNotTurnContiguousPCMIntoAnInterruption() async throws { + let source = try slightlyFastSourceTimebase() + let convert: (CMTime) -> CMTime = { CMSyncConvertTime($0, from: source, to: CMClockGetHostTimeClock()) } + let first = try CaptureClock.retime(syntheticAudioSample(at: 100), converting: convert) + let next = try CaptureClock.retime(syntheticAudioSample(at: 100.01), converting: convert) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try AudioWriter(url: url, firstSample: first) + try writer.append(first) + XCTAssertNoThrow(try writer.append(next)) + XCTAssertEqual(writer.sampleCount, 960) + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in writer.finish { continuation.resume(with: $0) } } + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/CaptureClockTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/CaptureClockTests.swift new file mode 100644 index 000000000..54f45f131 --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/CaptureClockTests.swift @@ -0,0 +1,41 @@ +import XCTest +import CoreMedia +@testable import IOSCaptureCore +final class CaptureClockTests: XCTestCase { + func testOffsetsPreserveDelayedAndEarlyAudio() { + let start = CMTime(value: 100000, timescale: 1000) + XCTAssertEqual(CaptureClock.offset(firstHostTime: CMTime(value: 100250, timescale: 1000), videoStartHostTime: start).seconds, 0.25, accuracy: 0.000001) + XCTAssertEqual(CaptureClock.offset(firstHostTime: CMTime(value: 99880, timescale: 1000), videoStartHostTime: start).seconds, -0.12, accuracy: 0.000001) + XCTAssertEqual(CaptureClock.offset(firstHostTime: CMTime(value: 6015, timescale: 60), videoStartHostTime: start).seconds, 0.25, accuracy: 0.000001) + } + func testRationalSerializationAndInvalidTime() throws { + let value = try NativeTime(CMTime(value: 9007199254740993, timescale: 1000000000)) + XCTAssertEqual(value.value, "9007199254740993") + XCTAssertThrowsError(try NativeTime(.invalid)) + } + func testRealHostClockRoundTrip() throws { + let clock = try CaptureClock(source: CMClockGetHostTimeClock()) + let original = CaptureClock.now + XCTAssertEqual(CMTimeCompare(try clock.toHost(original), original), 0) + } +} + +func slightlyFastSourceTimebase() throws -> CMTimebase { + var source: CMTimebase? + XCTAssertEqual(CMTimebaseCreateWithSourceClock(allocator: kCFAllocatorDefault, sourceClock: CMClockGetHostTimeClock(), timebaseOut: &source), noErr) + let timebase = try XCTUnwrap(source) + let anchor = CMTime(value: 100_000_000_000, timescale: 1_000_000_000) + XCTAssertEqual(CMTimebaseSetRateAndAnchorTime(timebase, rate: 1.0000005, anchorTime: anchor, immediateSourceTime: anchor), noErr) + return timebase +} +extension CaptureClockTests { + func testRateConversionMapsDurationAlongWithPTSAndDTS() throws { + let source = try slightlyFastSourceTimebase() + let sample = try syntheticAudioSample(at: 100) + let mapped = try CaptureClock.retime(sample) { CMSyncConvertTime($0, from: source, to: CMClockGetHostTimeClock()) } + let first = CMSampleBufferGetPresentationTimeStamp(mapped) + let end = CMSyncConvertTime(CMTime(value: 100_010_000_000, timescale: 1_000_000_000), from: source, to: CMClockGetHostTimeClock()) + XCTAssertLessThan(CMSampleBufferGetDuration(mapped).seconds, 0.01) + XCTAssertEqual(CMTimeAdd(first, CMSampleBufferGetDuration(mapped)).seconds, end.seconds, accuracy: 0.000001) + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/CaptureEngineTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/CaptureEngineTests.swift new file mode 100644 index 000000000..683d0641e --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/CaptureEngineTests.swift @@ -0,0 +1,58 @@ +import XCTest +@testable import IOSCaptureCore +final class CaptureEngineTests: XCTestCase { + func testHelloHasNoDiscoveryAndRejectsStartBeforePreparation() throws { + let engine = CaptureEngine() + var events = [(String, [String: Any])]() + engine.emit = { name, payload, _, _, _ in events.append((name, payload)) } + let hello = try ProtocolCommand.parse(Data(#"{"protocolVersion":1,"requestId":"hello","command":"hello"}"#.utf8)) + engine.queue.sync { engine.handle(hello) } + XCTAssertEqual(events.map(\.0), ["accepted"]) + XCTAssertEqual(events.first?.1["protocolVersion"] as? Int, 1) + let start = try ProtocolCommand.parse(Data("{\"protocolVersion\":1,\"requestId\":\"start\",\"command\":\"start\",\"sessionId\":\"\(UUID().uuidString)\"}".utf8)) + engine.queue.sync { engine.handle(start) } + XCTAssertEqual(events.last?.0, "error") + XCTAssertEqual(events.last?.1["code"] as? String, "INVALID_REQUEST") + } + func testTimingCheckpointAndPathBoundaries() throws { + let root = URL(fileURLWithPath: NSTemporaryDirectory()).resolvingSymlinksInPath().appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + let storage = try SessionStorage(["sessionRoot": root.path, "allowedRelativeNames": ["native-timing.json", "source-video.mov"]]) + XCTAssertThrowsError(try storage.file("../other.mov")) + let store = try NativeTimingStore(storage: storage) + try store.checkpoint(["version": 1, "phase": "recording"]) + try store.checkpoint(["version": 1, "phase": "completed"]) + let bytes = try Data(contentsOf: storage.file("native-timing.json")) + XCTAssertEqual((try JSONSerialization.jsonObject(with: bytes) as? [String: Any])?["phase"] as? String, "completed") + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("source-video.mov"), withDestinationURL: URL(fileURLWithPath: "/private/tmp/outside.mov")) + XCTAssertThrowsError(try storage.file("source-video.mov")) + } +} + +final class StorageReserveTests: XCTestCase { + func testReserveUsesWrittenMediaAndOnlyRequestedFinalCopy() { + let megabyte: Int64 = 1024 * 1024 + let videoBytes = 10 * megabyte + let audioBytes = megabyte + let noMix = StorageReservePolicy.requiredBytes(writtenBytes: videoBytes, videoBytes: videoBytes, elapsedSeconds: 10, mixAudio: false) + XCTAssertEqual(noMix, 316 * megabyte) + let mixed = StorageReservePolicy.requiredBytes(writtenBytes: videoBytes + audioBytes, videoBytes: videoBytes, elapsedSeconds: 10, mixAudio: true) + XCTAssertEqual(mixed, 332 * megabyte) + XCTAssertLessThan(mixed, 5 * 1024 * megabyte) + } +} +extension StorageReserveTests { + func testMeasuredBytesComeFromMediaFilesNotTimingOrPixelGeometry() throws { + let root = URL(fileURLWithPath: NSTemporaryDirectory()).resolvingSymlinksInPath().appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + let storage = try SessionStorage(["sessionRoot": root.path, "allowedRelativeNames": ["source-video.mov", "device-audio.mov", "microphone.mov", "native-timing.json"]]) + try Data(repeating: 0, count: 1024).write(to: storage.file("source-video.mov")) + try Data(repeating: 0, count: 512).write(to: storage.file("device-audio.mov")) + try Data(repeating: 0, count: 4096).write(to: storage.file("native-timing.json")) + let measured = try storage.writtenMediaBytes() + XCTAssertEqual(measured.video, 1024) + XCTAssertEqual(measured.total, 1536) + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/DeviceClassifierTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/DeviceClassifierTests.swift new file mode 100644 index 000000000..fd5140cc8 --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/DeviceClassifierTests.swift @@ -0,0 +1,20 @@ +import XCTest +@testable import IOSCaptureCore +final class DeviceClassifierTests: XCTestCase { + func testMuxedIOSScreenDoesNotRequireStandaloneVideoMediaType() { + // Observed USB screen-device metadata; muxed is the advertised output type. + XCTAssertTrue(DeviceClassifier.isEligible(.init(modelID: "iOS Device", hasMuxed: true, hasVideo: false))) + } + func testPositiveSignatureRequiresKnownModelAndMuxedMedia() { + XCTAssertTrue(DeviceClassifier.isEligible(.init(modelID: "iOS Device", hasMuxed: true, hasVideo: true))) + for facts in [DeviceFacts(modelID: "iOS Device", hasMuxed: false, hasVideo: true), .init(modelID: "Webcam", hasMuxed: true, hasVideo: true), .init(modelID: "Continuity Camera", hasMuxed: false, hasVideo: true), .init(modelID: "iPhone18,1", hasMuxed: false, hasVideo: true)] { XCTAssertFalse(DeviceClassifier.isEligible(facts)) } + } + func testStableDistinctTokensAndRemovalInvalidation() { + var inventory = TokenInventory() + let a = inventory.reconcile(["native-a", "native-b"]) + XCTAssertNotEqual(a["native-a"], a["native-b"]) + XCTAssertEqual(inventory.reconcile(["native-a", "native-b"]), a) + _ = inventory.reconcile(["native-b"]) + XCTAssertNotEqual(inventory.reconcile(["native-a", "native-b"])["native-a"], a["native-a"]) + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/DeviceDiscoveryLifecycleTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/DeviceDiscoveryLifecycleTests.swift new file mode 100644 index 000000000..ee34032cf --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/DeviceDiscoveryLifecycleTests.swift @@ -0,0 +1,115 @@ +import XCTest +@testable import IOSCaptureCore + +final class DeviceDiscoveryLifecycleTests: XCTestCase { + func testExplicitRefreshReopensTheDiscoveryBoundary() throws { + let harness = DiscoveryHarness() + try harness.queue.sync { + try harness.lifecycle.start(refresh: false) + try harness.lifecycle.start(refresh: true) + XCTAssertEqual(harness.snapshots, [1, 2]) + XCTAssertEqual(harness.cancelled, [1]) + harness.lifecycle.stop() + } + } + + func testPreparedSourceRefreshReusesItsExistingObservation() throws { + let harness = DiscoveryHarness() + try harness.queue.sync { + try harness.lifecycle.start(refresh: false) + try harness.lifecycle.start(refresh: false) + XCTAssertEqual(harness.snapshots, [1, 1]) + XCTAssertTrue(harness.cancelled.isEmpty) + harness.lifecycle.stop() + XCTAssertEqual(harness.cancelled, [1]) + } + } + + func testDeviceChangeBurstPublishesOneCoalescedSnapshot() throws { + let harness = DiscoveryHarness() + try harness.queue.sync { try harness.lifecycle.start(refresh: false) } + let notify = harness.queue.sync { harness.notifications[0] } + for _ in 0..<8 { notify() } + drain(harness) + harness.queue.sync { + XCTAssertEqual(harness.snapshots, [1, 1]) + harness.lifecycle.stop() + } + } + + func testOldObservationCannotPublishAfterRefresh() throws { + let harness = DiscoveryHarness() + try harness.queue.sync { try harness.lifecycle.start(refresh: false) } + let oldNotification = harness.queue.sync { harness.notifications[0] } + oldNotification() + try harness.queue.sync { try harness.lifecycle.start(refresh: true) } + oldNotification() + drain(harness) + harness.queue.sync { XCTAssertEqual(harness.snapshots, [1, 2]) } + harness.queue.sync { harness.notifications.last! }() + drain(harness) + harness.queue.sync { + XCTAssertEqual(harness.snapshots, [1, 2, 2]) + harness.lifecycle.stop() + } + } + + func testStopDiscardsAlreadyQueuedDeviceChanges() throws { + let harness = DiscoveryHarness() + try harness.queue.sync { try harness.lifecycle.start(refresh: false) } + let notify = harness.queue.sync { harness.notifications[0] } + notify() + harness.queue.sync { harness.lifecycle.stop() } + notify() + drain(harness) + harness.queue.sync { + XCTAssertEqual(harness.snapshots, [1]) + XCTAssertEqual(harness.cancelled, [1]) + } + } + + func testPollRepairsADeviceChangeWithoutNotification() throws { + let harness = DiscoveryHarness(pollInterval: 0.02) + let repaired = expectation(description: "Fallback poll publishes the current inventory") + harness.onSnapshot = { snapshots in + if snapshots == [1, 1] { repaired.fulfill() } + } + try harness.queue.sync { try harness.lifecycle.start(refresh: false) } + wait(for: [repaired], timeout: 1) + harness.queue.sync { harness.lifecycle.stop() } + } + + private func drain(_ harness: DiscoveryHarness) { + let drained = expectation(description: "Discovery coalescer drained") + harness.queue.asyncAfter(deadline: .now() + 0.05) { drained.fulfill() } + wait(for: [drained], timeout: 1) + } +} + +private final class DiscoveryHarness { + let queue = DispatchQueue(label: "recordly.tests.discovery") + let pollInterval: TimeInterval + var notifications: [() -> Void] = [] + var cancelled: [Int] = [] + var snapshots: [Int] = [] + var onSnapshot: (([Int]) -> Void)? + private var source = 0 + + init(pollInterval: TimeInterval = 30) { self.pollInterval = pollInterval } + + lazy var lifecycle = DiscoveryLifecycle( + queue: queue, + pollInterval: pollInterval, + coalescingInterval: 0.01, + observe: { [unowned self] notification in + self.source += 1 + let opened = self.source + self.notifications.append(notification) + return { [unowned self] in self.cancelled.append(opened) } + }, + reconcile: { [unowned self] in + self.snapshots.append(self.source) + self.onSnapshot?(self.snapshots) + } + ) +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/Fixtures/PROVENANCE.md b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/Fixtures/PROVENANCE.md new file mode 100644 index 000000000..9e9d728ed --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/Fixtures/PROVENANCE.md @@ -0,0 +1,6 @@ +Synthetic 64x96 testsrc2 animation generated locally, with no user media. +Reproduction (FFmpeg with libx264, engineering test dependency only): + +ffmpeg -f lavfi -i 'testsrc2=size=64x96:rate=10:duration=0.3' -c:v libx264 -profile:v baseline -bf 0 -g 30 -pix_fmt yuv420p -x264-params 'colorprim=bt709:transfer=bt709:colormatrix=bt709' -movflags +write_colr baseline.mov + +The fixture is original synthetic test data, dedicated to CC0-1.0. diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/Fixtures/baseline.mov b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/Fixtures/baseline.mov new file mode 100644 index 000000000..ea8bcf73a Binary files /dev/null and b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/Fixtures/baseline.mov differ diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/MediaInspectorTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/MediaInspectorTests.swift new file mode 100644 index 000000000..7082c6dfd --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/MediaInspectorTests.swift @@ -0,0 +1,186 @@ +import XCTest +import AVFoundation +import CoreVideo +@testable import IOSCaptureCore + +func syntheticVideoSample(width: Int = 64, height: Int = 96, seconds: Double = 100, primaries: CFString = kCVImageBufferColorPrimaries_ITU_R_709_2, transfer: CFString = kCVImageBufferTransferFunction_ITU_R_709_2, matrix: CFString = kCVImageBufferYCbCrMatrix_ITU_R_709_2) throws -> CMSampleBuffer { + var pixels: CVPixelBuffer? + XCTAssertEqual(CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, [kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary, &pixels), kCVReturnSuccess) + let buffer = try XCTUnwrap(pixels) + CVPixelBufferLockBaseAddress(buffer, []) + for plane in 0..) in + writer.finish(at: CMTime(value: 101, timescale: 1)) { continuation.resume(with: $0) } + } + let inspection = try await MediaInspector.inspect(url: url) + XCTAssertEqual(inspection["decodable"] as? Bool, true) + let video = try XCTUnwrap(inspection["video"] as? [String: Any]) + XCTAssertEqual(video["colorPrimaries"] as? String, "ITU_R_709_2") + XCTAssertEqual(video["transferFunction"] as? String, "IEC_sRGB") + XCTAssertEqual(video["ycbcrMatrix"] as? String, "ITU_R_709_2") + XCTAssertEqual(video["codedWidth"] as? Int, 1206) + XCTAssertEqual(video["codedHeight"] as? Int, 2622) + XCTAssertEqual(video["displayWidth"] as? Double, 1206) + XCTAssertEqual(video["displayHeight"] as? Double, 2622) + + let asset = AVURLAsset(url: url) + let tracks = try await asset.loadTracks(withMediaType: .video) + let track = try XCTUnwrap(tracks.first) + let reader = try AVAssetReader(asset: asset) + let output = AVAssetReaderTrackOutput(track: track, outputSettings: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]) + XCTAssertTrue(reader.canAdd(output)); reader.add(output) + XCTAssertTrue(reader.startReading()) + defer { reader.cancelReading() } + let decoded = try XCTUnwrap(output.copyNextSampleBuffer()) + let pixels = try XCTUnwrap(CMSampleBufferGetImageBuffer(decoded)) + XCTAssertEqual(CVPixelBufferGetWidth(pixels), 1206) + XCTAssertEqual(CVPixelBufferGetHeight(pixels), 2622) + XCTAssertEqual(CVPixelBufferGetPixelFormatType(pixels), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) + XCTAssertEqual(CVPixelBufferLockBaseAddress(pixels, .readOnly), kCVReturnSuccess) + defer { CVPixelBufferUnlockBaseAddress(pixels, .readOnly) } + let yBase = try XCTUnwrap(CVPixelBufferGetBaseAddressOfPlane(pixels, 0)).assumingMemoryBound(to: UInt8.self) + let uvBase = try XCTUnwrap(CVPixelBufferGetBaseAddressOfPlane(pixels, 1)).assumingMemoryBound(to: UInt8.self) + var decodedLevels: [Int] = [] + for index in levels.indices { + let x = width * (2 * index + 1) / (2 * levels.count) + let y = Int(yBase[1310 * CVPixelBufferGetBytesPerRowOfPlane(pixels, 0) + x]) + let uv = 655 * CVPixelBufferGetBytesPerRowOfPlane(pixels, 1) + x / 2 * 2 + decodedLevels.append(y) + // Flat patch centers avoid chroma edges; three code values allow H.264 quantization. + XCTAssertEqual(Double(y), Double(levels[index]), accuracy: 3) + XCTAssertEqual(Double(uvBase[uv]), Double(chroma[index].0), accuracy: 3) + XCTAssertEqual(Double(uvBase[uv + 1]), Double(chroma[index].1), accuracy: 3) + } + print("sRGB round trip: 1206x2622, \(video["colorPrimaries"]!), \(video["transferFunction"]!), \(video["ycbcrMatrix"]!), decoded luma \(decodedLevels)") + } + + func testSRGBSupportDoesNotAdmitUntestedColourTriplets() throws { + let rejected: [(CFString, CFString, CFString)] = [ + (kCVImageBufferColorPrimaries_P3_D65, kCVImageBufferTransferFunction_sRGB, kCVImageBufferYCbCrMatrix_ITU_R_709_2), + (kCVImageBufferColorPrimaries_ITU_R_709_2, kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ, kCVImageBufferYCbCrMatrix_ITU_R_709_2), + (kCVImageBufferColorPrimaries_ITU_R_709_2, kCVImageBufferTransferFunction_ITU_R_2100_HLG, kCVImageBufferYCbCrMatrix_ITU_R_709_2), + (kCVImageBufferColorPrimaries_ITU_R_709_2, "unknown" as CFString, kCVImageBufferYCbCrMatrix_ITU_R_709_2), + (kCVImageBufferColorPrimaries_ITU_R_709_2, kCVImageBufferTransferFunction_sRGB, kCVImageBufferYCbCrMatrix_ITU_R_601_4), + ] + for (primaries, transfer, matrix) in rejected { + let sample = try syntheticVideoSample(primaries: primaries, transfer: transfer, matrix: matrix) + XCTAssertThrowsError(try VideoFormatPolicy.inspect(sample)) { + XCTAssertEqual(($0 as? CaptureFailure)?.code, "UNSUPPORTED_FORMAT") + } + } + if #available(macOS 15, *) {} else { + XCTAssertThrowsError(try VideoFormatPolicy.inspect(syntheticVideoSample(transfer: kCVImageBufferTransferFunction_sRGB))) + } + } + + func testUntestedWideColourIsRejectedBeforeWriting() throws { + XCTAssertThrowsError(try VideoFormatPolicy.inspect(syntheticVideoSample(primaries: kCVImageBufferColorPrimaries_P3_D65))) + } + func testZeroFrameWriterNeverReportsSuccess() async throws { + let sample = try syntheticVideoSample() + let (format, mode) = try VideoFormatPolicy.inspect(sample) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try VideoWriter(url: url, format: format, mode: mode, description: CMSampleBufferGetFormatDescription(sample)!, boundary: .zero, recommended: [:]) + do { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in writer.finish(at: CMTime(value: 1, timescale: 1)) { continuation.resume(with: $0) } } + XCTFail("empty movie reported complete") + } catch { XCTAssertEqual((error as? CaptureFailure)?.code, "NO_VIDEO_SAMPLES") } + } + + func testOddGeometryPadsWithoutCroppingDisplayAperture() async throws { + let raw = try syntheticVideoSample(width: 63, height: 95) + let sample = try VideoFormatPolicy.normalizeRawGeometry(raw) + let (format, _) = try VideoFormatPolicy.inspect(sample) + XCTAssertEqual(format.codedWidth, 64) + XCTAssertEqual(format.codedHeight, 96) + XCTAssertEqual(format.displayWidth, 63) + XCTAssertEqual(format.displayHeight, 95) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try VideoWriter(url: url, format: format, mode: "h264-encode", description: CMSampleBufferGetFormatDescription(sample)!, boundary: .zero, recommended: [:]) + XCTAssertTrue(try writer.append(sample)) + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in writer.finish(at: CMTime(value: 101, timescale: 1)) { continuation.resume(with: $0) } } + let inspection = try await MediaInspector.inspect(url: url) + let video = inspection["video"] as! [String: Any] + XCTAssertEqual(video["displayWidth"] as? Double, 63) + XCTAssertEqual(video["displayHeight"] as? Double, 95) + } + + func testSparseSingleFrameRetainsFiveMinuteTimelineAndDecodes() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let sample = try syntheticVideoSample() + let (format, mode) = try VideoFormatPolicy.inspect(sample) + let url = directory.appendingPathComponent("source.mov") + let writer = try VideoWriter(url: url, format: format, mode: mode, description: CMSampleBufferGetFormatDescription(sample)!, boundary: CMTime(value: 100, timescale: 1), recommended: [:]) + XCTAssertTrue(try writer.append(sample)) + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in writer.finish(at: CMTime(value: 400, timescale: 1)) { continuation.resume(with: $0) } } + let inspected = try await MediaInspector.inspect(url: url) + XCTAssertEqual(inspected["decodable"] as? Bool, true) + let duration = inspected["duration"] as! [String: Any] + XCTAssertEqual(Double(duration["value"] as! String)! / Double(duration["timescale"] as! Int32), 300, accuracy: 0.01) + XCTAssertEqual((inspected["video"] as? [String: Any])?["codedWidth"] as? Int, 64) + } + func testRejectsTruncatedAndEmptyMedia() async throws { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + try Data("invalid movie".utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + do { _ = try await MediaInspector.inspect(url: url); XCTFail("corrupt media accepted") } catch {} + } + func testRejectsFormatChangeBeforeAcceptingSample() throws { + let sample = try syntheticVideoSample() + let (format, mode) = try VideoFormatPolicy.inspect(sample) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try VideoWriter(url: url, format: format, mode: mode, description: CMSampleBufferGetFormatDescription(sample)!, boundary: .zero, recommended: [:]) + XCTAssertThrowsError(try writer.append(syntheticVideoSample(width: 66))) + XCTAssertEqual(writer.sampleCount, 0) + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/PassthroughTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/PassthroughTests.swift new file mode 100644 index 000000000..fd096830c --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/PassthroughTests.swift @@ -0,0 +1,46 @@ +import XCTest +import AVFoundation +@testable import IOSCaptureCore + +final class PassthroughTests: XCTestCase { + private func samples(_ url: URL) async throws -> [CMSampleBuffer] { + let asset = AVURLAsset(url: url) + let tracks = try await asset.loadTracks(withMediaType: .video) + let track = try XCTUnwrap(tracks.first) + let reader = try AVAssetReader(asset: asset) + let output = AVAssetReaderTrackOutput(track: track, outputSettings: nil) + reader.add(output); XCTAssertTrue(reader.startReading()) + var values = [CMSampleBuffer]() + while let sample = output.copyNextSampleBuffer() { if CMSampleBufferGetNumSamples(sample) > 0 { values.append(sample) } } + return values + } + private func payload(_ sample: CMSampleBuffer) throws -> Data { + let buffer = try XCTUnwrap(CMSampleBufferGetDataBuffer(sample)) + var data = Data(count: CMBlockBufferGetDataLength(buffer)) + let length = data.count + let status = data.withUnsafeMutableBytes { CMBlockBufferCopyDataBytes(buffer, atOffset: 0, dataLength: length, destination: $0.baseAddress!) } + XCTAssertEqual(status, noErr); return data + } + func testBaselinePacketsSurviveWriterAndSparseTail() async throws { + let source = Bundle.module.url(forResource: "baseline", withExtension: "mov", subdirectory: "Fixtures")! + let input = try await samples(source) + let first = try XCTUnwrap(input.first) + let (format, mode) = try VideoFormatPolicy.inspect(first) + XCTAssertEqual(mode, "passthrough") + try MediaInspector.validateFirstSample(first) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".mov") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try VideoWriter(url: url, format: format, mode: mode, description: CMSampleBufferGetFormatDescription(first)!, boundary: .zero, recommended: nil) + for sample in input { XCTAssertTrue(try writer.append(sample)) } + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in writer.finish(at: CMTime(value: 300, timescale: 1)) { continuation.resume(with: $0) } } + let output = try await samples(url) + XCTAssertEqual(try input.map(payload), try output.map(payload)) + let inspected = try await MediaInspector.inspect(url: url) + XCTAssertEqual(inspected["decodable"] as? Bool, true) + let duration = inspected["duration"] as! [String: Any] + XCTAssertEqual(Double(duration["value"] as! String)! / Double(duration["timescale"] as! Int32), 300, accuracy: 0.01) + let generator = AVAssetImageGenerator(asset: AVURLAsset(url: url)) + generator.requestedTimeToleranceBefore = .zero; generator.requestedTimeToleranceAfter = .zero + _ = try await generator.image(at: CMTime(value: 299, timescale: 1)) + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/PreviewEncoderTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/PreviewEncoderTests.swift new file mode 100644 index 000000000..de81a4b59 --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/PreviewEncoderTests.swift @@ -0,0 +1,10 @@ +import XCTest +@testable import IOSCaptureCore +final class PreviewEncoderTests: XCTestCase { + func testExactBigEndianHeader() throws { + let record = try PreviewFraming.record(jpeg: Data([0xff, 0xd8]), generation: 0x01020304, sequence: 5) + XCTAssertEqual(Array(record.prefix(24)), [82,76,73,80,0,1,0,0,1,2,3,4,0,0,0,5,0,0,0,2,0,0,0,0]) + XCTAssertEqual(record.count, 26) + } + func testRejectOversizedPreview() { XCTAssertThrowsError(try PreviewFraming.record(jpeg: Data(repeating: 0, count: 131073), generation: 1, sequence: 1)) } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/ProtocolTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/ProtocolTests.swift new file mode 100644 index 000000000..6834a0a6b --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/ProtocolTests.swift @@ -0,0 +1,38 @@ +import XCTest +@testable import IOSCaptureCore +final class ProtocolTests: XCTestCase { + func testHello() throws { + let command = try ProtocolCommand.parse(Data(#"{"protocolVersion":1,"requestId":"r1","command":"hello"}"#.utf8)) + XCTAssertEqual(command.command, "hello") + } + func testSplitLines() throws { + var parser = BoundedLineReader() + XCTAssertEqual(try parser.push(Data("hel".utf8)).count, 0) + XCTAssertEqual(try parser.push(Data("lo\n".utf8)), [Data("hello".utf8)]) + } + func testOversizedLineAndInvalidUTF8() { + var parser = BoundedLineReader() + XCTAssertThrowsError(try parser.push(Data(repeating: 65, count: 65 * 1024))) + XCTAssertThrowsError(try ProtocolCommand.parse(Data([0xff]))) + } + func testRejectUnknownVersionCommandAndFields() { + for line in [#"{"protocolVersion":2,"requestId":"r1","command":"hello"}"#, + #"{"protocolVersion":1,"requestId":"r1","command":"pause"}"#, + #"{"protocolVersion":1,"requestId":"r1","command":"hello","outputPath":"/tmp/a"}"#] { + XCTAssertThrowsError(try ProtocolCommand.parse(Data(line.utf8))) + } + } + func testIdempotencyAndCollision() throws { + var cache = RequestCache() + let input = Data("a".utf8) + try cache.begin(id: "r", input: input) + cache.complete(id: "r", events: [Data("reply".utf8)]) + XCTAssertEqual(try cache.lookup(id: "r", input: input), [Data("reply".utf8)]) + XCTAssertThrowsError(try cache.lookup(id: "r", input: Data("b".utf8))) + } + func testEOFRejectsUnterminatedCommand() throws { + var parser = BoundedLineReader() + _ = try parser.push(Data("x".utf8)) + XCTAssertThrowsError(try parser.finish()) + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/RawVideoNegotiationTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/RawVideoNegotiationTests.swift new file mode 100644 index 000000000..9377ce01f --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/RawVideoNegotiationTests.swift @@ -0,0 +1,80 @@ +import XCTest +import AVFoundation +import CoreVideo +@testable import IOSCaptureCore + +final class RawVideoNegotiationTests: XCTestCase { + func testQueuedCompressedSamplesWaitForSupportedRawAfterOneRequest() throws { + var negotiation = RawVideoNegotiation() + XCTAssertEqual(try negotiation.observe(rawPixelFormat: nil, supportsPassthrough: false, availablePixelFormats: [kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]), .requestRaw(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)) + XCTAssertEqual(try negotiation.observe(rawPixelFormat: nil, supportsPassthrough: false, availablePixelFormats: []), .waitForRaw) + // Even an old, otherwise eligible packet must not choose passthrough after the request. + XCTAssertEqual(try negotiation.observe(rawPixelFormat: nil, supportsPassthrough: true, availablePixelFormats: []), .waitForRaw) + XCTAssertEqual(try negotiation.observe(rawPixelFormat: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, supportsPassthrough: false, availablePixelFormats: []), .inspectSample) + } + + func testUsesAdvertisedFullRangeWhenVideoRangeIsUnavailable() throws { + var negotiation = RawVideoNegotiation() + XCTAssertEqual(try negotiation.observe(rawPixelFormat: nil, supportsPassthrough: false, availablePixelFormats: [kCVPixelFormatType_32BGRA, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange]), .requestRaw(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) + XCTAssertEqual(try negotiation.observe(rawPixelFormat: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, supportsPassthrough: false, availablePixelFormats: []), .inspectSample) + } + + func testRejectsUnavailableConversionAndUnsupportedRawPixels() throws { + var negotiation = RawVideoNegotiation() + XCTAssertThrowsError(try negotiation.observe(rawPixelFormat: nil, supportsPassthrough: false, availablePixelFormats: [kCVPixelFormatType_32BGRA])) { + XCTAssertEqual(($0 as? CaptureFailure)?.code, "UNSUPPORTED_FORMAT") + } + XCTAssertThrowsError(try negotiation.observe(rawPixelFormat: kCVPixelFormatType_32BGRA, supportsPassthrough: false, availablePixelFormats: [])) { + XCTAssertEqual(($0 as? CaptureFailure)?.code, "UNSUPPORTED_FORMAT") + } + } + + func testNativeSupportedSamplesDoNotRequestConversion() throws { + var negotiation = RawVideoNegotiation() + XCTAssertEqual(try negotiation.observe(rawPixelFormat: nil, supportsPassthrough: true, availablePixelFormats: []), .inspectSample) + XCTAssertEqual(try negotiation.observe(rawPixelFormat: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, supportsPassthrough: false, availablePixelFormats: []), .inspectSample) + } + + func testPassthroughDoesNotQueryEncoderRecommendations() { + let output = RecommendationOutput(codecs: [.h264]) + XCTAssertNil(CaptureEngine.recommendedVideoSettings(mode: "passthrough", output: output)) + XCTAssertEqual(output.codecQueries, 0) + XCTAssertEqual(output.settingsQueries, 0) + } + + func testUnavailableH264DoesNotQueryEncoderRecommendations() { + let output = RecommendationOutput(codecs: [.hevc]) + XCTAssertNil(CaptureEngine.recommendedVideoSettings(mode: "h264-encode", output: output)) + XCTAssertEqual(output.codecQueries, 1) + XCTAssertEqual(output.settingsQueries, 0) + } + + func testAdvertisedH264RecommendationsRemainAvailableForEncoding() { + let output = RecommendationOutput(codecs: [.h264]) + XCTAssertEqual(CaptureEngine.recommendedVideoSettings(mode: "h264-encode", output: output)?[AVVideoWidthKey] as? Int, 64) + XCTAssertEqual(output.codecQueries, 1) + XCTAssertEqual(output.settingsQueries, 1) + } +} + +/// Replaces only AVFoundation capability queries; never opens a device or session. +private final class RecommendationOutput: AVCaptureVideoDataOutput { + let codecs: [AVVideoCodecType] + var codecQueries = 0 + var settingsQueries = 0 + + init(codecs: [AVVideoCodecType]) { self.codecs = codecs; super.init() } + + override func availableVideoCodecTypesForAssetWriter(writingTo outputFileType: AVFileType) -> [AVVideoCodecType] { + XCTAssertEqual(outputFileType, .mov) + codecQueries += 1 + return codecs + } + + override func recommendedVideoSettings(forVideoCodecType videoCodecType: AVVideoCodecType, assetWriterOutputFileType outputFileType: AVFileType) -> [String: Any]? { + XCTAssertEqual(videoCodecType, .h264) + XCTAssertEqual(outputFileType, .mov) + settingsQueries += 1 + return [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 64, AVVideoHeightKey: 96] + } +} diff --git a/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/VideoWriterTests.swift b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/VideoWriterTests.swift new file mode 100644 index 000000000..1d2ac1b95 --- /dev/null +++ b/electron/native/ios-device-capture/Tests/IOSCaptureCoreTests/VideoWriterTests.swift @@ -0,0 +1,17 @@ +import XCTest +import CoreMedia +@testable import IOSCaptureCore +final class VideoWriterTests: XCTestCase { + func testBoundaryWaitsForSyncSample() { + var policy = VideoBoundary(boundary: CMTime(value: 10, timescale: 1)) + XCTAssertFalse(policy.accept(pts: CMTime(value: 9, timescale: 1), isSync: true)) + XCTAssertFalse(policy.accept(pts: CMTime(value: 11, timescale: 1), isSync: false)) + XCTAssertTrue(policy.accept(pts: CMTime(value: 12, timescale: 1), isSync: true)) + XCTAssertTrue(policy.accept(pts: CMTime(value: 13, timescale: 1), isSync: false)) + XCTAssertTrue(policy.accept(pts: CMTime(value: 12, timescale: 1), isSync: false)) + } + func testQualityPolicyAndOddDimensions() { + XCTAssertEqual(VideoFormatPolicy.bitrate(width: 1170, height: 2532, fps: 30), 12000000) + XCTAssertThrowsError(try VideoFormatPolicy.validateGeometry(width: 1171, height: 2533)) + } +} diff --git a/electron/preload.ts b/electron/preload.ts index a30372314..97668914a 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,5 +1,6 @@ import { contextBridge, ipcRenderer } from "electron"; import type { RecordingSessionData } from "./ipc/types"; +import type { IOSCaptureAPI } from "../src/shared/iosCaptureAPI"; type NativeVideoExportWriteResult = { success: boolean; error?: string }; type NativeVideoAudioMuxMetrics = { @@ -164,6 +165,60 @@ function settleNativeVideoExportPendingRequests( } contextBridge.exposeInMainWorld("electronAPI", { + iosCapture: { + getCapabilities: () => ipcRenderer.invoke("ios-capture:capabilities"), + getSnapshot: () => ipcRenderer.invoke("ios-capture:snapshot"), + discover: () => ipcRenderer.invoke("ios-capture:discover"), + prepare: (input) => ipcRenderer.invoke("ios-capture:prepare", input), + start: (sessionId) => ipcRenderer.invoke("ios-capture:start", sessionId), + stop: (sessionId) => ipcRenderer.invoke("ios-capture:stop", sessionId), + cancel: (sessionId, discardAcceptedMedia) => + ipcRenderer.invoke("ios-capture:cancel", { sessionId, discardAcceptedMedia }), + release: (sessionId) => ipcRenderer.invoke("ios-capture:release", sessionId), + setPreviewEnabled: (enabled) => ipcRenderer.invoke("ios-capture:preview-enabled", enabled), + setDiscoveryActive: (active) => ipcRenderer.invoke("ios-capture:discovery-active", active), + onState: (callback) => { + const listener = ( + _event: Electron.IpcRendererEvent, + state: Parameters[0], + ) => callback(state); + ipcRenderer.on("ios-capture:state", listener); + void ipcRenderer.invoke("ios-capture:subscribe").catch(() => { + /* Failure is retained by the authoritative controller. */ + }); + return () => { + ipcRenderer.removeListener("ios-capture:state", listener); + void ipcRenderer.invoke("ios-capture:unsubscribe").catch(() => { + /* Failure is retained by the authoritative controller. */ + }); + }; + }, + onPreview: (callback) => { + const listener = ( + _event: Electron.IpcRendererEvent, + frame: Parameters[0], + ) => { + try { + callback(frame); + } finally { + ipcRenderer.send("ios-capture:preview-ack", { + generation: frame.generation, + sequence: frame.sequence, + }); + } + }; + ipcRenderer.on("ios-capture:preview", listener); + return () => ipcRenderer.removeListener("ios-capture:preview", listener); + }, + getRecoveryCandidates: () => ipcRenderer.invoke("ios-capture:recovery-list"), + recover: (sessionId, mode) => + ipcRenderer.invoke("ios-capture:recover", { sessionId, mode }), + openRecoveryFolder: (sessionId) => + ipcRenderer.invoke("ios-capture:recovery-folder", sessionId), + discardRecovery: (sessionId) => + ipcRenderer.invoke("ios-capture:recovery-discard", sessionId), + exportDiagnostics: (sessionId) => ipcRenderer.invoke("ios-capture:diagnostics", sessionId), + } satisfies IOSCaptureAPI, hudOverlaySetIgnoreMouse: (ignore: boolean) => { ipcRenderer.send("hud-overlay-set-ignore-mouse", ignore); }, @@ -485,7 +540,7 @@ contextBridge.exposeInMainWorld("electronAPI", { openSourceSelector: () => { return ipcRenderer.invoke("open-source-selector"); }, - selectSource: (source: ProcessedDesktopSource) => { + selectSource: (source: SelectedCaptureSource) => { return ipcRenderer.invoke("select-source", source); }, showSourceHighlight: (source: ProcessedDesktopSource) => { @@ -494,10 +549,10 @@ contextBridge.exposeInMainWorld("electronAPI", { getSelectedSource: () => { return ipcRenderer.invoke("get-selected-source"); }, - onSelectedSourceChanged: (callback: (source: ProcessedDesktopSource | null) => void) => { + onSelectedSourceChanged: (callback: (source: SelectedCaptureSource | null) => void) => { const listener = ( _event: Electron.IpcRendererEvent, - payload: ProcessedDesktopSource | null, + payload: SelectedCaptureSource | null, ) => callback(payload); ipcRenderer.on("selected-source-changed", listener); return () => ipcRenderer.removeListener("selected-source-changed", listener); diff --git a/package.json b/package.json index 6ef75b014..c12192cb0 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "url": "https://github.com/webadderallorg/Recordly/issues" }, "private": true, + "recordlyNativeIOSHelper": true, "version": "1.4.0", "type": "module", "scripts": { @@ -25,6 +26,9 @@ "preview": "vite preview --config vite.config.ts", "rebuild:native": "node ./node_modules/@electron/rebuild/lib/cli.js --force --only uiohook-napi", "build:native-helpers": "node scripts/build-native-helpers.mjs", + "build:ios-helper": "node scripts/build-ios-device-helper.mjs", + "test:ios-native": "node scripts/test-ios-device-helper.mjs", + "verify:ios-capture-fixture": "node scripts/verify-ios-capture-fixture.mjs", "build:whisper-runtime": "node scripts/build-whisper-runtime.mjs", "build:platform-native-helpers": "npm run build:whisper-runtime && npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:nvidia-cuda-compositor && npm run build:cursor-monitor", "build:windows-gpu-export": "node scripts/build-windows-gpu-export.mjs", diff --git a/public/third-party/Maya-LICENSE.txt b/public/third-party/Maya-LICENSE.txt new file mode 100644 index 000000000..819b48b4f --- /dev/null +++ b/public/third-party/Maya-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Ronaldo Avalos + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scripts/build-ios-device-helper.mjs b/scripts/build-ios-device-helper.mjs new file mode 100644 index 000000000..079752a34 --- /dev/null +++ b/scripts/build-ios-device-helper.mjs @@ -0,0 +1,89 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, copyFile, mkdir } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +if (process.platform !== "darwin") { + console.log("[ios-helper] SKIPPED: native builds require macOS."); + process.exit(0); +} +const root = process.cwd(); +const packagePath = path.join(root, "electron/native/ios-device-capture"); +const checkout = createHash("sha256").update(root).digest("hex").slice(0, 12); +const cache = path.join(os.tmpdir(), `recordly-ios-module-cache-${checkout}`); +function run(command, args) { + const result = spawnSync(command, args, { + encoding: "utf8", + env: { + ...process.env, + CLANG_MODULE_CACHE_PATH: cache, + SWIFTPM_MODULECACHE_OVERRIDE: cache, + }, + }); + if (result.status !== 0) + throw new Error( + result.error?.message ?? + ([result.stderr, result.stdout].filter(Boolean).join("\n") || `${command} failed`), + ); + return result.stdout.trim(); +} +for (const [arch, tag] of [ + ["arm64", "darwin-arm64"], + ["x86_64", "darwin-x64"], +]) { + const scratch = path.join(os.tmpdir(), `recordly-ios-build-${checkout}-${arch}`); + const args = [ + "build", + "--disable-sandbox", + "--package-path", + packagePath, + "--scratch-path", + scratch, + "-c", + "release", + "--triple", + `${arch}-apple-macosx14.0`, + "--product", + "recordly-ios-device-helper", + "-Xlinker", + "-sectcreate", + "-Xlinker", + "__TEXT", + "-Xlinker", + "__info_plist", + "-Xlinker", + path.join(packagePath, "Resources/Info.plist"), + ]; + console.log(`[ios-helper] Building ${tag}`); + run("swift", args); + const binPath = run("swift", [ + "build", + "--disable-sandbox", + "--package-path", + packagePath, + "--scratch-path", + scratch, + "-c", + "release", + "--triple", + `${arch}-apple-macosx14.0`, + "--show-bin-path", + ]); + const directory = path.join(root, "electron/native/bin", tag); + await mkdir(directory, { recursive: true }); + const binary = path.join(directory, "recordly-ios-device-helper"); + await copyFile(path.join(binPath, "recordly-ios-device-helper"), binary); + await chmod(binary, 0o755); + if (run("lipo", ["-archs", binary]) !== arch) + throw new Error(`Architecture mismatch for ${tag}`); + const info = run("vtool", ["-show-build", binary]); + if (!/minos\s+14\.0(?:\.0)?\b/.test(info)) + throw new Error(`Deployment target mismatch for ${tag}`); + const plist = run("otool", ["-s", "__TEXT", "__info_plist", binary]); + if (!plist.includes("__info_plist")) + throw new Error(`Missing embedded privacy metadata for ${tag}`); + console.log( + `[ios-helper] Staged ${tag} (compile evidence; signing and physical acceptance not established)`, + ); +} diff --git a/scripts/build-native-helpers.mjs b/scripts/build-native-helpers.mjs index 24d0bf953..96972692e 100644 --- a/scripts/build-native-helpers.mjs +++ b/scripts/build-native-helpers.mjs @@ -4,6 +4,10 @@ import os from "node:os"; import path from "node:path"; const projectRoot = process.cwd(); +const iosBuild = spawnSync(process.execPath, ["scripts/build-ios-device-helper.mjs"], { + stdio: "inherit", +}); +if (iosBuild.status !== 0) throw new Error("iOS device helper build failed"); const nativeRoot = path.join(projectRoot, "electron", "native"); const moduleCacheRoot = path.join(os.tmpdir(), "recordly-swift-module-cache"); diff --git a/scripts/fixtures/ios-capture/generate.mjs b/scripts/fixtures/ios-capture/generate.mjs new file mode 100644 index 000000000..7c2f81ecd --- /dev/null +++ b/scripts/fixtures/ios-capture/generate.mjs @@ -0,0 +1,305 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { resolveFixtureFFmpeg } from "../../verify-ios-capture-fixture.mjs"; +export const VARIANTS = [ + "portrait", + "landscape", + "odd-dimension", + "delayed-microphone", + "negative-offset", + "internal-gap", + "silent-static", + "timing-30m", +]; +function args(argv) { + const parsed = {}; + for (let i = 0; i < argv.length; i += 2) { + const key = argv[i]; + if (!["--output-dir", "--variant"].includes(key) || parsed[key] || !argv[i + 1]) + throw new Error("Use --output-dir /absolute/new-directory --variant name"); + parsed[key] = argv[i + 1]; + } + if (!path.isAbsolute(parsed["--output-dir"] ?? "") || !VARIANTS.includes(parsed["--variant"])) + throw new Error(`Choose an absolute output directory and variant: ${VARIANTS.join(", ")}`); + return parsed; +} +async function ffmpeg(binary, argv) { + await new Promise((resolve, reject) => { + const child = spawn( + binary, + ["-hide_banner", "-loglevel", "error", "-nostdin", "-n", ...argv], + { shell: false, stdio: ["ignore", "ignore", "pipe"] }, + ); + let error = ""; + child.stderr.on("data", (chunk) => { + error = (error + chunk.toString("utf8")).slice(-4096); + }); + child.once("error", reject); + child.once("close", (code) => (code === 0 ? resolve() : reject(new Error(error)))); + }); +} +export async function generateFixture(outputDir, variant) { + if (!path.isAbsolute(outputDir) || !VARIANTS.includes(variant)) + throw new Error("Invalid fixture output/variant"); + const binary = resolveFixtureFFmpeg(); + await fs.mkdir(outputDir, { mode: 0o700 }); + const directory = await fs.realpath(outputDir); + const width = variant === "landscape" ? 426 : variant === "odd-dimension" ? 241 : 240; + const height = variant === "landscape" ? 240 : variant === "odd-dimension" ? 427 : 426; + const durationMs = variant === "timing-30m" ? 1800000 : 4000; + const duration = durationMs / 1000; + const fps = 30; + const silent = variant === "silent-static"; + const offsetMs = + variant === "delayed-microphone" ? 250 : variant === "negative-offset" ? -120 : 0; + const localEvents = silent + ? [] + : variant === "timing-30m" + ? [500, durationMs - 500] + : variant === "internal-gap" + ? [500, 3500] + : [500, 2000, 3500]; + const eventTimesMs = localEvents + .map((t) => t + offsetMs) + .filter((t) => t >= 0 && t < durationMs); + const audioKind = ["delayed-microphone", "negative-offset"].includes(variant) + ? "microphone" + : "device-audio"; + const audioName = audioKind === "microphone" ? "microphone.mov" : "device-audio.mov"; + const source = path.join(directory, "source-video.mov"); + const audio = path.join(directory, audioName); + const movie = path.join(directory, "recording.mov"); + const fontCandidates = + process.platform === "darwin" + ? [ + "/System/Library/Fonts/Menlo.ttc", + "/System/Library/Fonts/Supplemental/Courier New.ttf", + ] + : ["/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf"]; + let font; + for (const candidate of fontCandidates) { + try { + await fs.access(candidate); + font = candidate; + break; + } catch { + /* Optional evidence unavailable; retain explicit fallback. */ + } + } + if (!font) throw new Error("Fixture generation requires an installed monospace font"); + const filters = [ + `format=yuv444p`, + `drawbox=x=0:y=0:w=iw:h=ih:color=white:t=1`, + `drawbox=x=10:y=45:w=60:h=40:color=red:t=fill`, + `drawbox=x=80:y=45:w=60:h=40:color=green:t=fill`, + `drawbox=x=150:y=45:w=60:h=40:color=blue:t=fill`, + `drawtext=fontfile='${font}':text='R G B':fontcolor=white:fontsize=12:x=15:y=58`, + `drawtext=fontfile='${font}':text='Fine UI 0123456789 AaBb':fontcolor=white:fontsize=10:x=${silent ? "12" : "12+10*sin(t)"}:y=100`, + ]; + if (!silent) + filters.push( + `drawbox=x=iw-25:y=ih-25:w=20:h=20:color=white:t=fill:enable='${eventTimesMs.map((t) => `between(t,${t / 1000},${t / 1000 + 1 / fps})`).join("+")}'`, + ); + const pixelFormat = variant === "odd-dimension" ? "yuv444p" : "yuv420p"; + await ffmpeg(binary, [ + "-f", + "lavfi", + "-i", + `color=c=0x18202a:s=${width}x${height}:r=${fps}:d=${duration},format=yuv444p`, + "-vf", + filters.join(","), + "-an", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "12", + "-pix_fmt", + pixelFormat, + "-x264-params", + "colorprim=bt709:transfer=bt709:colormatrix=bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-colorspace", + "bt709", + "-color_range", + "tv", + "-movflags", + "+faststart+write_colr", + source, + ]); + const pulses = localEvents + .map((t) => `between(t\\,${t / 1000}\\,${t / 1000 + 0.01})`) + .join("+"); + const expression = silent ? "0" : `0.8*sin(2*PI*1000*t)*(${pulses})`; + await ffmpeg(binary, [ + "-f", + "lavfi", + "-i", + `aevalsrc=${expression}:s=48000:d=${duration}`, + "-c:a", + "pcm_s16le", + audio, + ]); + const trim = Math.max(0, -offsetMs) / 1000, + delay = Math.max(0, offsetMs); + await ffmpeg(binary, [ + "-i", + source, + "-i", + audio, + "-filter_complex", + `[1:a:0]atrim=start=${trim},asetpts=PTS-STARTPTS,adelay=${delay}:all=1,aresample=48000,volume=1,apad,atrim=duration=${duration}[a]`, + "-map", + "0:v:0", + "-map", + "[a]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-ac", + "1", + "-t", + String(duration), + "-movflags", + "+faststart", + movie, + ]); + const hash = createHash("sha256").update(variant).digest("hex"); + const sessionId = `${hash.slice(0, 8)}-${hash.slice(8, 12)}-4${hash.slice(13, 16)}-8${hash.slice(17, 20)}-${hash.slice(20, 32)}`; + const sourceColour = { + colorPrimaries: "ITU_R_709_2", + transferFunction: "ITU_R_709_2", + ycbcrMatrix: "ITU_R_709_2", + fullRange: false, + }; + const format = { + codedWidth: width, + codedHeight: height, + displayWidth: width, + displayHeight: height, + codec: "avc1", + ...sourceColour, + transform: [1, 0, 0, 1, 0, 0], + observedFrameRate: fps, + fingerprint: `synthetic-${variant}`, + }; + const firstHostTime = { value: "1000000000000", timescale: 1000 }; + const audioStart = { + value: String(BigInt(firstHostTime.value) + BigInt(offsetMs)), + timescale: 1000, + }; + const mediaDuration = { value: String(durationMs), timescale: 1000 }; + const stream = { + mediaKind: audioKind, + firstHostTime: audioStart, + duration: mediaDuration, + rate: { numerator: "1", denominator: "1" }, + clockAnchor: { hostTime: audioStart, mediaTime: { value: "0", timescale: 1000 } }, + gaps: + variant === "internal-gap" + ? [ + { + start: { value: "1000", timescale: 1000 }, + duration: { value: "1500", timescale: 1000 }, + }, + ] + : [], + }; + const timing = { + version: 1, + timeline: "host-mapped", + gapsRepresentedInMedia: true, + streams: [stream], + }; + const nativeResult = { + sessionId, + stopReason: "user-stop", + mode: "passthrough", + format, + video: { + relativeName: "source-video.mov", + mediaKind: "video", + firstHostTime, + duration: mediaDuration, + sampleCount: duration * fps, + mediaFormat: { codec: "avc1", width, height }, + }, + [audioKind === "microphone" ? "microphone" : "deviceAudio"]: { + relativeName: audioName, + mediaKind: audioKind, + firstHostTime: audioStart, + duration: mediaDuration, + sampleCount: duration * 48000, + mediaFormat: { codec: "lpcm", sampleRate: 48000, channels: 1 }, + }, + timingFile: "native-timing.json", + timing, + }; + const expected = { + fixtureVersion: 1, + name: variant, + sessionId, + videoFile: "recording.mov", + videoMode: "passthrough", + durationMs, + durationToleranceMs: 50, + displayGeometry: { width, height, transform: [1, 0, 0, 1, 0, 0] }, + audio: { + required: true, + channels: 1, + eventToleranceMs: 80, + streams: [{ kind: audioKind, eventTimesMs }], + }, + expectedVideoSamples: duration * fps, + permittedSampleLoss: 0, + sourceColour, + terminalStatus: "completed", + provenance: + "Synthetic lavfi media generated by Recordly fixture utility; CC0-1.0. This is authored software fixture data, not a native writer or physical-device capture.", + compareSourceVideoPayload: true, + }; + await fs.writeFile(path.join(directory, "expected.json"), JSON.stringify(expected, null, 2)); + await fs.writeFile( + path.join(directory, "native-timing.json"), + JSON.stringify({ timing, result: nativeResult }, null, 2), + ); + await fs.writeFile( + path.join(directory, "capture-journal.json"), + JSON.stringify( + { + version: 1, + sessionId, + createdAt: "2026-09-08T00:00:00.000Z", + state: "committed", + nativeResult, + committedFile: "recording.mov", + syntheticFixture: true, + }, + null, + 2, + ), + ); + return { directory, expectedPath: path.join(directory, "expected.json") }; +} +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const parsed = args(process.argv.slice(2)); + const result = await generateFixture(parsed["--output-dir"], parsed["--variant"]); + process.stdout.write(`${JSON.stringify(result)}\n`); + } catch (error) { + process.stderr.write(`${String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/ios-helper-policy.mjs b/scripts/ios-helper-policy.mjs new file mode 100644 index 000000000..8283e2795 --- /dev/null +++ b/scripts/ios-helper-policy.mjs @@ -0,0 +1,115 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { accessSync, constants } from "node:fs"; + +export function validateIOSHelperFacts(facts, { architecture, signed = false }) { + const errors = []; + if (!facts.executable) errors.push("helper is not executable"); + if (facts.architecture !== architecture) errors.push("wrong helper architecture"); + if (!/^14\.0(?:\.0)?$/.test(facts.minimumOS ?? "")) errors.push("wrong deployment floor"); + if (facts.protocolVersion !== 1) errors.push("wrong control protocol"); + for (const key of ["NSCameraUsageDescription", "NSMicrophoneUsageDescription"]) { + if (typeof facts.privacy?.[key] !== "string" || !facts.privacy[key].trim()) + errors.push(`missing ${key}`); + } + if (signed) { + for (const key of [ + "com.apple.security.device.camera", + "com.apple.security.device.audio-input", + ]) { + if (facts.entitlements?.[key] !== true) errors.push(`missing signed ${key}`); + } + if (facts.entitlements?.["com.apple.security.get-task-allow"] === true) + errors.push("debug entitlement in distribution"); + } + return errors; +} + +function run(binary, args) { + return execFileSync(binary, args, { + encoding: "utf8", + timeout: 30_000, + maxBuffer: 1024 * 1024, + }); +} +function parsePlist(xml) { + const result = spawnSync("plutil", ["-convert", "json", "-o", "-", "-"], { + input: xml, + encoding: "utf8", + timeout: 10_000, + }); + if (result.status !== 0) throw new Error("Invalid helper privacy plist"); + return JSON.parse(result.stdout); +} + +export function decodeMachOPlistSection(section) { + const words = section + .split(/\r?\n/) + .flatMap((line) => + /^[0-9a-f]{8,16}\s/i.test(line) ? line.trim().split(/\s+/).slice(1) : [], + ); + return Buffer.concat( + words.map((word) => { + if (/^[0-9a-f]{2}$/i.test(word)) return Buffer.from(word, "hex"); + if (!/^[0-9a-f]{8}$/i.test(word)) throw new Error("Invalid Mach-O section bytes"); + const data = Buffer.alloc(4); + data.writeUInt32LE(Number.parseInt(word, 16)); + return data; + }), + ); +} + +export function inspectIOSHelper(binary, { architecture, signed = false, teamId } = {}) { + accessSync(binary, constants.X_OK); + const actualArch = run("lipo", ["-archs", binary]).trim(); + const minimumOS = run("vtool", ["-show-build", binary]).match(/minos\s+(\S+)/)?.[1]; + // otool prints bytes for Intel and little-endian 32-bit words for arm64. + const section = run("otool", ["-X", "-s", "__TEXT", "__info_plist", binary]); + const bytes = decodeMachOPlistSection(section); + const end = bytes.indexOf(""); + if (end < 0) throw new Error("Missing embedded helper privacy plist"); + const privacy = parsePlist(bytes.subarray(0, end + 8)); + let protocolVersion = privacy.RecordlyCaptureProtocolVersion; + const hostArch = process.arch === "arm64" ? "arm64" : "x86_64"; + if (actualArch === hostArch) { + const smoke = JSON.parse(run(binary, ["--self-test"])); + if (smoke.selfTest !== "passed") throw new Error("Helper self-test failed"); + protocolVersion = smoke.protocolVersion; + } + let entitlements; + if (signed) { + run("codesign", ["--verify", "--strict", binary]); + const signature = spawnSync("codesign", ["--display", "--verbose=4", binary], { + encoding: "utf8", + }); + if ( + signature.status !== 0 || + (teamId && !signature.stderr.includes(`TeamIdentifier=${teamId}`)) || + !signature.stderr.includes("runtime") + ) + throw new Error("Invalid helper distribution signature"); + const result = spawnSync( + "codesign", + ["--display", "--entitlements", "-", "--xml", binary], + { encoding: "utf8" }, + ); + const output = result.stdout + result.stderr; + const start = output.indexOf("", start); + if (result.status !== 0 || start < 0 || stop < 0) + throw new Error("Missing helper entitlements"); + entitlements = parsePlist(output.slice(start, stop + 8)); + } + const errors = validateIOSHelperFacts( + { + executable: true, + architecture: actualArch, + minimumOS, + protocolVersion, + privacy, + entitlements, + }, + { architecture, signed }, + ); + if (errors.length) throw new Error(errors.join("; ")); + return { architecture: actualArch, minimumOS, protocolVersion, signed }; +} diff --git a/scripts/smoke-packaged-binaries.mjs b/scripts/smoke-packaged-binaries.mjs index eeb01c17f..0bf97d6c0 100644 --- a/scripts/smoke-packaged-binaries.mjs +++ b/scripts/smoke-packaged-binaries.mjs @@ -1,6 +1,7 @@ import { execFileSync } from "node:child_process"; import { accessSync, constants, existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; +import { inspectIOSHelper } from "./ios-helper-policy.mjs"; const projectRoot = process.cwd(); const releaseRoot = path.join(projectRoot, "release"); @@ -187,6 +188,15 @@ function getExpectedNativeHelperFiles(archTag) { if (archTag.startsWith("darwin-")) { return [ + ...(packageJson.recordlyNativeIOSHelper + ? [ + { + name: "recordly-ios-device-helper", + label: "iOS device helper", + executable: true, + }, + ] + : []), { name: "recordly-screencapturekit-helper", label: "ScreenCaptureKit helper", @@ -259,6 +269,11 @@ function verifyNativeHelpers(unpackedRoot) { }, ); } + if (archTag.startsWith("darwin-") && packageJson.recordlyNativeIOSHelper) { + inspectIOSHelper(path.join(archDir, "recordly-ios-device-helper"), { + architecture: archTag === "darwin-arm64" ? "arm64" : "x86_64", + }); + } } } diff --git a/scripts/test-ios-device-helper.mjs b/scripts/test-ios-device-helper.mjs new file mode 100644 index 000000000..8d87b5dc4 --- /dev/null +++ b/scripts/test-ios-device-helper.mjs @@ -0,0 +1,33 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; + +if (process.platform !== "darwin") { + console.log("[ios-native] SKIPPED: native tests require macOS."); + process.exit(0); +} +const checkout = createHash("sha256").update(process.cwd()).digest("hex").slice(0, 12); +const cache = path.join(os.tmpdir(), `recordly-ios-module-cache-${checkout}`); +const result = spawnSync( + "swift", + [ + "test", + "--disable-sandbox", + "--package-path", + "electron/native/ios-device-capture", + "--scratch-path", + path.join(os.tmpdir(), `recordly-ios-swift-tests-${checkout}`), + ...process.argv.slice(2), + ], + { + stdio: "inherit", + env: { + ...process.env, + CLANG_MODULE_CACHE_PATH: cache, + SWIFTPM_MODULECACHE_OVERRIDE: cache, + }, + }, +); +if (result.error) throw result.error; +process.exit(result.status ?? 1); diff --git a/scripts/verify-ios-capture-fixture.mjs b/scripts/verify-ios-capture-fixture.mjs new file mode 100644 index 000000000..3ac031bd2 --- /dev/null +++ b/scripts/verify-ios-capture-fixture.mjs @@ -0,0 +1,495 @@ +import fs from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { spawn, spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +const require = createRequire(import.meta.url); +const repository = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +export function resolveFixtureFFmpeg() { + try { + const binary = require("ffmpeg-static"); + if (typeof binary === "string" && existsSync(binary)) return binary; + } catch { + /* Optional evidence unavailable; retain explicit fallback. */ + } + const lookup = spawnSync(process.platform === "win32" ? "where" : "which", ["ffmpeg"], { + encoding: "utf8", + shell: false, + }); + const candidates = [ + ...(lookup.status === 0 ? lookup.stdout.trim().split(/\r?\n/) : []), + "/opt/homebrew/bin/ffmpeg", + "/usr/local/bin/ffmpeg", + "/usr/bin/ffmpeg", + ]; + const binary = candidates.find((p) => p && existsSync(p)); + if (!binary) throw new Error("FFmpeg is unavailable"); + return binary; +} +export function parseArguments(args) { + const result = {}; + const allowed = new Set(["session-dir", "expected", "report"]); + for (let i = 0; i < args.length; i += 2) { + const key = args[i]?.replace(/^--/, ""); + if ( + !args[i]?.startsWith("--") || + !allowed.has(key) || + result[key] || + !args[i + 1] || + args[i + 1].startsWith("--") + ) + throw new Error("Expected --session-dir, --expected and --report exactly once"); + result[key] = args[i + 1]; + } + if ([...allowed].some((key) => !result[key] || !path.isAbsolute(result[key]))) + throw new Error("All three named arguments must be absolute paths"); + return result; +} +const finite = (n) => typeof n === "number" && Number.isFinite(n); +export function validateExpected(expected) { + if ( + !expected || + expected.fixtureVersion !== 1 || + typeof expected.name !== "string" || + !expected.name || + !["passthrough", "h264-encode"].includes(expected.videoMode) || + !finite(expected.durationMs) || + expected.durationMs <= 0 || + !finite(expected.durationToleranceMs) || + expected.durationToleranceMs < 0 + ) + throw new Error("Invalid fixture expectations"); + if ( + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + expected.sessionId, + ) || + !["recording.mov", "source-video.mov"].includes(expected.videoFile) + ) + throw new Error("Invalid fixture session"); + const geometry = expected.displayGeometry; + if ( + !geometry || + !finite(geometry.width) || + geometry.width <= 0 || + !finite(geometry.height) || + geometry.height <= 0 || + !Array.isArray(geometry.transform) || + geometry.transform.length !== 6 || + !geometry.transform.every(finite) + ) + throw new Error("Invalid display geometry"); + const audio = expected.audio; + if ( + !audio || + typeof audio.required !== "boolean" || + ![1, 2].includes(audio.channels) || + !finite(audio.eventToleranceMs) || + audio.eventToleranceMs < 0 || + audio.eventToleranceMs > 80 || + !Array.isArray(audio.streams) || + audio.streams.length > 2 + ) + throw new Error("Invalid audio expectations"); + for (const stream of audio.streams) + if ( + !["device-audio", "microphone"].includes(stream.kind) || + !Array.isArray(stream.eventTimesMs) || + !stream.eventTimesMs.every((n) => finite(n) && n >= 0 && n <= expected.durationMs) + ) + throw new Error("Invalid audio events"); + if ( + !Number.isInteger(expected.expectedVideoSamples) || + expected.expectedVideoSamples < 1 || + !Number.isInteger(expected.permittedSampleLoss) || + expected.permittedSampleLoss < 0 || + expected.permittedSampleLoss >= expected.expectedVideoSamples || + !expected.sourceColour || + !["completed", "interrupted"].includes(expected.terminalStatus) || + typeof expected.provenance !== "string" || + !expected.provenance + ) + throw new Error("Missing fixture provenance, media or terminal expectations"); + for (const key of ["colorPrimaries", "transferFunction", "ycbcrMatrix"]) + if (expected.sourceColour[key] !== null && typeof expected.sourceColour[key] !== "string") + throw new Error("Invalid colour expectation"); + if ( + expected.sourceColour.fullRange !== null && + typeof expected.sourceColour.fullRange !== "boolean" + ) + throw new Error("Invalid range expectation"); + return expected; +} +export function evaluateFixture(expected, observation) { + validateExpected(expected); + const checks = []; + const check = (name, passed, actual, requested) => + checks.push({ + name, + passed: Boolean(passed), + actual: actual ?? null, + expected: requested ?? null, + }); + const inspected = observation.inspection; + const video = inspected?.video; + check( + "decodable", + inspected?.decodable && video && observation.decodeSucceeded, + Boolean(inspected?.decodable && video && observation.decodeSucceeded), + true, + ); + check( + "display-geometry", + video?.displayWidth === expected.displayGeometry.width && + video?.displayHeight === expected.displayGeometry.height, + video ? { width: video.displayWidth, height: video.displayHeight } : null, + expected.displayGeometry, + ); + check( + "transform", + JSON.stringify(video?.transform) === JSON.stringify(expected.displayGeometry.transform), + video?.transform, + expected.displayGeometry.transform, + ); + let durationMs = null; + try { + durationMs = + (Number(BigInt(inspected.duration.value)) * 1000) / inspected.duration.timescale; + } catch { + /* Optional evidence unavailable; retain explicit fallback. */ + } + check( + "duration", + finite(durationMs) && + Math.abs(durationMs - expected.durationMs) <= expected.durationToleranceMs, + durationMs, + expected.durationMs, + ); + check( + "soundtrack", + expected.audio.required + ? Boolean( + inspected?.audio && + inspected.audio.channels === expected.audio.channels && + inspected.audio.sampleRate === 48000, + ) + : !inspected?.audio, + inspected?.audio ?? null, + expected.audio.required, + ); + const events = [...new Set(expected.audio.streams.flatMap((s) => s.eventTimesMs))].sort( + (a, b) => a - b, + ); + const observed = observation.audioEventTimesMs ?? []; + check( + "audio-event-timing", + events.length === observed.length && + events.every( + (time, index) => + Math.abs(time - observed[index]) <= expected.audio.eventToleranceMs, + ), + observed, + events, + ); + check( + "sample-loss", + Number.isInteger(observation.decodedVideoSamples) && + observation.decodedVideoSamples >= + expected.expectedVideoSamples - expected.permittedSampleLoss, + observation.decodedVideoSamples, + expected.expectedVideoSamples, + ); + check( + "colour", + Object.entries(expected.sourceColour).every(([key, value]) => video?.[key] === value), + video + ? Object.fromEntries(Object.keys(expected.sourceColour).map((key) => [key, video[key]])) + : null, + expected.sourceColour, + ); + check( + "video-mode", + observation.videoMode === expected.videoMode, + observation.videoMode, + expected.videoMode, + ); + check( + "terminal-status", + observation.terminalStatus === expected.terminalStatus, + observation.terminalStatus, + expected.terminalStatus, + ); + if (expected.compareSourceVideoPayload) + check( + "copied-video-payload", + Boolean(observation.sourcePayloadHash) && + observation.sourcePayloadHash === observation.finalPayloadHash, + observation.finalPayloadHash, + observation.sourcePayloadHash, + ); + return { + fixtureVersion: 1, + fixtureName: expected.name, + passed: checks.every((c) => c.passed), + checks, + }; +} +async function safeFile(root, name) { + if (name !== path.basename(name) || name.includes("\\")) + throw new Error("Unsafe fixture filename"); + const file = path.join(root, name); + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink() || (await fs.realpath(file)) !== file) + throw new Error("Unsafe fixture file"); + return file; +} +export async function inspectFixtureMedia(root, name, sessionId) { + if (process.platform !== "darwin") + throw new Error("Native media inspection requires macOS; fixture was not verified"); + const helper = path.join( + repository, + "electron", + "native", + "bin", + `darwin-${process.arch}`, + "recordly-ios-device-helper", + ); + await fs.access(helper); + return new Promise((resolve, reject) => { + const child = spawn(helper, [], { + shell: false, + stdio: ["pipe", "pipe", "pipe", "ignore"], + }); + let buffer = ""; + let settled = false; + let inspection; + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + finish(new Error("Native inspection timed out")); + }, 30000); + function finish(error) { + if (settled) return; + settled = true; + clearTimeout(timeout); + if (error) reject(error); + else resolve(inspection); + } + child.stderr.on("data", () => undefined); + child.stdout.on("data", (chunk) => { + buffer += chunk.toString("utf8"); + if (buffer.length > 65536) { + child.kill("SIGKILL"); + finish(new Error("Oversized inspector response")); + return; + } + while (true) { + const index = buffer.indexOf("\n"); + if (index < 0) break; + const line = buffer.slice(0, index); + buffer = buffer.slice(index + 1); + try { + const event = JSON.parse(line); + if (event.requestId === "fixture-inspect" && event.event === "error") { + child.stdin.end(); + finish( + new Error( + `Native inspection rejected media: ${event.payload?.code ?? "UNKNOWN"}`, + ), + ); + } + if (event.requestId === "fixture-inspect" && event.payload?.inspection) { + inspection = event.payload.inspection; + child.stdin.end(); + } + } catch { + child.kill("SIGKILL"); + finish(new Error("Malformed native inspector response")); + } + } + }); + child.once("error", finish); + child.once("close", (code) => + finish( + code === 0 && inspection + ? undefined + : new Error("Native inspection did not complete"), + ), + ); + child.stdin.write( + `${JSON.stringify({ protocolVersion: 1, requestId: "fixture-inspect", command: "inspectMedia", sessionId, payload: { relativeName: name }, storage: { sessionRoot: root, allowedRelativeNames: [name] } })}\n`, + ); + }); +} +async function runFFmpeg(binary, args, onStdout) { + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { shell: false, stdio: ["ignore", "pipe", "pipe"] }); + let diagnostics = ""; + child.stdout.on("data", onStdout); + child.stderr.on("data", (c) => { + diagnostics = (diagnostics + c.toString("utf8")).slice(-4096); + }); + child.once("error", reject); + child.once("close", (code) => + code === 0 ? resolve() : reject(new Error(`Fixture FFmpeg failed: ${diagnostics}`)), + ); + }); +} +export async function decodeFixtureAudio(binary, file) { + let carry = Buffer.alloc(0), + sampleIndex = 0, + lastEvent = -Infinity; + const events = []; + await runFFmpeg( + binary, + [ + "-hide_banner", + "-loglevel", + "error", + "-copyts", + "-i", + file, + "-map", + "0:a:0", + "-af", + "aresample=48000:async=1:first_pts=0", + "-f", + "f32le", + "-ac", + "1", + "-ar", + "48000", + "pipe:1", + ], + (chunk) => { + const data = carry.length ? Buffer.concat([carry, chunk]) : chunk; + const end = data.length - (data.length % 4); + for (let offset = 0; offset < end; offset += 4) { + const amplitude = Math.abs(data.readFloatLE(offset)); + const time = sampleIndex++ / 48; + if (amplitude > 0.06 && time - lastEvent > 100) { + events.push(time); + lastEvent = time; + } + } + carry = data.subarray(end); + }, + ); + return events; +} +export async function verifyFixture({ sessionDir, expectedPath, reportPath }, deps = {}) { + const expected = validateExpected(JSON.parse(await fs.readFile(expectedPath, "utf8"))); + const root = await fs.realpath(sessionDir); + if ((await fs.lstat(sessionDir)).isSymbolicLink()) throw new Error("Unsafe fixture directory"); + const observation = { decodeSucceeded: false, decodedVideoSamples: 0, audioEventTimesMs: [] }; + const errors = []; + try { + const movie = await safeFile(root, expected.videoFile); + const binary = deps.ffmpeg ?? resolveFixtureFFmpeg(); + observation.inspection = await (deps.inspectMedia ?? inspectFixtureMedia)( + root, + expected.videoFile, + expected.sessionId, + ); + let progress = ""; + await runFFmpeg( + binary, + [ + "-hide_banner", + "-loglevel", + "error", + "-xerror", + "-i", + movie, + "-map", + "0:v:0", + "-progress", + "pipe:1", + "-f", + "null", + "-", + ], + (chunk) => { + progress += chunk.toString("utf8"); + while (true) { + const newline = progress.indexOf("\n"); + if (newline < 0) break; + const line = progress.slice(0, newline); + progress = progress.slice(newline + 1); + const match = /^frame=(\d+)$/.exec(line); + if (match) observation.decodedVideoSamples = Number(match[1]); + } + }, + ); + observation.decodeSucceeded = true; + if (observation.inspection.audio) + observation.audioEventTimesMs = await decodeFixtureAudio(binary, movie); + if (expected.compareSourceVideoPayload) { + for (const [key, file] of [ + ["sourcePayloadHash", await safeFile(root, "source-video.mov")], + ["finalPayloadHash", movie], + ]) { + let hash = ""; + await runFFmpeg( + binary, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + file, + "-map", + "0:v:0", + "-c:v", + "copy", + "-f", + "hash", + "-hash", + "sha256", + "pipe:1", + ], + (chunk) => { + hash += chunk.toString("utf8"); + }, + ); + observation[key] = hash.trim(); + } + } + const journal = JSON.parse( + await fs.readFile(await safeFile(root, "capture-journal.json"), "utf8"), + ); + observation.videoMode = journal.nativeResult?.mode ?? journal.mode; + const reason = journal.nativeResult?.stopReason ?? journal.stopReason; + observation.terminalStatus = + journal.state === "committed" + ? ["user-stop", "userStop", "stopped", "stop", "completed"].includes(reason) + ? "completed" + : "interrupted" + : journal.state; + } catch (error) { + errors.push(String(error)); + } + const report = { + ...evaluateFixture(expected, observation), + errors, + inspectionSource: deps.inspectMedia ? "injected-test-inspector" : "native-helper", + physicalDeviceEvidence: false, + }; + report.passed = report.passed && errors.length === 0; + await fs.writeFile(reportPath, JSON.stringify(report, null, 2)); + return report; +} +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const args = parseArguments(process.argv.slice(2)); + const report = await verifyFixture({ + sessionDir: args["session-dir"], + expectedPath: args.expected, + reportPath: args.report, + }); + process.stdout.write(`${report.passed ? "PASS" : "FAIL"}: ${report.fixtureName}\n`); + process.exitCode = report.passed ? 0 : 1; + } catch (error) { + process.stderr.write(`${String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/verify-macos-distribution.mjs b/scripts/verify-macos-distribution.mjs index bd8e4bbcf..0fee66e25 100644 --- a/scripts/verify-macos-distribution.mjs +++ b/scripts/verify-macos-distribution.mjs @@ -17,6 +17,7 @@ import { import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { inspectIOSHelper } from "./ios-helper-policy.mjs"; import { assertValidAppleTeamId, collectArchitectureErrors, @@ -333,6 +334,27 @@ function verifyAppBundle(appPath, label, { arch, check, full, teamId, tempRoot } if (full) { verifyMachOBinaries(appPath, arch, check); + if (packageJson.recordlyNativeIOSHelper) + check("iOS capture helper privacy, protocol and signed entitlements", () => { + const binary = path.join( + appPath, + "Contents", + "Resources", + "app.asar.unpacked", + "electron", + "native", + "bin", + `darwin-${arch}`, + "recordly-ios-device-helper", + ); + return JSON.stringify( + inspectIOSHelper(binary, { + architecture: arch === "arm64" ? "arm64" : "x86_64", + signed: true, + teamId, + }), + ); + }); } check(`${label}: stapled notarization ticket`, () => { diff --git a/src/assets/device-frames/README.md b/src/assets/device-frames/README.md new file mode 100644 index 000000000..41e0e9b9b --- /dev/null +++ b/src/assets/device-frames/README.md @@ -0,0 +1,22 @@ +# iPhone frames + +The iPhone 16 Pro Black Titanium and White Titanium PNGs are from +[Maya](https://github.com/ronaldo-avalos/Maya/tree/ec960151bfdc3f1dfad67e41f934b75991a47002), +commit `ec960151bfdc3f1dfad67e41f934b75991a47002`, copyright 2026 Ronaldo Avalos. +They are used under the [MIT license](../../../public/third-party/Maya-LICENSE.txt), +also included in the built application under `third-party/Maya-LICENSE.txt`. + +Each original PNG is 450 × 920 with a transparent screen and an opaque Dynamic +Island. Maya defines a 402 × 874 screen at (24, 23), including one pixel of edge +bleed, and a 60-pixel screen radius. The fully transparent interior measures +400 × 872 at (25, 24). Recordly fits the recording into this aperture without +stretching it; other source aspects receive black letterboxing. Landscape rotates +the entire frame and its aperture together. + +These are raster assets. Export scales the bezel with the composition; the screen +recording retains its own rendering resolution. No device status bar is added. + +Original files: + +- [Black Titanium](https://raw.githubusercontent.com/ronaldo-avalos/Maya/ec960151bfdc3f1dfad67e41f934b75991a47002/Maya/Assets.xcassets/iphone%20frames/iPhone%2016%20Pro%20-%20Black%20Titanium.imageset/iPhone%2016%20Pro%20-%20Black%20Titanium.png) +- [White Titanium](https://raw.githubusercontent.com/ronaldo-avalos/Maya/ec960151bfdc3f1dfad67e41f934b75991a47002/Maya/Assets.xcassets/iphone%20frames/iPhone%2016%20Pro%20-%20White%20Titanium.imageset/iPhone%2016%20Pro%20-%20White%20Titanium.png) diff --git a/src/assets/device-frames/iphone-16-pro-black-titanium.png b/src/assets/device-frames/iphone-16-pro-black-titanium.png new file mode 100644 index 000000000..dfe1fd786 Binary files /dev/null and b/src/assets/device-frames/iphone-16-pro-black-titanium.png differ diff --git a/src/assets/device-frames/iphone-16-pro-white-titanium.png b/src/assets/device-frames/iphone-16-pro-white-titanium.png new file mode 100644 index 000000000..b724a1b47 Binary files /dev/null and b/src/assets/device-frames/iphone-16-pro-white-titanium.png differ diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 66cbe608b..93b9d6d82 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,6 +1,9 @@ +import { IOSCaptureStatus } from "./ios/IOSCaptureStatus"; +import { getIOSCapturePresentation } from "@/lib/iosCapturePresentation"; import { ArrowClockwiseIcon, CaretUpIcon, + DeviceMobileIcon, DotsThreeVerticalIcon, MicrophoneIcon, MicrophoneSlashIcon, @@ -58,6 +61,9 @@ function LaunchWindowContent() { const { openId, requestClose, requestOpen } = useLaunchPopoverCoordinator(); const { + ios, + mobile, + sourceSelectionLoaded, recording, paused, finalizing, @@ -81,7 +87,7 @@ function LaunchWindowContent() { preparePermissions, } = useScreenRecorder(); - const { elapsed, formatTime } = useRecordingTimer(recording, paused); + const { elapsed, formatTime } = useRecordingTimer(!mobile && recording, paused); const hudContentRef = useRef(null); const hudBarRef = useRef(null); @@ -96,16 +102,17 @@ function LaunchWindowContent() { refreshProjectLibrary, } = useLaunchWindowActions(); - const showWebcamControls = webcamEnabled && !recording; + const desktopControlsEnabled = sourceSelectionLoaded && !mobile; + const showWebcamControls = desktopControlsEnabled && webcamEnabled && !recording; const { devices, selectedDeviceId, setSelectedDeviceId } = useMicrophoneDevices( - microphoneEnabled || openId === "mic", + desktopControlsEnabled && (microphoneEnabled || openId === "mic"), microphoneDeviceId, ); const { devices: videoDevices, selectedDeviceId: selectedVideoDeviceId, setSelectedDeviceId: setSelectedVideoDeviceId, - } = useVideoDevices(webcamEnabled || openId === "webcam"); + } = useVideoDevices(desktopControlsEnabled && (webcamEnabled || openId === "webcam")); const { hudOverlayMousePassthroughSupported, @@ -146,7 +153,7 @@ function LaunchWindowContent() { setWebcamPreviewNode, setRecordingWebcamPreviewNode, } = useWebcamPreviewOverlay({ - webcamEnabled, + webcamEnabled: desktopControlsEnabled && webcamEnabled, webcamDeviceId, showWebcamControls, webcamPopoverOpen: openId === "webcam", @@ -207,7 +214,13 @@ function LaunchWindowContent() { ease: [0.22, 1, 0.36, 1] as const, }; - const recordingControls = ( + const recordingControls = mobile ? ( + void ios.stop().catch(() => undefined)} + onCancel={cancelRecording} + /> + ) : ( - + {mobile ? ( +