From 43658ae96f4ef3351c06c66324621088071a12bd Mon Sep 17 00:00:00 2001 From: Kevin Date: Sat, 3 Oct 2026 10:20:01 -0700 Subject: [PATCH 1/2] Crop window recordings natively and start still windows Window recordings captured the whole display and cropped every frame to the window with a CoreImage render on the sample queue. At display resolution that render took up to 200 ms per frame, which dropped video frames and delayed everything else on the queue. ScreenCaptureKit now crops the display to the window with sourceRect, and window tracking updates that rectangle as the window moves or resizes. A window that is not changing sends one complete frame when capture starts, and it could arrive before the writer was ready. The recording then never started. The newest frame is now kept until the writer can take it, and it starts the timeline when it is written. Co-Authored-By: Claude Opus 5.5 --- .../native/ScreenCaptureKitRecorder.swift | 264 +++++++++--------- .../native/ScreenCaptureKitRecorder.test.ts | 23 +- 2 files changed, 146 insertions(+), 141 deletions(-) diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index 002b1a67e..5c7749720 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -2,7 +2,6 @@ import Foundation import ScreenCaptureKit import AVFoundation import CoreGraphics -import CoreImage struct CaptureConfig: Codable { let fps: Int? @@ -38,17 +37,18 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private let queue = DispatchQueue(label: "recordly.screencapturekit.video") private var assetWriter: AVAssetWriter? private var videoInput: AVAssetWriterInput? - private var videoPixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor? - private var windowCropRect: CGRect? - private var windowCropDisplayId: CGDirectDisplayID? + /// Window captures crop the display natively with `sourceRect`. + private var streamConfiguration: SCStreamConfiguration? + private var captureFrame: CGRect? + private var captureDisplayId: CGDirectDisplayID? + private var trackedWindowInitialFrame: CGRect? private var excludedProcessIds = Set() - private var lastCroppedPixelBuffer: CVPixelBuffer? - private let imageContext = CIContext(options: [.cacheIntermediates: false]) private var systemAudioWriter: AVAssetWriter? private var systemAudioInput: AVAssetWriterInput? private var microphoneOnlyWriter: AVAssetWriter? private var microphoneOnlyInput: AVAssetWriterInput? private var stream: SCStream? + private var pendingFirstFrame: CMSampleBuffer? private var firstSampleTime: CMTime = .zero private var firstSystemAudioSampleTime: CMTime? private var firstMicrophoneSampleTime: CMTime? @@ -158,22 +158,21 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { excludingApplications: excludedApplications, exceptingWindows: [] ) - windowCropRect = CGRect( - x: (captureRect.minX - display.frame.minX) / display.frame.width, - y: (captureRect.minY - display.frame.minY) / display.frame.height, - width: captureRect.width / display.frame.width, - height: captureRect.height / display.frame.height - ) - windowCropDisplayId = display.displayID + // ScreenCaptureKit crops the display to the window natively, so menus and + // popovers over the window are still captured as they appear on screen. + streamConfig.sourceRect = Self.sourceRect(for: captureRect, on: display) + trackedWindowInitialFrame = window.frame + captureFrame = visibleFrame + captureDisplayId = display.displayID outputWidth = max(2, Int(captureRect.width) * scaleFactor) & ~1 outputHeight = max(2, Int(captureRect.height) * scaleFactor) & ~1 - streamConfig.width = max(2, Int(display.frame.width) * scaleFactor) - streamConfig.height = max(2, Int(display.frame.height) * scaleFactor) - streamConfig.pixelFormat = kCVPixelFormatType_32BGRA + streamConfig.width = outputWidth + streamConfig.height = outputHeight } else { trackedWindowId = nil - windowCropRect = nil - windowCropDisplayId = nil + trackedWindowInitialFrame = nil + captureFrame = nil + captureDisplayId = nil let displayId = config.displayId ?? CGMainDisplayID() guard let display = availableContent.displays.first(where: { $0.displayID == displayId }) else { throw NSError(domain: "RecordlyCapture", code: 4, userInfo: [NSLocalizedDescriptionKey: "Display not found"]) @@ -191,6 +190,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { streamConfig.width = outputWidth streamConfig.height = outputHeight } + streamConfiguration = streamConfig let destinationURL: URL if let outputPath = config.outputPath, !outputPath.isEmpty { @@ -215,9 +215,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { let sourceVideoFormat = try CMVideoFormatDescription( videoCodecType: CMFormatDescription.MediaSubType( - rawValue: windowCropRect == nil - ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange - : kCVPixelFormatType_32BGRA + rawValue: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ), width: outputWidth, height: outputHeight @@ -249,16 +247,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { assetWriter.add(videoInput) self.videoInput = videoInput - videoPixelBufferAdaptor = windowCropRect.map { _ in - AVAssetWriterInputPixelBufferAdaptor( - assetWriterInput: videoInput, - sourcePixelBufferAttributes: [ - kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, - kCVPixelBufferWidthKey as String: outputWidth, - kCVPixelBufferHeightKey as String: outputHeight, - ] - ) - } // Add inline audio track directly to the video so the .mp4 always contains audio. // This eliminates the dependency on the post-recording ffmpeg mux step. @@ -355,6 +343,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { firstSampleTime = .zero lastVideoPresentationTime = .zero lastVideoDuration = .zero + queue.async { + self.appendPendingFirstFrame(attemptsRemaining: 100) + } startWindowValidationIfNeeded() } @@ -393,7 +384,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) { - guard sessionStarted, sampleBuffer.isValid, isRecording else { return } + guard sampleBuffer.isValid else { return } if outputType == .screen { guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]], @@ -404,47 +395,27 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return } - guard let videoInput = videoInput, + guard sessionStarted, isRecording, + let videoInput = videoInput, assetWriter?.status == .writing, - videoInput.isReadyForMoreMediaData else { return } - - // Only a complete frame that the writer can accept may establish time zero. - guard let presentationTime = adjustedPresentationTime(for: sampleBuffer, outputType: outputType) else { return } - if frameCount > 0 && CMTimeCompare(presentationTime, lastVideoPresentationTime) <= 0 { + videoInput.isReadyForMoreMediaData else { + // A still screen sends one complete frame and then nothing until it + // changes, so keep the newest frame until the writer can take it. + if frameCount == 0 && !isFinalizing { + pendingFirstFrame = sampleBuffer + } return } - lastSampleBuffer = sampleBuffer - let appended: Bool - if videoPixelBufferAdaptor != nil { - appended = appendCroppedVideoFrame(sampleBuffer, at: presentationTime) - } else { - let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp) - if let retimed = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) { - appended = videoInput.append(retimed) - } else { - appended = false - } - } - if appended { - lastVideoPresentationTime = presentationTime - lastVideoDuration = sampleBuffer.duration - frameCount += 1 - if frameCount == 1 { - // Signal readiness only after AVAssetWriter has accepted a - // real frame, so countdown warm-start cannot pause too early. - print("Recording started") - fflush(stdout) - } - } else if frameCount == 0 { - // A failed crop/append must not leave an empty interval before frame one. - firstSampleTime = .zero - } + // Only a complete frame that the writer can accept may establish time zero. + guard let presentationTime = adjustedPresentationTime(for: sampleBuffer.presentationTimeStamp, outputType: outputType) else { return } + appendVideoFrame(sampleBuffer, at: presentationTime, to: videoInput) return } + guard sessionStarted, isRecording else { return } guard frameCount > 0, - let presentationTime = adjustedPresentationTime(for: sampleBuffer, outputType: outputType) else { return } + let presentationTime = adjustedPresentationTime(for: sampleBuffer.presentationTimeStamp, outputType: outputType) else { return } if outputType == .audio { guard let systemAudioInput else { return } @@ -470,46 +441,54 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return } - private func appendCroppedVideoFrame(_ sampleBuffer: CMSampleBuffer, at presentationTime: CMTime) -> Bool { - guard let crop = windowCropRect, - let adaptor = videoPixelBufferAdaptor, - let pool = adaptor.pixelBufferPool, - let source = CMSampleBufferGetImageBuffer(sampleBuffer) else { return false } - - var destination: CVPixelBuffer? - guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &destination) == kCVReturnSuccess, - let destination else { return false } - - let sourceWidth = CGFloat(CVPixelBufferGetWidth(source)) - let sourceHeight = CGFloat(CVPixelBufferGetHeight(source)) - let sourceRect = CGRect( - x: crop.minX * sourceWidth, - y: (1 - crop.maxY) * sourceHeight, - width: crop.width * sourceWidth, - height: crop.height * sourceHeight - ) - let destinationSize = CGSize( - width: CVPixelBufferGetWidth(destination), - height: CVPixelBufferGetHeight(destination) - ) - let image = CIImage(cvPixelBuffer: source) - .cropped(to: sourceRect) - .transformed(by: CGAffineTransform(translationX: -sourceRect.minX, y: -sourceRect.minY)) - .transformed(by: CGAffineTransform( - scaleX: destinationSize.width / sourceRect.width, - y: destinationSize.height / sourceRect.height - )) - let bounds = CGRect(origin: .zero, size: destinationSize) - imageContext.render( - image, - to: destination, - bounds: bounds, - colorSpace: CGColorSpace(name: CGColorSpace.sRGB) - ) + /// Appends one complete frame at its timeline time. Runs on the video queue. + private func appendVideoFrame(_ sampleBuffer: CMSampleBuffer, at presentationTime: CMTime, to videoInput: AVAssetWriterInput) { + if frameCount > 0 && CMTimeCompare(presentationTime, lastVideoPresentationTime) <= 0 { + return + } - let appended = adaptor.append(destination, withPresentationTime: presentationTime) - if appended { lastCroppedPixelBuffer = destination } - return appended + lastSampleBuffer = sampleBuffer + let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp) + let appended: Bool + if let retimed = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) { + appended = videoInput.append(retimed) + } else { + appended = false + } + if appended { + pendingFirstFrame = nil + lastVideoPresentationTime = presentationTime + lastVideoDuration = sampleBuffer.duration + frameCount += 1 + if frameCount == 1 { + // Signal readiness only after AVAssetWriter has accepted a + // real frame, so countdown warm-start cannot pause too early. + print("Recording started") + fflush(stdout) + } + } else if frameCount == 0 { + // A failed append must not leave an empty interval before frame one. + firstSampleTime = .zero + } + } + + /// Writes the frame that arrived before the writer was ready, so recording a still + /// window starts without waiting for something on screen to change. The frame shows + /// the screen as it is now, so it starts the timeline now. Runs on the video queue + /// and retries until the writer accepts a frame. + private func appendPendingFirstFrame(attemptsRemaining: Int) { + guard isRecording, frameCount == 0 else { return } + if let pendingFirstFrame, + let videoInput, + assetWriter?.status == .writing, + videoInput.isReadyForMoreMediaData, + let presentationTime = adjustedPresentationTime(for: CMClockGetTime(CMClockGetHostTimeClock()), outputType: .screen) { + appendVideoFrame(pendingFirstFrame, at: presentationTime, to: videoInput) + } + guard frameCount == 0, attemptsRemaining > 0 else { return } + queue.asyncAfter(deadline: .now() + .milliseconds(50)) { + self.appendPendingFirstFrame(attemptsRemaining: attemptsRemaining - 1) + } } func stream(_ stream: SCStream, didStopWithError error: Error) { @@ -596,13 +575,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { let videoInput = videoInput, await waitUntilReady(videoInput, of: assetWriter) { let additionalTime = lastVideoPresentationTime + frameDuration(for: originalBuffer) - if let adaptor = videoPixelBufferAdaptor, let pixelBuffer = lastCroppedPixelBuffer { - adaptor.append(pixelBuffer, withPresentationTime: additionalTime) - } else { - let timing = CMSampleTimingInfo(duration: originalBuffer.duration, presentationTimeStamp: additionalTime, decodeTimeStamp: originalBuffer.decodeTimeStamp) - if let additionalSampleBuffer = try? CMSampleBuffer(copying: originalBuffer, withNewTiming: [timing]) { + let timing = CMSampleTimingInfo(duration: originalBuffer.duration, presentationTimeStamp: additionalTime, decodeTimeStamp: originalBuffer.decodeTimeStamp) + if let additionalSampleBuffer = try? CMSampleBuffer(copying: originalBuffer, withNewTiming: [timing]) { videoInput.append(additionalSampleBuffer) - } } } @@ -639,11 +614,12 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { let path = outputURL?.path ?? "" assetWriter = nil videoInput = nil - videoPixelBufferAdaptor = nil - windowCropRect = nil - windowCropDisplayId = nil + streamConfiguration = nil + captureFrame = nil + captureDisplayId = nil + trackedWindowInitialFrame = nil + pendingFirstFrame = nil excludedProcessIds.removeAll() - lastCroppedPixelBuffer = nil systemAudioWriter = nil systemAudioInput = nil microphoneOnlyWriter = nil @@ -714,12 +690,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { ]) } - private func adjustedPresentationTime(for sampleBuffer: CMSampleBuffer, outputType: SCStreamOutputType) -> CMTime? { + private func adjustedPresentationTime(for sampleTime: CMTime, outputType: SCStreamOutputType) -> CMTime? { if isPaused { return nil } - let sampleTime = sampleBuffer.presentationTimeStamp if pendingResumeAdjustment { // Audio and video callbacks share this queue but their timestamps can be // offset slightly. Anchor the post-countdown adjustment to video and drop @@ -853,8 +828,14 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return supportsConfigSelector && supportsDeviceSelector && supportsOutputType } + /// Follows the recorded window as it moves, resizes or changes display. The crop + /// keeps the accessibility inset measured at start by moving with the window frame. private func startWindowValidationIfNeeded() { - guard let trackedWindowId else { + guard let trackedWindowId, + let initialWindowFrame = trackedWindowInitialFrame, + let initialCaptureFrame = captureFrame, + let initialDisplayId = captureDisplayId, + let streamConfiguration else { windowValidationTask?.cancel() windowValidationTask = nil return @@ -862,11 +843,12 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { windowValidationTask?.cancel() windowValidationTask = Task.detached(priority: .utility) { [weak self] in - guard let self else { return } + var currentDisplayId = initialDisplayId + var currentSourceRect = streamConfiguration.sourceRect while !Task.isCancelled { try? await Task.sleep(nanoseconds: 500_000_000) if Task.isCancelled { return } - guard self.isRecording else { return } + guard let self, self.isRecording else { return } let availableContent: SCShareableContent do { @@ -895,19 +877,20 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return } + let frame = CGRect( + x: initialCaptureFrame.minX + (window.frame.minX - initialWindowFrame.minX), + y: initialCaptureFrame.minY + (window.frame.minY - initialWindowFrame.minY), + width: max(2, initialCaptureFrame.width + (window.frame.width - initialWindowFrame.width)), + height: max(2, initialCaptureFrame.height + (window.frame.height - initialWindowFrame.height)) + ) guard let display = Self.captureDisplay(for: window.frame, from: availableContent.displays) else { continue } - let captureRect = window.frame.intersection(display.frame) - guard captureRect.width > 0, captureRect.height > 0 else { continue } - let cropRect = CGRect( - x: (captureRect.minX - display.frame.minX) / display.frame.width, - y: (captureRect.minY - display.frame.minY) / display.frame.height, - width: captureRect.width / display.frame.width, - height: captureRect.height / display.frame.height - ) + let captureRect = frame.intersection(display.frame) + guard captureRect.width > 0, captureRect.height > 0, let activeStream = self.stream else { continue } + let sourceRect = Self.sourceRect(for: captureRect, on: display) - if self.windowCropDisplayId != display.displayID, let activeStream = self.stream { + if currentDisplayId != display.displayID { let excludedApplications = availableContent.applications.filter { self.excludedProcessIds.contains($0.processID) } @@ -918,24 +901,35 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { ) do { try await activeStream.updateContentFilter(filter) + currentDisplayId = display.displayID } catch { continue } } - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.queue.async { - if self.isRecording { - self.windowCropRect = cropRect - self.windowCropDisplayId = display.displayID - } - continuation.resume() + if !Self.rect(sourceRect, isCloseTo: currentSourceRect) { + streamConfiguration.sourceRect = sourceRect + do { + try await activeStream.updateConfiguration(streamConfiguration) + currentSourceRect = sourceRect + } catch { + continue } } } } } + /// `sourceRect` is in points relative to the display's top-left corner. + private static func sourceRect(for captureRect: CGRect, on display: SCDisplay) -> CGRect { + captureRect.offsetBy(dx: -display.frame.minX, dy: -display.frame.minY) + } + + private static func rect(_ lhs: CGRect, isCloseTo rhs: CGRect) -> Bool { + abs(lhs.minX - rhs.minX) < 0.5 && abs(lhs.minY - rhs.minY) < 0.5 + && abs(lhs.width - rhs.width) < 0.5 && abs(lhs.height - rhs.height) < 0.5 + } + private static func captureDisplay(for frame: CGRect, from displays: [SCDisplay]) -> SCDisplay? { let midpoint = CGPoint(x: frame.midX, y: frame.midY) return displays.first(where: { $0.frame.contains(midpoint) }) diff --git a/electron/native/ScreenCaptureKitRecorder.test.ts b/electron/native/ScreenCaptureKitRecorder.test.ts index bf787842e..f6ab004a0 100644 --- a/electron/native/ScreenCaptureKitRecorder.test.ts +++ b/electron/native/ScreenCaptureKitRecorder.test.ts @@ -74,16 +74,18 @@ describe("ScreenCaptureKitRecorder colour metadata", () => { }); describe("ScreenCaptureKitRecorder window capture", () => { - it("records the display and crops it to the selected window bounds", () => { - expect(recorderSource).not.toContain("streamConfig.sourceRect"); + it("crops the display natively to the selected window bounds", () => { expect(recorderSource).not.toContain("desktopIndependentWindow"); + expect(recorderSource).not.toContain("CIContext"); expect(recorderSource).toContain( "visibleFrame = CGRect(x: x, y: y, width: width, height: height)", ); expect(recorderSource).toContain( "let captureRect = visibleFrame.intersection(display.frame)", ); - expect(recorderSource).toContain("appendCroppedVideoFrame(sampleBuffer"); + expect(recorderSource).toContain( + "streamConfig.sourceRect = Self.sourceRect(for: captureRect, on: display)", + ); }); it("refreshes the crop and capture display while the window moves or resizes", () => { @@ -91,13 +93,22 @@ describe("ScreenCaptureKitRecorder window capture", () => { "guard let display = Self.captureDisplay(for: window.frame", ); expect(recorderSource).toContain("try await activeStream.updateContentFilter(filter)"); - expect(recorderSource).toContain("self.windowCropRect = cropRect"); + expect(recorderSource).toContain( + "try await activeStream.updateConfiguration(streamConfiguration)", + ); }); -}); + it("writes the frame that arrived before the writer was ready", () => { + expect(recorderSource).toContain("pendingFirstFrame = sampleBuffer"); + expect(recorderSource).toContain("self.appendPendingFirstFrame(attemptsRemaining: 100)"); + }); +}); describe("ScreenCaptureKitRecorder first frame timing", () => { - const callback = recorderSource.slice(recorderSource.indexOf("func stream(_ stream:"), recorderSource.indexOf("func stream(_ stream:") + 5000); + const callback = recorderSource.slice( + recorderSource.indexOf("func stream(_ stream:"), + recorderSource.indexOf("func stream(_ stream:") + 5000, + ); it("validates a complete frame and writer readiness before setting time zero", () => { const clock = callback.indexOf("adjustedPresentationTime(for:"); expect(clock).toBeGreaterThan(callback.indexOf("status == .complete")); From cc2a812f8f44851822f24b75303286d57005c0f0 Mon Sep 17 00:00:00 2001 From: Kevin Date: Sat, 3 Oct 2026 11:13:16 -0700 Subject: [PATCH 2/2] Record window pixels one to one and size the bitrate for 60 fps The crop rectangle followed the window frame in points, and the output size was rounded down to an even number of pixels. On a 1x display a window with an odd width or height was therefore scaled by a fraction of a pixel, which turned one-pixel detail into gray. The crop now snaps to whole pixels and trims to an even size, so the recording holds the screen's own pixels. AVOutputSettingsAssistant sizes its bitrate and keyframe interval for 30 fps, but recordings run at 60. Scale the bitrate to the capture rate and keep one keyframe per second, so text stays sharp while the screen scrolls. Co-Authored-By: Claude Opus 5.5 --- .../native/ScreenCaptureKitRecorder.swift | 43 ++++++++++++++++--- .../native/ScreenCaptureKitRecorder.test.ts | 11 ++++- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index 5c7749720..cd35521d7 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -160,12 +160,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { ) // ScreenCaptureKit crops the display to the window natively, so menus and // popovers over the window are still captured as they appear on screen. - streamConfig.sourceRect = Self.sourceRect(for: captureRect, on: display) + let sourceRect = Self.sourceRect(for: captureRect, on: display, scale: scaleFactor) + streamConfig.sourceRect = sourceRect trackedWindowInitialFrame = window.frame captureFrame = visibleFrame captureDisplayId = display.displayID - outputWidth = max(2, Int(captureRect.width) * scaleFactor) & ~1 - outputHeight = max(2, Int(captureRect.height) * scaleFactor) & ~1 + outputWidth = Int((sourceRect.width * CGFloat(scaleFactor)).rounded()) + outputHeight = Int((sourceRect.height * CGFloat(scaleFactor)).rounded()) streamConfig.width = outputWidth streamConfig.height = outputHeight } else { @@ -228,6 +229,17 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { outputSettings[AVVideoWidthKey] = outputWidth outputSettings[AVVideoHeightKey] = outputHeight + // The assistant sizes the bitrate for 30 fps. Scale it to the capture rate so + // text stays sharp while the screen scrolls or animates. + if var compression = outputSettings[AVVideoCompressionPropertiesKey] as? [String: Any] { + let assistantFPS = max(1, compression[AVVideoExpectedSourceFrameRateKey] as? Int ?? 30) + if let averageBitRate = compression[AVVideoAverageBitRateKey] as? Int { + compression[AVVideoAverageBitRateKey] = averageBitRate * requestedFPS / assistantFPS + } + compression[AVVideoExpectedSourceFrameRateKey] = requestedFPS + compression[AVVideoMaxKeyFrameIntervalKey] = requestedFPS + outputSettings[AVVideoCompressionPropertiesKey] = compression + } outputSettings[AVVideoColorPropertiesKey] = [ AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2, @@ -888,7 +900,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } let captureRect = frame.intersection(display.frame) guard captureRect.width > 0, captureRect.height > 0, let activeStream = self.stream else { continue } - let sourceRect = Self.sourceRect(for: captureRect, on: display) + let sourceRect = Self.sourceRect( + for: captureRect, + on: display, + scale: Self.scaleFactor(for: display.displayID) + ) if currentDisplayId != display.displayID { let excludedApplications = availableContent.applications.filter { @@ -920,9 +936,22 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } - /// `sourceRect` is in points relative to the display's top-left corner. - private static func sourceRect(for captureRect: CGRect, on display: SCDisplay) -> CGRect { - captureRect.offsetBy(dx: -display.frame.minX, dy: -display.frame.minY) + /// `sourceRect` is in points relative to the display's top-left corner. Its edges + /// sit on whole pixels and its size is an even number of pixels, so the recording + /// gets the screen's own pixels instead of a resampled copy. + private static func sourceRect(for captureRect: CGRect, on display: SCDisplay, scale: Int) -> CGRect { + let pixelsPerPoint = CGFloat(scale) + let local = captureRect.offsetBy(dx: -display.frame.minX, dy: -display.frame.minY) + let left = (local.minX * pixelsPerPoint).rounded() + let top = (local.minY * pixelsPerPoint).rounded() + let width = max(2, Int((local.maxX * pixelsPerPoint).rounded() - left) & ~1) + let height = max(2, Int((local.maxY * pixelsPerPoint).rounded() - top) & ~1) + return CGRect( + x: left / pixelsPerPoint, + y: top / pixelsPerPoint, + width: CGFloat(width) / pixelsPerPoint, + height: CGFloat(height) / pixelsPerPoint + ) } private static func rect(_ lhs: CGRect, isCloseTo rhs: CGRect) -> Bool { diff --git a/electron/native/ScreenCaptureKitRecorder.test.ts b/electron/native/ScreenCaptureKitRecorder.test.ts index f6ab004a0..c647ee9c2 100644 --- a/electron/native/ScreenCaptureKitRecorder.test.ts +++ b/electron/native/ScreenCaptureKitRecorder.test.ts @@ -84,7 +84,16 @@ describe("ScreenCaptureKitRecorder window capture", () => { "let captureRect = visibleFrame.intersection(display.frame)", ); expect(recorderSource).toContain( - "streamConfig.sourceRect = Self.sourceRect(for: captureRect, on: display)", + "let sourceRect = Self.sourceRect(for: captureRect, on: display, scale: scaleFactor)", + ); + expect(recorderSource).toContain("streamConfig.sourceRect = sourceRect"); + }); + + it("records whole screen pixels at a bitrate sized for the frame rate", () => { + expect(recorderSource).toMatch(/let width = max\(2, Int\(.*\) & ~1\)/); + expect(recorderSource).toContain("averageBitRate * requestedFPS / assistantFPS"); + expect(recorderSource).toContain( + "compression[AVVideoExpectedSourceFrameRateKey] = requestedFPS", ); });