diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index 002b1a67e..cd35521d7 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -2,7 +2,6 @@ import Foundation import ScreenCaptureKit import AVFoundation import CoreGraphics -import CoreImage struct CaptureConfig: Codable { let fps: Int? @@ -38,17 +37,18 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private let queue = DispatchQueue(label: "recordly.screencapturekit.video") private var assetWriter: AVAssetWriter? private var videoInput: AVAssetWriterInput? - private var videoPixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor? - private var windowCropRect: CGRect? - private var windowCropDisplayId: CGDirectDisplayID? + /// Window captures crop the display natively with `sourceRect`. + private var streamConfiguration: SCStreamConfiguration? + private var captureFrame: CGRect? + private var captureDisplayId: CGDirectDisplayID? + private var trackedWindowInitialFrame: CGRect? private var excludedProcessIds = Set() - private var lastCroppedPixelBuffer: CVPixelBuffer? - private let imageContext = CIContext(options: [.cacheIntermediates: false]) private var systemAudioWriter: AVAssetWriter? private var systemAudioInput: AVAssetWriterInput? private var microphoneOnlyWriter: AVAssetWriter? private var microphoneOnlyInput: AVAssetWriterInput? private var stream: SCStream? + private var pendingFirstFrame: CMSampleBuffer? private var firstSampleTime: CMTime = .zero private var firstSystemAudioSampleTime: CMTime? private var firstMicrophoneSampleTime: CMTime? @@ -158,22 +158,22 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { excludingApplications: excludedApplications, exceptingWindows: [] ) - windowCropRect = CGRect( - x: (captureRect.minX - display.frame.minX) / display.frame.width, - y: (captureRect.minY - display.frame.minY) / display.frame.height, - width: captureRect.width / display.frame.width, - height: captureRect.height / display.frame.height - ) - windowCropDisplayId = display.displayID - outputWidth = max(2, Int(captureRect.width) * scaleFactor) & ~1 - outputHeight = max(2, Int(captureRect.height) * scaleFactor) & ~1 - streamConfig.width = max(2, Int(display.frame.width) * scaleFactor) - streamConfig.height = max(2, Int(display.frame.height) * scaleFactor) - streamConfig.pixelFormat = kCVPixelFormatType_32BGRA + // ScreenCaptureKit crops the display to the window natively, so menus and + // popovers over the window are still captured as they appear on screen. + let sourceRect = Self.sourceRect(for: captureRect, on: display, scale: scaleFactor) + streamConfig.sourceRect = sourceRect + trackedWindowInitialFrame = window.frame + captureFrame = visibleFrame + captureDisplayId = display.displayID + outputWidth = Int((sourceRect.width * CGFloat(scaleFactor)).rounded()) + outputHeight = Int((sourceRect.height * CGFloat(scaleFactor)).rounded()) + streamConfig.width = outputWidth + streamConfig.height = outputHeight } else { trackedWindowId = nil - windowCropRect = nil - windowCropDisplayId = nil + trackedWindowInitialFrame = nil + captureFrame = nil + captureDisplayId = nil let displayId = config.displayId ?? CGMainDisplayID() guard let display = availableContent.displays.first(where: { $0.displayID == displayId }) else { throw NSError(domain: "RecordlyCapture", code: 4, userInfo: [NSLocalizedDescriptionKey: "Display not found"]) @@ -191,6 +191,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { streamConfig.width = outputWidth streamConfig.height = outputHeight } + streamConfiguration = streamConfig let destinationURL: URL if let outputPath = config.outputPath, !outputPath.isEmpty { @@ -215,9 +216,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { let sourceVideoFormat = try CMVideoFormatDescription( videoCodecType: CMFormatDescription.MediaSubType( - rawValue: windowCropRect == nil - ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange - : kCVPixelFormatType_32BGRA + rawValue: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ), width: outputWidth, height: outputHeight @@ -230,6 +229,17 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { outputSettings[AVVideoWidthKey] = outputWidth outputSettings[AVVideoHeightKey] = outputHeight + // The assistant sizes the bitrate for 30 fps. Scale it to the capture rate so + // text stays sharp while the screen scrolls or animates. + if var compression = outputSettings[AVVideoCompressionPropertiesKey] as? [String: Any] { + let assistantFPS = max(1, compression[AVVideoExpectedSourceFrameRateKey] as? Int ?? 30) + if let averageBitRate = compression[AVVideoAverageBitRateKey] as? Int { + compression[AVVideoAverageBitRateKey] = averageBitRate * requestedFPS / assistantFPS + } + compression[AVVideoExpectedSourceFrameRateKey] = requestedFPS + compression[AVVideoMaxKeyFrameIntervalKey] = requestedFPS + outputSettings[AVVideoCompressionPropertiesKey] = compression + } outputSettings[AVVideoColorPropertiesKey] = [ AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2, @@ -249,16 +259,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { assetWriter.add(videoInput) self.videoInput = videoInput - videoPixelBufferAdaptor = windowCropRect.map { _ in - AVAssetWriterInputPixelBufferAdaptor( - assetWriterInput: videoInput, - sourcePixelBufferAttributes: [ - kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, - kCVPixelBufferWidthKey as String: outputWidth, - kCVPixelBufferHeightKey as String: outputHeight, - ] - ) - } // Add inline audio track directly to the video so the .mp4 always contains audio. // This eliminates the dependency on the post-recording ffmpeg mux step. @@ -355,6 +355,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { firstSampleTime = .zero lastVideoPresentationTime = .zero lastVideoDuration = .zero + queue.async { + self.appendPendingFirstFrame(attemptsRemaining: 100) + } startWindowValidationIfNeeded() } @@ -393,7 +396,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) { - guard sessionStarted, sampleBuffer.isValid, isRecording else { return } + guard sampleBuffer.isValid else { return } if outputType == .screen { guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]], @@ -404,47 +407,27 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return } - guard let videoInput = videoInput, + guard sessionStarted, isRecording, + let videoInput = videoInput, assetWriter?.status == .writing, - videoInput.isReadyForMoreMediaData else { return } - - // Only a complete frame that the writer can accept may establish time zero. - guard let presentationTime = adjustedPresentationTime(for: sampleBuffer, outputType: outputType) else { return } - if frameCount > 0 && CMTimeCompare(presentationTime, lastVideoPresentationTime) <= 0 { + videoInput.isReadyForMoreMediaData else { + // A still screen sends one complete frame and then nothing until it + // changes, so keep the newest frame until the writer can take it. + if frameCount == 0 && !isFinalizing { + pendingFirstFrame = sampleBuffer + } return } - lastSampleBuffer = sampleBuffer - let appended: Bool - if videoPixelBufferAdaptor != nil { - appended = appendCroppedVideoFrame(sampleBuffer, at: presentationTime) - } else { - let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp) - if let retimed = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) { - appended = videoInput.append(retimed) - } else { - appended = false - } - } - if appended { - lastVideoPresentationTime = presentationTime - lastVideoDuration = sampleBuffer.duration - frameCount += 1 - if frameCount == 1 { - // Signal readiness only after AVAssetWriter has accepted a - // real frame, so countdown warm-start cannot pause too early. - print("Recording started") - fflush(stdout) - } - } else if frameCount == 0 { - // A failed crop/append must not leave an empty interval before frame one. - firstSampleTime = .zero - } + // Only a complete frame that the writer can accept may establish time zero. + guard let presentationTime = adjustedPresentationTime(for: sampleBuffer.presentationTimeStamp, outputType: outputType) else { return } + appendVideoFrame(sampleBuffer, at: presentationTime, to: videoInput) return } + guard sessionStarted, isRecording else { return } guard frameCount > 0, - let presentationTime = adjustedPresentationTime(for: sampleBuffer, outputType: outputType) else { return } + let presentationTime = adjustedPresentationTime(for: sampleBuffer.presentationTimeStamp, outputType: outputType) else { return } if outputType == .audio { guard let systemAudioInput else { return } @@ -470,46 +453,54 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return } - private func appendCroppedVideoFrame(_ sampleBuffer: CMSampleBuffer, at presentationTime: CMTime) -> Bool { - guard let crop = windowCropRect, - let adaptor = videoPixelBufferAdaptor, - let pool = adaptor.pixelBufferPool, - let source = CMSampleBufferGetImageBuffer(sampleBuffer) else { return false } - - var destination: CVPixelBuffer? - guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &destination) == kCVReturnSuccess, - let destination else { return false } - - let sourceWidth = CGFloat(CVPixelBufferGetWidth(source)) - let sourceHeight = CGFloat(CVPixelBufferGetHeight(source)) - let sourceRect = CGRect( - x: crop.minX * sourceWidth, - y: (1 - crop.maxY) * sourceHeight, - width: crop.width * sourceWidth, - height: crop.height * sourceHeight - ) - let destinationSize = CGSize( - width: CVPixelBufferGetWidth(destination), - height: CVPixelBufferGetHeight(destination) - ) - let image = CIImage(cvPixelBuffer: source) - .cropped(to: sourceRect) - .transformed(by: CGAffineTransform(translationX: -sourceRect.minX, y: -sourceRect.minY)) - .transformed(by: CGAffineTransform( - scaleX: destinationSize.width / sourceRect.width, - y: destinationSize.height / sourceRect.height - )) - let bounds = CGRect(origin: .zero, size: destinationSize) - imageContext.render( - image, - to: destination, - bounds: bounds, - colorSpace: CGColorSpace(name: CGColorSpace.sRGB) - ) + /// Appends one complete frame at its timeline time. Runs on the video queue. + private func appendVideoFrame(_ sampleBuffer: CMSampleBuffer, at presentationTime: CMTime, to videoInput: AVAssetWriterInput) { + if frameCount > 0 && CMTimeCompare(presentationTime, lastVideoPresentationTime) <= 0 { + return + } + + lastSampleBuffer = sampleBuffer + let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp) + let appended: Bool + if let retimed = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) { + appended = videoInput.append(retimed) + } else { + appended = false + } + if appended { + pendingFirstFrame = nil + lastVideoPresentationTime = presentationTime + lastVideoDuration = sampleBuffer.duration + frameCount += 1 + if frameCount == 1 { + // Signal readiness only after AVAssetWriter has accepted a + // real frame, so countdown warm-start cannot pause too early. + print("Recording started") + fflush(stdout) + } + } else if frameCount == 0 { + // A failed append must not leave an empty interval before frame one. + firstSampleTime = .zero + } + } - let appended = adaptor.append(destination, withPresentationTime: presentationTime) - if appended { lastCroppedPixelBuffer = destination } - return appended + /// Writes the frame that arrived before the writer was ready, so recording a still + /// window starts without waiting for something on screen to change. The frame shows + /// the screen as it is now, so it starts the timeline now. Runs on the video queue + /// and retries until the writer accepts a frame. + private func appendPendingFirstFrame(attemptsRemaining: Int) { + guard isRecording, frameCount == 0 else { return } + if let pendingFirstFrame, + let videoInput, + assetWriter?.status == .writing, + videoInput.isReadyForMoreMediaData, + let presentationTime = adjustedPresentationTime(for: CMClockGetTime(CMClockGetHostTimeClock()), outputType: .screen) { + appendVideoFrame(pendingFirstFrame, at: presentationTime, to: videoInput) + } + guard frameCount == 0, attemptsRemaining > 0 else { return } + queue.asyncAfter(deadline: .now() + .milliseconds(50)) { + self.appendPendingFirstFrame(attemptsRemaining: attemptsRemaining - 1) + } } func stream(_ stream: SCStream, didStopWithError error: Error) { @@ -596,13 +587,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { let videoInput = videoInput, await waitUntilReady(videoInput, of: assetWriter) { let additionalTime = lastVideoPresentationTime + frameDuration(for: originalBuffer) - if let adaptor = videoPixelBufferAdaptor, let pixelBuffer = lastCroppedPixelBuffer { - adaptor.append(pixelBuffer, withPresentationTime: additionalTime) - } else { - let timing = CMSampleTimingInfo(duration: originalBuffer.duration, presentationTimeStamp: additionalTime, decodeTimeStamp: originalBuffer.decodeTimeStamp) - if let additionalSampleBuffer = try? CMSampleBuffer(copying: originalBuffer, withNewTiming: [timing]) { + let timing = CMSampleTimingInfo(duration: originalBuffer.duration, presentationTimeStamp: additionalTime, decodeTimeStamp: originalBuffer.decodeTimeStamp) + if let additionalSampleBuffer = try? CMSampleBuffer(copying: originalBuffer, withNewTiming: [timing]) { videoInput.append(additionalSampleBuffer) - } } } @@ -639,11 +626,12 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { let path = outputURL?.path ?? "" assetWriter = nil videoInput = nil - videoPixelBufferAdaptor = nil - windowCropRect = nil - windowCropDisplayId = nil + streamConfiguration = nil + captureFrame = nil + captureDisplayId = nil + trackedWindowInitialFrame = nil + pendingFirstFrame = nil excludedProcessIds.removeAll() - lastCroppedPixelBuffer = nil systemAudioWriter = nil systemAudioInput = nil microphoneOnlyWriter = nil @@ -714,12 +702,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { ]) } - private func adjustedPresentationTime(for sampleBuffer: CMSampleBuffer, outputType: SCStreamOutputType) -> CMTime? { + private func adjustedPresentationTime(for sampleTime: CMTime, outputType: SCStreamOutputType) -> CMTime? { if isPaused { return nil } - let sampleTime = sampleBuffer.presentationTimeStamp if pendingResumeAdjustment { // Audio and video callbacks share this queue but their timestamps can be // offset slightly. Anchor the post-countdown adjustment to video and drop @@ -853,8 +840,14 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return supportsConfigSelector && supportsDeviceSelector && supportsOutputType } + /// Follows the recorded window as it moves, resizes or changes display. The crop + /// keeps the accessibility inset measured at start by moving with the window frame. private func startWindowValidationIfNeeded() { - guard let trackedWindowId else { + guard let trackedWindowId, + let initialWindowFrame = trackedWindowInitialFrame, + let initialCaptureFrame = captureFrame, + let initialDisplayId = captureDisplayId, + let streamConfiguration else { windowValidationTask?.cancel() windowValidationTask = nil return @@ -862,11 +855,12 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { windowValidationTask?.cancel() windowValidationTask = Task.detached(priority: .utility) { [weak self] in - guard let self else { return } + var currentDisplayId = initialDisplayId + var currentSourceRect = streamConfiguration.sourceRect while !Task.isCancelled { try? await Task.sleep(nanoseconds: 500_000_000) if Task.isCancelled { return } - guard self.isRecording else { return } + guard let self, self.isRecording else { return } let availableContent: SCShareableContent do { @@ -895,19 +889,24 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return } + let frame = CGRect( + x: initialCaptureFrame.minX + (window.frame.minX - initialWindowFrame.minX), + y: initialCaptureFrame.minY + (window.frame.minY - initialWindowFrame.minY), + width: max(2, initialCaptureFrame.width + (window.frame.width - initialWindowFrame.width)), + height: max(2, initialCaptureFrame.height + (window.frame.height - initialWindowFrame.height)) + ) guard let display = Self.captureDisplay(for: window.frame, from: availableContent.displays) else { continue } - let captureRect = window.frame.intersection(display.frame) - guard captureRect.width > 0, captureRect.height > 0 else { continue } - let cropRect = CGRect( - x: (captureRect.minX - display.frame.minX) / display.frame.width, - y: (captureRect.minY - display.frame.minY) / display.frame.height, - width: captureRect.width / display.frame.width, - height: captureRect.height / display.frame.height + let captureRect = frame.intersection(display.frame) + guard captureRect.width > 0, captureRect.height > 0, let activeStream = self.stream else { continue } + let sourceRect = Self.sourceRect( + for: captureRect, + on: display, + scale: Self.scaleFactor(for: display.displayID) ) - if self.windowCropDisplayId != display.displayID, let activeStream = self.stream { + if currentDisplayId != display.displayID { let excludedApplications = availableContent.applications.filter { self.excludedProcessIds.contains($0.processID) } @@ -918,24 +917,48 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { ) do { try await activeStream.updateContentFilter(filter) + currentDisplayId = display.displayID } catch { continue } } - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.queue.async { - if self.isRecording { - self.windowCropRect = cropRect - self.windowCropDisplayId = display.displayID - } - continuation.resume() + if !Self.rect(sourceRect, isCloseTo: currentSourceRect) { + streamConfiguration.sourceRect = sourceRect + do { + try await activeStream.updateConfiguration(streamConfiguration) + currentSourceRect = sourceRect + } catch { + continue } } } } } + /// `sourceRect` is in points relative to the display's top-left corner. Its edges + /// sit on whole pixels and its size is an even number of pixels, so the recording + /// gets the screen's own pixels instead of a resampled copy. + private static func sourceRect(for captureRect: CGRect, on display: SCDisplay, scale: Int) -> CGRect { + let pixelsPerPoint = CGFloat(scale) + let local = captureRect.offsetBy(dx: -display.frame.minX, dy: -display.frame.minY) + let left = (local.minX * pixelsPerPoint).rounded() + let top = (local.minY * pixelsPerPoint).rounded() + let width = max(2, Int((local.maxX * pixelsPerPoint).rounded() - left) & ~1) + let height = max(2, Int((local.maxY * pixelsPerPoint).rounded() - top) & ~1) + return CGRect( + x: left / pixelsPerPoint, + y: top / pixelsPerPoint, + width: CGFloat(width) / pixelsPerPoint, + height: CGFloat(height) / pixelsPerPoint + ) + } + + private static func rect(_ lhs: CGRect, isCloseTo rhs: CGRect) -> Bool { + abs(lhs.minX - rhs.minX) < 0.5 && abs(lhs.minY - rhs.minY) < 0.5 + && abs(lhs.width - rhs.width) < 0.5 && abs(lhs.height - rhs.height) < 0.5 + } + private static func captureDisplay(for frame: CGRect, from displays: [SCDisplay]) -> SCDisplay? { let midpoint = CGPoint(x: frame.midX, y: frame.midY) return displays.first(where: { $0.frame.contains(midpoint) }) diff --git a/electron/native/ScreenCaptureKitRecorder.test.ts b/electron/native/ScreenCaptureKitRecorder.test.ts index bf787842e..c647ee9c2 100644 --- a/electron/native/ScreenCaptureKitRecorder.test.ts +++ b/electron/native/ScreenCaptureKitRecorder.test.ts @@ -74,16 +74,27 @@ describe("ScreenCaptureKitRecorder colour metadata", () => { }); describe("ScreenCaptureKitRecorder window capture", () => { - it("records the display and crops it to the selected window bounds", () => { - expect(recorderSource).not.toContain("streamConfig.sourceRect"); + it("crops the display natively to the selected window bounds", () => { expect(recorderSource).not.toContain("desktopIndependentWindow"); + expect(recorderSource).not.toContain("CIContext"); expect(recorderSource).toContain( "visibleFrame = CGRect(x: x, y: y, width: width, height: height)", ); expect(recorderSource).toContain( "let captureRect = visibleFrame.intersection(display.frame)", ); - expect(recorderSource).toContain("appendCroppedVideoFrame(sampleBuffer"); + expect(recorderSource).toContain( + "let sourceRect = Self.sourceRect(for: captureRect, on: display, scale: scaleFactor)", + ); + expect(recorderSource).toContain("streamConfig.sourceRect = sourceRect"); + }); + + it("records whole screen pixels at a bitrate sized for the frame rate", () => { + expect(recorderSource).toMatch(/let width = max\(2, Int\(.*\) & ~1\)/); + expect(recorderSource).toContain("averageBitRate * requestedFPS / assistantFPS"); + expect(recorderSource).toContain( + "compression[AVVideoExpectedSourceFrameRateKey] = requestedFPS", + ); }); it("refreshes the crop and capture display while the window moves or resizes", () => { @@ -91,13 +102,22 @@ describe("ScreenCaptureKitRecorder window capture", () => { "guard let display = Self.captureDisplay(for: window.frame", ); expect(recorderSource).toContain("try await activeStream.updateContentFilter(filter)"); - expect(recorderSource).toContain("self.windowCropRect = cropRect"); + expect(recorderSource).toContain( + "try await activeStream.updateConfiguration(streamConfiguration)", + ); }); -}); + it("writes the frame that arrived before the writer was ready", () => { + expect(recorderSource).toContain("pendingFirstFrame = sampleBuffer"); + expect(recorderSource).toContain("self.appendPendingFirstFrame(attemptsRemaining: 100)"); + }); +}); describe("ScreenCaptureKitRecorder first frame timing", () => { - const callback = recorderSource.slice(recorderSource.indexOf("func stream(_ stream:"), recorderSource.indexOf("func stream(_ stream:") + 5000); + const callback = recorderSource.slice( + recorderSource.indexOf("func stream(_ stream:"), + recorderSource.indexOf("func stream(_ stream:") + 5000, + ); it("validates a complete frame and writer readiness before setting time zero", () => { const clock = callback.indexOf("adjustedPresentationTime(for:"); expect(clock).toBeGreaterThan(callback.indexOf("status == .complete"));