diff --git a/packages/browser-rum-core/src/domain/configuration/configuration.spec.ts b/packages/browser-rum-core/src/domain/configuration/configuration.spec.ts index 3eb13b6235..dff3622565 100644 --- a/packages/browser-rum-core/src/domain/configuration/configuration.spec.ts +++ b/packages/browser-rum-core/src/domain/configuration/configuration.spec.ts @@ -355,25 +355,68 @@ describe('validateAndBuildRumConfiguration', () => { sessionReplayCanvasRecording: { enable: true }, })! - expect(configuration.sessionReplayCanvasRecording).toEqual({ enable: true, maxFramesPerSecond: 1 }) + expect(configuration.sessionReplayCanvasRecording).toEqual({ + enable: true, + maxFramesPerSecond: 1, + hashingMaxDimension: 100, + maxImageDimension: 1000, + }) }) it('uses the configured frame rate', () => { const configuration = validateAndBuildRumConfiguration({ ...DEFAULT_INIT_CONFIGURATION, - sessionReplayCanvasRecording: { enable: true, maxFramesPerSecond: 2.5 }, + sessionReplayCanvasRecording: { + enable: true, + maxFramesPerSecond: 2.5, + hashingMaxDimension: 50, + maxImageDimension: 500, + }, })! - expect(configuration.sessionReplayCanvasRecording).toEqual({ enable: true, maxFramesPerSecond: 2.5 }) + expect(configuration.sessionReplayCanvasRecording).toEqual({ + enable: true, + maxFramesPerSecond: 2.5, + hashingMaxDimension: 50, + maxImageDimension: 500, + }) }) - it('preserves the configured frame rate when disabled', () => { + it('preserves the configured options when disabled', () => { const configuration = validateAndBuildRumConfiguration({ ...DEFAULT_INIT_CONFIGURATION, - sessionReplayCanvasRecording: { enable: false, maxFramesPerSecond: 2.5 }, + sessionReplayCanvasRecording: { + enable: false, + maxFramesPerSecond: 2.5, + hashingMaxDimension: 50, + maxImageDimension: 500, + }, })! - expect(configuration.sessionReplayCanvasRecording).toEqual({ enable: false, maxFramesPerSecond: 2.5 }) + expect(configuration.sessionReplayCanvasRecording).toEqual({ + enable: false, + maxFramesPerSecond: 2.5, + hashingMaxDimension: 50, + maxImageDimension: 500, + }) + }) + + it('rejects a hashing dimension above 100 pixels', () => { + expect( + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplayCanvasRecording: { enable: true, hashingMaxDimension: 101 }, + }) + ).toBeUndefined() + }) + + it('rejects an image dimension above 1000 pixels', () => { + expect( + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplayCanvasRecording: { enable: true, maxImageDimension: 1001 }, + }) + ).toBeUndefined() }) it('rejects invalid canvas recording options', () => { diff --git a/packages/browser-rum-core/src/domain/configuration/configuration.ts b/packages/browser-rum-core/src/domain/configuration/configuration.ts index 49999e317e..e78ef674f8 100644 --- a/packages/browser-rum-core/src/domain/configuration/configuration.ts +++ b/packages/browser-rum-core/src/domain/configuration/configuration.ts @@ -232,6 +232,22 @@ export interface RumInitConfiguration extends InitConfiguration { * @defaultValue 1 */ maxFramesPerSecond?: number | undefined + + /** + * The maximum width or height, in pixels, of the image used for canvas change detection, between 1 and 100. + * Images are downscaled proportionally to fit within this bound and smaller images are not upscaled. + * + * @defaultValue 100 + */ + hashingMaxDimension?: number | undefined + + /** + * The maximum width or height, in pixels, of recorded canvas images, between 1 and 1000. Images are downscaled proportionally + * to fit within this bound and smaller images are not upscaled. + * + * @defaultValue 1000 + */ + maxImageDimension?: number | undefined } | undefined @@ -423,6 +439,8 @@ export const RUM_SCHEMA = { schema: { enable: { type: 'boolean', required: true }, maxFramesPerSecond: { type: 'number', min: 0, max: 5, default: 1 }, + hashingMaxDimension: { type: 'number', min: 1, max: 100, default: 100 }, + maxImageDimension: { type: 'number', min: 1, max: 1000, default: 1000 }, }, }, diff --git a/packages/browser-rum/src/domain/record/canvas/canvasHash.spec.ts b/packages/browser-rum/src/domain/record/canvas/canvasHash.spec.ts new file mode 100644 index 0000000000..cdfd8598d5 --- /dev/null +++ b/packages/browser-rum/src/domain/record/canvas/canvasHash.spec.ts @@ -0,0 +1,72 @@ +import { registerCleanupTask, replaceMockable } from '@datadog/browser-core/test' +import { globalObject } from '@datadog/js-core/util' +import type { CanvasSnapshot } from './canvasSnapshot' +import { computeImageHash } from './canvasHash' + +describe('computeImageHash', () => { + it('returns the same hash for the same content', async () => { + const first = await computeImageHash(createSnapshot('red'), 100) + const second = await computeImageHash(createSnapshot('red'), 100) + + expect(first).toBeDefined() + expect(second).toBe(first) + }) + + it('returns a different hash when the content changes', async () => { + const red = await computeImageHash(createSnapshot('red'), 100) + const blue = await computeImageHash(createSnapshot('blue'), 100) + + expect(blue).not.toBe(red) + }) + + it('returns a different hash when the canvas dimensions change but the downscaled pixels do not', async () => { + const small = await computeImageHash(createSnapshot('red', 2), 1) + const large = await computeImageHash(createSnapshot('red', 4), 1) + + expect(large).not.toBe(small) + }) + + it('returns undefined when no 2d context is available', async () => { + const snapshot = createSnapshot('red') + spyOn(HTMLCanvasElement.prototype, 'getContext').and.returnValue(null) + + expect(await computeImageHash(snapshot, 100)).toBeUndefined() + }) + + it('hashes the image when SubtleCrypto is unavailable', async () => { + replaceMockable(globalObject.crypto?.subtle, undefined) + + const red = await computeImageHash(createSnapshot('red'), 100) + const sameRed = await computeImageHash(createSnapshot('red'), 100) + const blue = await computeImageHash(createSnapshot('blue'), 100) + + expect(red).toBeDefined() + expect(sameRed).toBe(red) + expect(blue).not.toBe(red) + }) + + it('hashes the image when the digest fails', async () => { + replaceMockable(globalObject.crypto?.subtle, { + digest: () => Promise.reject(new Error('unsupported')), + } as unknown as SubtleCrypto) + + const red = await computeImageHash(createSnapshot('red'), 100) + const blue = await computeImageHash(createSnapshot('blue'), 100) + + expect(red).toBeDefined() + expect(blue).not.toBe(red) + }) +}) + +function createSnapshot(color: string, size = 2): CanvasSnapshot { + const source = document.createElement('canvas') + source.width = size + source.height = size + const context = source.getContext('2d')! + context.fillStyle = color + context.fillRect(0, 0, size, size) + document.body.appendChild(source) + registerCleanupTask(() => source.remove()) + + return { canvasHeight: size, canvasWidth: size, source } +} diff --git a/packages/browser-rum/src/domain/record/canvas/canvasHash.ts b/packages/browser-rum/src/domain/record/canvas/canvasHash.ts new file mode 100644 index 0000000000..96245b456a --- /dev/null +++ b/packages/browser-rum/src/domain/record/canvas/canvasHash.ts @@ -0,0 +1,54 @@ +import { mockable } from '@datadog/browser-core' +import { globalObject } from '@datadog/js-core/util' +import type { CanvasSnapshot } from './canvasSnapshot' + +export function computeImageHash(snapshot: CanvasSnapshot, maxHashDimension: number): Promise { + const { height: sourceHeight, width: sourceWidth } = snapshot.source + const scale = Math.min(1, maxHashDimension / Math.max(sourceWidth, sourceHeight)) + const width = Math.max(1, Math.round(sourceWidth * scale)) + const height = Math.max(1, Math.round(sourceHeight * scale)) + + const thumbnail = document.createElement('canvas') + thumbnail.width = width + thumbnail.height = height + + const context = thumbnail.getContext('2d') + if (!context) { + return Promise.resolve(undefined) + } + context.imageSmoothingQuality = 'low' + context.drawImage(snapshot.source, 0, 0, width, height) + + const data = context.getImageData(0, 0, width, height).data + // crypto.subtle is only exposed in secure contexts, so it is missing on plain HTTP pages. + const subtleCrypto = mockable(globalObject.crypto?.subtle) + if (!subtleCrypto) { + return Promise.resolve(createChangeHash(snapshot.canvasWidth, snapshot.canvasHeight, fnv1aHash(data))) + } + + return subtleCrypto.digest('SHA-256', data).then( + (buffer) => createChangeHash(snapshot.canvasWidth, snapshot.canvasHeight, arrayBufferToHex(buffer)), + () => createChangeHash(snapshot.canvasWidth, snapshot.canvasHeight, fnv1aHash(data)) + ) +} + +function arrayBufferToHex(buffer: ArrayBuffer): string { + return Array.from(new Uint8Array(buffer), (byte) => byte.toString(16).padStart(2, '0')).join('') +} + +function createChangeHash(width: number, height: number, pixelHash: string): string { + return `${width}x${height}:${pixelHash}` +} + +/* eslint-disable no-bitwise */ +function fnv1aHash(data: ArrayLike): string { + let hash = 0x811c9dc5 + + for (let index = 0; index < data.length; index += 1) { + hash ^= data[index] + hash = Math.imul(hash, 0x01000193) + } + + return (hash >>> 0).toString(16).padStart(8, '0') +} +/* eslint-enable no-bitwise */ diff --git a/packages/browser-rum/src/domain/record/canvas/canvasManager.spec.ts b/packages/browser-rum/src/domain/record/canvas/canvasManager.spec.ts index 2d49271abd..d00a1f539b 100644 --- a/packages/browser-rum/src/domain/record/canvas/canvasManager.spec.ts +++ b/packages/browser-rum/src/domain/record/canvas/canvasManager.spec.ts @@ -1,21 +1,22 @@ import { registerCleanupTask } from '@datadog/browser-core/test' -import { createCanvasManager } from './canvasManager' +import { CanvasStatus, createCanvasManager } from './canvasManager' +import type { CanvasCaptureAttempt } from './canvasManager' describe('CanvasManager', () => { - it('tracks whether a canvas is dirty', () => { + it('tracks whether a canvas is capturable', () => { const canvasManager = createCanvasManager() const canvas = appendCanvas() - expect(canvasManager.isCanvasDirty(canvas)).toBeFalse() + expect(canvasManager.getCapturableCanvases()).toEqual([]) - canvasManager.markCanvasDirty(canvas) - expect(canvasManager.isCanvasDirty(canvas)).toBeTrue() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) - canvasManager.markCanvasClean(canvas) - expect(canvasManager.isCanvasDirty(canvas)).toBeFalse() + canvasManager.markCanvas(canvas, CanvasStatus.Clean) + expect(canvasManager.getCapturableCanvases()).toEqual([]) - canvasManager.markCanvasDirty(canvas) - expect(canvasManager.isCanvasDirty(canvas)).toBeTrue() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) }) it('tracks canvases independently', () => { @@ -23,46 +24,278 @@ describe('CanvasManager', () => { const dirtyCanvas = appendCanvas() const cleanCanvas = appendCanvas() - canvasManager.markCanvasDirty(dirtyCanvas) + canvasManager.markCanvas(dirtyCanvas, CanvasStatus.Dirty) - expect(canvasManager.isCanvasDirty(dirtyCanvas)).toBeTrue() - expect(canvasManager.isCanvasDirty(cleanCanvas)).toBeFalse() + expect(canvasManager.getCapturableCanvases()).toEqual([dirtyCanvas]) + expect(canvasManager.getCapturableCanvases().includes(cleanCanvas)).toBe(false) }) - it('returns connected dirty canvases', () => { + it('does not retain detached canvases', () => { + const canvasManager = createCanvasManager() + const canvas = document.createElement('canvas') + + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + expect(canvasManager.getCapturableCanvases()).toEqual([]) + + document.body.appendChild(canvas) + registerCleanupTask(() => canvas.remove()) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + it('does not return tainted canvases for capture', () => { const canvasManager = createCanvasManager() const canvas = appendCanvas() - canvasManager.markCanvasDirty(canvas) + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + canvasManager.markCanvas(canvas, CanvasStatus.Tainted) - expect(canvasManager.getDirtyCanvases()).toEqual([canvas]) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) - canvasManager.markCanvasClean(canvas) - expect(canvasManager.getDirtyCanvases()).toEqual([]) + it('does not start a capture for a tainted canvas', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + + canvasManager.markCanvas(canvas, CanvasStatus.Tainted) + + const run = jasmine.createSpy().and.returnValue(Promise.resolve()) + await canvasManager.capture(canvas, run) + + expect(run).not.toHaveBeenCalled() }) - it('does not retain detached canvases', () => { + it('does not start a second capture while one is in flight', async () => { const canvasManager = createCanvasManager() - const canvas = document.createElement('canvas') + const canvas = appendCanvas() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) - canvasManager.markCanvasDirty(canvas) + let resolveFirst!: () => void + const firstCapture = canvasManager.capture( + canvas, + () => + new Promise((resolve) => { + resolveFirst = resolve + }) + ) + await Promise.resolve() - expect(canvasManager.getDirtyCanvases()).toEqual([]) + expect(canvasManager.getCapturableCanvases()).toEqual([]) - document.body.appendChild(canvas) - registerCleanupTask(() => canvas.remove()) - expect(canvasManager.getDirtyCanvases()).toEqual([]) + const secondRun = jasmine.createSpy().and.returnValue(Promise.resolve()) + await canvasManager.capture(canvas, secondRun) + expect(secondRun).not.toHaveBeenCalled() + + resolveFirst() + await firstCapture }) - it('clears dirty canvases', () => { + it('releases the in-flight capture once the attempt settles', async () => { const canvasManager = createCanvasManager() const canvas = appendCanvas() - canvasManager.markCanvasDirty(canvas) + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + let attempt!: CanvasCaptureAttempt + await canvasManager.capture(canvas, (currentAttempt) => { + attempt = currentAttempt + return Promise.resolve() + }) + + expect(attempt.isCurrent()).toBeFalse() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + }) + + it('marks the canvas clean when the attempt settles without a draw in between', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + await canvasManager.capture(canvas, (attempt) => { + attempt.settle('hash') + return Promise.resolve() + }) + + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + it('keeps the canvas dirty when a draw happens while the attempt is in flight', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + await canvasManager.capture(canvas, (attempt) => { + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + attempt.settle('hash') + return Promise.resolve() + }) + + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + }) + + it('exposes the changeHash emitted by the previous attempt', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + await canvasManager.capture(canvas, (attempt) => { + expect(attempt.lastChangeHash).toBeUndefined() + attempt.settle('hash') + return Promise.resolve() + }) + + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + let lastChangeHash: string | undefined + await canvasManager.capture(canvas, (attempt) => { + lastChangeHash = attempt.lastChangeHash + return Promise.resolve() + }) + + expect(lastChangeHash).toBe('hash') + }) + + it('leaves the canvas dirty when the attempt fails with a non-security error', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + await canvasManager.capture(canvas, (attempt) => { + attempt.fail(new Error('boom')) + return Promise.resolve() + }) + + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + }) + + const securityErrorFailureModes: Array<{ + description: string + run: (attempt: CanvasCaptureAttempt) => Promise + }> = [ + { + description: 'when the attempt fails with a SecurityError', + run: (attempt) => { + attempt.fail(new DOMException('tainted', 'SecurityError')) + return Promise.resolve() + }, + }, + { + description: 'when the run callback rejects with a SecurityError', + run: () => Promise.reject(new DOMException('tainted', 'SecurityError')), + }, + ] + + securityErrorFailureModes.forEach(({ description, run }) => { + it(`taints the canvas ${description}`, async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + await canvasManager.capture(canvas, run) + + expect(canvasManager.getCapturableCanvases()).toEqual([]) + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + }) + + it('discards the in-flight capture and the last hash when the bitmap is reset', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + let attempt!: CanvasCaptureAttempt + // Fire-and-forget: this capture is meant to stay in flight for the rest of the test. + void canvasManager.capture(canvas, (currentAttempt) => { + attempt = currentAttempt + attempt.settle('hash') + return new Promise(() => { + // never resolves: simulates a capture still in flight + }) + }) + await Promise.resolve() + + canvasManager.resetCanvasBitmap(canvas) + + expect(attempt.isCurrent()).toBeFalse() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + + let lastChangeHash: string | undefined + await canvasManager.capture(canvas, (currentAttempt) => { + lastChangeHash = currentAttempt.lastChangeHash + return Promise.resolve() + }) + expect(lastChangeHash).toBeUndefined() + }) + + it('keeps a tainted canvas tainted after its bitmap is reset', () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + + canvasManager.markCanvas(canvas, CanvasStatus.Tainted) + + canvasManager.resetCanvasBitmap(canvas) + + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + it('forgets capture state but keeps the taint when a canvas node is removed', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + let attempt!: CanvasCaptureAttempt + // Fire-and-forget: this capture is meant to stay in flight for the rest of the test. + void canvasManager.capture(canvas, (currentAttempt) => { + attempt = currentAttempt + attempt.settle('hash') + return new Promise(() => { + // never resolves: simulates a capture still in flight + }) + }) + await Promise.resolve() + canvasManager.markCanvas(canvas, CanvasStatus.Tainted) + + canvasManager.forgetCanvas(canvas) + + expect(attempt.isCurrent()).toBeFalse() + expect(canvasManager.getCapturableCanvases()).toEqual([]) + + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + it('resets capture hashes', async () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + await canvasManager.capture(canvas, (attempt) => { + attempt.settle('hash') + return Promise.resolve() + }) + + canvasManager.reset() + + let lastChangeHash: string | undefined + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + await canvasManager.capture(canvas, (attempt) => { + lastChangeHash = attempt.lastChangeHash + return Promise.resolve() + }) + expect(lastChangeHash).toBeUndefined() + }) + + it('does not forget tainted canvases on reset', () => { + const canvasManager = createCanvasManager() + const canvas = appendCanvas() + + canvasManager.markCanvas(canvas, CanvasStatus.Tainted) - canvasManager.clearDirtyCanvases() + canvasManager.reset() - expect(canvasManager.getDirtyCanvases()).toEqual([]) - expect(canvasManager.isCanvasDirty(canvas)).toBeFalse() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + expect(canvasManager.getCapturableCanvases()).toEqual([]) }) }) diff --git a/packages/browser-rum/src/domain/record/canvas/canvasManager.ts b/packages/browser-rum/src/domain/record/canvas/canvasManager.ts index da21bd0c02..611e60e3e9 100644 --- a/packages/browser-rum/src/domain/record/canvas/canvasManager.ts +++ b/packages/browser-rum/src/domain/record/canvas/canvasManager.ts @@ -1,35 +1,179 @@ +export const enum CanvasStatus { + /** The canvas is clean, meaning it has not been marked as dirty or tainted */ + Clean, + /** The canvas is dirty, meaning it has been marked as dirty by a draw operation */ + Dirty, + /** The canvas is tainted, meaning it has been marked as tainted by a SecurityError */ + Tainted, +} + +export interface CanvasCaptureAttempt { + /** changeHash emitted the last time this canvas was captured, taken when the attempt started */ + readonly lastChangeHash: string | undefined + /** false if the canvas was forgotten/reset, or if another attempt took its place */ + isCurrent: () => boolean + /** stores the changeHash and marks the canvas clean only if no draw happened since the attempt started */ + settle: (changeHash: string) => void + /** SecurityError taints the canvas; any other error leaves it dirty for the next tick */ + fail: (error: unknown) => void +} + export interface CanvasManager { - clearDirtyCanvases: () => void - getDirtyCanvases: () => HTMLCanvasElement[] - isCanvasDirty: (canvas: HTMLCanvasElement) => boolean - markCanvasClean: (canvas: HTMLCanvasElement) => void - markCanvasDirty: (canvas: HTMLCanvasElement) => void + /** Single entry point for the canvas status */ + markCanvas: (canvas: HTMLCanvasElement, status: CanvasStatus) => void + /** The node left the DOM: forget its capture state, but not its taint */ + forgetCanvas: (canvas: HTMLCanvasElement) => void + /** width/height were assigned: the bitmap was cleared, so drop the last hash and mark dirty */ + resetCanvasBitmap: (canvas: HTMLCanvasElement) => void + /** Dirty, connected, not tainted, no capture in flight (disconnected canvases are forgotten as a side effect) */ + getCapturableCanvases: () => HTMLCanvasElement[] + /** Runs `run` with a capture attempt; the in-flight state is released only when the returned promise settles */ + capture: (canvas: HTMLCanvasElement, run: (attempt: CanvasCaptureAttempt) => Promise) => Promise + /** New record stream: discards the per-stream state (not the taint) */ + reset: () => void +} + +/** Identifies a specific capture() call so a stale attempt can tell it no longer owns the canvas */ +type CaptureToken = object + +interface CanvasState { + version: number + capture?: CaptureToken + lastChangeHash?: string } export function createCanvasManager(): CanvasManager { const dirtyCanvases = new Set() + const taintedCanvases = new WeakSet() + let canvasStates = new WeakMap() + + function getState(canvas: HTMLCanvasElement): CanvasState { + let state = canvasStates.get(canvas) + if (!state) { + state = { version: 0 } + canvasStates.set(canvas, state) + } + return state + } + + function markDirty(canvas: HTMLCanvasElement) { + if (!canvas.isConnected || taintedCanvases.has(canvas)) { + return + } + getState(canvas).version++ + dirtyCanvases.add(canvas) + } + + function markClean(canvas: HTMLCanvasElement) { + dirtyCanvases.delete(canvas) + } + + function markTainted(canvas: HTMLCanvasElement) { + taintedCanvases.add(canvas) + dirtyCanvases.delete(canvas) + } return { - clearDirtyCanvases: () => dirtyCanvases.clear(), - getDirtyCanvases: () => { - const connectedCanvases: HTMLCanvasElement[] = [] + markCanvas: (canvas, status) => { + switch (status) { + case CanvasStatus.Dirty: + markDirty(canvas) + return + case CanvasStatus.Clean: + markClean(canvas) + return + case CanvasStatus.Tainted: + markTainted(canvas) + return + default: + status satisfies never + return + } + }, + + forgetCanvas: (canvas) => { + dirtyCanvases.delete(canvas) + canvasStates.delete(canvas) + }, + + resetCanvasBitmap: (canvas) => { + canvasStates.delete(canvas) + markDirty(canvas) + }, + + getCapturableCanvases: () => { + const capturableCanvases: HTMLCanvasElement[] = [] dirtyCanvases.forEach((canvas) => { - if (canvas.isConnected) { - connectedCanvases.push(canvas) - } else { + if (!canvas.isConnected) { dirtyCanvases.delete(canvas) + } else if (!taintedCanvases.has(canvas) && !canvasStates.get(canvas)?.capture) { + capturableCanvases.push(canvas) } }) - return connectedCanvases + return capturableCanvases }, - isCanvasDirty: (canvas) => dirtyCanvases.has(canvas), - markCanvasClean: (canvas) => dirtyCanvases.delete(canvas), - markCanvasDirty: (canvas) => { - if (canvas.isConnected) { - dirtyCanvases.add(canvas) + + capture: (canvas, run) => { + // getCapturableCanvases() already filters out tainted/in-flight canvases; these guards + // just make capture() safe to call directly (as the specs do) without going through it. + if (taintedCanvases.has(canvas)) { + return Promise.resolve() + } + + const state = getState(canvas) + if (state.capture) { + return Promise.resolve() } + + const startVersion = state.version + const token: CaptureToken = {} + state.capture = token + + const isCurrent = () => canvasStates.get(canvas)?.capture === token + + const attempt: CanvasCaptureAttempt = { + lastChangeHash: state.lastChangeHash, + isCurrent, + settle: (changeHash) => { + if (!isCurrent()) { + return + } + state.lastChangeHash = changeHash + if (state.version === startVersion) { + markClean(canvas) + } + }, + fail: (error) => { + if (!isCurrent()) { + return + } + if (isSecurityError(error)) { + markTainted(canvas) + } else { + markDirty(canvas) + } + }, + } + + return Promise.resolve() + .then(() => run(attempt)) + .catch((error: unknown) => attempt.fail(error)) + .finally(() => { + if (state.capture === token) { + state.capture = undefined + } + }) + }, + + reset: () => { + dirtyCanvases.clear() + canvasStates = new WeakMap() }, } } + +function isSecurityError(error: unknown): boolean { + return error instanceof DOMException && error.name === 'SecurityError' +} diff --git a/packages/browser-rum/src/domain/record/canvas/canvasSnapshot.spec.ts b/packages/browser-rum/src/domain/record/canvas/canvasSnapshot.spec.ts new file mode 100644 index 0000000000..b8a09e3af1 --- /dev/null +++ b/packages/browser-rum/src/domain/record/canvas/canvasSnapshot.spec.ts @@ -0,0 +1,122 @@ +import { registerCleanupTask } from '@datadog/browser-core/test' +import type { CanvasSnapshot } from './canvasSnapshot' +import { captureCanvasImage, createCanvasSnapshot } from './canvasSnapshot' + +describe('createCanvasSnapshot', () => { + it('downscales the snapshot to the configured maximum dimension', () => { + const snapshot = createSnapshot(createCanvas(4, 2), 2) + + expect([snapshot.source.width, snapshot.source.height]).toEqual([2, 1]) + }) + + it('does not upscale a canvas smaller than the maximum dimension', () => { + const snapshot = createSnapshot(createCanvas(2, 2), 1000) + + expect([snapshot.source.width, snapshot.source.height]).toEqual([2, 2]) + }) + + it('keeps the dimensions of the canvas it was taken from', () => { + const snapshot = createSnapshot(createCanvas(4, 2), 1) + + expect([snapshot.canvasWidth, snapshot.canvasHeight]).toEqual([4, 2]) + }) + + it('does not take a snapshot when no 2d context is available', () => { + const canvas = createCanvas(2, 2) + spyOn(HTMLCanvasElement.prototype, 'getContext').and.returnValue(null) + + expect(createCanvasSnapshot(canvas, 1000)).toBeUndefined() + }) + + it('is not affected by later draws on the canvas', async () => { + const canvas = createCanvas(1, 1, ['red']) + const snapshot = createSnapshot(canvas, 1000) + + fill(canvas, 'blue') + + expect(await imagePixels((await captureCanvasImage(snapshot))!)).toEqual([255, 0, 0, 255]) + }) +}) + +describe('captureCanvasImage', () => { + it('encodes the snapshot as a PNG image', async () => { + const snapshot = createSnapshot(createCanvas(2, 2), 1000) + + const image = await captureCanvasImage(snapshot) + + expect(image?.type).toBe('image/png') + }) + + it('encodes an image with the dimensions of the snapshot', async () => { + const snapshot = createSnapshot(createCanvas(4, 2), 2) + + const image = await captureCanvasImage(snapshot) + + expect(await imageSize(image!)).toEqual([2, 1]) + }) + + it('downscales the whole canvas instead of cropping it', async () => { + // Left half red, right half blue: a cropped image would be fully red. + const snapshot = createSnapshot(createCanvas(4, 2, ['red', 'blue']), 2) + + const image = await captureCanvasImage(snapshot) + + expect(await imagePixels(image!)).toEqual([255, 0, 0, 255, 0, 0, 255, 255]) + }) +}) + +function createCanvas(width: number, height: number, colors: string[] = ['red']): HTMLCanvasElement { + const canvas = document.createElement('canvas') + canvas.width = width + canvas.height = height + const context = canvas.getContext('2d')! + const stripeWidth = width / colors.length + colors.forEach((color, index) => { + context.fillStyle = color + context.fillRect(index * stripeWidth, 0, stripeWidth, height) + }) + document.body.appendChild(canvas) + registerCleanupTask(() => canvas.remove()) + return canvas +} + +function fill(canvas: HTMLCanvasElement, color: string) { + const context = canvas.getContext('2d')! + context.fillStyle = color + context.fillRect(0, 0, canvas.width, canvas.height) +} + +function createSnapshot(canvas: HTMLCanvasElement, maxImageDimension: number): CanvasSnapshot { + return createCanvasSnapshot(canvas, maxImageDimension)! +} + +function decodeImage(image: Blob): Promise<{ pixels: number[]; size: [number, number] }> { + return new Promise((resolve, reject) => { + const url = URL.createObjectURL(image) + const element = new Image() + element.onload = () => { + const width = element.naturalWidth + const height = element.naturalHeight + const canvas = document.createElement('canvas') + canvas.width = width + canvas.height = height + const context = canvas.getContext('2d')! + context.drawImage(element, 0, 0) + URL.revokeObjectURL(url) + resolve({ pixels: Array.from(context.getImageData(0, 0, width, height).data), size: [width, height] }) + } + element.onerror = () => { + URL.revokeObjectURL(url) + reject(new Error('failed to decode the image')) + } + element.src = url + }) +} + +async function imageSize(image: Blob): Promise<[number, number]> { + return (await decodeImage(image)).size +} + +async function imagePixels(image: Blob): Promise { + return (await decodeImage(image)).pixels +} diff --git a/packages/browser-rum/src/domain/record/canvas/canvasSnapshot.ts b/packages/browser-rum/src/domain/record/canvas/canvasSnapshot.ts new file mode 100644 index 0000000000..df42418349 --- /dev/null +++ b/packages/browser-rum/src/domain/record/canvas/canvasSnapshot.ts @@ -0,0 +1,37 @@ +export interface CanvasSnapshot { + canvasHeight: number + canvasWidth: number + source: HTMLCanvasElement +} + +export function createCanvasSnapshot(canvas: HTMLCanvasElement, maxImageDimension: number): CanvasSnapshot | undefined { + const canvasWidth = canvas.width + const canvasHeight = canvas.height + const scale = Math.min(1, maxImageDimension / Math.max(canvasWidth, canvasHeight)) + const width = Math.max(1, Math.round(canvasWidth * scale)) + const height = Math.max(1, Math.round(canvasHeight * scale)) + + const source = document.createElement('canvas') + source.width = width + source.height = height + + const context = source.getContext('2d') + if (!context) { + return undefined + } + + context.imageSmoothingQuality = 'low' + context.drawImage(canvas, 0, 0, width, height) + + return { canvasHeight, canvasWidth, source } +} + +export function captureCanvasImage(snapshot: CanvasSnapshot): Promise { + return new Promise((resolve) => { + try { + snapshot.source.toBlob((blob) => resolve(blob ?? undefined), 'image/png') + } catch { + resolve(undefined) + } + }) +} diff --git a/packages/browser-rum/src/domain/record/record.spec.ts b/packages/browser-rum/src/domain/record/record.spec.ts index 47b8a626fc..794c8d0304 100644 --- a/packages/browser-rum/src/domain/record/record.spec.ts +++ b/packages/browser-rum/src/domain/record/record.spec.ts @@ -76,7 +76,14 @@ describe('record', () => { it('instruments canvas drawing when canvas recording is enabled', () => { const originalFillRect = Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value - startRecording({ sessionReplayCanvasRecording: { enable: true, maxFramesPerSecond: 1 } }) + startRecording({ + sessionReplayCanvasRecording: { + enable: true, + maxFramesPerSecond: 1, + hashingMaxDimension: 100, + maxImageDimension: 1000, + }, + }) expect(Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value).not.toBe( originalFillRect @@ -96,7 +103,14 @@ describe('record', () => { it('does not instrument canvas drawing when the maximum frame rate is zero', () => { const originalFillRect = Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value - startRecording({ sessionReplayCanvasRecording: { enable: true, maxFramesPerSecond: 0 } }) + startRecording({ + sessionReplayCanvasRecording: { + enable: true, + maxFramesPerSecond: 0, + hashingMaxDimension: 100, + maxImageDimension: 1000, + }, + }) expect(Object.getOwnPropertyDescriptor(CanvasRenderingContext2D.prototype, 'fillRect')!.value).toBe( originalFillRect diff --git a/packages/browser-rum/src/domain/record/record.ts b/packages/browser-rum/src/domain/record/record.ts index 3883a15419..ebb6fdd19c 100644 --- a/packages/browser-rum/src/domain/record/record.ts +++ b/packages/browser-rum/src/domain/record/record.ts @@ -16,6 +16,7 @@ import { trackViewportResize, trackVisualViewportResize, trackCanvasContent, + trackCanvasCapture, } from './trackers' import { createElementsScrollPositions } from './elementsScrollPositions' import type { ShadowRootsController } from './shadowRootsController' @@ -83,6 +84,7 @@ export function record(options: RecordOptions): RecordAPI { trackVisualViewportResize(processRecord), trackViewEnd(lifeCycle, processRecord, flushMutations), trackCanvasContent(scope), + trackCanvasCapture(scope), ] return { diff --git a/packages/browser-rum/src/domain/record/recordingScope.ts b/packages/browser-rum/src/domain/record/recordingScope.ts index f566771185..2ef0981327 100644 --- a/packages/browser-rum/src/domain/record/recordingScope.ts +++ b/packages/browser-rum/src/domain/record/recordingScope.ts @@ -42,6 +42,7 @@ export function createRecordingScope( scope.nodeIds.clear() scope.stringIds.clear() scope.styleSheetIds.clear() + scope.canvasManager.reset() }, canvasManager, diff --git a/packages/browser-rum/src/domain/record/serialization/serializeMutations.ts b/packages/browser-rum/src/domain/record/serialization/serializeMutations.ts index 0ff5f64b05..f53a19248f 100644 --- a/packages/browser-rum/src/domain/record/serialization/serializeMutations.ts +++ b/packages/browser-rum/src/domain/record/serialization/serializeMutations.ts @@ -114,7 +114,7 @@ function processRemovedNodes(nodes: Set, transaction: SerializationTransac forNodeAndDescendants(node, (node: Node) => { if (isCanvasElement(node)) { - transaction.scope.canvasManager.markCanvasClean(node) + transaction.scope.canvasManager.forgetCanvas(node) } if (isNodeShadowHost(node)) { @@ -261,12 +261,14 @@ function processAttributeMutations( const change: RoleAnnotatedAttributeChange = [nodeId] for (const [domAttributeName, oldValue] of attributeNames) { - if (node.getAttribute(domAttributeName) === oldValue) { - continue // No change since the last snapshot. + if (isCanvasElement(node) && isCanvasSizeAttribute(domAttributeName)) { + // Assigning either dimension resets the bitmap even when the attribute value does not change, + // so this must run before the "no change since the last snapshot" check below. + transaction.scope.canvasManager.resetCanvasBitmap(node) } - if (isCanvasElement(node) && isCanvasSizeAttribute(domAttributeName)) { - transaction.scope.canvasManager.markCanvasDirty(node) + if (node.getAttribute(domAttributeName) === oldValue) { + continue // No change since the last snapshot. } if (domAttributeName === 'value') { diff --git a/packages/browser-rum/src/domain/record/serialization/serializeNode.node.spec.ts b/packages/browser-rum/src/domain/record/serialization/serializeNode.node.spec.ts index 1c20d72596..9acb1c07c5 100644 --- a/packages/browser-rum/src/domain/record/serialization/serializeNode.node.spec.ts +++ b/packages/browser-rum/src/domain/record/serialization/serializeNode.node.spec.ts @@ -131,7 +131,7 @@ describe('serializeNode for DOM nodes', () => { await serializeHtml('
', { scope, after: (target) => { - expect(scope.canvasManager.getDirtyCanvases()).toEqual( + expect(scope.canvasManager.getCapturableCanvases()).toEqual( Array.from((target as Element).querySelectorAll('canvas')) ) }, diff --git a/packages/browser-rum/src/domain/record/serialization/serializeNode.ts b/packages/browser-rum/src/domain/record/serialization/serializeNode.ts index 6d1b69a738..de6d45b4f8 100644 --- a/packages/browser-rum/src/domain/record/serialization/serializeNode.ts +++ b/packages/browser-rum/src/domain/record/serialization/serializeNode.ts @@ -16,6 +16,7 @@ import { MediaInteractionType, StringRole } from '../../../types' import type { NodeId, StyleSheetId } from '../encoding' import { createAttributeAssignment, createString } from '../encoding' import { isCanvasElement } from '../canvas/canvasUtils' +import { CanvasStatus } from '../canvas/canvasManager' import type { InsertionCursor } from './insertionCursor' import type { SerializationTransaction } from './serializationTransaction' import { serializeDOMAttributes, serializeVirtualAttributes } from './serializeAttributes' @@ -149,7 +150,7 @@ function serializeElementNode( transaction.addNode(insertionPoint, encodedElementName(element), ...domAttributes) if (isCanvasElement(element)) { - transaction.scope.canvasManager.markCanvasDirty(element) + transaction.scope.canvasManager.markCanvas(element, CanvasStatus.Dirty) } const { diff --git a/packages/browser-rum/src/domain/record/trackers/index.ts b/packages/browser-rum/src/domain/record/trackers/index.ts index e2e4786cd8..768c4426ef 100644 --- a/packages/browser-rum/src/domain/record/trackers/index.ts +++ b/packages/browser-rum/src/domain/record/trackers/index.ts @@ -9,4 +9,5 @@ export { trackViewEnd } from './trackViewEnd' export { trackInput } from './trackInput' export { trackMutation } from './trackMutation' export { trackCanvasContent } from './trackCanvasContent' +export { trackCanvasCapture } from './trackCanvasCapture' export type { Tracker } from './tracker.types' diff --git a/packages/browser-rum/src/domain/record/trackers/trackCanvasCapture.spec.ts b/packages/browser-rum/src/domain/record/trackers/trackCanvasCapture.spec.ts new file mode 100644 index 0000000000..1575b0beb6 --- /dev/null +++ b/packages/browser-rum/src/domain/record/trackers/trackCanvasCapture.spec.ts @@ -0,0 +1,428 @@ +import { globalObject } from '@datadog/js-core/util' +import { + collectAsyncCalls, + registerCleanupTask, + mockClock, + replaceMockable, + waitAfterNextPaint, +} from '@datadog/browser-core/test' +import type { Clock } from '@datadog/browser-core/test' +import { NodePrivacyLevel, PRIVACY_ATTR_NAME, PRIVACY_ATTR_VALUE_MASK } from '@datadog/browser-rum-core' +import type { CanvasManager } from '../canvas/canvasManager' +import { CanvasStatus, createCanvasManager } from '../canvas/canvasManager' +import type { NodeId } from '../encoding' +import { createRecordingScopeForTesting } from '../test/recordingScope.specHelper' +import type { Tracker } from './tracker.types' +import type { CanvasCaptureCallback } from './trackCanvasCapture' +import { trackCanvasCapture } from './trackCanvasCapture' + +describe('trackCanvasCapture', () => { + let canvas: HTMLCanvasElement + let canvasContext: CanvasRenderingContext2D + let canvasManager: CanvasManager + let scope: ReturnType + let tracker: Tracker + let clock: Clock + let toBlobSpy: jasmine.Spy + const privacyLevels = Object.values(NodePrivacyLevel).filter( + (privacyLevel) => privacyLevel !== NodePrivacyLevel.IGNORE + ) + + beforeEach(() => { + clock = mockClock() + canvas = document.createElement('canvas') + canvas.width = 2 + canvas.height = 2 + canvasContext = canvas.getContext('2d')! + canvasManager = createCanvasManager() + document.body.appendChild(canvas) + toBlobSpy = spyOn(HTMLCanvasElement.prototype, 'toBlob').and.callFake((callback) => { + callback(new Blob([], { type: 'image/png' })) + }) + + registerCleanupTask(() => { + tracker?.stop() + canvas.remove() + }) + }) + + function startTracking( + onCanvasCapture: CanvasCaptureCallback = jasmine.createSpy(), + maxImageDimension = 1000, + hashingMaxDimension = 100 + ) { + scope = createRecordingScopeForTesting({ + canvasManager, + configuration: { + sessionReplayCanvasRecording: { + enable: true, + maxFramesPerSecond: 1, + hashingMaxDimension, + maxImageDimension, + }, + }, + }) + scope.nodeIds.getOrInsert(canvas) + tracker = trackCanvasCapture(scope, onCanvasCapture) + return onCanvasCapture as jasmine.Spy + } + + function markCanvasDirtyAndWaitForCapture() { + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + clock.tick(1000) + } + + async function waitForCanvasCapture() { + await waitAfterNextPaint() + await Promise.resolve() + } + + function draw(color: string) { + canvasContext.fillStyle = color + canvasContext.fillRect(0, 0, canvas.width, canvas.height) + } + + function replaceCanvasWithNewNodeId(): NodeId { + canvas.remove() + canvasManager.forgetCanvas(canvas) + scope.nodeIds.delete(canvas) + document.body.appendChild(canvas) + return scope.nodeIds.getOrInsert(canvas) + } + + function firstPixelOf(image: Blob) { + return new Promise((resolve, reject) => { + const url = URL.createObjectURL(image) + const element = new Image() + element.onload = () => { + const decodeCanvas = document.createElement('canvas') + decodeCanvas.width = element.naturalWidth + decodeCanvas.height = element.naturalHeight + const context = decodeCanvas.getContext('2d')! + context.drawImage(element, 0, 0) + URL.revokeObjectURL(url) + resolve(Array.from(context.getImageData(0, 0, 1, 1).data)) + } + element.onerror = () => { + URL.revokeObjectURL(url) + reject(new Error('failed to decode the image')) + } + element.src = url + }) + } + + it('captures a dirty canvas the first time it is seen', async () => { + draw('red') + const onCanvasCapture = startTracking() + + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(onCanvasCapture).toHaveBeenCalledOnceWith({ + nodeId: jasmine.any(Number), + changeHash: jasmine.any(String), + image: jasmine.any(Blob), + }) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + it('looks up the node ID before reading canvas pixels', async () => { + const drawImageSpy = spyOn(CanvasRenderingContext2D.prototype, 'drawImage').and.callThrough() + startTracking() + scope.nodeIds.delete(canvas) + + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(drawImageSpy).not.toHaveBeenCalled() + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + it('discards an unchanged canvas and marks it clean', async () => { + draw('red') + const onCanvasCapture = startTracking() + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(onCanvasCapture).toHaveBeenCalledTimes(1) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + it('hashes and emits the same immutable canvas snapshot', async () => { + let resolveFirstDigest!: () => void + let isFirstDigest = true + const digestSpy = jasmine.createSpy().and.callFake((_algorithm: AlgorithmIdentifier, data: BufferSource) => { + const bytes = ArrayBuffer.isView(data) + ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + : new Uint8Array(data) + const result = Uint8Array.of(bytes[0]).buffer + + if (isFirstDigest) { + isFirstDigest = false + return new Promise((resolve) => { + resolveFirstDigest = () => resolve(result) + }) + } + return Promise.resolve(result) + }) + replaceMockable(globalObject.crypto?.subtle, { digest: digestSpy } as unknown as SubtleCrypto) + // The emitted image is the assertion here, so it has to be a real PNG rather than the empty + // blob the suite stubs in. + toBlobSpy.and.callThrough() + + draw('red') + const onCanvasCapture = startTracking() + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(digestSpy).toHaveBeenCalledTimes(1) + draw('blue') + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + resolveFirstDigest() + await collectAsyncCalls(onCanvasCapture, 1) + + expect(await firstPixelOf(onCanvasCapture.calls.argsFor(0)[0].image)).toEqual([255, 0, 0, 255]) + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + + clock.tick(1000) + await collectAsyncCalls(onCanvasCapture, 2) + + expect(await firstPixelOf(onCanvasCapture.calls.argsFor(1)[0].image)).toEqual([0, 0, 255, 255]) + expect(onCanvasCapture.calls.argsFor(1)[0].changeHash).not.toBe(onCanvasCapture.calls.argsFor(0)[0].changeHash) + }) + + const nodeIdentityChanges: Array<{ description: string; change: () => NodeId | undefined }> = [ + { + description: 'after the recording scope is reset', + change: () => { + scope.resetIds() + scope.nodeIds.getOrInsert(canvas) + return undefined + }, + }, + { + description: 'when it receives a new node ID', + change: () => replaceCanvasWithNewNodeId(), + }, + ] + + nodeIdentityChanges.forEach(({ description, change }) => { + it(`captures a canvas again ${description}`, async () => { + draw('red') + const onCanvasCapture = startTracking() + const previousNodeId = scope.nodeIds.get(canvas)! + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + const currentNodeId = change() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + clock.tick(1000) + await waitForCanvasCapture() + + expect(onCanvasCapture).toHaveBeenCalledTimes(2) + if (currentNodeId !== undefined) { + expect(currentNodeId).not.toBe(previousNodeId) + expect(onCanvasCapture.calls.argsFor(1)[0].nodeId).toBe(currentNodeId) + } + }) + }) + + it('does not capture a tainted canvas that is reset while detached and then reinserted', async () => { + const onCanvasCapture = startTracking() + canvasManager.markCanvas(canvas, CanvasStatus.Tainted) + + canvas.remove() + canvasManager.forgetCanvas(canvas) + scope.nodeIds.delete(canvas) + canvas.width = 4 + draw('red') + document.body.appendChild(canvas) + scope.nodeIds.getOrInsert(canvas) + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + clock.tick(1000) + await waitForCanvasCapture() + + expect(onCanvasCapture).not.toHaveBeenCalled() + }) + + function deferFirstDigest(): () => void { + let resolveFirstDigest!: (value: ArrayBuffer) => void + const firstDigestPromise = new Promise((resolve) => { + resolveFirstDigest = resolve + }) + let isFirstDigest = true + const digestSpy = jasmine.createSpy().and.callFake(() => { + if (isFirstDigest) { + isFirstDigest = false + return firstDigestPromise + } + return Promise.resolve(new ArrayBuffer(32)) + }) + replaceMockable(globalObject.crypto?.subtle, { digest: digestSpy } as unknown as SubtleCrypto) + return () => resolveFirstDigest(new ArrayBuffer(32)) + } + + function deferFirstBlob(): () => void { + let resolveFirstBlob!: BlobCallback + let isFirstBlob = true + toBlobSpy.and.callFake((callback: BlobCallback) => { + if (isFirstBlob) { + isFirstBlob = false + resolveFirstBlob = callback + } else { + callback(new Blob([], { type: 'image/png' })) + } + }) + return () => resolveFirstBlob(new Blob([], { type: 'image/png' })) + } + + const nodeIdChangeCaptureStages: Array<{ description: string; deferCapture: () => () => void }> = [ + { description: 'while hashing', deferCapture: deferFirstDigest }, + { description: 'while encoding', deferCapture: deferFirstBlob }, + ] + + nodeIdChangeCaptureStages.forEach(({ description, deferCapture }) => { + it(`discards an in-flight capture when the canvas receives a new node ID ${description}`, async () => { + const resumeCapture = deferCapture() + draw('red') + const onCanvasCapture = startTracking() + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + const currentNodeId = replaceCanvasWithNewNodeId() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + resumeCapture() + await waitForCanvasCapture() + + expect(onCanvasCapture).not.toHaveBeenCalled() + + clock.tick(1000) + await waitForCanvasCapture() + + expect(onCanvasCapture).toHaveBeenCalledOnceWith({ + nodeId: currentNodeId, + changeHash: jasmine.any(String), + image: jasmine.any(Blob), + }) + }) + }) + + it('leaves the canvas dirty when the capture callback fails', async () => { + draw('red') + const onCanvasCapture = jasmine.createSpy().and.throwError('capture failed') + startTracking(onCanvasCapture) + + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + }) + + it('leaves the canvas dirty when hashing is unavailable', async () => { + spyOn(HTMLCanvasElement.prototype, 'getContext').and.returnValue(null) + const onCanvasCapture = startTracking() + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + + clock.tick(1000) + await waitForCanvasCapture() + + expect(onCanvasCapture).not.toHaveBeenCalled() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + }) + + it('stops trying to capture a canvas when taking the snapshot throws', async () => { + const drawImageSpy = spyOn(CanvasRenderingContext2D.prototype, 'drawImage').and.callFake(() => { + throw new DOMException('canvas is tainted', 'SecurityError') + }) + startTracking() + + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(drawImageSpy).toHaveBeenCalledTimes(1) + expect(drawImageSpy.calls.argsFor(0).slice(0, 5)).toEqual([canvas, 0, 0, 2, 2]) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + + canvasManager.markCanvas(canvas, CanvasStatus.Dirty) + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(drawImageSpy).toHaveBeenCalledTimes(1) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + for (const privacyLevel of privacyLevels) { + it(`only captures canvases with the allow privacy level, when the privacy level is ${privacyLevel}`, async () => { + canvas.setAttribute(PRIVACY_ATTR_NAME, privacyLevel) + const onCanvasCapture = startTracking() + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + if (privacyLevel === NodePrivacyLevel.ALLOW) { + expect(onCanvasCapture).toHaveBeenCalled() + expect(canvasManager.getCapturableCanvases()).toEqual([]) + } else { + expect(onCanvasCapture).not.toHaveBeenCalled() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + } + }) + } + + it('captures a dirty canvas after its privacy level becomes allow', async () => { + canvas.setAttribute(PRIVACY_ATTR_NAME, PRIVACY_ATTR_VALUE_MASK) + draw('red') + const onCanvasCapture = startTracking() + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(onCanvasCapture).not.toHaveBeenCalled() + + canvas.setAttribute(PRIVACY_ATTR_NAME, NodePrivacyLevel.ALLOW) + clock.tick(1000) + await waitForCanvasCapture() + + expect(onCanvasCapture).toHaveBeenCalled() + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + + const maskingCaptureStages: Array<{ description: string; deferCapture: () => () => void }> = [ + { description: 'while hashing', deferCapture: deferFirstDigest }, + { description: 'while encoding', deferCapture: deferFirstBlob }, + ] + + maskingCaptureStages.forEach(({ description, deferCapture }) => { + it(`emits the snapshot taken before the canvas becomes masked ${description}`, async () => { + const resumeCapture = deferCapture() + draw('red') + const onCanvasCapture = startTracking() + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + canvas.setAttribute(PRIVACY_ATTR_NAME, PRIVACY_ATTR_VALUE_MASK) + resumeCapture() + await waitForCanvasCapture() + + expect(onCanvasCapture).toHaveBeenCalledOnceWith({ + nodeId: jasmine.any(Number), + changeHash: jasmine.any(String), + image: jasmine.any(Blob), + }) + expect(canvasManager.getCapturableCanvases()).toEqual([]) + }) + }) + + it('stops capturing after the tracker is stopped', async () => { + draw('red') + const onCanvasCapture = startTracking() + tracker.stop() + + markCanvasDirtyAndWaitForCapture() + await waitForCanvasCapture() + + expect(onCanvasCapture).not.toHaveBeenCalled() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + }) +}) diff --git a/packages/browser-rum/src/domain/record/trackers/trackCanvasCapture.ts b/packages/browser-rum/src/domain/record/trackers/trackCanvasCapture.ts new file mode 100644 index 0000000000..9776b3cf68 --- /dev/null +++ b/packages/browser-rum/src/domain/record/trackers/trackCanvasCapture.ts @@ -0,0 +1,93 @@ +import { clearInterval, noop, setInterval } from '@datadog/browser-core' +import { getNodePrivacyLevel, NodePrivacyLevel } from '@datadog/browser-rum-core' +import { ONE_SECOND } from '@datadog/js-core/time' +import type { RecordingScope } from '../recordingScope' +import type { NodeId } from '../encoding' +import type { CanvasCaptureAttempt } from '../canvas/canvasManager' +import { CanvasStatus } from '../canvas/canvasManager' +import { captureCanvasImage, createCanvasSnapshot } from '../canvas/canvasSnapshot' +import { computeImageHash } from '../canvas/canvasHash' +import type { Tracker } from './tracker.types' + +export interface CanvasCapture { + nodeId: NodeId + changeHash: string + image: Blob +} + +export type CanvasCaptureCallback = (capture: CanvasCapture) => void + +export const trackCanvasCapture = (scope: RecordingScope, onCanvasCapture: CanvasCaptureCallback = noop): Tracker => { + const canvasManager = scope.canvasManager + const configuration = scope.configuration.sessionReplayCanvasRecording + const maxFramesPerSecond = configuration?.maxFramesPerSecond ?? 0 + + if (!configuration?.enable || maxFramesPerSecond === 0) { + return { stop: noop } + } + + let stopped = false + const captureIntervalId = setInterval(captureDirtyCanvases, ONE_SECOND / maxFramesPerSecond) + + function captureDirtyCanvases() { + const capturableCanvases = canvasManager.getCapturableCanvases() + capturableCanvases.forEach((canvas) => { + const nodeId = scope.nodeIds.get(canvas) + if (nodeId === undefined) { + canvasManager.markCanvas(canvas, CanvasStatus.Clean) + return + } + + const nodePrivacyLevel = getNodePrivacyLevel(canvas, scope.configuration.defaultPrivacyLevel) + if (nodePrivacyLevel !== NodePrivacyLevel.ALLOW) { + return // Keep it dirty so it can be captured if its privacy level becomes allow + } + + void canvasManager.capture(canvas, (attempt) => captureCanvas(canvas, nodeId, attempt)) + }) + } + + async function captureCanvas(canvas: HTMLCanvasElement, nodeId: NodeId, attempt: CanvasCaptureAttempt) { + try { + const snapshot = createCanvasSnapshot(canvas, configuration?.maxImageDimension ?? 1000) + if (!snapshot) { + return // snapshot failed; leave it dirty + } + + const hash = await computeImageHash(snapshot, configuration?.hashingMaxDimension ?? 100) + if (stopped || !attempt.isCurrent()) { + return + } + if (hash === undefined) { + return // hashing failed; leave it dirty + } + + if (hash === attempt.lastChangeHash) { + attempt.settle(hash) + return // unchanged: no capture/output + } + + const image = await captureCanvasImage(snapshot) + if (stopped || !attempt.isCurrent()) { + return + } + if (!image) { + return // encoding failed; leave it dirty + } + + onCanvasCapture({ nodeId, changeHash: hash, image }) + attempt.settle(hash) + } catch (error) { + if (!stopped) { + attempt.fail(error) + } + } + } + + return { + stop: () => { + stopped = true + clearInterval(captureIntervalId) + }, + } +} diff --git a/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.spec.ts b/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.spec.ts index 84d73786ff..f567250657 100644 --- a/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.spec.ts +++ b/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.spec.ts @@ -1,6 +1,6 @@ import { registerCleanupTask } from '@datadog/browser-core/test' import type { CanvasManager } from '../canvas/canvasManager' -import { createCanvasManager } from '../canvas/canvasManager' +import { CanvasStatus, createCanvasManager } from '../canvas/canvasManager' import { createRecordingScopeForTesting } from '../test/recordingScope.specHelper' import type { Tracker } from './tracker.types' import { trackCanvasContent } from './trackCanvasContent' @@ -8,7 +8,7 @@ import { trackCanvasContent } from './trackCanvasContent' describe('trackCanvasContent', () => { let canvas: HTMLCanvasElement let context: CanvasRenderingContext2D - let markCanvasDirtySpy: jasmine.Spy<(canvas: HTMLCanvasElement) => void> + let markCanvasDirtySpy: jasmine.Spy let canvasManager: CanvasManager let tracker: Tracker | undefined @@ -16,15 +16,22 @@ describe('trackCanvasContent', () => { canvas = document.createElement('canvas') context = canvas.getContext('2d')! markCanvasDirtySpy = jasmine.createSpy() - canvasManager = { ...createCanvasManager(), markCanvasDirty: markCanvasDirtySpy } + canvasManager = { ...createCanvasManager(), markCanvas: markCanvasDirtySpy } registerCleanupTask(() => tracker?.stop()) }) - function startTracking(enable = true, maxFramesPerSecond = 1): Tracker { + function startTracking( + enable = true, + maxFramesPerSecond = 1, + hashingMaxDimension = 100, + maxImageDimension = 1000 + ): Tracker { const scope = createRecordingScopeForTesting({ canvasManager, - configuration: { sessionReplayCanvasRecording: { enable, maxFramesPerSecond } }, + configuration: { + sessionReplayCanvasRecording: { enable, maxFramesPerSecond, hashingMaxDimension, maxImageDimension }, + }, }) scope.nodeIds.getOrInsert(canvas) tracker = trackCanvasContent(scope) @@ -57,7 +64,7 @@ describe('trackCanvasContent', () => { .forEach(({ draw }) => { markCanvasDirtySpy.calls.reset() draw() - expect(markCanvasDirtySpy).toHaveBeenCalledOnceWith(canvas) + expect(markCanvasDirtySpy).toHaveBeenCalledOnceWith(canvas, CanvasStatus.Dirty) }) }) @@ -74,7 +81,14 @@ describe('trackCanvasContent', () => { it('does not mark an unserialized canvas dirty', () => { const scope = createRecordingScopeForTesting({ canvasManager, - configuration: { sessionReplayCanvasRecording: { enable: true, maxFramesPerSecond: 1 } }, + configuration: { + sessionReplayCanvasRecording: { + enable: true, + maxFramesPerSecond: 1, + hashingMaxDimension: 100, + maxImageDimension: 1000, + }, + }, }) tracker = trackCanvasContent(scope) diff --git a/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.ts b/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.ts index 822bd56555..8219dc7e54 100644 --- a/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.ts +++ b/packages/browser-rum/src/domain/record/trackers/trackCanvasContent.ts @@ -1,9 +1,8 @@ import { instrumentMethod, noop } from '@datadog/browser-core' import type { RecordingScope } from '../recordingScope' +import { CanvasStatus } from '../canvas/canvasManager' import type { Tracker } from './tracker.types' -export type MarkCanvasDirty = (canvas: HTMLCanvasElement) => void - const CANVAS_2D_DRAWING_METHODS = [ 'clearRect', 'fillRect', @@ -34,7 +33,7 @@ export function trackCanvasContent(scope: RecordingScope): Tracker { instrumentMethod(CanvasRenderingContext2D.prototype, method, ({ target: context, onPostCall }) => { onPostCall(() => { if (scope.nodeIds.get(context.canvas) !== undefined) { - scope.canvasManager.markCanvasDirty(context.canvas) + scope.canvasManager.markCanvas(context.canvas, CanvasStatus.Dirty) } }) }) diff --git a/packages/browser-rum/src/domain/record/trackers/trackMutation.spec.ts b/packages/browser-rum/src/domain/record/trackers/trackMutation.spec.ts index 43a37735e9..5206233650 100644 --- a/packages/browser-rum/src/domain/record/trackers/trackMutation.spec.ts +++ b/packages/browser-rum/src/domain/record/trackers/trackMutation.spec.ts @@ -10,7 +10,7 @@ import { import type { BrowserChangeRecord, BrowserFullSnapshotChangeRecord, BrowserRecord } from '../../../types' import { ChangeType } from '../../../types' import type { RecordingScope } from '../recordingScope' -import { createCanvasManager } from '../canvas/canvasManager' +import { CanvasStatus, createCanvasManager } from '../canvas/canvasManager' import type { AddShadowRootCallBack, RemoveShadowRootCallBack } from '../shadowRootsController' import type { ChangeDecoder } from '../encoding' import type { SerializationStats } from '../serialization' @@ -444,6 +444,7 @@ describe('trackMutation', () => { it('removes a canvas from dirty canvases when it is removed', async () => { const canvasManager = createCanvasManager() + const forgetCanvasSpy = spyOn(canvasManager, 'forgetCanvas').and.callThrough() const scope = createRecordingScopeForTesting({ canvasManager }) let canvas!: HTMLCanvasElement @@ -451,14 +452,15 @@ describe('trackMutation', () => { '', (sandbox) => { canvas = sandbox as HTMLCanvasElement - expect(canvasManager.isCanvasDirty(canvas)).toBeTrue() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) canvas.remove() }, { scope } ) - expect(canvasManager.isCanvasDirty(canvas)).toBeFalse() + expect(forgetCanvasSpy).toHaveBeenCalledOnceWith(canvas) + expect(canvasManager.getCapturableCanvases()).toEqual([]) }) }) @@ -579,13 +581,32 @@ describe('trackMutation', () => { '', (sandbox) => { canvas = sandbox as HTMLCanvasElement - canvasManager.markCanvasClean(canvas) + canvasManager.markCanvas(canvas, CanvasStatus.Clean) canvas.setAttribute('width', '101') }, { scope } ) - expect(canvasManager.isCanvasDirty(canvas)).toBeTrue() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) + }) + + it('marks a canvas bitmap as reset when a size attribute is assigned its current value', async () => { + const canvasManager = createCanvasManager() + const scope = createRecordingScopeForTesting({ canvasManager }) + let canvas!: HTMLCanvasElement + + const { mutation } = await recordMutationOf( + '', + (sandbox) => { + canvas = sandbox as HTMLCanvasElement + canvasManager.markCanvas(canvas, CanvasStatus.Clean) + canvas.setAttribute('width', '101') + }, + { scope } + ) + + expect(mutation).toBeUndefined() + expect(canvasManager.getCapturableCanvases()).toEqual([canvas]) }) it('does not mark a canvas dirty when an unrelated attribute changes', async () => { @@ -597,13 +618,13 @@ describe('trackMutation', () => { '', (sandbox) => { canvas = sandbox as HTMLCanvasElement - canvasManager.markCanvasClean(canvas) + canvasManager.markCanvas(canvas, CanvasStatus.Clean) canvas.setAttribute('class', 'foo') }, { scope } ) - expect(canvasManager.isCanvasDirty(canvas)).toBeFalse() + expect(canvasManager.getCapturableCanvases()).toEqual([]) }) })