@remotion/player: Fix audio hiccups on rapid play/pause toggling

packages/core/src/audio/shared-audio-tags.tsx

@@ -90,21 +90,21 @@ type SharedAudioContextValue = {
 	unscheduleAudioNode: (node: AudioBufferSourceNode) => void;
 };
 
+type RegisterAudioOptions = {
+	aud: AudioHTMLAttributes<HTMLAudioElement>;
+	audioId: string;
+	premounting: boolean;
+	postmounting: boolean;
+};
+
+type UpdateAudioOptions = RegisterAudioOptions & {
+	id: number;
+};
+
 type SharedAudioTagsContextValue = {
-	registerAudio: (options: {
-		aud: AudioHTMLAttributes<HTMLAudioElement>;
-		audioId: string;
-		premounting: boolean;
-		postmounting: boolean;
-	}) => AudioElem;
+	registerAudio: (options: RegisterAudioOptions) => AudioElem;
 	unregisterAudio: (id: number) => void;
-	updateAudio: (options: {
-		id: number;
-		aud: AudioHTMLAttributes<HTMLAudioElement>;
-		audioId: string;
-		premounting: boolean;
-		postmounting: boolean;
-	}) => void;
+	updateAudio: (options: UpdateAudioOptions) => void;
 	playAllAudios: () => void;
 	numberOfAudioTags: number;
 };
@@ -191,12 +191,16 @@ const shouldSaveForLater = (
 	throw new Error(`Unexpected audio context state: ${state satisfies never}`);
 };
 
-export const SharedAudioContextProvider: React.FC<{
+type SharedAudioContextProviderProps = {
 	readonly children: React.ReactNode;
 	readonly audioLatencyHint: AudioContextLatencyCategory;
 	readonly audioEnabled: boolean;
 	readonly previewSampleRate: number | null;
-}> = ({children, audioLatencyHint, audioEnabled, previewSampleRate}) => {
+};
+
+export const SharedAudioContextProvider: React.FC<
+	SharedAudioContextProviderProps
+> = ({children, audioLatencyHint, audioEnabled, previewSampleRate}) => {
 	const logLevel = useLogLevel();
 	const sampleRate = previewSampleRate ?? 48000;
 
@@ -364,22 +368,30 @@ export const SharedAudioContextProvider: React.FC<{
 
 		audioContextIsPlayingEventually.current = true;
 
-		ctxAndGain.gainNode.gain.cancelScheduledValues(
-			ctxAndGain.audioContext.currentTime,
-		);
-		ctxAndGain.gainNode.gain.setValueAtTime(
-			0,
-			ctxAndGain.audioContext.currentTime,
-		);
-		ctxAndGain.gainNode.gain.linearRampToValueAtTime(
-			1,
-			ctxAndGain.audioContext.currentTime + 0.03,
-		);
+		// Nodes that were scheduled while the context was not running start
+		// mid-waveform, which can produce an audible click - mask it with a
+		// short fade-in. If there are no nodes to start, resume() merely
+		// unfreezes the nodes that suspend() froze in place, which continues
+		// sample-exact - fading would cause an audible dip on every
+		// play/pause toggle.
+		if (nodesToResume.current.size > 0) {
+			ctxAndGain.gainNode.gain.cancelScheduledValues(
+				ctxAndGain.audioContext.currentTime,
+			);
+			ctxAndGain.gainNode.gain.setValueAtTime(
+				0,
+				ctxAndGain.audioContext.currentTime,
+			);
+			ctxAndGain.gainNode.gain.linearRampToValueAtTime(
+				1,
+				ctxAndGain.audioContext.currentTime + 0.03,
+			);
 
-		nodesToResume.current.forEach((r, node) => {
-			node.start(r.scheduledTime, r.offset, r.duration);
-		});
-		nodesToResume.current.clear();
+			nodesToResume.current.forEach((r, node) => {
+				node.start(r.scheduledTime, r.offset, r.duration);
+			});
+			nodesToResume.current.clear();
+		}
 
 		const resumePromise = ctxAndGain.resume();
 
@@ -451,10 +463,14 @@ export const SharedAudioContextProvider: React.FC<{
 	);
 };
 
-export const SharedAudioTagsContextProvider: React.FC<{
+type SharedAudioTagsContextProviderProps = {
 	readonly numberOfAudioTags: number;
 	readonly children: React.ReactNode;
-}> = ({children, numberOfAudioTags}) => {
+};
+
+export const SharedAudioTagsContextProvider: React.FC<
+	SharedAudioTagsContextProviderProps
+> = ({children, numberOfAudioTags}) => {
 	const audios = useRef<AudioElem[]>([]);
 	const [initialNumberOfAudioTags] = useState(numberOfAudioTags);
 
@@ -544,12 +560,7 @@ export const SharedAudioTagsContextProvider: React.FC<{
 	}, [refs]);
 
 	const registerAudio = useCallback(
-		(options: {
-			aud: AudioHTMLAttributes<HTMLAudioElement>;
-			audioId: string;
-			premounting: boolean;
-			postmounting: boolean;
-		}) => {
+		(options: RegisterAudioOptions) => {
 			const {aud, audioId, premounting, postmounting} = options;
 			const found = audios.current?.find((a) => a.audioId === audioId);
 			if (found) {
@@ -609,19 +620,7 @@ export const SharedAudioTagsContextProvider: React.FC<{
 	);
 
 	const updateAudio = useCallback(
-		({
-			aud,
-			audioId,
-			id,
-			premounting,
-			postmounting,
-		}: {
-			id: number;
-			aud: AudioHTMLAttributes<HTMLAudioElement>;
-			audioId: string;
-			premounting: boolean;
-			postmounting: boolean;
-		}) => {
+		({aud, audioId, id, premounting, postmounting}: UpdateAudioOptions) => {
 			let changed = false;
 
 			audios.current = audios.current?.map((prevA): AudioElem => {
@@ -720,12 +719,7 @@ export const useSharedAudio = ({
 	audioId,
 	premounting,
 	postmounting,
-}: {
-	aud: AudioHTMLAttributes<HTMLAudioElement>;
-	audioId: string;
-	premounting: boolean;
-	postmounting: boolean;
-}) => {
+}: RegisterAudioOptions) => {
 	const audioCtx = useContext(SharedAudioContext);
 	const tagsCtx = useContext(SharedAudioTagsContext);
 

packages/player/src/set-global-time-anchor.ts

@@ -2,21 +2,21 @@ import {Internals, type LogLevel} from 'remotion';
 
 export const ALLOWED_GLOBAL_TIME_ANCHOR_SHIFT = 0.1;
 
+type SetGlobalTimeAnchorOptions = {
+	audioContext: AudioContext;
+	audioSyncAnchor: {value: number};
+	absoluteTimeInSeconds: number;
+	globalPlaybackRate: number;
+	logLevel: LogLevel;
+};
+
 export const setGlobalTimeAnchor = ({
 	audioContext,
 	audioSyncAnchor,
 	absoluteTimeInSeconds,
 	globalPlaybackRate,
 	logLevel,
-	force,
-}: {
-	audioContext: AudioContext;
-	audioSyncAnchor: {value: number};
-	absoluteTimeInSeconds: number;
-	globalPlaybackRate: number;
-	logLevel: LogLevel;
-	force: boolean;
-}): boolean => {
+}: SetGlobalTimeAnchorOptions): boolean => {
 	const newAnchor =
 		audioContext.currentTime - absoluteTimeInSeconds / globalPlaybackRate;
 	const shift = newAnchor - audioSyncAnchor.value;
@@ -25,20 +25,13 @@ export const setGlobalTimeAnchor = ({
 	const latency = audioContext.baseLatency + safeOutputLatency;
 
 	// Skip small shifts to avoid audio glitches from frame-quantized re-anchoring
-	if (Math.abs(shift) < ALLOWED_GLOBAL_TIME_ANCHOR_SHIFT + latency && !force) {
-		return false;
-	}
-
-	// If force is true, but shift is zero, no change is needed
-	if (Math.abs(shift) < Number.EPSILON) {
+	if (Math.abs(shift) < ALLOWED_GLOBAL_TIME_ANCHOR_SHIFT + latency) {
 		return false;
 	}
 
 	Internals.Log.verbose(
 		{logLevel, tag: 'audio-scheduling'},
-		'Anchor ' +
-			(force ? 'forcibly ' : '') +
-			'changed from %s to %s with shift %s',
+		'Anchor changed from %s to %s with shift %s',
 		audioSyncAnchor.value,
 		newAnchor,
 		shift,

packages/player/src/use-playback.ts

@@ -10,7 +10,7 @@ import {useIsBackgrounded} from './is-backgrounded.js';
 import {setGlobalTimeAnchor} from './set-global-time-anchor.js';
 import {usePlayer} from './use-player.js';
 
-const shouldForceAnchorChange = (newState: RemotionAudioContextState) => {
+const shouldReanchorOnStateChange = (newState: RemotionAudioContextState) => {
 	if (newState === 'suspended' || newState === 'running-to-suspended') {
 		return true;
 	}
@@ -29,6 +29,27 @@ const shouldForceAnchorChange = (newState: RemotionAudioContextState) => {
 	);
 };
 
+type QueuedFrameCall =
+	| {
+			type: 'raf';
+			id: number;
+	  }
+	| {
+			type: 'timeout';
+			id: Timer;
+	  };
+
+type UsePlaybackOptions = {
+	loop: boolean;
+	playbackRate: number;
+	moveToBeginningWhenEnded: boolean;
+	inFrame: number | null;
+	outFrame: number | null;
+	browserMediaControlsBehavior: BrowserMediaControlsBehavior;
+	getCurrentFrame: ReturnType<typeof usePlayer>['getCurrentFrame'];
+	muted: boolean;
+};
+
 export const usePlayback = ({
 	loop,
 	playbackRate,
@@ -38,16 +59,7 @@ export const usePlayback = ({
 	browserMediaControlsBehavior,
 	getCurrentFrame,
 	muted,
-}: {
-	loop: boolean;
-	playbackRate: number;
-	moveToBeginningWhenEnded: boolean;
-	inFrame: number | null;
-	outFrame: number | null;
-	browserMediaControlsBehavior: BrowserMediaControlsBehavior;
-	getCurrentFrame: ReturnType<typeof usePlayer>['getCurrentFrame'];
-	muted: boolean;
-}) => {
+}: UsePlaybackOptions) => {
 	const config = Internals.useUnsafeVideoConfig();
 	const frame = Internals.Timeline.useTimelinePosition();
 	const {playing, pause, emitter, isPlaying} = usePlayer();
@@ -100,15 +112,18 @@ export const usePlayback = ({
 			absoluteTimeInSeconds: frame / config.fps,
 			globalPlaybackRate: playbackRate,
 			logLevel,
-			force: false,
 		});
 		if (changed) {
 			sharedAudioContext.audioSyncAnchorEmitter.dispatch('changed');
 		}
 	}, [config, frame, logLevel, playbackRate, sharedAudioContext, muted]);
 
 	// When the audio context is suspended, we use the opportunity to
-	// re-anchor the time to be exact.
+	// re-anchor the time if it has drifted beyond the allowed shift.
+	// Small drifts are left alone on purpose: suspend() freezes the
+	// scheduled audio nodes in place and resume() unfreezes them
+	// sample-exact, so keeping the anchor lets a plain pause/play cycle
+	// reuse the queued nodes without tearing them down.
 	useLayoutEffect(() => {
 		const audioContext = sharedAudioContext?.audioContext;
 		if (!audioContext) {
@@ -125,15 +140,19 @@ export const usePlayback = ({
 
 		const callback = () => {
 			const newState = sharedAudioContext?.getAudioContextState();
-			if (newState && shouldForceAnchorChange(newState)) {
-				setGlobalTimeAnchor({
+			if (newState && shouldReanchorOnStateChange(newState)) {
+				const changed = setGlobalTimeAnchor({
 					audioContext,
 					audioSyncAnchor: sharedAudioContext.audioSyncAnchor,
 					absoluteTimeInSeconds: getCurrentFrame() / config.fps,
 					globalPlaybackRate: playbackRate,
 					logLevel,
-					force: true,
 				});
+				// The nodes queued so far were scheduled against the old anchor,
+				// so they have to be rebuilt if the anchor moved.
+				if (changed) {
+					sharedAudioContext.audioSyncAnchorEmitter.dispatch('changed');
+				}
 			}
 		};
 
@@ -161,16 +180,7 @@ export const usePlayback = ({
 		}
 
 		let hasBeenStopped = false;
-		let reqAnimFrameCall:
-			| {
-					type: 'raf';
-					id: number;
-			  }
-			| {
-					type: 'timeout';
-					id: Timer;
-			  }
-			| null = null;
+		let reqAnimFrameCall: QueuedFrameCall | null = null;
 		let startedTime = performance.now();
 		let framesAdvanced = 0;