From e5223b29b14fbec2b295fee17f7b912edac5f8c4 Mon Sep 17 00:00:00 2001 From: hashmapenjoyer Date: Fri, 17 Apr 2026 12:06:47 -0400 Subject: [PATCH 1/3] BANGGGGGGG --- src/audio/AudioEngine.ts | 8 +- src/audio/BufferCache.ts | 17 ++- src/audio/WaveformCache.ts | 160 +++++++++++++++++++++++ src/audio/__tests__/webAudioMock.ts | 7 + src/components/Timeline/ClipWaveform.tsx | 92 +++++++++++++ src/components/Timeline/Timeline.tsx | 6 + src/components/Timeline/TimelineClip.tsx | 21 +++ src/styles/timeline.css | 9 ++ 8 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 src/audio/WaveformCache.ts create mode 100644 src/components/Timeline/ClipWaveform.tsx diff --git a/src/audio/AudioEngine.ts b/src/audio/AudioEngine.ts index dad1eb2..baaf726 100644 --- a/src/audio/AudioEngine.ts +++ b/src/audio/AudioEngine.ts @@ -1,5 +1,6 @@ import { EventEmitter } from './EventEmitter'; import { BufferCache } from './BufferCache'; +import { WaveformCache } from './WaveformCache'; import { PlaylistManager } from './PlaylistManager'; import { Scheduler } from './Scheduler'; import { TransportController } from './TransportController'; @@ -29,6 +30,7 @@ export class AudioEngine extends EventEmitter { readonly sfx: SfxController; readonly volume: VolumeController; readonly buffers: BufferCache; + readonly waveforms: WaveformCache; private readonly scheduler: Scheduler; private readonly playlistManager: PlaylistManager; @@ -41,7 +43,11 @@ export class AudioEngine extends EventEmitter { // subsystems this.volume = new VolumeController(this.ctx); - this.buffers = new BufferCache(this.ctx); + this.waveforms = new WaveformCache(); + this.buffers = new BufferCache(this.ctx, { + onAdd: (id, buffer) => this.waveforms.add(id, buffer), + onRemove: (id) => this.waveforms.remove(id), + }); this.playlistManager = new PlaylistManager(); this.scheduler = new Scheduler( diff --git a/src/audio/BufferCache.ts b/src/audio/BufferCache.ts index b26faee..891372d 100644 --- a/src/audio/BufferCache.ts +++ b/src/audio/BufferCache.ts @@ -1,20 +1,31 @@ import type { ID, Seconds } from './types'; +export interface BufferCacheHooks { + onAdd?: (id: ID, buffer: AudioBuffer) => void; + onRemove?: (id: ID) => void; +} + /** * decodes and caches AudioBuffers by ID. * single source of truth for decoded audio data. + * + * hooks let the mediator wire derived caches (e.g. WaveformCache) without + * BufferCache knowing about them. */ export class BufferCache { private readonly buffers = new Map(); private readonly ctx: AudioContext; + private readonly hooks: BufferCacheHooks; - constructor(ctx: AudioContext) { + constructor(ctx: AudioContext, hooks: BufferCacheHooks = {}) { this.ctx = ctx; + this.hooks = hooks; } async add(id: ID, arrayBuffer: ArrayBuffer): Promise { const audioBuffer = await this.ctx.decodeAudioData(arrayBuffer); this.buffers.set(id, audioBuffer); + this.hooks.onAdd?.(id, audioBuffer); return audioBuffer; } @@ -27,7 +38,9 @@ export class BufferCache { } remove(id: ID): void { - this.buffers.delete(id); + if (this.buffers.delete(id)) { + this.hooks.onRemove?.(id); + } } getDuration(id: ID): Seconds | undefined { diff --git a/src/audio/WaveformCache.ts b/src/audio/WaveformCache.ts new file mode 100644 index 0000000..49292a3 --- /dev/null +++ b/src/audio/WaveformCache.ts @@ -0,0 +1,160 @@ +import type { ID } from './types'; + +/** + * precomputed min/max peaks for one zoom level + * peaks are interleaved [min0, max0, min1, max1, ...] normalized to -127..127 + */ +export interface PeakLevel { + samplesPerBucket: number; + peaks: Int8Array; +} + +export interface PeakSlice { + peaks: Int8Array; + samplesPerBucket: number; + sampleRate: number; +} + +// smallest-bucket level is the finest detail, each subsequent level is 4x coarser +// covers zoom from ~1 bucket per 6ms up to ~1 bucket per 1.5s at 44.1kHz +const BASE_SAMPLES_PER_BUCKET = 256; +const LEVEL_COUNT = 4; +const LEVEL_STEP = 4; + +/** + * caches mipmapped waveform peaks per bufferId. + * computed once on buffer add, reads are O(visible pixels) regardless of clip length + * + * lifecycle is driven by BufferCache via callback injection in AudioEngine, + * so peaks exist if the buffer exists. + */ +export class WaveformCache { + private readonly levelsById = new Map(); + private readonly sampleRateById = new Map(); + + add(id: ID, buffer: AudioBuffer): void { + const mono = downmixToMono(buffer); + const levels: PeakLevel[] = []; + + let prev: Int8Array | null = null; + + for (let i = 0; i < LEVEL_COUNT; i++) { + const samplesPerBucket = BASE_SAMPLES_PER_BUCKET * Math.pow(LEVEL_STEP, i); + const peaks: Int8Array = prev + ? downsamplePeaks(prev, LEVEL_STEP) + : computePeaksFromSamples(mono, samplesPerBucket); + levels.push({ samplesPerBucket, peaks }); + prev = peaks; + } + + this.levelsById.set(id, levels); + this.sampleRateById.set(id, buffer.sampleRate); + } + + remove(id: ID): void { + this.levelsById.delete(id); + this.sampleRateById.delete(id); + } + + has(id: ID): boolean { + return this.levelsById.has(id); + } + + /** + * returns the finest level whose bucket count still fits within targetBuckets + * (i.e. one peak-pair covers at least one pixel). callers ask for pixel width, + * the cache picks the right resolution. + */ + get(id: ID, targetBuckets: number): PeakSlice | undefined { + const levels = this.levelsById.get(id); + const sampleRate = this.sampleRateById.get(id); + if (!levels || !sampleRate || targetBuckets <= 0) return undefined; + + let chosen = levels[0]; + for (const level of levels) { + const bucketCount = level.peaks.length / 2; + if (bucketCount >= targetBuckets) { + chosen = level; + } else { + break; + } + } + + return { + peaks: chosen.peaks, + samplesPerBucket: chosen.samplesPerBucket, + sampleRate, + }; + } +} + +/** average all channels into a single Float32Array */ +function downmixToMono(buffer: AudioBuffer): Float32Array { + const length = buffer.length; + const channelCount = buffer.numberOfChannels; + if (channelCount === 1) return buffer.getChannelData(0); + + const out = new Float32Array(length); + for (let ch = 0; ch < channelCount; ch++) { + const data = buffer.getChannelData(ch); + for (let i = 0; i < length; i++) { + out[i] += data[i]; + } + } + const inv = 1 / channelCount; + for (let i = 0; i < length; i++) out[i] *= inv; + return out; +} + +/** build the base mipmap level directly from sample data */ +function computePeaksFromSamples( + samples: Float32Array, + samplesPerBucket: number, +): Int8Array { + const bucketCount = Math.ceil(samples.length / samplesPerBucket); + const out = new Int8Array(bucketCount * 2); + + for (let b = 0; b < bucketCount; b++) { + const start = b * samplesPerBucket; + const end = Math.min(start + samplesPerBucket, samples.length); + let min = 1; + let max = -1; + for (let i = start; i < end; i++) { + const s = samples[i]; + if (s < min) min = s; + if (s > max) max = s; + } + out[b * 2] = clampToInt8(min); + out[b * 2 + 1] = clampToInt8(max); + } + return out; +} + +function downsamplePeaks(source: Int8Array, step: number): Int8Array { + const sourceBuckets = source.length / 2; + const bucketCount = Math.ceil(sourceBuckets / step); + const out = new Int8Array(bucketCount * 2); + + for (let b = 0; b < bucketCount; b++) { + const start = b * step; + const end = Math.min(start + step, sourceBuckets); + let min = 127; + let max = -128; + for (let i = start; i < end; i++) { + const mn = source[i * 2]; + const mx = source[i * 2 + 1]; + if (mn < min) min = mn; + if (mx > max) max = mx; + } + out[b * 2] = min; + out[b * 2 + 1] = max; + } + return out; +} + +function clampToInt8(v: number): number { + const scaled = Math.round(v * 127); + if (scaled > 127) return 127; + if (scaled < -128) return -128; + return scaled; +} diff --git a/src/audio/__tests__/webAudioMock.ts b/src/audio/__tests__/webAudioMock.ts index 8b028a7..6e78a02 100644 --- a/src/audio/__tests__/webAudioMock.ts +++ b/src/audio/__tests__/webAudioMock.ts @@ -98,11 +98,18 @@ export class MockAudioBuffer { readonly duration: number; readonly sampleRate: number; readonly numberOfChannels: number; + readonly length: number; constructor(duration: number, sampleRate = 44100, numberOfChannels = 2) { this.duration = duration; this.sampleRate = sampleRate; this.numberOfChannels = numberOfChannels; + // WaveformCache iterates over samples; keep this tiny so tests stay fast. + this.length = Math.min(Math.floor(duration * sampleRate), 1024); + } + + getChannelData(_channel: number): Float32Array { + return new Float32Array(this.length); } } diff --git a/src/components/Timeline/ClipWaveform.tsx b/src/components/Timeline/ClipWaveform.tsx new file mode 100644 index 0000000..dd9f349 --- /dev/null +++ b/src/components/Timeline/ClipWaveform.tsx @@ -0,0 +1,92 @@ +import { useEffect, useRef } from 'react'; +import { useAudioEngine } from '../../audio/UseAudioEngine'; + +interface ClipWaveformProps { + bufferId: string; + /** seconds into the source buffer where this clip's playback starts */ + bufferOffset: number; + /** how many seconds of the buffer this clip actually plays */ + playDuration: number; + widthPx: number; + heightPx: number; + color: string; +} + +/** + * canvas-backed waveform that reads pre-computed peaks from engine.waveforms. + * redraw cost is O(widthPx) regardless of clip length or zoom level — the + * mipmap level is chosen by WaveformCache to match the requested bucket count. + */ +export default function ClipWaveform({ + bufferId, + bufferOffset, + playDuration, + widthPx, + heightPx, + color, +}: ClipWaveformProps) { + const { engine } = useAudioEngine(); + const canvasRef = useRef(null); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || widthPx <= 0 || heightPx <= 0) return; + + const dpr = window.devicePixelRatio || 1; + canvas.width = Math.max(1, Math.floor(widthPx * dpr)); + canvas.height = Math.max(1, Math.floor(heightPx * dpr)); + canvas.style.width = `${widthPx}px`; + canvas.style.height = `${heightPx}px`; + + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + ctx.clearRect(0, 0, canvas.width, canvas.height); + + const slice = engine.waveforms.get(bufferId, Math.ceil(widthPx)); + if (!slice) return; + + const { peaks, samplesPerBucket, sampleRate } = slice; + const totalBuckets = peaks.length / 2; + + const startBucket = Math.max(0, Math.floor((bufferOffset * sampleRate) / samplesPerBucket)); + const endBucket = Math.min( + totalBuckets, + Math.ceil(((bufferOffset + playDuration) * sampleRate) / samplesPerBucket), + ); + const bucketSpan = Math.max(1, endBucket - startBucket); + + const columnCount = Math.max(1, Math.floor(widthPx * dpr)); + const mid = canvas.height / 2; + const amp = canvas.height / 2; + + ctx.fillStyle = color; + + // for each output column, find the min/max across the peaks it covers. + // bucketsPerColumn is usually < 1 when zoomed in (one bucket spans many columns) + // and >= 1 when zoomed out. + for (let col = 0; col < columnCount; col++) { + const bucketStart = startBucket + (col / columnCount) * bucketSpan; + const bucketEnd = startBucket + ((col + 1) / columnCount) * bucketSpan; + const bStart = Math.floor(bucketStart); + const bEnd = Math.max(bStart + 1, Math.ceil(bucketEnd)); + + let min = 127; + let max = -128; + for (let b = bStart; b < bEnd && b < totalBuckets; b++) { + const mn = peaks[b * 2]; + const mx = peaks[b * 2 + 1]; + if (mn < min) min = mn; + if (mx > max) max = mx; + } + if (min > max) continue; + + const yTop = mid - (max / 127) * amp; + const yBot = mid - (min / 127) * amp; + const h = Math.max(1, yBot - yTop); + ctx.fillRect(col, yTop, 1, h); + } + }, [engine, bufferId, bufferOffset, playDuration, widthPx, heightPx, color]); + + return