diff --git a/chrome/player/modules/analyzer/AudioAnalyzer.mjs b/chrome/player/modules/analyzer/AudioAnalyzer.mjs index 1fef8073..e7659bdb 100644 --- a/chrome/player/modules/analyzer/AudioAnalyzer.mjs +++ b/chrome/player/modules/analyzer/AudioAnalyzer.mjs @@ -26,6 +26,8 @@ export class AudioAnalyzer extends EventEmitter { this.backgroundAnalyzerStatus = AnalyzerStatus.IDLE; this.backgroundDoneRanges = []; this.backgroundAnalyzerEnabled = true; + // Counts startBackgroundAnalyzer's loads (see there). + this.backgroundStarts = 0; } onVadFrameProcessed(time, isSpeechProb) { @@ -184,27 +186,34 @@ export class AudioAnalyzer extends EventEmitter { console.log('[AudioAnalyzer] Starting background analyzer'); + // A stop and a new start while this one loads (the tool closed and opened again, or a + // new video) make this start stale: only the latest one may keep its player. Two + // starts for the same source used to both keep theirs, and the first ran on for good. + const start = ++this.backgroundStarts; let backgroundAnalyzerPlayer; try { backgroundAnalyzerPlayer = await this.loadPlayer(this.backgroundAnalyzerSource, this.backgroundDoneRanges, (completed) => { - if (this.backgroundAnalyzerPlayer === backgroundAnalyzerPlayer) { - console.log('[AudioAnalyzer] Background analyzer finished', completed ? 'successfully' : 'with errors'); - this.backgroundAnalyzerStatus = completed ? AnalyzerStatus.FINISHED : AnalyzerStatus.FAILED; - this.client.interfaceController.updateMarkers(); + // Only the running player's end counts: a stale player, destroyed below, cleared + // the running one's place, and nothing could stop that one any more. + if (this.backgroundAnalyzerPlayer !== backgroundAnalyzerPlayer) { + return; } + console.log('[AudioAnalyzer] Background analyzer finished', completed ? 'successfully' : 'with errors'); + this.backgroundAnalyzerStatus = completed ? AnalyzerStatus.FINISHED : AnalyzerStatus.FAILED; + this.client.interfaceController.updateMarkers(); this.backgroundAnalyzerPlayer = null; }); } catch (e) { console.warn('[AudioAnalyzer] The background analyzer did not start', e); // Forget the source, or the check above refuses every later start for it: the // next play tries again. - if (this.backgroundAnalyzerSource === newSource) { + if (start === this.backgroundStarts && this.backgroundAnalyzerSource === newSource) { this.backgroundAnalyzerSource = null; } return; } - if (newSource !== this.backgroundAnalyzerSource) { + if (start !== this.backgroundStarts || newSource !== this.backgroundAnalyzerSource) { backgroundAnalyzerPlayer.destroy(); return; } diff --git a/chrome/player/modules/crosstalk/convolution.mjs b/chrome/player/modules/crosstalk/convolution.mjs index 81eb923b..73132017 100644 --- a/chrome/player/modules/crosstalk/convolution.mjs +++ b/chrome/player/modules/crosstalk/convolution.mjs @@ -81,7 +81,9 @@ export class ConvolutionXTC { configure(options) { const g = options.g; - const tc = Math.round(options.tc); + // In samples, and not rounded to whole ones: at 48 kHz that moved the microdelay knob in + // steps of 20.8 us, so most of its range changed nothing. + const tc = options.tc; let y = options.y; if (this.cachedOptions.g === g && this.cachedOptions.tc === tc && this.cachedOptions.y === y) { @@ -104,7 +106,9 @@ export class ConvolutionXTC { y = Math.max(min_y, y); const valid = isFinite(y) && y <= max_y; - for (let k = 0; k < n; k++) { + // The positive frequencies; the negative ones mirror them below, as they must for a + // real impulse response. A delay of whole samples gave that by itself, a fraction does not. + for (let k = 0; k <= n / 2; k++) { const omegatc = 2 * Math.PI * k / n * tc; const cos = Math.cos(omegatc); const cm_I = Math.sqrt(gg - 2*g*cos + 1); @@ -126,7 +130,16 @@ export class ConvolutionXTC { H_CIS[k * 2 + 1] = H[1]; H_CROSS[k * 2] = H[2]; H_CROSS[k * 2 + 1] = H[3]; + if (k > 0 && k < n / 2) { + H_CIS[(n - k) * 2] = H[0]; + H_CIS[(n - k) * 2 + 1] = -H[1]; + H_CROSS[(n - k) * 2] = H[2]; + H_CROSS[(n - k) * 2 + 1] = -H[3]; + } } + // A real filter's response at the Nyquist frequency is real. + H_CIS[n + 1] = 0; + H_CROSS[n + 1] = 0; this.H_CIS = H_CIS; this.H_CROSS = H_CROSS; @@ -150,28 +163,33 @@ export class ConvolutionXTC { this.convolvers_XTC[other].buffer = this.buffer_XTC; if (!this.switchTimeout) { - this.getInputNode().connect(this.convolvers_XTC[other], 1); + // Connected with no output index, as it is disconnected: connected with one (which + // meant nothing here), the disconnect below never matched it and threw into an + // empty catch, so both convolvers ran for good, at twice the CPU. + this.getInputNode().connect(this.convolvers_XTC[other]); } clearTimeout(this.switchTimeout); this.switchTimeout = setTimeout(() => { this.getOutputNode().connectFrom(this.convolvers_XTC[other]); - try { - this.getInputNode().disconnect(this.convolvers_XTC[current]); - } catch (e) { - - } - try { - this.getOutputNode().disconnectFrom(this.convolvers_XTC[current]); - } catch (e) { - - } + this.detachConvolver(this.convolvers_XTC[current]); this.currentConvolver = other; this.switchTimeout = null; }, 100); } + // Takes a convolver off the input and the output where it is on them: the first switch + // has no convolver to retire, and while a switch is pending both are on the input. + detachConvolver(convolver) { + if (this.getInputNode().indexConnectedTo(convolver) !== -1) { + this.getInputNode().disconnect(convolver); + } + if (this.getOutputNode().indexConnectedFrom(convolver) !== -1) { + this.getOutputNode().disconnectFrom(convolver); + } + } + async init() { this.inputNode = new VirtualAudioNode('ConvolutionXTC Input'); this.inputBypassNode = new VirtualAudioNode('ConvolutionXTC Bypass Input'); @@ -208,14 +226,7 @@ export class ConvolutionXTC { this.getOutputNode().disconnectFrom(this.convolver_BYPASS); this.convolvers_XTC.forEach((convolver) => { - try { - this.getInputNode().disconnect(convolver); - } catch (e) { - } - try { - this.getOutputNode().disconnectFrom(convolver); - } catch (e) { - } + this.detachConvolver(convolver); }); this.convolver_BYPASS = null; this.convolvers_XTC = null; diff --git a/chrome/player/modules/vad/vad.mjs b/chrome/player/modules/vad/vad.mjs index 2be4fc37..645be8ee 100644 --- a/chrome/player/modules/vad/vad.mjs +++ b/chrome/player/modules/vad/vad.mjs @@ -57,6 +57,12 @@ const Message = { SpeechEnd: 4, }; +// Silero v5 and v6 score each 512-sample frame against the last 64 samples of the frame +// before it, as silero-vad's own OnnxWrapper.__call__ does (ricky0123/vad aa048997, "Give +// Silero v5 its context window"). Fed the bare frame, the model saw every frame as the first +// of an utterance: it scored quiet and starting speech too low. +const CONTEXT_SAMPLES = 64; + class Silero { static async new(ort, modelFetcher) { const model = new Silero(ort, modelFetcher); @@ -66,6 +72,7 @@ class Silero { constructor(ort, modelFetcher) { this.ort = ort; this.modelFetcher = modelFetcher; + this.context = new Float32Array(CONTEXT_SAMPLES); this.init = async () => { console.debug('initializing vad'); const modelArrayBuffer = await this.modelFetcher(); @@ -87,9 +94,15 @@ class Silero { this.reset_state = () => { const zeroes = Array(2 * 1 * 128).fill(0); this.state = new this.ort.Tensor('float32', zeroes, [2, 1, 128]); + this.context = new Float32Array(CONTEXT_SAMPLES); }; this.process = async (audioFrame) => { - const t = new this.ort.Tensor('float32', audioFrame, [1, audioFrame.length]); + const withContext = new Float32Array(CONTEXT_SAMPLES + audioFrame.length); + withContext.set(this.context, 0); + withContext.set(audioFrame, CONTEXT_SAMPLES); + // A copy: the caller may reuse its frame. + this.context = audioFrame.slice(-CONTEXT_SAMPLES); + const t = new this.ort.Tensor('float32', withContext, [1, withContext.length]); const inputs = { input: t, state: this.state, @@ -223,13 +236,27 @@ class AudioNodeVAD { async init() { await this.ctx.audioWorklet.addModule(assetPath('vad.worklet.mjs')); - const vadNode = new AudioWorkletNode(this.ctx, 'vad-helper-worklet', { - processorOptions: { - frameSamples: this.options.frameSamples, - }, - }); - this.entryNode = vadNode; + // The model before the node: a model that did not load (a missing file, ONNX Runtime + // not starting) left behind a node whose processor ran for the rest of the context. const model = await createModel(this.options.ort); + let vadNode; + try { + vadNode = new AudioWorkletNode(this.ctx, 'vad-helper-worklet', { + processorOptions: { + frameSamples: this.options.frameSamples, + }, + // One channel, mixed down by Web Audio (for 5.1: 0.707 (L + R) + C + 0.5 (SL + SR)): + // the processor reads the first channel only, which was the left one, so the + // dialogue in a 5.1 film's centre channel went unheard. + channelCount: 1, + channelCountMode: 'explicit', + channelInterpretation: 'speakers', + }); + } catch (e) { + model.release().catch(() => {}); + throw e; + } + this.entryNode = vadNode; this.model = model; this.frameProcessor = new FrameProcessor(model.process, model.reset_state, { frameSamples: this.options.frameSamples, diff --git a/chrome/player/ui/audio/AudioChannelMixer.mjs b/chrome/player/ui/audio/AudioChannelMixer.mjs index e0a28f2f..614adf01 100644 --- a/chrome/player/ui/audio/AudioChannelMixer.mjs +++ b/chrome/player/ui/audio/AudioChannelMixer.mjs @@ -10,6 +10,20 @@ import {AudioEqualizer} from './AudioEqualizer.mjs'; import {MAX_AUDIO_CHANNELS, CHANNEL_NAMES} from './config/AudioProfile.mjs'; import {VirtualAudioNode} from './VirtualAudioNode.mjs'; +/** + * Sizes a meter's canvas when its size has changed. Setting a canvas's size, even to the + * size it has, clears it and resets its context: that was done every frame on all seven + * meters, which are cleared before they are drawn anyway. + * @param {HTMLCanvasElement} canvas + * @param {number} width + * @param {number} height + */ +function sizeCanvas(canvas, width, height) { + // The size is a whole number: a fraction is dropped when it is set. + if (canvas.width !== Math.trunc(width)) canvas.width = width; + if (canvas.height !== Math.trunc(height)) canvas.height = height; +} + export class AudioChannelMixer extends AbstractAudioModule { constructor(configManager) { super('AudioChannelMixer'); @@ -125,8 +139,7 @@ export class AudioChannelMixer extends AbstractAudioModule { const height = canvas.clientHeight * window.devicePixelRatio; if (width === 0 || height === 0) return; - canvas.width = width; - canvas.height = height; + sizeCanvas(canvas, width, height); ctx.clearRect(0, 0, width, height); @@ -202,8 +215,7 @@ export class AudioChannelMixer extends AbstractAudioModule { const height = canvas.clientHeight * window.devicePixelRatio; if (width === 0 || height === 0) return; - canvas.width = width; - canvas.height = height; + sizeCanvas(canvas, width, height); ctx.clearRect(0, 0, width, height); @@ -434,7 +446,7 @@ export class AudioChannelMixer extends AbstractAudioModule { els.muteButton.classList.toggle('active', channel.mono); } else { channel.muted = !channel.muted; - els.muteButton.classList.toggle('active', channel.mute); + els.muteButton.classList.toggle('active', channel.muted); } this.updateNodes(); }; @@ -764,6 +776,13 @@ export class AudioChannelMixer extends AbstractAudioModule { return nodes.equalizer.hasNodes() || nodes.compressor.isEnabled(); }); + // A compressor delays what it compresses by its look-ahead: while any channel has one + // on, the channels without wait as long, or they reached the speakers 6 ms early. + const anyCompressed = activeChannels.some((i) => this.channelNodes[i].compressor.isEnabled()); + this.channelNodes.forEach((nodes, i) => { + nodes.compressor.setLatencyMatch(anyCompressed && activeChannels.includes(i)); + }); + const needsMerger = hasNonUnityChannelGains || hasActiveNodes || needsAnalyzer; const needsSplitter = needsMerger; // numberOfChannels > 1 && needsMerger; if (needsMasterGain) { diff --git a/chrome/player/ui/audio/AudioCompressor.mjs b/chrome/player/ui/audio/AudioCompressor.mjs index 4e236047..ee85b817 100644 --- a/chrome/player/ui/audio/AudioCompressor.mjs +++ b/chrome/player/ui/audio/AudioCompressor.mjs @@ -4,6 +4,47 @@ import {WebUtils} from '../../utils/WebUtils.mjs'; import {createKnob} from '../components/Knob.mjs'; import {AbstractAudioModule} from './AbstractAudioModule.mjs'; +// The DynamicsCompressorNode's look-ahead in seconds: the pre-delay of Blink's +// DynamicsCompressor, which Firefox's node is built on. +const COMPRESSOR_PRE_DELAY = 0.006; + +/** + * A DelayNode that holds the sound back exactly as long as a compressor does: Firefox + * rounds the look-ahead down to whole frames (288 at 48 kHz, 264 at 44.1 kHz). + * @param {BaseAudioContext} audioContext + * @return {DelayNode} + */ +function createLookAheadDelay(audioContext) { + const delay = audioContext.createDelay(COMPRESSOR_PRE_DELAY * 2); + delay.delayTime.value = Math.floor(COMPRESSOR_PRE_DELAY * audioContext.sampleRate) / audioContext.sampleRate; + return delay; +} + +/** + * The channels each compressor takes, for a source of more than two channels, in Web + * Audio's order (5.1: L R C LFE SL SR). A DynamicsCompressorNode takes two at most. A + * pair of speakers shares one, so both sides are turned down together and the image stays + * put; the centre and the LFE get one each, so a bass hit does not turn the dialogue down. + * @param {number} count - The number of channels, 3 or more. + * @return {number[][]} + */ +function compressorGroups(count) { + const groups = [[0, 1]]; + let next = 2; + // A centre in every layout but quad (L R SL SR); an LFE from 5.1 on. + if (count !== 4) { + groups.push([next++]); + } + if (count >= 6) { + groups.push([next++]); + } + while (next < count) { + groups.push(next + 1 < count ? [next, next + 1] : [next]); + next += 2; + } + return groups; +} + export class AudioCompressor extends AbstractAudioModule { constructor(customTitlePrepend, numberOfChannelsGetter) { super('AudioCompressor'); @@ -101,13 +142,14 @@ export class AudioCompressor extends AbstractAudioModule { if (compressor.enabled) { await this.createCompressorNodes(); - if (this.compressorNode) { - this.compressorNode.threshold.value = compressor.threshold; - this.compressorNode.knee.value = compressor.knee; - this.compressorNode.ratio.value = compressor.ratio; - this.compressorNode.attack.value = compressor.attack; - this.compressorNode.release.value = compressor.release; - this.compressorGain.gain.value = compressor.gain; + // Every compressor, with the same settings. + for (const stage of this.compressorStages || []) { + stage.compressor.threshold.value = compressor.threshold; + stage.compressor.knee.value = compressor.knee; + stage.compressor.ratio.value = compressor.ratio; + stage.compressor.attack.value = compressor.attack; + stage.compressor.release.value = compressor.release; + stage.gain.gain.value = compressor.gain; } } else { this.destroyCompressorNodes(); @@ -120,40 +162,56 @@ export class AudioCompressor extends AbstractAudioModule { if (numChannels === 0 || this.compressorNode || !this.compressorConfig?.enabled) return; const audioContext = this.audioContext; - - this.compressorNode = audioContext.createDynamicsCompressor(); - this.compressorGain = audioContext.createGain(); - - this.compressorNode.connect(this.compressorGain); + this.removeBypassDelay(); const shouldUseSplitterMerger = numChannels > 2; if (shouldUseSplitterMerger) { + // A DynamicsCompressorNode takes two channels at most: one compressor per group of + // channels (compressorGroups), all with the same settings. Only the front pair used to + // be compressed, the other channels went around it. Every compressor delays by the + // same look-ahead, so the channels stay in time. this.splitterNode = audioContext.createChannelSplitter(numChannels); this.mergerNode = audioContext.createChannelMerger(numChannels); - this.compressorMerger = audioContext.createChannelMerger(2); - this.compressorSplitter = audioContext.createChannelSplitter(2); - - for (let i = 2; i < numChannels; i++) { - this.splitterNode.connect(this.mergerNode, i, i); - } - - this.splitterNode.connect(this.compressorMerger, 0, 0); - this.splitterNode.connect(this.compressorMerger, 1, 1); - - this.compressorMerger.connect(this.compressorNode); - this.compressorGain.connect(this.compressorSplitter); - - this.compressorSplitter.connect(this.mergerNode, 0, 0); - this.compressorSplitter.connect(this.mergerNode, 1, 1); + this.compressorStages = compressorGroups(numChannels).map((channels) => { + const stage = { + compressor: audioContext.createDynamicsCompressor(), + gain: audioContext.createGain(), + }; + stage.compressor.connect(stage.gain); + if (channels.length === 1) { + this.splitterNode.connect(stage.compressor, channels[0], 0); + stage.gain.connect(this.mergerNode, 0, channels[0]); + } else { + stage.merger = audioContext.createChannelMerger(2); + stage.splitter = audioContext.createChannelSplitter(2); + channels.forEach((channel, i) => { + this.splitterNode.connect(stage.merger, channel, i); + stage.splitter.connect(this.mergerNode, i, channel); + }); + stage.merger.connect(stage.compressor); + stage.gain.connect(stage.splitter); + } + return stage; + }); this.getInputNode().disconnect(this.getOutputNode()); this.getInputNode().connect(this.splitterNode); this.getOutputNode().connectFrom(this.mergerNode); } else { + const stage = { + compressor: audioContext.createDynamicsCompressor(), + gain: audioContext.createGain(), + }; + stage.compressor.connect(stage.gain); + this.compressorStages = [stage]; + this.getInputNode().disconnect(this.getOutputNode()); - this.getInputNode().connect(this.compressorNode); - this.getOutputNode().connectFrom(this.compressorGain); + this.getInputNode().connect(stage.compressor); + this.getOutputNode().connectFrom(stage.gain); } + // The front pair's: the graph shows its gain reduction. + this.compressorNode = this.compressorStages[0].compressor; + this.compressorGain = this.compressorStages[0].gain; } destroyCompressorNodes(skipDisconnect = false) { @@ -162,32 +220,72 @@ export class AudioCompressor extends AbstractAudioModule { if (!skipDisconnect) { if (this.splitterNode) { this.getInputNode().disconnect(this.splitterNode); - this.splitterNode.disconnect(this.compressorMerger); - this.compressorMerger.disconnect(this.compressorNode); - this.compressorGain.disconnect(this.compressorSplitter); - this.compressorSplitter.disconnect(this.mergerNode); this.getOutputNode().disconnectFrom(this.mergerNode); + this.splitterNode.disconnect(); + for (const stage of this.compressorStages) { + stage.merger?.disconnect(); + stage.compressor.disconnect(); + stage.gain.disconnect(); + stage.splitter?.disconnect(); + } } else { this.getInputNode().disconnect(this.compressorNode); this.getOutputNode().disconnectFrom(this.compressorGain); + this.compressorNode.disconnect(this.compressorGain); } - this.compressorNode.disconnect(this.compressorGain); this.getInputNode().connect(this.getOutputNode()); } this.splitterNode = null; this.mergerNode = null; - this.compressorMerger = null; + this.compressorStages = null; this.compressorNode = null; this.compressorGain = null; - this.compressorSplitter = null; + + if (!skipDisconnect) { + this.updateBypassDelay(); + } + } + + /** + * Has the sound wait as long as the compressor would make it wait, while the compressor + * is off: the mixer asks it of a channel whose compressor is off while another channel's + * is on, or that channel reached the speakers 6 ms before the compressed ones. + * @param {boolean} match - Whether to wait. + */ + setLatencyMatch(match) { + this.latencyMatch = match; + this.updateBypassDelay(); + } + + updateBypassDelay() { + if (!this.audioContext || !this.getInputNode()) return; + const wanted = !!this.latencyMatch && !this.compressorNode; + if (wanted && !this.bypassDelay) { + this.bypassDelay = createLookAheadDelay(this.audioContext); + this.getInputNode().disconnect(this.getOutputNode()); + this.getInputNode().connect(this.bypassDelay); + this.getOutputNode().connectFrom(this.bypassDelay); + } else if (!wanted) { + this.removeBypassDelay(); + } + } + + removeBypassDelay() { + if (!this.bypassDelay) return; + this.getInputNode().disconnect(this.bypassDelay); + this.getOutputNode().disconnectFrom(this.bypassDelay); + this.getInputNode().connect(this.getOutputNode()); + this.bypassDelay = null; } setupNodes(audioContext) { super.setupNodes(audioContext); this.getInputNode().connect(this.getOutputNode()); this.destroyCompressorNodes(true); + this.bypassDelay = null; + this.updateBypassDelay(); this.updateCompressor(); } diff --git a/chrome/player/ui/audio/AudioConfigManager.mjs b/chrome/player/ui/audio/AudioConfigManager.mjs index 6bfcea63..aa1a8343 100644 --- a/chrome/player/ui/audio/AudioConfigManager.mjs +++ b/chrome/player/ui/audio/AudioConfigManager.mjs @@ -15,6 +15,9 @@ import {OutputConvolver} from './OutputConvolver.mjs'; import {OutputMeter} from './OutputMeter.mjs'; import {AudioProfile} from './config/AudioProfile.mjs'; +// The count a recount replaced ends with this; it is not a failure. +const RECOUNTED = 'Channel counter node changed'; + export class AudioConfigManager extends AbstractAudioModule { constructor(client) { super('AudioConfigManager'); @@ -508,8 +511,13 @@ export class AudioConfigManager extends AbstractAudioModule { this.audioUpmixer.updateChannelCount(count, this.audioContext.destination.channelCount); this.audioChannelMixer.updateChannelCount(); - this.outputConvolver.updateChannelCount(); + // There is none where IndexedDB is not available. + this.outputConvolver?.updateChannelCount(); }).catch((e) => { + // Anything but a recount that replaced this one used to vanish here. + if (e?.message !== RECOUNTED) { + console.warn('The audio channels were not updated', e); + } }); } @@ -542,12 +550,27 @@ export class AudioConfigManager extends AbstractAudioModule { try { await node.init(); } catch (e) { - resolve(2); // Fallback to 2 channels if init fails for ppl who have older browsers + // Fallback to 2 channels if init fails for ppl who have older browsers. Kept, or + // the next caller waited on this settled promise and got null. + if (this.channelCounterNode === node) { + this.cachedChannelCount = 2; + this.channelCounterPromise = null; + this.channelCounterNode = null; + } + resolve(2); + return; + } + // A recount while its worklet loaded (each video and each level switch asks for + // one) discarded it: it must not run. It used to connect itself anyway and stayed + // connected and counting for the life of the audio context. + if (this.channelCounterNode !== node) { + node.destroy(); + reject(new Error(RECOUNTED)); return; } node.once('channelcount', (count) => { if (this.channelCounterNode !== node) { - reject(new Error('Channel counter node changed')); + reject(new Error(RECOUNTED)); return; } this.cachedChannelCount = count; diff --git a/chrome/player/ui/audio/AudioCrosstalk.mjs b/chrome/player/ui/audio/AudioCrosstalk.mjs index c45484d7..f8580ba4 100644 --- a/chrome/player/ui/audio/AudioCrosstalk.mjs +++ b/chrome/player/ui/audio/AudioCrosstalk.mjs @@ -79,7 +79,10 @@ export class AudioCrosstalk extends AbstractAudioModule { return { microdelay: isNaN(this.crosstalkConfig.microdelay) ? predicted.microdelay : this.crosstalkConfig.microdelay, decay: AudioUtils.dbToGain(isNaN(this.crosstalkConfig.decay) ? predicted.decay : this.crosstalkConfig.decay), - colorgain: this.crosstalkConfig.colorgain === 20 ? Infinity : AudioUtils.dbToGain(this.crosstalkConfig.colorgain), + // The knob's top is 20 dB, as it says. It used to mean no limit at all: with the decay + // near 0 dB the filter then boosted some frequencies by over 50 dB (clipping), with + // an impulse response far longer than the 1024 samples kept of it (ringing). + colorgain: AudioUtils.dbToGain(this.crosstalkConfig.colorgain), highbypass: this.crosstalkConfig.highbypass, lowbypass: this.crosstalkConfig.lowbypass, }; diff --git a/chrome/player/ui/audio/AudioEqualizer.mjs b/chrome/player/ui/audio/AudioEqualizer.mjs index cd05fe5a..8350c63f 100644 --- a/chrome/player/ui/audio/AudioEqualizer.mjs +++ b/chrome/player/ui/audio/AudioEqualizer.mjs @@ -166,6 +166,26 @@ export class AudioEqualizer extends AbstractAudioModule { } refreshEQNodes() { + // The new filters first: a value the browser refuses (NaN) threw after the old ones + // were taken out, and the channel stayed silent until the next profile. + const equalizerNodes = this.equalizerConfig.map((node) => { + if (this.audioContext) { + const eqNode = this.audioContext.createBiquadFilter(); + eqNode.type = node.type; + eqNode.frequency.value = node.frequency; + eqNode.gain.value = node.gain; + eqNode.Q.value = node.q; + return eqNode; + } else { + return { + type: node.type, + frequency: {value: node.frequency}, + gain: {value: node.gain}, + Q: {value: node.q}, + }; + } + }); + try { this.getInputNode().disconnect(this.getOutputNode()); } catch (e) { @@ -182,26 +202,7 @@ export class AudioEqualizer extends AbstractAudioModule { node.disconnect(); }); - this.equalizerNodes = []; - - this.equalizerConfig.forEach((node) => { - if (this.audioContext) { - const eqNode = this.audioContext.createBiquadFilter(); - eqNode.type = node.type; - eqNode.frequency.value = node.frequency; - eqNode.gain.value = node.gain; - eqNode.Q.value = node.q; - - this.equalizerNodes.push(eqNode); - } else { - this.equalizerNodes.push({ - type: node.type, - frequency: {value: node.frequency}, - gain: {value: node.gain}, - Q: {value: node.q}, - }); - } - }); + this.equalizerNodes = equalizerNodes; this.equalizerNodes.forEach((node, index) => { diff --git a/chrome/player/ui/audio/ChannelUpmixer.mjs b/chrome/player/ui/audio/ChannelUpmixer.mjs index b7450054..88dd36a9 100644 --- a/chrome/player/ui/audio/ChannelUpmixer.mjs +++ b/chrome/player/ui/audio/ChannelUpmixer.mjs @@ -30,7 +30,14 @@ export class ChannelUpmixer extends AbstractAudioModule { } if (this.outputChannelCount !== outputChannelCount) { - this.mixerNode.channelCount = outputChannelCount; + // Kept, as enable() keeps it: it was never updated, so each call set the count + // again, and a count the node refused threw out of the whole channel update. + this.outputChannelCount = outputChannelCount; + try { + this.mixerNode.channelCount = outputChannelCount; + } catch (e) { + console.warn('Could not set channel count on upmixer', e); + } } } diff --git a/chrome/player/ui/audio/OutputConvolver.mjs b/chrome/player/ui/audio/OutputConvolver.mjs index ee8d467d..9afff672 100644 --- a/chrome/player/ui/audio/OutputConvolver.mjs +++ b/chrome/player/ui/audio/OutputConvolver.mjs @@ -108,7 +108,7 @@ export class OutputConvolver extends AbstractAudioModule { const arrayBuffer = await file.arrayBuffer(); const audioData = await this.audioContext.decodeAudioData(arrayBuffer).catch((e) => null); if (!audioData) { - throw new Error('Could not decode impulse response: ' + name); + throw new Error('Could not decode impulse response: ' + file.name); } // audiobuffer, get first channel @@ -284,8 +284,12 @@ export class OutputConvolver extends AbstractAudioModule { ch.convolverNode.connect(this.mergerNode, 0, idx); } - ch.convolverNode.normalize = channelConfig.normalize; - ch.convolverNode.buffer = ch.impulseBuffer; + // Only when they change: a buffer given again restarts the convolution (its tail cut + // off, a click), and this runs at every quality switch and every checkbox. + if (ch.convolverNode.buffer !== ch.impulseBuffer || ch.convolverNode.normalize !== channelConfig.normalize) { + ch.convolverNode.normalize = channelConfig.normalize; + ch.convolverNode.buffer = ch.impulseBuffer; + } }); } } diff --git a/chrome/player/ui/audio/config/AudioChannelControl.mjs b/chrome/player/ui/audio/config/AudioChannelControl.mjs index 2ace9580..9a9cce2e 100644 --- a/chrome/player/ui/audio/config/AudioChannelControl.mjs +++ b/chrome/player/ui/audio/config/AudioChannelControl.mjs @@ -1,5 +1,9 @@ import {AudioEQNode} from './AudioEQNode.mjs'; import {AudioCompressionControl} from './AudioCompressionControl.mjs'; +import {finiteOr} from './ConfigNumbers.mjs'; + +// +10 dB, the top of a channel's fader. +const MAX_CHANNEL_GAIN = Math.pow(10, 10 / 20); export class AudioChannelControl { constructor(channelId, gain, mutedOrMono, solo, equalizerNodes, compressor) { @@ -22,15 +26,18 @@ export class AudioChannelControl { } static fromObj(obj) { - const equalizerNodes = obj.equalizerNodes?.map((nodeObj) => { + const equalizerNodes = Array.isArray(obj.equalizerNodes) ? obj.equalizerNodes.map((nodeObj) => { return AudioEQNode.fromObj(nodeObj); - }) || []; + }).filter((node) => node) : []; const compressor = obj.compressor ? AudioCompressionControl.fromObj(obj.compressor) : AudioCompressionControl.default(); + // The fader's range, -∞ to +10 dB; a gain that is not a number (it threw in the + // mixer, which then left the other channels as they were) is unity. + const gain = finiteOr(obj.gain, 1, 0, MAX_CHANNEL_GAIN); if (obj.id === 'master') { - return new AudioChannelControl(obj.id, obj.gain, obj.mono, null, equalizerNodes, compressor); + return new AudioChannelControl(obj.id, gain, !!obj.mono, null, equalizerNodes, compressor); } else { - return new AudioChannelControl(obj.id, obj.gain, obj.muted, obj.solo, equalizerNodes, compressor); + return new AudioChannelControl(obj.id, gain, !!obj.muted, !!obj.solo, equalizerNodes, compressor); } } diff --git a/chrome/player/ui/audio/config/AudioCompressionControl.mjs b/chrome/player/ui/audio/config/AudioCompressionControl.mjs index e98c66a9..b09fb48d 100644 --- a/chrome/player/ui/audio/config/AudioCompressionControl.mjs +++ b/chrome/player/ui/audio/config/AudioCompressionControl.mjs @@ -1,3 +1,7 @@ +import {finiteOr} from './ConfigNumbers.mjs'; + +// +20 dB, the top of the compressor's gain knob. +const MAX_MAKEUP_GAIN = 10; export class AudioCompressionControl { constructor(enabled, attack, knee, ratio, release, threshold, gain) { @@ -11,7 +15,16 @@ export class AudioCompressionControl { } static fromObj(obj) { - return new AudioCompressionControl(obj.enabled, obj.attack, obj.knee, obj.ratio, obj.release, obj.threshold, obj.gain); + // Within the knobs' ranges; a value that is not a number threw when the compressor was + // built, so it takes the default. + const def = AudioCompressionControl.default(); + return new AudioCompressionControl(obj.enabled, + finiteOr(obj.attack, def.attack, 0, 1), + finiteOr(obj.knee, def.knee, 0, 40), + finiteOr(obj.ratio, def.ratio, 1, 20), + finiteOr(obj.release, def.release, 0, 1), + finiteOr(obj.threshold, def.threshold, -80, 0), + finiteOr(obj.gain, def.gain, 1, MAX_MAKEUP_GAIN)); } static default() { diff --git a/chrome/player/ui/audio/config/AudioConvolverControl.mjs b/chrome/player/ui/audio/config/AudioConvolverControl.mjs index dafceb13..a762ad71 100644 --- a/chrome/player/ui/audio/config/AudioConvolverControl.mjs +++ b/chrome/player/ui/audio/config/AudioConvolverControl.mjs @@ -1,4 +1,5 @@ import {MAX_AUDIO_CHANNELS} from './AudioProfile.mjs'; +import {finiteOr} from './ConfigNumbers.mjs'; export class AudioConvolverChannel { constructor(id, enabled, normalize) { @@ -51,7 +52,10 @@ export class AudioConvolverProfile { newChannels.push(AudioConvolverChannel.default(i)); } } - return new AudioConvolverProfile(obj.id, obj.label, obj.downmix, obj.bufferSize, newChannels); + // As the impulse length field allows: a stored size that was not a number kept the whole + // file, minutes of it, as the impulse response. + const bufferSize = Math.floor(finiteOr(obj.bufferSize, 4096, 128, 16384)); + return new AudioConvolverProfile(obj.id, obj.label, obj.downmix, bufferSize, newChannels); } static default(id) { diff --git a/chrome/player/ui/audio/config/AudioCrosstalkControl.mjs b/chrome/player/ui/audio/config/AudioCrosstalkControl.mjs index d653feb6..20b86d2f 100644 --- a/chrome/player/ui/audio/config/AudioCrosstalkControl.mjs +++ b/chrome/player/ui/audio/config/AudioCrosstalkControl.mjs @@ -1,3 +1,5 @@ +import {finiteOr} from './ConfigNumbers.mjs'; + export class AudioCrosstalkControl { constructor(enabled, decay, colorgain, microdelay, lowbypass, highbypass) { this.enabled = !!enabled; @@ -9,7 +11,16 @@ export class AudioCrosstalkControl { } static fromObj(obj) { - return new AudioCrosstalkControl(obj.enabled, obj.decay || obj.decaygain / 1000, obj.colorgain, obj.microdelay, obj.lowbypass, obj.highbypass); + // Within the knobs' ranges. The decay and the microdelay may be NaN (stored as null): + // that means the value worked out from the distances. The others must be numbers: a + // crossover frequency that was not one made the filter throw when it was built. + const def = AudioCrosstalkControl.default(); + return new AudioCrosstalkControl(obj.enabled, + finiteOr(obj.decay || obj.decaygain / 1000, NaN, -5, -0.01), + finiteOr(obj.colorgain, def.colorgain, 0, 20), + finiteOr(obj.microdelay, NaN, 30, 200), + finiteOr(obj.lowbypass, def.lowbypass, 20, 2000), + finiteOr(obj.highbypass, def.highbypass, 2000, 20000)); } static default() { diff --git a/chrome/player/ui/audio/config/AudioEQNode.mjs b/chrome/player/ui/audio/config/AudioEQNode.mjs index d4e2cd45..7135f774 100644 --- a/chrome/player/ui/audio/config/AudioEQNode.mjs +++ b/chrome/player/ui/audio/config/AudioEQNode.mjs @@ -1,3 +1,9 @@ +import {finiteOr} from './ConfigNumbers.mjs'; + +// What a BiquadFilterNode can be. It ignores any other type it is given, so a node of an +// unknown type stayed a lowpass at its frequency. +const FILTER_TYPES = ['lowpass', 'highpass', 'bandpass', 'lowshelf', 'highshelf', 'peaking', 'notch', 'allpass']; + export class AudioEQNode { constructor(type, frequency, gain, q) { this.type = type; @@ -6,8 +12,17 @@ export class AudioEQNode { this.q = parseFloat(q); } + /** + * @param {Object} obj - A node as stored or imported. + * @return {AudioEQNode|null} null for a node with no usable type or frequency. + */ static fromObj(obj) { - return new AudioEQNode(obj.type, obj.frequency, obj.gainDb === undefined ? obj.gain : obj.gainDb, obj.q ?? 1); + const frequency = parseFloat(obj?.frequency); + if (!FILTER_TYPES.includes(obj?.type) || !Number.isFinite(frequency) || frequency <= 0) { + return null; + } + const gain = obj.gainDb === undefined ? obj.gain : obj.gainDb; + return new AudioEQNode(obj.type, frequency, finiteOr(gain, 0), finiteOr(obj.q, 1)); } toObj() { @@ -19,4 +34,3 @@ export class AudioEQNode { }; } } - diff --git a/chrome/player/ui/audio/config/AudioProfile.mjs b/chrome/player/ui/audio/config/AudioProfile.mjs index 5c20aab3..4ff18bcb 100644 --- a/chrome/player/ui/audio/config/AudioProfile.mjs +++ b/chrome/player/ui/audio/config/AudioProfile.mjs @@ -25,8 +25,8 @@ export class AudioProfile { const profile = new AudioProfile(obj.id); profile.label = obj.label; - if (obj.channels && obj.channels.length <= MAX_AUDIO_CHANNELS) { - profile.channels = obj.channels.map((channel) => { + if (Array.isArray(obj.channels) && obj.channels.length <= MAX_AUDIO_CHANNELS) { + profile.channels = obj.channels.filter((channel) => channel && typeof channel === 'object').map((channel) => { return AudioChannelControl.fromObj(channel); }); } else if (obj.mixerChannels && obj.mixerChannels.length === 7) { // Legacy @@ -36,36 +36,26 @@ export class AudioProfile { profile.master = profile.channels.pop(); } - // Sort channels by ID, increasing - profile.channels.sort((a, b) => a.id - b.id); - - // fill remaining with defaults if less than MAX_CHANNELS - if (profile.channels.length < MAX_AUDIO_CHANNELS) { - const newChannels = []; - for (let i = 0; i < MAX_AUDIO_CHANNELS; i++) { - const existingChannel = profile.channels.find((ch) => ch.id === i); - if (existingChannel) { - newChannels.push(existingChannel); - } else { - newChannels.push(AudioChannelControl.default(i)); - } - } - profile.channels = newChannels; - } + // One channel per ID, 0 to MAX_AUDIO_CHANNELS - 1, in order, the missing ones default: + // a full list with an ID out of range (7) or twice had no channel for some ID, and the + // mixer threw on it. The first of an ID wins; other IDs are dropped. + const channels = profile.channels; + profile.channels = Array.from({length: MAX_AUDIO_CHANNELS}, (_, i) => { + return channels.find((ch) => ch.id === i) || AudioChannelControl.default(i); + }); if (obj.master) { - const masterChannel = AudioChannelControl.fromObj(obj.master); - profile.master = masterChannel; + profile.master = AudioChannelControl.fromObj({...obj.master, id: 'master'}); } if (!profile.master) { profile.master = AudioChannelControl.default('master'); } - if (obj.equalizerNodes) { + if (Array.isArray(obj.equalizerNodes)) { profile.master.equalizerNodes = obj.equalizerNodes.map((node) => { return AudioEQNode.fromObj(node); - }) || []; + }).filter((node) => node); } if (obj.compressor) { diff --git a/chrome/player/ui/audio/config/ConfigNumbers.mjs b/chrome/player/ui/audio/config/ConfigNumbers.mjs new file mode 100644 index 00000000..e3a2d204 --- /dev/null +++ b/chrome/player/ui/audio/config/ConfigNumbers.mjs @@ -0,0 +1,16 @@ +/** + * A number from a stored or imported audio profile, made safe to hand to Web Audio: an + * AudioParam throws on NaN and Infinity, and a profile file can hold anything. + * @param {*} value - The value as stored. + * @param {number} fallback - What to use when it is not a finite number. + * @param {number} [min=-Infinity] - The smallest value allowed. + * @param {number} [max=Infinity] - The largest value allowed. + * @return {number} + */ +export function finiteOr(value, fallback, min = -Infinity, max = Infinity) { + const number = parseFloat(value); + if (!Number.isFinite(number)) { + return fallback; + } + return Math.min(max, Math.max(min, number)); +} diff --git a/chrome/player/utils/AudioUtils.mjs b/chrome/player/utils/AudioUtils.mjs index a984aa1a..657a7311 100644 --- a/chrome/player/utils/AudioUtils.mjs +++ b/chrome/player/utils/AudioUtils.mjs @@ -71,7 +71,9 @@ export class AudioUtils { } static isClipping(analyser) { - const bufferLength = analyser.frequencyBinCount; + // The whole window: an array of frequencyBinCount (half of it) got the older half only, + // so a clip in the newer half went unseen. + const bufferLength = analyser.fftSize; const dataArray = new Float32Array(bufferLength); analyser.getFloatTimeDomainData(dataArray); diff --git a/docs/vendored-libraries.md b/docs/vendored-libraries.md index aa54a7b1..3ed1c628 100644 --- a/docs/vendored-libraries.md +++ b/docs/vendored-libraries.md @@ -560,10 +560,15 @@ only place it executes; the VAD is reached only from `AudioAnalyzerNode`, behind subtitle syncing. It scores a fixed 600-frame signal (six kinds x 100 frames: silence, noise, sweep, tremolo+noise, harmonic buzz, loud noise; seeded) through `VadJS.createModel()`, the same entry `AudioNodeVAD` uses, -and compares with `tests/e2e/vad-reference.json`: 600 scores recorded in -Firefox from the old custom runtime, 105 of the 600 frames speech. The test -asserts a max difference <= 1e-4 and that no frame crosses the 0.5 speech -threshold differently from the reference. +and compares with `tests/e2e/vad-reference.json`. The test asserts a max +difference <= 1e-4 and that no frame crosses the 0.5 speech threshold +differently from the reference. Until 2026-10-03 the file held 600 scores +recorded in Firefox from the old custom runtime, 105 of the 600 frames +speech. Since `vad.mjs` gives the model the previous frame's last 64 samples +with each frame (ricky0123/vad aa048997, issue #127), it holds the scores of +that path on onnxruntime-web 1.30.0 in Node, 112 frames speech: on the same +runtime Node gave the old Firefox scores to within 3.4e-6, with no decision +changed. Old runtime vs 1.30.0, measured in Firefox 156 on Windows (old: 3 runs; 1.30.0: 10 runs, each with the same scores): diff --git a/tests/e2e/ext-specs/vad.e2e.mjs b/tests/e2e/ext-specs/vad.e2e.mjs index 3457673a..ba14f96b 100644 --- a/tests/e2e/ext-specs/vad.e2e.mjs +++ b/tests/e2e/ext-specs/vad.e2e.mjs @@ -31,11 +31,12 @@ import {EXTENSION_UUID, OPENER_URL} from '../wdio.extension.conf.mjs'; const ORIGIN = `moz-extension://${EXTENSION_UUID}`; const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); -// The scores the runtime shipped until 1.3.82.44 gave for the signal sequence below: -// onnxruntime-web 1.20.0's loader driving a custom build of ONNX Runtime at -// 5c74539ab7, recorded in Firefox before that runtime was replaced by the stock -// onnxruntime-web files. VAD_WRITE_REFERENCE=1 rewrites the file from whatever -// runtime the build now ships - only do that on purpose. +// The scores vad.mjs gives for the signal sequence below since it feeds the model the +// previous frame's last 64 samples with each frame (ricky0123/vad aa048997), computed on +// onnxruntime-web 1.30.0 in Node. On the same runtime, Node gave the scores recorded here +// in Firefox before that change (from the custom runtime shipped until 1.3.82.44) to +// within 3.4e-6, with no decision changed. VAD_WRITE_REFERENCE=1 rewrites the file from +// whatever runtime the build now ships - only do that on purpose. const REFERENCE = path.join(__dirname, '../vad-reference.json'); describe('the voice activity detector', function() { @@ -65,9 +66,9 @@ describe('the voice activity detector', function() { }); const session = await ort.InferenceSession.create(model); - // The same call AudioAnalyzerNode makes: 512 samples at 16 kHz, and a - // zeroed [2,1,128] state. `sr` is commented out in vad.mjs because - // this model has the rate baked in. + // Close to the call vad.mjs makes: 512 samples at 16 kHz (vad.mjs puts the + // previous frame's last 64 in front of them), and a zeroed [2,1,128] state. + // `sr` is commented out in vad.mjs because this model has the rate baked in. const run = async (frame, state) => { const out = await session.run({ input: new ort.Tensor('float32', frame, [1, frame.length]), @@ -200,9 +201,9 @@ describe('the voice activity detector', function() { if ((score >= 0.5) !== (reference[i] >= 0.5)) flips++; }); console.log(` vad: largest difference from the reference ${maxDiff}, ${flips} decisions changed`); - // A newer runtime may round differently (1.30.0 against the reference: 3.4e-6 + // A newer runtime may round differently (1.30.0 against the old custom runtime: 3.4e-6 // at most, measured over 3000 frames), but no frame may change its speech decision. - // The reference frame nearest the 0.5 threshold is 5.3e-4 away, so a runtime + // The reference frame nearest the 0.5 threshold is 1.6e-3 away, so a runtime // within 1e-4 cannot flip one, and a broken model misses by far more. expect(maxDiff).toBeLessThanOrEqual(1e-4); expect(flips).toBe(0); diff --git a/tests/e2e/vad-reference.json b/tests/e2e/vad-reference.json index 4b6cf80b..12c3770b 100644 --- a/tests/e2e/vad-reference.json +++ b/tests/e2e/vad-reference.json @@ -1 +1 @@ -{"scores":[0.04426264762878418,0.03361243009567261,0.0221235454082489,0.016314268112182617,0.012152045965194702,0.009266972541809082,0.00727573037147522,0.005945742130279541,0.005024522542953491,0.004377901554107666,0.0039028823375701904,0.0035454928874969482,0.003265947103500366,0.0030364394187927246,0.002840697765350342,0.0026682615280151367,0.0025125443935394287,0.0023693740367889404,0.002236485481262207,0.0021126866340637207,0.0019974112510681152,0.0018902122974395752,0.0017913877964019775,0.0017004013061523438,0.001616448163986206,0.001536250114440918,0.0014613866806030273,0.001392662525177002,0.001329660415649414,0.0012719333171844482,0.0012189745903015137,0.001170337200164795,0.0011256635189056396,0.0010859966278076172,0.0010502040386199951,0.001017153263092041,0.0009867548942565918,0.0009585916996002197,0.0009326040744781494,0.0009084939956665039,0.0008861422538757324,0.0008639097213745117,0.0008423030376434326,0.0008222758769989014,0.0008035898208618164,0.0007860958576202393,0.0007698535919189453,0.0007547438144683838,0.00074043869972229,0.0007270872592926025,0.0007145702838897705,0.0007027685642242432,0.0006917417049407959,0.0006813108921051025,0.0006714463233947754,0.0006621479988098145,0.000653386116027832,0.000645071268081665,0.0006372034549713135,0.0006297528743743896,0.0006226301193237305,0.0006158947944641113,0.0006094574928283691,0.0006033480167388916,0.0005975663661956787,0.000591963529586792,0.0005867481231689453,0.0005816221237182617,0.0005767941474914551,0.0005721151828765869,0.000567704439163208,0.0005634427070617676,0.0005593597888946533,0.0005553364753723145,0.0005516111850738525,0.0005480349063873291,0.000544428825378418,0.0005410909652709961,0.0005379617214202881,0.0005348026752471924,0.0005317330360412598,0.000528872013092041,0.0005259811878204346,0.000523298978805542,0.0005206465721130371,0.0005181431770324707,0.0005156993865966797,0.0005132853984832764,0.0005109906196594238,0.0005087554454803467,0.0005066096782684326,0.0005045235157012939,0.0005025267601013184,0.0005005002021789551,0.0004986226558685303,0.0004967451095581055,0.0004950165748596191,0.0004932582378387451,0.0004915595054626465,0.0004899501800537109,0.000280529260635376,0.0002352893352508545,0.00020816922187805176,0.0003294646739959717,0.00044649839401245117,0.00023412704467773438,0.0007701218128204346,0.0006906092166900635,0.0002975165843963623,0.0007514357566833496,0.0004387795925140381,0.00029781460762023926,0.0003911256790161133,0.00039514899253845215,0.00036591291427612305,0.000509798526763916,0.0016371607780456543,0.00037872791290283203,0.0005443692207336426,0.0002085566520690918,0.00035569071769714355,0.0005967915058135986,0.0009030699729919434,0.0002493560314178467,0.0003338456153869629,0.00040978193283081055,0.0003426671028137207,0.00041610002517700195,0.0005152225494384766,0.00041028857231140137,0.00029143691062927246,0.00032848119735717773,0.0006676018238067627,0.0006451904773712158,0.000502854585647583,0.0004943907260894775,0.0002886354923248291,0.00030356645584106445,0.0006278455257415771,0.00026556849479675293,0.000496596097946167,0.00026220083236694336,0.00039070844650268555,0.0004126429557800293,0.0004073977470397949,0.0008152127265930176,0.0003261566162109375,0.0003250539302825928,0.00024113059043884277,0.00043573975563049316,0.0006508827209472656,0.00041812658309936523,0.0005345642566680908,0.00029155611991882324,0.00040215253829956055,0.000534743070602417,0.0005028843879699707,0.0006400346755981445,0.0005637705326080322,0.00031322240829467773,0.00043782591819763184,0.0008508563041687012,0.0007560253143310547,0.0005782544612884521,0.0003986656665802002,0.0003501474857330322,0.0005432367324829102,0.0006785690784454346,0.0007443726062774658,0.00046634674072265625,0.0006253421306610107,0.00040906667709350586,0.0004870295524597168,0.0003662109375,0.0005242526531219482,0.0005266070365905762,0.0004024803638458252,0.000999540090560913,0.0005761981010437012,0.0005726218223571777,0.0009427368640899658,0.0005815625190734863,0.000387340784072876,0.0005991458892822266,0.0005819797515869141,0.0006287097930908203,0.0007088780403137207,0.0013613700866699219,0.000762641429901123,0.0007155835628509521,0.0007409751415252686,0.0006592869758605957,0.0008808672428131104,0.0010702013969421387,0.0005277693271636963,0.0007278323173522949,0.0007666647434234619,0.0006475448608398438,0.0008795261383056641,0.000896066427230835,0.0008035600185394287,0.012467175722122192,0.00908505916595459,0.03214120864868164,0.032052814960479736,0.0319305956363678,0.10611802339553833,0.1972711980342865,0.21052545309066772,0.1222984790802002,0.1440516710281372,0.1294560432434082,0.1291925311088562,0.20221689343452454,0.06969058513641357,0.019384682178497314,0.014617115259170532,0.005248516798019409,0.004913777112960815,0.005455523729324341,0.004832327365875244,0.004020869731903076,0.003595978021621704,0.002728074789047241,0.0015239715576171875,0.0020419657230377197,0.0012610852718353271,0.0012173056602478027,0.0034584105014801025,0.13903549313545227,0.14074376225471497,0.15839415788650513,0.1924954354763031,0.062997967004776,0.07023832201957703,0.027933835983276367,0.03550717234611511,0.03526264429092407,0.04018750786781311,0.02934950590133667,0.026507943868637085,0.027344226837158203,0.0424523651599884,0.04171904921531677,0.06723561882972717,0.0877496600151062,0.0432010293006897,0.024874985218048096,0.013879895210266113,0.009854704141616821,0.00814354419708252,0.005283266305923462,0.001922011375427246,0.0011453330516815186,0.001430124044418335,0.0014149248600006104,0.00125160813331604,0.0017453134059906006,0.0024900436401367188,0.0027844011783599854,0.03048574924468994,0.039834678173065186,0.03173971176147461,0.022761374711990356,0.0076965391635894775,0.0035931766033172607,0.002367943525314331,0.0022437572479248047,0.0025293827056884766,0.002426058053970337,0.0018527805805206299,0.0017022788524627686,0.0015384256839752197,0.001866757869720459,0.0024281442165374756,0.0028335750102996826,0.0031022727489471436,0.002794027328491211,0.0024118125438690186,0.0020422637462615967,0.0017963647842407227,0.000754624605178833,0.0008837580680847168,0.0006856918334960938,0.0006917417049407959,0.0007860362529754639,0.0008540749549865723,0.0008137226104736328,0.0009023547172546387,0.0010724067687988281,0.0015714764595031738,0.001501321792602539,0.001413196325302124,0.0013950765132904053,0.001658797264099121,0.001969367265701294,0.0025510191917419434,0.0023665428161621094,0.0019764602184295654,0.001634150743484497,0.017277568578720093,0.10238721966743469,0.07813394069671631,0.07716038823127747,0.09054115414619446,0.36181342601776123,0.1092241108417511,0.10487452149391174,0.09083902835845947,0.14875876903533936,0.11405313014984131,0.3151179552078247,0.12118583917617798,0.1427289843559265,0.1132509708404541,0.3258451223373413,0.11850568652153015,0.13971209526062012,0.12032002210617065,0.3689267039299011,0.13318389654159546,0.13616955280303955,0.11436048150062561,0.5217306613922119,0.10879844427108765,0.12328401207923889,0.10047104954719543,0.08021271228790283,0.43793654441833496,0.10077130794525146,0.09975284337997437,0.1604977250099182,0.5603173971176147,0.12825697660446167,0.11752280592918396,0.19925037026405334,0.4938255846500397,0.15522998571395874,0.14045575261116028,0.15070906281471252,0.5005322098731995,0.1535453498363495,0.1404176652431488,0.12657427787780762,0.19099485874176025,0.14423882961273193,0.13992652297019958,0.09876009821891785,0.19545337557792664,0.15226948261260986,0.29352515935897827,0.11600729823112488,0.18773630261421204,0.14650890231132507,0.379223108291626,0.14670562744140625,0.17140263319015503,0.14015626907348633,0.4831201136112213,0.15364447236061096,0.15105247497558594,0.129347026348114,0.5414308309555054,0.09899771213531494,0.14082270860671997,0.11659976840019226,0.5102393627166748,0.1108982264995575,0.13015401363372803,0.1064402163028717,0.1824110448360443,0.4626973569393158,0.13572248816490173,0.12268170714378357,0.18563178181648254,0.4959312379360199,0.1570112705230713,0.13942211866378784,0.16903114318847656,0.4754716157913208,0.1471881866455078,0.13610512018203735,0.15219548344612122,0.4743066430091858,0.14727839827537537,0.14357930421829224,0.1218685507774353,0.4164915084838867,0.14826583862304688,0.35806864500045776,0.12808656692504883,0.19141000509262085,0.14678463339805603,0.38064882159233093,0.14942795038223267,0.18718701601028442,0.14782091975212097,0.3849732279777527,0.19237816333770752,0.17454814910888672,0.9232373237609863,0.9456128478050232,0.923130989074707,0.7805612087249756,0.4625827670097351,0.6127474308013916,0.7825641632080078,0.8517621755599976,0.8168652057647705,0.838385820388794,0.8976709246635437,0.9185230731964111,0.9333450794219971,0.9393789768218994,0.819932222366333,0.7564963102340698,0.6872079968452454,0.8655126094818115,0.9252445697784424,0.9188910722732544,0.9343234300613403,0.950434148311615,0.9541885852813721,0.9585597515106201,0.951445460319519,0.8312418460845947,0.8355081081390381,0.826471209526062,0.8250129222869873,0.9026691913604736,0.9354641437530518,0.9271295666694641,0.9461213946342468,0.9632974863052368,0.9509962797164917,0.7233214378356934,0.7276747822761536,0.8297522068023682,0.8967413902282715,0.8685028553009033,0.8902546763420105,0.9012893438339233,0.9300196170806885,0.9452201128005981,0.940500020980835,0.8410181999206543,0.687976062297821,0.605289876461029,0.7462476491928101,0.8988511562347412,0.9050636291503906,0.9118446111679077,0.9343488216400146,0.9402833580970764,0.9460069537162781,0.9480079412460327,0.782203197479248,0.8236589431762695,0.8091512322425842,0.7824221849441528,0.7940798997879028,0.9100990891456604,0.9032817482948303,0.9275792837142944,0.9472370147705078,0.9361611008644104,0.8140461444854736,0.7496094107627869,0.8111400604248047,0.8615120053291321,0.8670157790184021,0.8975076079368591,0.9106007814407349,0.933722734451294,0.94832444190979,0.9505101442337036,0.927918553352356,0.7534503936767578,0.6347561478614807,0.7121369242668152,0.8916966915130615,0.8934404850006104,0.9050190448760986,0.9387825727462769,0.9476673007011414,0.9421846866607666,0.9417272806167603,0.7889217138290405,0.808903694152832,0.8403158783912659,0.8092901706695557,0.7826581001281738,0.907044529914856,0.9048027396202087,0.9337646961212158,0.9462253451347351,0.9544730186462402,0.9109150171279907,0.7660410404205322,0.7861652374267578,0.5954439043998718,0.3379449248313904,0.17538979649543762,0.19470202922821045,0.1389811933040619,0.09987139701843262,0.18215975165367126,0.11516577005386353,0.04875907301902771,0.09366518259048462,0.13927555084228516,0.06272342801094055,0.10533198714256287,0.1373630166053772,0.04801374673843384,0.1859237551689148,0.11163592338562012,0.11630609631538391,0.07277965545654297,0.21708819270133972,0.1852976381778717,0.11214697360992432,0.1635431945323944,0.04090172052383423,0.0653482973575592,0.0996628999710083,0.042926669120788574,0.05344778299331665,0.07750532031059265,0.1328551471233368,0.07993754744529724,0.12982797622680664,0.14214763045310974,0.07475066184997559,0.20357981324195862,0.061999887228012085,0.08291122317314148,0.04724803566932678,0.03421333432197571,0.03755667805671692,0.06629377603530884,0.15968173742294312,0.08666309714317322,0.12651947140693665,0.06436705589294434,0.05963855981826782,0.11346849799156189,0.15542739629745483,0.09663704037666321,0.06478184461593628,0.09779596328735352,0.08419451117515564,0.06367221474647522,0.05166226625442505,0.024181127548217773,0.05056107044219971,0.024588584899902344,0.09644943475723267,0.0976790189743042,0.10267552733421326,0.0882524847984314,0.036702752113342285,0.07604891061782837,0.07869204878807068,0.05198565125465393,0.08601769804954529,0.0388149619102478,0.04104799032211304,0.06763559579849243,0.03467079997062683,0.1415744423866272,0.04902368783950806,0.09605416655540466,0.0366629958152771,0.05941307544708252,0.10148024559020996,0.0631156861782074,0.09227639436721802,0.07513812184333801,0.024425774812698364,0.05642393231391907,0.04534587264060974,0.06785833835601807,0.16380971670150757,0.0607510507106781,0.04690265655517578,0.14689144492149353,0.14343440532684326,0.19799700379371643,0.11087444424629211,0.09388649463653564,0.11715355515480042,0.12333992123603821,0.16498810052871704,0.03317055106163025,0.08260032534599304,0.08325603604316711,0.025661349296569824,0.04480057954788208,0.0388471782207489]} +{"scores":[0.012012064456939697,0.007816582918167114,0.005424201488494873,0.004969954490661621,0.004763931035995483,0.004032492637634277,0.003262549638748169,0.0027583837509155273,0.0025595128536224365,0.002436131238937378,0.0023582279682159424,0.0023499727249145508,0.0024065375328063965,0.002463698387145996,0.002504408359527588,0.0025522708892822266,0.002610117197036743,0.0026641786098480225,0.0027118325233459473,0.002758026123046875,0.0028040707111358643,0.002848297357559204,0.002890974283218384,0.0029337704181671143,0.002975374460220337,0.003016054630279541,0.0030519962310791016,0.003078639507293701,0.003103882074356079,0.00312766432762146,0.003149747848510742,0.0031698644161224365,0.003188103437423706,0.003204256296157837,0.0032186508178710938,0.003231137990951538,0.003241926431655884,0.003251314163208008,0.0032592713832855225,0.0032659173011779785,0.00327146053314209,0.0032761096954345703,0.00327986478805542,0.0032827556133270264,0.003284931182861328,0.003286510705947876,0.0032877326011657715,0.0032885372638702393,0.0032889246940612793,0.0032888054847717285,0.00328826904296875,0.0032873153686523438,0.0032860636711120605,0.0032843947410583496,0.0032824277877807617,0.0032801032066345215,0.0032775402069091797,0.0032745301723480225,0.0032711923122406006,0.003267735242843628,0.003263920545578003,0.0032597780227661133,0.003255516290664673,0.0032508373260498047,0.0032460391521453857,0.003240913152694702,0.003235548734664917,0.003230154514312744,0.0032243728637695312,0.003218322992324829,0.003212273120880127,0.0032059550285339355,0.0031995177268981934,0.0031932592391967773,0.0031867921352386475,0.003180176019668579,0.0031733810901641846,0.003166407346725464,0.0031593143939971924,0.0031520426273345947,0.0031447112560272217,0.0031372010707855225,0.003129512071609497,0.0031218528747558594,0.003113985061645508,0.00310593843460083,0.00309792160987854,0.0030897557735443115,0.00308150053024292,0.0030732154846191406,0.003064870834350586,0.0030564069747924805,0.003047853708267212,0.0030393004417419434,0.003030627965927124,0.0030219554901123047,0.0030132830142974854,0.0030044913291931152,0.0029956698417663574,0.0029868483543395996,0.03816753625869751,0.01722654700279236,0.009419113397598267,0.00718960165977478,0.006668031215667725,0.005145043134689331,0.005670994520187378,0.005742847919464111,0.004341572523117065,0.07559290528297424,0.018985658884048462,0.009428054094314575,0.010650426149368286,0.005805015563964844,0.005720585584640503,0.005162328481674194,0.0038298070430755615,0.005061358213424683,0.004683703184127808,0.004808336496353149,0.00423043966293335,0.04712232947349548,0.01584094762802124,0.009939968585968018,0.00910213589668274,0.004684358835220337,0.005525171756744385,0.003927111625671387,0.04630228877067566,0.013787686824798584,0.009774893522262573,0.009409427642822266,0.01009988784790039,0.018088102340698242,0.011069536209106445,0.008645206689834595,0.008141547441482544,0.006127715110778809,0.006486803293228149,0.005269348621368408,0.00792694091796875,0.005607724189758301,0.005846947431564331,0.004242241382598877,0.018168658018112183,0.010078907012939453,0.007524043321609497,0.008545726537704468,0.008300602436065674,0.00626903772354126,0.006842613220214844,0.004329413175582886,0.005861073732376099,0.004518061876296997,0.004844963550567627,0.004957973957061768,0.0025760233402252197,0.006557971239089966,0.010337620973587036,0.009451240301132202,0.008681267499923706,0.011517554521560669,0.006478607654571533,0.004159152507781982,0.014917463064193726,0.007405966520309448,0.00583416223526001,0.005343616008758545,0.004761546850204468,0.005018711090087891,0.007939934730529785,0.005634576082229614,0.006717652082443237,0.00789603590965271,0.00617942214012146,0.004684031009674072,0.002999424934387207,0.004438817501068115,0.004482269287109375,0.003753185272216797,0.024243205785751343,0.008219599723815918,0.006851226091384888,0.006004601716995239,0.0033975541591644287,0.004468739032745361,0.004895389080047607,0.002485722303390503,0.0023785531520843506,0.0030267834663391113,0.00389862060546875,0.0032938718795776367,0.015068501234054565,0.006275951862335205,0.009092777967453003,0.007496267557144165,0.006128638982772827,0.008082956075668335,0.006980627775192261,0.014891892671585083,0.05498245358467102,0.17415210604667664,0.5888603925704956,0.7433733344078064,0.6584667563438416,0.8459290862083435,0.9060503244400024,0.7552506923675537,0.7210556268692017,0.6479096412658691,0.49839383363723755,0.3687772750854492,0.3729599714279175,0.47015029191970825,0.297778844833374,0.31302040815353394,0.18562954664230347,0.05313059687614441,0.11314228177070618,0.031509727239608765,0.039510369300842285,0.04927036166191101,0.014393806457519531,0.029615163803100586,0.029167473316192627,0.007513284683227539,0.05126166343688965,0.01627117395401001,0.005814969539642334,0.029527097940444946,0.011159002780914307,0.0035352706909179688,0.010105729103088379,0.004011034965515137,0.005406171083450317,0.0028472840785980225,0.007918745279312134,0.004113256931304932,0.003496408462524414,0.002588331699371338,0.003298550844192505,0.0044512152671813965,0.00800827145576477,0.0108872652053833,0.008396178483963013,0.006539881229400635,0.004095733165740967,0.0029164552688598633,0.009186655282974243,0.009292930364608765,0.001878976821899414,0.0071017444133758545,0.0008233785629272461,0.0003649294376373291,0.00030669569969177246,0.0001748204231262207,0.00016227364540100098,0.00016394257545471191,0.00034928321838378906,0.00015723705291748047,0.00017437338829040527,0.000326305627822876,0.0002262890338897705,0.0003243386745452881,0.00023311376571655273,0.003007948398590088,0.0009798109531402588,0.0005395114421844482,0.0016696155071258545,0.0006997585296630859,0.001407712697982788,0.0013041794300079346,0.0006301701068878174,0.0008629262447357178,0.0005390048027038574,0.0004367232322692871,0.0006929337978363037,0.0005476176738739014,0.000731348991394043,0.0008575618267059326,0.0008417963981628418,0.0006485283374786377,0.001417607069015503,0.0008109509944915771,0.0002824366092681885,0.00036066770553588867,0.0004533827304840088,0.0005088150501251221,0.0004189908504486084,0.00036600232124328613,0.00041294097900390625,0.0006042122840881348,0.0007426440715789795,0.000567317008972168,0.0006493031978607178,0.0005719661712646484,0.0003282129764556885,0.0006926655769348145,0.0004534423351287842,0.0003959536552429199,0.015317022800445557,0.021366596221923828,0.011258602142333984,0.01050800085067749,0.025231927633285522,0.06563934683799744,0.006252944469451904,0.004114478826522827,0.0211237370967865,0.030828356742858887,0.013185679912567139,0.016650080680847168,0.04537615180015564,0.027416914701461792,0.01104581356048584,0.013581424951553345,0.034161269664764404,0.0193861722946167,0.01001688838005066,0.0131072998046875,0.04085242748260498,0.012769460678100586,0.01338893175125122,0.0233650803565979,0.10370376706123352,0.027494311332702637,0.01806163787841797,0.05124324560165405,0.11395472288131714,0.03283888101577759,0.028063923120498657,0.07614389061927795,0.10178625583648682,0.030175983905792236,0.048006415367126465,0.08234956860542297,0.06161335110664368,0.033801108598709106,0.0262603759765625,0.05112758278846741,0.06010407209396362,0.03163209557533264,0.02292400598526001,0.0681963562965393,0.04937741160392761,0.022628039121627808,0.011105954647064209,0.054887861013412476,0.02048039436340332,0.011886626482009888,0.009146571159362793,0.0464017391204834,0.016219109296798706,0.007589936256408691,0.009001374244689941,0.02379581332206726,0.02171918749809265,0.010971903800964355,0.014903098344802856,0.03117045760154724,0.02675488591194153,0.012766510248184204,0.037695080041885376,0.1112591028213501,0.04352766275405884,0.02018722891807556,0.08196043968200684,0.12262731790542603,0.061264097690582275,0.0526946485042572,0.14023280143737793,0.1672254502773285,0.08882054686546326,0.044610053300857544,0.1464816927909851,0.14201584458351135,0.10365426540374756,0.05676710605621338,0.13697388768196106,0.18257489800453186,0.11876100301742554,0.16883698105812073,0.20033758878707886,0.16814598441123962,0.09082984924316406,0.08337479829788208,0.2409522533416748,0.09561753273010254,0.07907810807228088,0.07653176784515381,0.1615498960018158,0.06152331829071045,0.041939765214920044,0.03811725974082947,0.09819856286048889,0.09105953574180603,0.03610950708389282,0.041266947984695435,0.12431588768959045,0.16908606886863708,0.8783588409423828,0.8523179292678833,0.8637770414352417,0.6867486834526062,0.828519344329834,0.9693354368209839,0.9746501445770264,0.985417366027832,0.9857873320579529,0.9802793264389038,0.9756191968917847,0.9664293527603149,0.9471850395202637,0.9586745500564575,0.8914963006973267,0.9894343018531799,0.9910632371902466,0.9935771226882935,0.9937795400619507,0.9951835870742798,0.9913797378540039,0.9863985776901245,0.9837948679924011,0.9797943830490112,0.9383604526519775,0.9820331335067749,0.9938508868217468,0.9932963848114014,0.9924983978271484,0.9932867288589478,0.9943439960479736,0.9937725067138672,0.986248254776001,0.981883704662323,0.9767674207687378,0.9736368656158447,0.9960067868232727,0.9940594434738159,0.9959986209869385,0.9950988292694092,0.9931716918945312,0.9918347001075745,0.9889426231384277,0.9808589220046997,0.9816438555717468,0.9610747694969177,0.9948509931564331,0.9952807426452637,0.9952412843704224,0.9955497980117798,0.9962206482887268,0.995356559753418,0.9913790225982666,0.9904806613922119,0.9861980676651001,0.9755676984786987,0.9771641492843628,0.9960482120513916,0.9951426982879639,0.9941935539245605,0.9946128726005554,0.9958383440971375,0.9941928386688232,0.9905720353126526,0.9880802035331726,0.9885327816009521,0.9822808504104614,0.9968860149383545,0.9949486255645752,0.9973607659339905,0.9961792826652527,0.995972216129303,0.9933240413665771,0.9918946623802185,0.9893010854721069,0.984634518623352,0.9712132811546326,0.9899642467498779,0.9964091777801514,0.9955111742019653,0.9958993196487427,0.9961754679679871,0.9957305192947388,0.9922305345535278,0.990932047367096,0.9875543713569641,0.9841341972351074,0.9810841083526611,0.9967012405395508,0.9956602454185486,0.9955430626869202,0.9954653978347778,0.9952539205551147,0.9950008988380432,0.9937803745269775,0.9908759593963623,0.9903257489204407,0.981259822845459,0.9955655336380005,0.994831919670105,0.9736359119415283,0.9064651727676392,0.7473030090332031,0.6289181709289551,0.4153292775154114,0.25487303733825684,0.1601373553276062,0.12954995036125183,0.09194928407669067,0.10546278953552246,0.08366993069648743,0.11196452379226685,0.12083274126052856,0.08098837733268738,0.05104994773864746,0.03846624493598938,0.08231264352798462,0.05239206552505493,0.051449596881866455,0.030320197343826294,0.027673184871673584,0.03620484471321106,0.04021275043487549,0.03014436364173889,0.03536984324455261,0.02334517240524292,0.018496453762054443,0.015662312507629395,0.0266457200050354,0.0227566659450531,0.022283226251602173,0.029027074575424194,0.022390216588974,0.01801156997680664,0.09768745303153992,0.034563809633255005,0.023249894380569458,0.03901803493499756,0.01761770248413086,0.014397084712982178,0.01642945408821106,0.010726720094680786,0.014456778764724731,0.011142939329147339,0.009669274091720581,0.011903911828994751,0.10916715860366821,0.06537619233131409,0.04073143005371094,0.02705356478691101,0.012898653745651245,0.021140456199645996,0.017198890447616577,0.030946075916290283,0.023236781358718872,0.015082031488418579,0.011476010084152222,0.010728150606155396,0.010260045528411865,0.011281251907348633,0.011193424463272095,0.011101007461547852,0.007141709327697754,0.008009225130081177,0.010236918926239014,0.011374175548553467,0.009715855121612549,0.00646856427192688,0.0074959397315979,0.02556455135345459,0.01726260781288147,0.014219671487808228,0.010909438133239746,0.01175546646118164,0.007347464561462402,0.014222979545593262,0.009312331676483154,0.007020860910415649,0.007466822862625122,0.007276564836502075,0.007938891649246216,0.007605165243148804,0.014298200607299805,0.014850109815597534,0.012043178081512451,0.011300891637802124,0.005235433578491211,0.006916463375091553,0.006863832473754883,0.009281814098358154,0.006519615650177002,0.00485876202583313,0.006672292947769165,0.00631183385848999,0.007124900817871094,0.010616183280944824,0.007171839475631714,0.00554049015045166,0.0035542547702789307,0.007384449243545532]} diff --git a/tests/unit/audioChannelCount.test.mjs b/tests/unit/audioChannelCount.test.mjs new file mode 100644 index 00000000..6da6dd60 --- /dev/null +++ b/tests/unit/audioChannelCount.test.mjs @@ -0,0 +1,166 @@ +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; + +import {FakeAudioContext} from './fakeWebAudio.mjs'; + +// How the audio settings learn the video's channel count: a small worklet counts them, and +// every video and every quality switch asks again. A recount that came while the previous +// counter's worklet was still loading left that counter connected and running for the life +// of the audio context; a counter that could not start answered null to the next caller; +// and a missing convolver (no IndexedDB) threw, into a catch that swallowed everything. + +const counters = []; + +vi.mock('../../chrome/player/modules/channelcounter/channelcounter.mjs', () => ({ + ChannelCounterNode: class { + constructor(ctx) { + this.ctx = ctx; + this.listeners = {}; + this.destroyed = false; + this.started = new Promise((resolve, reject) => { + this.finishInit = () => { + this.counterNode = ctx.node('counter'); + resolve(this); + }; + this.failInit = reject; + }); + counters.push(this); + } + init() { + return this.started; + } + getNode() { + return this.counterNode; + } + destroy() { + if (!this.counterNode) return; + this.destroyed = true; + this.counterNode.disconnect(); + this.counterNode = undefined; + } + once(name, fn) { + (this.listeners[name] ||= []).push(fn); + } + emit(name, ...args) { + const fns = this.listeners[name] || []; + this.listeners[name] = []; + fns.forEach((fn) => fn(...args)); + } + }, +})); +vi.mock('../../chrome/player/ui/DOMElements.mjs', () => ({DOMElements: {}})); +vi.mock('../../chrome/player/modules/Localize.mjs', () => ({Localize: {getMessage: (key) => key}})); +vi.mock('../../chrome/player/utils/AlertPolyfill.mjs', () => ({AlertPolyfill: {}})); +vi.mock('../../chrome/player/utils/InterfaceUtils.mjs', () => ({InterfaceUtils: {}})); +vi.mock('../../chrome/player/ui/components/Dropdown.mjs', () => ({})); +vi.mock('../../chrome/player/ui/audio/AudioChannelMixer.mjs', () => ({AudioChannelMixer: class {}})); +vi.mock('../../chrome/player/ui/audio/AudioCrosstalk.mjs', () => ({AudioCrosstalk: class {}})); +vi.mock('../../chrome/player/ui/audio/OutputConvolver.mjs', () => ({OutputConvolver: class {}})); +vi.mock('../../chrome/player/ui/audio/OutputMeter.mjs', () => ({OutputMeter: class {}})); + +const {AudioConfigManager} = await import('../../chrome/player/ui/audio/AudioConfigManager.mjs'); +const {ChannelUpmixer} = await import('../../chrome/player/ui/audio/ChannelUpmixer.mjs'); +const {VirtualAudioNode} = await import('../../chrome/player/ui/audio/VirtualAudioNode.mjs'); + +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +/** + * The audio settings' channel counting, on a stand-in graph fed by a source node: the + * rest of the settings (their UI) left out. + * @return {{manager: AudioConfigManager, ctx: FakeAudioContext, source: Object}} + */ +function setup() { + const ctx = new FakeAudioContext({maxChannelCount: 6}); + const manager = Object.create(AudioConfigManager.prototype); + manager.audioContext = ctx; + manager.inputNode = new VirtualAudioNode('AudioConfigManager input'); + manager.outputNode = new VirtualAudioNode('AudioConfigManager output'); + const source = ctx.node('source'); + manager.inputNode.connectFrom(source); + manager.audioUpmixer = {updateChannelCount: vi.fn()}; + manager.audioChannelMixer = {updateChannelCount: vi.fn()}; + return {manager, ctx, source}; +} + +describe('AudioConfigManager: counting the channels', () => { + let warn; + + beforeEach(() => { + counters.length = 0; + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + warn.mockRestore(); + }); + + it('leaves no counter running when a recount comes while the last one starts', async () => { + const {manager, ctx, source} = setup(); + manager.outputConvolver = {updateChannelCount: vi.fn()}; + manager.updateChannelCount(); + manager.updateChannelCount(); + expect(counters).toHaveLength(2); + const [first, second] = counters; + + first.finishInit(); + second.finishInit(); + await settle(); + + // The first, discarded before it started, is not wired in, and stopped. + expect(first.destroyed).toBe(true); + expect(manager.inputNode.outputNodes.map(([node]) => node)).not.toContain(first.getNode()); + expect(ctx.outputsOf(source).map((edge) => edge.to.kind)).toEqual(['counter']); + + // The second counts, and is removed once it has. + second.emit('channelcount', 6); + await settle(); + expect(second.destroyed).toBe(true); + expect(ctx.outputsOf(source)).toEqual([]); + expect(await manager.getInputChannelCount()).toBe(6); + expect(manager.audioChannelMixer.updateChannelCount).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + }); + + it('answers 2 every time when the counter cannot start', async () => { + const {manager} = setup(); + const count = manager.getInputChannelCount(); + counters[0].failInit(new Error('no AudioWorklet')); + expect(await count).toBe(2); + expect(await manager.getInputChannelCount()).toBe(2); + }); + + it('updates the channels with no convolver, where IndexedDB is not available', async () => { + const {manager} = setup(); + manager.outputConvolver = undefined; + manager.updateChannelCount(); + counters[0].finishInit(); + await settle(); + counters[0].emit('channelcount', 2); + await settle(); + expect(manager.audioChannelMixer.updateChannelCount).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + }); +}); + +describe('ChannelUpmixer', () => { + it('sets a new output channel count once', () => { + const ctx = new FakeAudioContext({maxChannelCount: 6}); + const upmixer = new ChannelUpmixer(); + upmixer.setupNodes(ctx); + upmixer.updateChannelCount(1, 2); + const mixer = upmixer.mixerNode; + let sets = 0; + let channelCount = mixer.channelCount; + Object.defineProperty(mixer, 'channelCount', { + get: () => channelCount, + set: (v) => { + sets++; + channelCount = v; + }, + }); + + upmixer.updateChannelCount(2, 6); + upmixer.updateChannelCount(2, 6); + expect(channelCount).toBe(6); + expect(sets).toBe(1); + }); +}); diff --git a/tests/unit/audioCompressor.test.mjs b/tests/unit/audioCompressor.test.mjs new file mode 100644 index 00000000..60d1ef68 --- /dev/null +++ b/tests/unit/audioCompressor.test.mjs @@ -0,0 +1,146 @@ +import {describe, expect, it, vi} from 'vitest'; + +import {FakeAudioContext} from './fakeWebAudio.mjs'; + +// The master compressor on a video with more than two channels, on a stand-in Web Audio +// graph. A DynamicsCompressorNode takes two channels at most, so only the front pair went +// through it: the centre, the LFE and the surrounds were not compressed at all, and as the +// compressor delays what it compresses by its 6 ms look-ahead, they also reached the +// speakers 6 ms before the front pair. Every channel now has a compressor (a pair of +// speakers shares one, the centre and the LFE have one each). A channel compressor that is +// off can wait as long as one that is on (setLatencyMatch, for the mixer). Stereo is built +// as before. + +const el = () => ({ + style: {}, + classList: {add() {}, remove() {}, toggle() {}}, + appendChild() {}, + replaceChildren() {}, + addEventListener() {}, + getContext: () => ({}), +}); +vi.mock('../../chrome/player/ui/DOMElements.mjs', () => ({DOMElements: {}})); +vi.mock('../../chrome/player/modules/Localize.mjs', () => ({Localize: {getMessage: (key) => key}})); +vi.mock('../../chrome/player/utils/WebUtils.mjs', () => ({WebUtils: {create: () => el(), setupTabIndex() {}}})); +vi.mock('../../chrome/player/ui/components/Knob.mjs', () => ({createKnob: () => ({container: el(), knob: {val() {}}})})); + +const {AudioCompressor} = await import('../../chrome/player/ui/audio/AudioCompressor.mjs'); +const {AudioCompressionControl} = await import('../../chrome/player/ui/audio/config/AudioCompressionControl.mjs'); + +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +/** + * A compressor between a source and the destination of a stand-in graph. + * @param {number} channels - What its channel count getter answers. + * @param {number} [sampleRate=48000] + * @return {Promise<{compressor: AudioCompressor, ctx: FakeAudioContext, source: Object}>} + */ +async function compressorOn(channels, sampleRate = 48000) { + const ctx = new FakeAudioContext({sampleRate, maxChannelCount: 8}); + const compressor = new AudioCompressor('Master ', async () => channels); + compressor.setupNodes(ctx); + const source = ctx.node('source'); + compressor.getInputNode().connectFrom(source); + compressor.getOutputNode().connect(ctx.destination); + const config = AudioCompressionControl.default(); + config.enabled = true; + config.threshold = -30; + config.gain = 2; + compressor.setConfig(config); + await settle(); + return {compressor, ctx, source}; +} + +describe('AudioCompressor: the master compressor on more than two channels', () => { + it('compresses every channel of 5.1, the centre and the LFE on their own', async () => { + const {compressor, ctx} = await compressorOn(6); + const paths = ctx.channelPaths(compressor.splitterNode, compressor.mergerNode); + expect(paths.map((path) => [path.from, path.to])).toEqual([[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]]); + expect(paths.every((path) => path.compressors.length === 1)).toBe(true); + // Which channels share a compressor. + const compressors = ctx.nodes.filter((node) => node.kind === 'compressor'); + const groups = compressors.map((node) => paths.filter((path) => path.compressors[0] === node).map((path) => path.from)); + expect(groups).toEqual([[0, 1], [2], [3], [4, 5]]); + }); + + for (const [channels, groups] of [[3, [[0, 1], [2]]], [4, [[0, 1], [2, 3]]], [5, [[0, 1], [2], [3, 4]]]]) { + it(`compresses every channel of a ${channels}-channel source`, async () => { + const {compressor, ctx} = await compressorOn(channels); + const paths = ctx.channelPaths(compressor.splitterNode, compressor.mergerNode); + expect(paths.map((path) => [path.from, path.to])).toEqual(Array.from({length: channels}, (_, i) => [i, i])); + const compressors = ctx.nodes.filter((node) => node.kind === 'compressor'); + expect(compressors.map((node) => paths.filter((path) => path.compressors.includes(node)).map((path) => path.from))).toEqual(groups); + }); + } + + for (const sampleRate of [48000, 44100]) { + it(`keeps all six channels in time at ${sampleRate} Hz`, async () => { + const {compressor, ctx} = await compressorOn(6, sampleRate); + const paths = ctx.channelPaths(compressor.splitterNode, compressor.mergerNode); + expect(paths).toHaveLength(6); + for (const path of paths) { + expect(path.delay).toBe(ctx.compressorLookAhead); + } + }); + } + + it('gives every compressor the settings', async () => { + const {ctx} = await compressorOn(6); + const compressors = ctx.nodes.filter((node) => node.kind === 'compressor'); + expect(compressors.map((node) => node.threshold.value)).toEqual([-30, -30, -30, -30]); + const gains = ctx.nodes.filter((node) => node.kind === 'gain' && ctx.inputsOf(node).some((edge) => edge.from.kind === 'compressor')); + expect(gains.map((node) => node.gain.value)).toEqual([2, 2, 2, 2]); + }); + + it('goes back to a straight path when it is switched off', async () => { + const {compressor, ctx, source} = await compressorOn(6); + compressor.compressorConfig.enabled = false; + await compressor.updateCompressor(); + expect(ctx.outputsOf(source).map((edge) => edge.to)).toEqual([ctx.destination]); + }); +}); + +describe('AudioCompressor: stereo', () => { + it('is built as before: source, compressor, gain, out', async () => { + const {ctx, source} = await compressorOn(2); + const kinds = (edges) => edges.map((edge) => `${edge.from.kind}:${edge.output} -> ${edge.to.kind}:${edge.input}`).sort(); + expect(kinds(ctx.edges)).toEqual([ + 'compressor:0 -> gain:0', + 'gain:0 -> destination:0', + 'source:0 -> compressor:0', + ]); + expect(ctx.outputsOf(source)[0].to.threshold.value).toBe(-30); + }); +}); + +describe('AudioCompressor: a channel compressor that is off, next to one that is on', () => { + it('waits as long as a compressor would, and only while asked to and off', async () => { + // 44.1 kHz: the look-ahead is whole frames, 264, not 6 ms. + const ctx = new FakeAudioContext({sampleRate: 44100}); + const compressor = new AudioCompressor('Center '); + compressor.setupNodes(ctx); + const source = ctx.node('source'); + compressor.getInputNode().connectFrom(source); + compressor.getOutputNode().connect(ctx.destination); + const config = AudioCompressionControl.default(); + compressor.setConfig(config); + await settle(); + const path = () => ctx.outputsOf(source).map((edge) => edge.to.kind); + + expect(path()).toEqual(['destination']); + compressor.setLatencyMatch(true); + expect(path()).toEqual(['delay']); + expect(ctx.outputsOf(source)[0].to.delayTime.value).toBe(ctx.compressorLookAhead); + + // Switched on, the compressor itself waits; switched off again, the delay is back. + config.enabled = true; + await compressor.updateCompressor(); + expect(path()).toEqual(['compressor']); + config.enabled = false; + await compressor.updateCompressor(); + expect(path()).toEqual(['delay']); + + compressor.setLatencyMatch(false); + expect(path()).toEqual(['destination']); + }); +}); diff --git a/tests/unit/audioCrosstalk.test.mjs b/tests/unit/audioCrosstalk.test.mjs new file mode 100644 index 00000000..066744bf --- /dev/null +++ b/tests/unit/audioCrosstalk.test.mjs @@ -0,0 +1,136 @@ +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; + +import {FakeAudioContext} from './fakeWebAudio.mjs'; + +// The crosstalk cancellation filter (CrosstalkNode -> ConvolutionXTC), on a stand-in Web +// Audio graph. Each change of its settings builds a new impulse response into the idle one +// of two convolvers and switches to it; the retired convolver was never taken off the +// input, so both ran from the second change on. The colour gain knob's top (20 dB) meant +// no limit at all, over 50 dB of boost with the decay near 0 dB; and the microdelay was +// rounded to whole samples, so most of its range changed nothing. + +vi.mock('../../chrome/player/ui/DOMElements.mjs', () => ({DOMElements: {}})); +vi.mock('../../chrome/player/modules/Localize.mjs', () => ({Localize: {getMessage: (key) => key}})); +vi.mock('../../chrome/player/ui/components/Knob.mjs', () => ({createKnob: () => ({})})); + +const {CrosstalkNode} = await import('../../chrome/player/modules/crosstalk/crosstalk.mjs'); +const {ConvolutionXTC} = await import('../../chrome/player/modules/crosstalk/convolution.mjs'); +const {AudioCrosstalk} = await import('../../chrome/player/ui/audio/AudioCrosstalk.mjs'); + +const N = 4096; // the filter's frequency grid +const SHIFT = 256; // how far the impulse response is delayed to make it causal + +/** + * The response of a filter's impulse response at one frequency bin, the shift taken off. + * @param {Float32Array} h - The impulse response (N samples). + * @param {number} k - The bin. + * @return {number[]} re, im + */ +function dft(h, k) { + let re = 0; + let im = 0; + for (let i = 0; i < N; i++) { + const x = h[(i + SHIFT) % N]; + re += x * Math.cos(2 * Math.PI * k * i / N); + im -= x * Math.sin(2 * Math.PI * k * i / N); + } + return [re, im]; +} + +/** + * The largest gain of a filter's cis and cross responses, in dB. + * @param {ConvolutionXTC} xtc + * @return {number} + */ +function peakDb(xtc) { + let peak = 0; + for (const H of [xtc.H_CIS, xtc.H_CROSS]) { + for (let k = 0; k < N; k++) { + peak = Math.max(peak, Math.hypot(H[2 * k], H[2 * k + 1])); + } + } + return 20 * Math.log10(peak); +} + +describe('ConvolutionXTC: the filter', () => { + const ctx = {sampleRate: 48000}; + const g = Math.pow(10, -3 / 20); + + it('has the response asked for with a delay that is not a whole number of samples', () => { + const tc = 4.5; + const xtc = new ConvolutionXTC(ctx, {g, tc, y: Infinity}); + for (const k of [10, 85, 170, 341, 500, 1023]) { + const [a, b, c, d] = xtc.calculateH(g, 2 * Math.PI * k / N * tc, 0); + const [re, im] = dft(xtc.h_CIS, k); + const [reX, imX] = dft(xtc.h_CROSS, k); + expect(Math.hypot(re - a, im - b)).toBeLessThan(1e-3 * Math.hypot(a, b) + 1e-4); + expect(Math.hypot(reX - c, imX - d)).toBeLessThan(1e-3 * Math.hypot(c, d) + 1e-4); + } + }); + + it('keeps the filter it had for a delay of whole samples', () => { + const tc = 6; + const xtc = new ConvolutionXTC(ctx, {g, tc, y: Infinity}); + // What the formula gave over the whole grid before. + for (let k = 0; k < N; k += 7) { + const [a, b, c, d] = xtc.calculateH(g, 2 * Math.PI * k / N * tc, 0); + expect(xtc.H_CIS[2 * k]).toBeCloseTo(a, 5); + expect(xtc.H_CIS[2 * k + 1]).toBeCloseTo(b, 5); + expect(xtc.H_CROSS[2 * k]).toBeCloseTo(c, 5); + expect(xtc.H_CROSS[2 * k + 1]).toBeCloseTo(d, 5); + } + }); +}); + +describe('AudioCrosstalk: the settings the filter gets', () => { + it('boosts no frequency by more than the colour gain knob\'s top, 20 dB', () => { + const crosstalk = Object.create(AudioCrosstalk.prototype); + crosstalk.speakerDistance = 30; + crosstalk.headDistance = 60; + crosstalk.crosstalkConfig = {enabled: true, decay: -0.01, colorgain: 20, microdelay: 125, lowbypass: 200, highbypass: 6000}; + const options = crosstalk.getCrosstalkConfigObj(); + const xtc = new ConvolutionXTC({sampleRate: 48000}, {g: options.decay, tc: options.microdelay * 48000 * 1e-6, y: options.colorgain}); + expect(peakDb(xtc)).toBeLessThanOrEqual(20); + }); +}); + +describe('CrosstalkNode: switching convolvers', () => { + let ctx; + + beforeEach(() => { + vi.useFakeTimers(); + ctx = new FakeAudioContext(); + // Built with `new IIRFilterNode(ctx, options)`; a constructor that returns an object + // gives that object. + vi.stubGlobal('IIRFilterNode', function(context, options) { + return context.node('iir', options); + }); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + const settings = (colorgain) => ({microdelay: 90, decay: 0.9, colorgain, lowbypass: 250, highbypass: 5000}); + + it('feeds only the convolver in use, after any number of changes', async () => { + const node = new CrosstalkNode(ctx, settings(3)); + await node.init(); + node.getOutputNode().connect(ctx.destination); + vi.advanceTimersByTime(500); + const convolvers = node.xtc.convolvers_XTC; + const fed = () => convolvers.filter((convolver) => ctx.inputsOf(convolver).length > 0); + const heard = () => convolvers.filter((convolver) => ctx.outputsOf(convolver).length > 0); + + for (const colorgain of [4, 5, 6]) { + node.configure(settings(colorgain)); + vi.advanceTimersByTime(500); + expect(fed()).toEqual([convolvers[node.xtc.currentConvolver]]); + expect(heard()).toEqual([convolvers[node.xtc.currentConvolver]]); + } + + node.destroy(); + expect(convolvers.flatMap((convolver) => ctx.inputsOf(convolver))).toEqual([]); + }); +}); diff --git a/tests/unit/audioMixer.test.mjs b/tests/unit/audioMixer.test.mjs new file mode 100644 index 00000000..aaf263a6 --- /dev/null +++ b/tests/unit/audioMixer.test.mjs @@ -0,0 +1,217 @@ +import {describe, expect, it, vi} from 'vitest'; + +import {FakeAudioContext} from './fakeWebAudio.mjs'; + +// The audio mixer's channel strips, on stand-in elements. The mute button's lit state was +// toggled from `channel.mute`, a field that does not exist: classList.toggle(name, +// undefined) flips, so it only matched while nothing else changed `muted`. +// The meters: each frame set the size of all seven canvases, which clears a canvas and +// resets its context even when the size is the same; and the clip check read half of the +// analyser's window, the older half, so a clip in the newer half went unseen. + +const {el} = vi.hoisted(() => { + const el = () => { + const classes = new Set(); + const listeners = {}; + return { + listeners, + style: {}, + dataset: {}, + classList: { + add: (name) => classes.add(name), + remove: (name) => classes.delete(name), + contains: (name) => classes.has(name), + toggle: (name, force) => { + const on = force === undefined ? !classes.has(name) : !!force; + if (on) classes.add(name); else classes.delete(name); + return on; + }, + }, + addEventListener(type, fn) { + (listeners[type] ||= []).push(fn); + }, + appendChild() {}, + replaceChildren() {}, + getContext: () => ({}), + }; + }; + return {el}; +}); + +vi.mock('../../chrome/player/ui/DOMElements.mjs', () => ({DOMElements: {playerContainer: el()}})); +vi.mock('../../chrome/player/modules/Localize.mjs', () => ({Localize: {getMessage: (key) => key}})); +vi.mock('../../chrome/player/utils/WebUtils.mjs', () => ({WebUtils: {create: () => el(), setupTabIndex() {}, setLabels() {}}})); +vi.mock('../../chrome/player/ui/components/Knob.mjs', () => ({createKnob: () => ({container: el(), knob: {val() {}}})})); +// The equalizers' drawing is not what these tests are about: a pass-through stand-in. +vi.mock('../../chrome/player/ui/audio/AudioEqualizer.mjs', async () => { + const {AbstractAudioModule} = await import('../../chrome/player/ui/audio/AbstractAudioModule.mjs'); + return {AudioEqualizer: class extends AbstractAudioModule { + setupNodes(ctx) { + super.setupNodes(ctx); + this.getInputNode().connect(this.getOutputNode()); + } + setConfig(config) { + this.config = config; + } + hasNodes() { + return false; + } + getElement() { + return el(); + } + render() {} + }}; +}); + +const {AudioChannelMixer} = await import('../../chrome/player/ui/audio/AudioChannelMixer.mjs'); +const {AudioChannelControl} = await import('../../chrome/player/ui/audio/config/AudioChannelControl.mjs'); +const {AudioUtils} = await import('../../chrome/player/utils/AudioUtils.mjs'); +const {AudioProfile} = await import('../../chrome/player/ui/audio/config/AudioProfile.mjs'); + +describe('AudioChannelMixer: a channel strip', () => { + it('lights the mute button when, and only when, the channel is muted', () => { + const mixer = Object.create(AudioChannelMixer.prototype); + mixer.updateNodes = vi.fn(); + const channel = AudioChannelControl.default(0); + const els = mixer.createMixerChannel(channel); + const click = () => els.muteButton.listeners.click.forEach((fn) => fn({})); + + click(); + expect(channel.muted).toBe(true); + expect(els.muteButton.classList.contains('active')).toBe(true); + + // Unmuted elsewhere while the strip is shown, then muted with the button. + channel.muted = false; + click(); + expect(channel.muted).toBe(true); + expect(els.muteButton.classList.contains('active')).toBe(true); + }); +}); + +/** + * An AnalyserNode of 256 samples holding the given window, as the browser hands it out: + * an array shorter than the window gets its first (oldest) samples. + * @param {Float32Array} samples - The last 256 samples, oldest first. + * @return {Object} + */ +function analyser(samples = new Float32Array(256)) { + return { + fftSize: 256, + frequencyBinCount: 128, + minDecibels: -100, + maxDecibels: -30, + context: {sampleRate: 48000}, + getFloatTimeDomainData: (array) => array.set(samples.subarray(0, array.length)), + getByteFrequencyData: (array) => array.fill(100), + }; +} + +/** + * A canvas of 10.5 x 100 CSS pixels that counts how often its size is set. + * @return {Object} + */ +function canvas() { + const c = {clientWidth: 10.5, clientHeight: 100, sets: 0, w: 300, h: 150}; + Object.defineProperty(c, 'width', { + get: () => c.w, + set: (v) => { + c.sets++; + c.w = Math.trunc(v); + }, + }); + Object.defineProperty(c, 'height', { + get: () => c.h, + set: (v) => { + c.sets++; + c.h = Math.trunc(v); + }, + }); + return c; +} + +const context2d = () => ({clearRect() {}, fillRect() {}}); + +describe('AudioChannelMixer: the meters', () => { + it('sizes each meter canvas once, not every frame', () => { + vi.stubGlobal('window', {devicePixelRatio: 1}); + try { + const mixer = Object.create(AudioChannelMixer.prototype); + const els = {volumeMeter: canvas(), volumeMeterCtx: context2d()}; + const nodes = {analyzer: analyser()}; + const master = {volumeMeter: canvas(), volumeMeterCtx: context2d()}; + mixer.outputMeterCache = []; + mixer.configManager = {getOutputMeter: () => ({ + getMeterData: () => [{volume: -60, isClipping: false}, {volume: -50, isClipping: false}], + minDecibels: -100, + maxDecibels: -30, + })}; + for (let frame = 0; frame < 3; frame++) { + mixer.renderChannel(nodes, els, true); + mixer.renderMaster(master); + } + expect([els.volumeMeter.w, els.volumeMeter.h]).toEqual([10, 100]); + expect(els.volumeMeter.sets).toBe(2); + // The master meter is drawn at twice the width. + expect([master.volumeMeter.w, master.volumeMeter.h]).toEqual([21, 100]); + expect(master.volumeMeter.sets).toBe(2); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('sees a clip anywhere in the analyser\'s window', () => { + const samples = new Float32Array(256); + samples[200] = 1.2; + expect(AudioUtils.isClipping(analyser(samples))).toBe(true); + samples[200] = 0.9; + samples[20] = -1.2; + expect(AudioUtils.isClipping(analyser(samples))).toBe(true); + samples[20] = 0; + expect(AudioUtils.isClipping(analyser(samples))).toBe(false); + }); +}); + +describe('AudioChannelMixer: channels with and without a compressor', () => { + /** + * A mixer on a 5.1 stand-in graph, between a source and the destination. + * @return {{mixer: AudioChannelMixer, ctx: FakeAudioContext}} + */ + function mixerOn51() { + const ctx = new FakeAudioContext({maxChannelCount: 6}); + const mixer = new AudioChannelMixer({ + getChannelCount: async () => 6, + getOutputMeter: () => ({updateChannelCount() {}, createAnalysers() {}, destroyAnalysers() {}}), + }); + mixer.setupUI(el(), el()); + // Hidden: no meters. + mixer.ui.mixer.offsetParent = null; + mixer.setupNodes(ctx); + mixer.getInputNode().connectFrom(ctx.node('source')); + mixer.getOutputNode().connect(ctx.destination); + return {mixer, ctx}; + } + + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + + it('keeps every channel in time when only the centre is compressed', async () => { + const {mixer, ctx} = mixerOn51(); + const profile = new AudioProfile(1); + profile.channels[2].compressor.enabled = true; + mixer.setConfig(profile); + await settle(); + + const paths = ctx.channelPaths(mixer.channelSplitter, mixer.channelMerger); + expect(paths.map((path) => [path.from, path.to])).toEqual([[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]]); + expect(paths.map((path) => path.compressors.length)).toEqual([0, 0, 1, 0, 0, 0]); + for (const path of paths) { + expect(path.delay).toBe(ctx.compressorLookAhead); + } + + // With no compressor on, no channel waits. + profile.channels[2].compressor.enabled = false; + await mixer.channelNodes[2].compressor.updateCompressor(); + mixer.channelNodes[2].compressor.emit('change'); + await settle(); + expect(ctx.edges.some((edge) => edge.to.kind === 'delay' || edge.from.kind === 'delay')).toBe(false); + }); +}); diff --git a/tests/unit/audioProfiles.test.mjs b/tests/unit/audioProfiles.test.mjs new file mode 100644 index 00000000..f3c740ac --- /dev/null +++ b/tests/unit/audioProfiles.test.mjs @@ -0,0 +1,137 @@ +import {describe, expect, it, vi} from 'vitest'; + +import {FakeAudioContext} from './fakeWebAudio.mjs'; + +// Audio profiles come from storage and from imported .fsprofile.json files, and nothing +// checked them. A value that was not a number reached an AudioParam, which throws on NaN: +// the equalizer had already taken its old filters out, so the channel went silent; a +// channel ID out of range left the mixer without a channel and it threw; an unknown filter +// type stayed a lowpass. The default profiles load exactly as before. + +vi.mock('../../chrome/player/ui/DOMElements.mjs', () => ({DOMElements: {}})); +vi.mock('../../chrome/player/modules/Localize.mjs', () => ({Localize: {getMessage: (key) => key}})); + +const {AudioProfile, MAX_AUDIO_CHANNELS} = await import('../../chrome/player/ui/audio/config/AudioProfile.mjs'); +const {AudioCrosstalkControl} = await import('../../chrome/player/ui/audio/config/AudioCrosstalkControl.mjs'); +const {AudioEQNode} = await import('../../chrome/player/ui/audio/config/AudioEQNode.mjs'); +const {DefaultProfilesData} = await import('../../chrome/player/ui/audio/config/DefaultProfiles.mjs'); +const {AudioEqualizer} = await import('../../chrome/player/ui/audio/AudioEqualizer.mjs'); +const {AbstractAudioModule} = await import('../../chrome/player/ui/audio/AbstractAudioModule.mjs'); + +/** + * An equalizer on a stand-in graph, between a source and the destination, its UI left out. + * @return {{equalizer: AudioEqualizer, ctx: FakeAudioContext, source: Object}} + */ +function equalizerOnGraph() { + const ctx = new FakeAudioContext(); + const equalizer = Object.create(AudioEqualizer.prototype); + AbstractAudioModule.prototype.setupNodes.call(equalizer, ctx); + equalizer.equalizerNodes = []; + equalizer.renderEqualizerResponse = () => {}; + equalizer.updateEqualizerNodeMarkers = () => {}; + const source = ctx.node('source'); + equalizer.getInputNode().connectFrom(source); + equalizer.getOutputNode().connect(ctx.destination); + equalizer.getInputNode().connect(equalizer.getOutputNode()); + return {equalizer, ctx, source}; +} + +/** + * Whether sound gets from one node to another on the stand-in graph. + * @param {FakeAudioContext} ctx + * @param {Object} from + * @param {Object} to + * @return {boolean} + */ +function reaches(ctx, from, to) { + if (from === to) return true; + return ctx.outputsOf(from).some((edge) => reaches(ctx, edge.to, to)); +} + +const broken = { + id: 5, + label: 'Broken', + channels: [ + {id: 0, gain: 'loud', equalizerNodes: [ + {type: 'peaking', frequency: 'abc', gainDb: 3, q: 1}, + {type: 'bogus', frequency: 1000, gainDb: 3, q: 1}, + {type: 'lowshelf', frequency: 100, q: 1}, + {type: 'highshelf', frequency: 8000, gainDb: 2, q: 'wide'}, + ]}, + {id: 1, gain: 0.5}, + {id: 1, gain: 2}, + {id: 3, gain: 1e9, muted: 'yes'}, + {id: 4, gain: -1}, + {id: 7, gain: 1}, + ], + master: {gain: null, mono: 1, equalizerNodes: 'none', compressor: {enabled: true, threshold: 'x', ratio: 50, attack: null, gain: 1e6}}, + crosstalk: {enabled: true, decay: null, colorgain: null, microdelay: 'far', lowbypass: 'x', highbypass: 1e9}, +}; + +describe('AudioProfile.fromObj: profiles from storage and files', () => { + it('loads the default profiles as they are', () => { + for (const data of DefaultProfilesData.profiles) { + const profile = AudioProfile.fromObj(data); + expect(AudioProfile.fromObj(profile.toObj()).toObj()).toEqual(profile.toObj()); + if (data.master) { + expect(profile.master.toObj()).toEqual({...data.master, id: 'master'}); + } + } + }); + + it('gives a broken profile one channel per ID with usable values', () => { + const profile = AudioProfile.fromObj(broken); + expect(profile.channels.map((channel) => channel.id)).toEqual(Array.from({length: MAX_AUDIO_CHANNELS}, (_, i) => i)); + expect(profile.channels.map((channel) => channel.gain)).toEqual([1, 0.5, 1, Math.pow(10, 10 / 20), 0, 1]); + expect(profile.channels[3].muted).toBe(true); + expect(profile.channels[0].equalizerNodes.map((node) => node.toObj())).toEqual([ + {type: 'lowshelf', frequency: 100, gainDb: 0, q: 1}, + {type: 'highshelf', frequency: 8000, gainDb: 2, q: 1}, + ]); + + expect(profile.master.isMaster()).toBe(true); + expect(profile.master.gain).toBe(1); + expect(profile.master.mono).toBe(true); + expect(profile.master.equalizerNodes).toEqual([]); + expect(profile.master.compressor.toObj()).toEqual({ + enabled: true, attack: 0.003, knee: 30, ratio: 20, release: 0.25, threshold: -24, gain: 10, + }); + + const crosstalk = profile.crosstalk; + expect(Number.isNaN(crosstalk.decay)).toBe(true); + expect(Number.isNaN(crosstalk.microdelay)).toBe(true); + expect([crosstalk.colorgain, crosstalk.lowbypass, crosstalk.highbypass]).toEqual([5, 200, 20000]); + }); + + it('builds every equalizer of a broken profile without a value the browser refuses', () => { + const profile = AudioProfile.fromObj(broken); + for (const channel of [...profile.channels, profile.master]) { + const {equalizer} = equalizerOnGraph(); + equalizer.equalizerConfig = channel.equalizerNodes; + expect(() => equalizer.refreshEQNodes()).not.toThrow(); + } + }); + + it('keeps the crosstalk decay and microdelay worked out from the distances through a save', () => { + const control = AudioCrosstalkControl.default(); + control.enabled = true; + const saved = JSON.parse(JSON.stringify(control.toObj())); + expect(saved.decay).toBeNull(); + const loaded = AudioCrosstalkControl.fromObj(saved); + expect(Number.isNaN(loaded.decay)).toBe(true); + expect(Number.isNaN(loaded.microdelay)).toBe(true); + }); +}); + +describe('AudioEqualizer.refreshEQNodes', () => { + it('leaves the sound on when a filter value is refused', () => { + const {equalizer, ctx, source} = equalizerOnGraph(); + equalizer.equalizerConfig = [new AudioEQNode('peaking', 1000, 3, 1)]; + equalizer.refreshEQNodes(); + expect(reaches(ctx, source, ctx.destination)).toBe(true); + + equalizer.equalizerConfig = [new AudioEQNode('peaking', NaN, 3, 1)]; + expect(() => equalizer.refreshEQNodes()).toThrow(TypeError); + expect(reaches(ctx, source, ctx.destination)).toBe(true); + }); +}); diff --git a/tests/unit/fakeWebAudio.mjs b/tests/unit/fakeWebAudio.mjs new file mode 100644 index 00000000..96a38405 --- /dev/null +++ b/tests/unit/fakeWebAudio.mjs @@ -0,0 +1,194 @@ +// A stand-in for the Web Audio graph, for the audio layer's unit tests (Node has no +// AudioContext). Each node records its connections in the context's edge list, so a test +// can follow the real signal path the VirtualAudioNodes expand into. An AudioParam's value, +// like the browser's (a WebIDL float), refuses NaN and Infinity with a TypeError. + +/** + * An AudioParam whose value refuses what the browser refuses. + * @param {number} initial - The default value. + * @return {{value: number}} + */ +export function fakeParam(initial) { + let value = initial; + return { + get value() { + return value; + }, + set value(v) { + if (!Number.isFinite(v)) { + throw new TypeError('AudioParam.value: Value being assigned is not a finite floating-point value.'); + } + value = v; + }, + }; +} + +export class FakeNode { + /** + * @param {FakeAudioContext} context - The context it belongs to. + * @param {string} kind - What it is ('gain', 'splitter', ...). + * @param {Object} props - Its own properties (params, counts). + */ + constructor(context, kind, props = {}) { + this.context = context; + this.kind = kind; + this.channelCount = 2; + this.channelCountMode = 'max'; + this.channelInterpretation = 'speakers'; + Object.assign(this, props); + } + + connect(to, output, input) { + this.context.edges.push({from: this, to, output: output ?? 0, input: input ?? 0}); + return to; + } + + disconnect(to, output, input) { + const edges = this.context.edges; + const matches = (edge) => edge.from === this && + (to === undefined || edge.to === to) && + (output === undefined || edge.output === output) && + (input === undefined || edge.input === input); + const before = edges.length; + this.context.edges = edges.filter((edge) => !matches(edge)); + if (to !== undefined && this.context.edges.length === before) { + throw new Error(`InvalidAccessError: ${this.kind} is not connected to ${to.kind}`); + } + } +} + +export class FakeAudioContext { + /** + * @param {{sampleRate?: number, maxChannelCount?: number}} options + */ + constructor({sampleRate = 48000, maxChannelCount = 2} = {}) { + this.sampleRate = sampleRate; + this.edges = []; + this.nodes = []; + this.destination = this.node('destination', {channelCount: 2, maxChannelCount}); + } + + node(kind, props) { + const node = new FakeNode(this, kind, props); + this.nodes.push(node); + return node; + } + + createGain() { + return this.node('gain', {gain: fakeParam(1)}); + } + + createDelay(maxDelayTime = 1) { + return this.node('delay', {delayTime: fakeParam(0), maxDelayTime}); + } + + createChannelSplitter(numberOfOutputs = 6) { + return this.node('splitter', {numberOfOutputs}); + } + + createChannelMerger(numberOfInputs = 6) { + return this.node('merger', {numberOfInputs}); + } + + createDynamicsCompressor() { + return this.node('compressor', { + threshold: fakeParam(-24), knee: fakeParam(30), ratio: fakeParam(12), + attack: fakeParam(0.003), release: fakeParam(0.25), reduction: 0, + }); + } + + createBiquadFilter() { + const node = this.node('biquad', {frequency: fakeParam(350), gain: fakeParam(0), Q: fakeParam(1)}); + // An enum attribute: a value it does not know is ignored, as in the browser. + let type = 'lowpass'; + Object.defineProperty(node, 'type', { + get: () => type, + set: (v) => { + if (['lowpass', 'highpass', 'bandpass', 'lowshelf', 'highshelf', 'peaking', 'notch', 'allpass'].includes(v)) { + type = v; + } + }, + }); + return node; + } + + createConvolver() { + // Counts what restarts the convolution: each buffer it is given. + const node = this.node('convolver', {normalize: true, bufferSets: 0}); + let buffer = null; + Object.defineProperty(node, 'buffer', { + get: () => buffer, + set: (v) => { + node.bufferSets++; + buffer = v; + }, + }); + return node; + } + + createBuffer(numberOfChannels, length, sampleRate) { + const channels = Array.from({length: numberOfChannels}, () => new Float32Array(length)); + return { + numberOfChannels, length, sampleRate, + getChannelData: (i) => channels[i], + copyToChannel: (data, i) => channels[i].set(data.subarray(0, length)), + }; + } + + /** + * The edges into a node. + * @param {FakeNode} node + * @return {Array<{from: FakeNode, to: FakeNode, output: number, input: number}>} + */ + inputsOf(node) { + return this.edges.filter((edge) => edge.to === node); + } + + /** + * The edges out of a node. + * @param {FakeNode} node + * @return {Array<{from: FakeNode, to: FakeNode, output: number, input: number}>} + */ + outputsOf(node) { + return this.edges.filter((edge) => edge.from === node); + } + + /** + * How long a DynamicsCompressorNode holds the sound back, as Firefox's does: its 6 ms + * pre-delay, rounded down to whole frames. + * @return {number} seconds + */ + get compressorLookAhead() { + return Math.floor(0.006 * this.sampleRate) / this.sampleRate; + } + + /** + * Every path from one splitter's outputs to one merger's inputs, following a channel + * through the mergers and splitters on the way. + * @param {FakeNode} splitter + * @param {FakeNode} merger + * @return {Array<{from: number, to: number, delay: number, compressors: FakeNode[]}>} + * sorted by channel; `delay` adds up the DelayNodes and the compressors' look-ahead. + */ + channelPaths(splitter, merger) { + const found = []; + const walk = (node, channel, delay, compressors, from) => { + for (const edge of this.outputsOf(node)) { + // A splitter passes one channel on each output. + if (node !== splitter && node.kind === 'splitter' && channel !== null && edge.output !== channel) continue; + const start = node === splitter ? edge.output : from; + if (edge.to === merger) { + found.push({from: start, to: edge.input, delay, compressors}); + continue; + } + const to = edge.to; + let next = node.kind === 'splitter' ? null : channel; + if (to.kind === 'merger') next = edge.input; + const added = to.kind === 'delay' ? to.delayTime.value : to.kind === 'compressor' ? this.compressorLookAhead : 0; + walk(to, next, delay + added, to.kind === 'compressor' ? [...compressors, to] : compressors, start); + } + }; + walk(splitter, null, 0, [], null); + return found.sort((a, b) => a.from - b.from || a.to - b.to); + } +} diff --git a/tests/unit/outputConvolver.test.mjs b/tests/unit/outputConvolver.test.mjs new file mode 100644 index 00000000..5b52593b --- /dev/null +++ b/tests/unit/outputConvolver.test.mjs @@ -0,0 +1,77 @@ +import {describe, expect, it, vi} from 'vitest'; + +import {FakeAudioContext} from './fakeWebAudio.mjs'; + +// The output convolver (impulse responses for the output device), on a stand-in Web Audio +// graph. Every quality switch and every checkbox rebuilt its settings and gave each +// convolver its impulse response again, which restarts the convolution (a click); a stored +// impulse length that was not a number kept the whole file as the impulse response; and a +// file that did not decode was reported with the window's name instead of its own. + +vi.mock('../../chrome/player/ui/DOMElements.mjs', () => ({DOMElements: {}})); +vi.mock('../../chrome/player/modules/Localize.mjs', () => ({Localize: {getMessage: (key) => key}})); +vi.mock('../../chrome/player/utils/AlertPolyfill.mjs', () => ({AlertPolyfill: {}})); +vi.mock('../../chrome/player/network/IndexedDBManager.mjs', () => ({IndexedDBManager: class {}})); +vi.mock('../../chrome/player/ui/components/Dropdown.mjs', () => ({})); + +const {OutputConvolver} = await import('../../chrome/player/ui/audio/OutputConvolver.mjs'); +const {AbstractAudioModule} = await import('../../chrome/player/ui/audio/AbstractAudioModule.mjs'); +const {AudioConvolverControl, AudioConvolverProfile} = await import('../../chrome/player/ui/audio/config/AudioConvolverControl.mjs'); + +/** + * An output convolver on a stereo stand-in graph with an impulse response on both + * channels, its UI left out. + * @return {{convolver: OutputConvolver, ctx: FakeAudioContext}} + */ +function stereoConvolver() { + const ctx = new FakeAudioContext(); + const convolver = Object.create(OutputConvolver.prototype); + AbstractAudioModule.prototype.setupNodes.call(convolver, ctx); + convolver.getInputNode().connect(convolver.getOutputNode()); + convolver.configManager = {getChannelCount: async () => 2}; + convolver.config = AudioConvolverControl.default(); + convolver.config.enabled = true; + convolver.currentProfile = convolver.config.profiles[0]; + convolver.currentProfile.channels[0].enabled = true; + convolver.currentProfile.channels[1].enabled = true; + convolver.convolverChannels = Array.from({length: 6}, (_, id) => ({ + id, + label: {classList: {add() {}, remove() {}}}, + impulseBuffer: id < 2 ? ctx.createBuffer(1, 512, 48000) : null, + })); + return {convolver, ctx}; +} + +describe('OutputConvolver', () => { + it('gives each convolver its impulse response only when it changes', async () => { + const {convolver, ctx} = stereoConvolver(); + await convolver.updateNodes(); + await convolver.updateNodes(); + await convolver.updateChannelCount(); + const nodes = ctx.nodes.filter((node) => node.kind === 'convolver'); + expect(nodes).toHaveLength(2); + expect(nodes.map((node) => node.bufferSets)).toEqual([1, 1]); + + // A new setting takes effect. + convolver.currentProfile.channels[0].normalize = true; + await convolver.updateNodes(); + expect(nodes.map((node) => node.bufferSets)).toEqual([2, 1]); + expect(nodes[0].normalize).toBe(true); + }); + + it('keeps a stored impulse length within what the field allows', () => { + const length = (bufferSize) => AudioConvolverProfile.fromObj({id: 0, label: 'P', bufferSize}).bufferSize; + expect(length('abc')).toBe(4096); + expect(length(null)).toBe(4096); + expect(length(10)).toBe(128); + expect(length(1e9)).toBe(16384); + expect(length(2048)).toBe(2048); + }); + + it('names the file that did not decode', async () => { + const convolver = Object.create(OutputConvolver.prototype); + convolver.audioContext = {decodeAudioData: () => Promise.reject(new Error('EncodingError'))}; + const file = {name: 'room.wav', arrayBuffer: async () => new ArrayBuffer(4)}; + await expect(convolver.getImpulseResponse(file)).rejects.toThrow('Could not decode impulse response: room.wav'); + }); +}); diff --git a/tests/unit/vadLifecycle.test.mjs b/tests/unit/vadLifecycle.test.mjs index 67781eba..9c60f452 100644 --- a/tests/unit/vadLifecycle.test.mjs +++ b/tests/unit/vadLifecycle.test.mjs @@ -5,7 +5,9 @@ import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; // reopening the syncer) used to leave two detectors running for good; a failed start // stayed failed; the ONNX session was never released, so each video and each start kept // another model in wasm memory; a failed background analyzer could never retry its -// source; and the first ~20 ms of audio were written to vadBuffer[-1]. +// source; the first ~20 ms of audio were written to vadBuffer[-1]; and a background +// analyzer player (8x playback, downloads, its own audio context) could be left running +// with nothing able to stop it, when a stop and a new start came while one loaded. const newVad = vi.fn(); @@ -191,6 +193,101 @@ describe('AudioAnalyzer', () => { expect(seen).toEqual([[-0.02, 200], [0.25, 100]]); }); + /** + * An analyzer whose background players load when the test says so. Each player's + * destroy() reports its end, as a running analyzer player does. + * @param {Object} c - The client. + * @return {{analyzer: AudioAnalyzer, loads: Array}} + */ + function withLoads(c) { + const analyzer = new AudioAnalyzer(c); + const loads = []; + analyzer.loadPlayer = (source, doneRanges, onDone) => { + const load = deferred(); + const player = { + source, + destroy: vi.fn(() => onDone(false)), + }; + loads.push({player, finish: () => load.resolve(player), fail: (e) => load.reject(e)}); + return load.promise; + }; + return {analyzer, loads}; + } + + it('keeps one background player when it is stopped and started again for the same video while loading', async () => { + const c = client(); + const {analyzer, loads} = withLoads(c); + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + try { + const startA = analyzer.startBackgroundAnalyzer(); + analyzer.stopBackgroundAnalyzer(); + const startB = analyzer.startBackgroundAnalyzer(); + expect(loads).toHaveLength(2); + loads[0].finish(); + loads[1].finish(); + await Promise.all([startA, startB]); + + const [a, b] = loads.map((load) => load.player); + expect(a.destroy).toHaveBeenCalledTimes(1); + expect(analyzer.backgroundAnalyzerPlayer).toBe(b); + expect(b.destroy).not.toHaveBeenCalled(); + + // And a stop stops the one that runs. + analyzer.stopBackgroundAnalyzer(); + expect(b.destroy).toHaveBeenCalledTimes(1); + } finally { + log.mockRestore(); + } + }); + + it('keeps the new video\'s background player when the old video\'s finishes loading after it', async () => { + const c = client(); + const {analyzer, loads} = withLoads(c); + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + try { + const startA = analyzer.startBackgroundAnalyzer(); + analyzer.reset(); + c.player.getSource = () => 'source-2'; + const startB = analyzer.startBackgroundAnalyzer(); + loads[1].finish(); + await startB; + loads[0].finish(); + await startA; + + const [a, b] = loads.map((load) => load.player); + expect(a.destroy).toHaveBeenCalledTimes(1); + // The old player's end did not clear the new one's place. + expect(analyzer.backgroundAnalyzerPlayer).toBe(b); + analyzer.stopBackgroundAnalyzer(); + expect(b.destroy).toHaveBeenCalledTimes(1); + } finally { + log.mockRestore(); + } + }); + + it('keeps a newer start for the same video when an older load fails after it', async () => { + const c = client(); + const {analyzer, loads} = withLoads(c); + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + const startA = analyzer.startBackgroundAnalyzer(); + analyzer.stopBackgroundAnalyzer(); + const startB = analyzer.startBackgroundAnalyzer(); + loads[0].fail(new Error('no audio')); + await startA; + loads[1].finish(); + await startB; + + const b = loads[1].player; + expect(b.destroy).not.toHaveBeenCalled(); + expect(analyzer.backgroundAnalyzerPlayer).toBe(b); + } finally { + log.mockRestore(); + warn.mockRestore(); + } + }); + it('tries the background analyzer again after a failed start, and destroys the failed player', async () => { const c = client(); const analyzer = new AudioAnalyzer(c); diff --git a/tests/unit/vadModel.test.mjs b/tests/unit/vadModel.test.mjs index 757d32ab..b8579bf4 100644 --- a/tests/unit/vadModel.test.mjs +++ b/tests/unit/vadModel.test.mjs @@ -4,9 +4,12 @@ import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; // in by the build, which CI runs after the unit tests. The session keeps the model in wasm // memory until it is released, and every video and every start of the voice detector // loads a new one; nothing released it. A failed model download went on to parse the -// error page as a model. +// error page as a model. The model was fed each frame without the previous frame's tail +// it was trained with; the worklet node took every channel of a 5.1 source and scored +// the left one only; and a model that did not load left a worklet node running. const sessions = []; +const workletNodes = []; const ort = { Tensor: class { @@ -31,10 +34,13 @@ const {VadJS} = await import('../../chrome/player/modules/vad/vad.mjs'); describe('the voice detector model', () => { beforeEach(() => { sessions.length = 0; + workletNodes.length = 0; vi.stubGlobal('fetch', vi.fn(async () => ({ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(8)}))); vi.stubGlobal('AudioWorkletNode', class { - constructor() { + constructor(context, name, options) { + this.options = options; this.port = {postMessage: vi.fn(), onmessage: null}; + workletNodes.push(this); } }); vi.spyOn(console, 'debug').mockImplementation(() => {}); @@ -83,4 +89,41 @@ describe('the voice detector model', () => { await expect(VadJS.createModel(ort)).rejects.toThrow(/model did not load \(404\)/); expect(sessions).toHaveLength(0); }); + + it('leaves no worklet node running when the model does not load', async () => { + vi.stubGlobal('fetch', vi.fn(async () => ({ok: false, status: 404, arrayBuffer: async () => new ArrayBuffer(0)}))); + await expect(VadJS.AudioNodeVAD.new({audioWorklet: {addModule: async () => {}}}, {ort})) + .rejects.toThrow(/model did not load/); + // A node's processor runs until it is told to close. + const running = workletNodes.filter((node) => !node.port.postMessage.mock.calls.some(([message]) => message === 'close')); + expect(running).toHaveLength(0); + }); + + it('has Web Audio mix every channel down to the one channel it scores', async () => { + await VadJS.AudioNodeVAD.new({audioWorklet: {addModule: async () => {}}}, {ort}); + expect(workletNodes).toHaveLength(1); + expect(workletNodes[0].options).toMatchObject({ + channelCount: 1, + channelCountMode: 'explicit', + channelInterpretation: 'speakers', + }); + }); + + it('scores each frame with the last 64 samples of the frame before it in front', async () => { + const model = await VadJS.createModel(ort); + const frame = (start) => Float32Array.from({length: 512}, (_, i) => start + i); + await model.process(frame(1000)); + await model.process(frame(2000)); + model.reset_state(); + await model.process(frame(3000)); + + const inputs = sessions[0].run.mock.calls.map(([feeds]) => feeds.input); + expect(inputs.map((input) => input.dims)).toEqual([[1, 576], [1, 576], [1, 576]]); + // The first frame and the first after a reset have silence in front of them. + expect(Array.from(inputs[0].data.subarray(0, 64))).toEqual(new Array(64).fill(0)); + expect(Array.from(inputs[0].data.subarray(64))).toEqual(Array.from(frame(1000))); + expect(Array.from(inputs[1].data.subarray(0, 64))).toEqual(Array.from(frame(1000).subarray(448))); + expect(Array.from(inputs[1].data.subarray(64))).toEqual(Array.from(frame(2000))); + expect(Array.from(inputs[2].data.subarray(0, 64))).toEqual(new Array(64).fill(0)); + }); }); diff --git a/tools/vendored-sources.json b/tools/vendored-sources.json index 23ee963c..ac5c5187 100644 --- a/tools/vendored-sources.json +++ b/tools/vendored-sources.json @@ -27,7 +27,7 @@ "copy": "chrome/player/modules/vad/vad.mjs", "repo": "ricky0123/vad", "path": "packages/web/src", - "ref": "7c2f259b9977125e6b8665118f24cdc1ed59c8ff" + "ref": "43e20495ffdfbf4320c88bc910febaf032a06829" } ] }