fix(audio): 5.1 voice detection, compressor timing, crosstalk limits, safe profiles, no leftover audio nodes

chrome/player/modules/analyzer/AudioAnalyzer.mjs

@@ -26,6 +26,8 @@ export class AudioAnalyzer extends EventEmitter {
     this.backgroundAnalyzerStatus = AnalyzerStatus.IDLE;
     this.backgroundDoneRanges = [];
     this.backgroundAnalyzerEnabled = true;
+    // Counts startBackgroundAnalyzer's loads (see there).
+    this.backgroundStarts = 0;
   }
 
   onVadFrameProcessed(time, isSpeechProb) {
@@ -184,27 +186,34 @@ export class AudioAnalyzer extends EventEmitter {
 
     console.log('[AudioAnalyzer] Starting background analyzer');
 
+    // A stop and a new start while this one loads (the tool closed and opened again, or a
+    // new video) make this start stale: only the latest one may keep its player. Two
+    // starts for the same source used to both keep theirs, and the first ran on for good.
+    const start = ++this.backgroundStarts;
     let backgroundAnalyzerPlayer;
     try {
       backgroundAnalyzerPlayer = await this.loadPlayer(this.backgroundAnalyzerSource, this.backgroundDoneRanges, (completed) => {
-        if (this.backgroundAnalyzerPlayer === backgroundAnalyzerPlayer) {
-          console.log('[AudioAnalyzer] Background analyzer finished', completed ? 'successfully' : 'with errors');
-          this.backgroundAnalyzerStatus = completed ? AnalyzerStatus.FINISHED : AnalyzerStatus.FAILED;
-          this.client.interfaceController.updateMarkers();
+        // Only the running player's end counts: a stale player, destroyed below, cleared
+        // the running one's place, and nothing could stop that one any more.
+        if (this.backgroundAnalyzerPlayer !== backgroundAnalyzerPlayer) {
+          return;
         }
+        console.log('[AudioAnalyzer] Background analyzer finished', completed ? 'successfully' : 'with errors');
+        this.backgroundAnalyzerStatus = completed ? AnalyzerStatus.FINISHED : AnalyzerStatus.FAILED;
+        this.client.interfaceController.updateMarkers();
         this.backgroundAnalyzerPlayer = null;
       });
     } catch (e) {
       console.warn('[AudioAnalyzer] The background analyzer did not start', e);
       // Forget the source, or the check above refuses every later start for it: the
       // next play tries again.
-      if (this.backgroundAnalyzerSource === newSource) {
+      if (start === this.backgroundStarts && this.backgroundAnalyzerSource === newSource) {
         this.backgroundAnalyzerSource = null;
       }
       return;
     }
 
-    if (newSource !== this.backgroundAnalyzerSource) {
+    if (start !== this.backgroundStarts || newSource !== this.backgroundAnalyzerSource) {
       backgroundAnalyzerPlayer.destroy();
       return;
     }

chrome/player/modules/crosstalk/convolution.mjs

@@ -81,7 +81,9 @@ export class ConvolutionXTC {
 
   configure(options) {
     const g = options.g;
-    const tc = Math.round(options.tc);
+    // In samples, and not rounded to whole ones: at 48 kHz that moved the microdelay knob in
+    // steps of 20.8 us, so most of its range changed nothing.
+    const tc = options.tc;
     let y = options.y;
 
     if (this.cachedOptions.g === g && this.cachedOptions.tc === tc && this.cachedOptions.y === y) {
@@ -104,7 +106,9 @@ export class ConvolutionXTC {
     y = Math.max(min_y, y);
     const valid = isFinite(y) && y <= max_y;
 
-    for (let k = 0; k < n; k++) {
+    // The positive frequencies; the negative ones mirror them below, as they must for a
+    // real impulse response. A delay of whole samples gave that by itself, a fraction does not.
+    for (let k = 0; k <= n / 2; k++) {
       const omegatc = 2 * Math.PI * k / n * tc;
       const cos = Math.cos(omegatc);
       const cm_I = Math.sqrt(gg - 2*g*cos + 1);
@@ -126,7 +130,16 @@ export class ConvolutionXTC {
       H_CIS[k * 2 + 1] = H[1];
       H_CROSS[k * 2] = H[2];
       H_CROSS[k * 2 + 1] = H[3];
+      if (k > 0 && k < n / 2) {
+        H_CIS[(n - k) * 2] = H[0];
+        H_CIS[(n - k) * 2 + 1] = -H[1];
+        H_CROSS[(n - k) * 2] = H[2];
+        H_CROSS[(n - k) * 2 + 1] = -H[3];
+      }
     }
+    // A real filter's response at the Nyquist frequency is real.
+    H_CIS[n + 1] = 0;
+    H_CROSS[n + 1] = 0;
 
     this.H_CIS = H_CIS;
     this.H_CROSS = H_CROSS;
@@ -150,28 +163,33 @@ export class ConvolutionXTC {
     this.convolvers_XTC[other].buffer = this.buffer_XTC;
 
     if (!this.switchTimeout) {
-      this.getInputNode().connect(this.convolvers_XTC[other], 1);
+      // Connected with no output index, as it is disconnected: connected with one (which
+      // meant nothing here), the disconnect below never matched it and threw into an
+      // empty catch, so both convolvers ran for good, at twice the CPU.
+      this.getInputNode().connect(this.convolvers_XTC[other]);
     }
 
     clearTimeout(this.switchTimeout);
     this.switchTimeout = setTimeout(() => {
       this.getOutputNode().connectFrom(this.convolvers_XTC[other]);
-      try {
-        this.getInputNode().disconnect(this.convolvers_XTC[current]);
-      } catch (e) {
-
-      }
-      try {
-        this.getOutputNode().disconnectFrom(this.convolvers_XTC[current]);
-      } catch (e) {
-
-      }
+      this.detachConvolver(this.convolvers_XTC[current]);
 
       this.currentConvolver = other;
       this.switchTimeout = null;
     }, 100);
   }
 
+  // Takes a convolver off the input and the output where it is on them: the first switch
+  // has no convolver to retire, and while a switch is pending both are on the input.
+  detachConvolver(convolver) {
+    if (this.getInputNode().indexConnectedTo(convolver) !== -1) {
+      this.getInputNode().disconnect(convolver);
+    }
+    if (this.getOutputNode().indexConnectedFrom(convolver) !== -1) {
+      this.getOutputNode().disconnectFrom(convolver);
+    }
+  }
+
   async init() {
     this.inputNode = new VirtualAudioNode('ConvolutionXTC Input');
     this.inputBypassNode = new VirtualAudioNode('ConvolutionXTC Bypass Input');
@@ -208,14 +226,7 @@ export class ConvolutionXTC {
     this.getOutputNode().disconnectFrom(this.convolver_BYPASS);
 
     this.convolvers_XTC.forEach((convolver) => {
-      try {
-        this.getInputNode().disconnect(convolver);
-      } catch (e) {
-      }
-      try {
-        this.getOutputNode().disconnectFrom(convolver);
-      } catch (e) {
-      }
+      this.detachConvolver(convolver);
     });
     this.convolver_BYPASS = null;
     this.convolvers_XTC = null;

chrome/player/modules/vad/vad.mjs

@@ -57,6 +57,12 @@ const Message = {
   SpeechEnd: 4,
 };
 
+// Silero v5 and v6 score each 512-sample frame against the last 64 samples of the frame
+// before it, as silero-vad's own OnnxWrapper.__call__ does (ricky0123/vad aa048997, "Give
+// Silero v5 its context window"). Fed the bare frame, the model saw every frame as the first
+// of an utterance: it scored quiet and starting speech too low.
+const CONTEXT_SAMPLES = 64;
+
 class Silero {
   static async new(ort, modelFetcher) {
     const model = new Silero(ort, modelFetcher);
@@ -66,6 +72,7 @@ class Silero {
   constructor(ort, modelFetcher) {
     this.ort = ort;
     this.modelFetcher = modelFetcher;
+    this.context = new Float32Array(CONTEXT_SAMPLES);
     this.init = async () => {
       console.debug('initializing vad');
       const modelArrayBuffer = await this.modelFetcher();
@@ -87,9 +94,15 @@ class Silero {
     this.reset_state = () => {
       const zeroes = Array(2 * 1 * 128).fill(0);
       this.state = new this.ort.Tensor('float32', zeroes, [2, 1, 128]);
+      this.context = new Float32Array(CONTEXT_SAMPLES);
     };
     this.process = async (audioFrame) => {
-      const t = new this.ort.Tensor('float32', audioFrame, [1, audioFrame.length]);
+      const withContext = new Float32Array(CONTEXT_SAMPLES + audioFrame.length);
+      withContext.set(this.context, 0);
+      withContext.set(audioFrame, CONTEXT_SAMPLES);
+      // A copy: the caller may reuse its frame.
+      this.context = audioFrame.slice(-CONTEXT_SAMPLES);
+      const t = new this.ort.Tensor('float32', withContext, [1, withContext.length]);
       const inputs = {
         input: t,
         state: this.state,
@@ -223,13 +236,27 @@ class AudioNodeVAD {
 
   async init() {
     await this.ctx.audioWorklet.addModule(assetPath('vad.worklet.mjs'));
-    const vadNode = new AudioWorkletNode(this.ctx, 'vad-helper-worklet', {
-      processorOptions: {
-        frameSamples: this.options.frameSamples,
-      },
-    });
-    this.entryNode = vadNode;
+    // The model before the node: a model that did not load (a missing file, ONNX Runtime
+    // not starting) left behind a node whose processor ran for the rest of the context.
     const model = await createModel(this.options.ort);
+    let vadNode;
+    try {
+      vadNode = new AudioWorkletNode(this.ctx, 'vad-helper-worklet', {
+        processorOptions: {
+          frameSamples: this.options.frameSamples,
+        },
+        // One channel, mixed down by Web Audio (for 5.1: 0.707 (L + R) + C + 0.5 (SL + SR)):
+        // the processor reads the first channel only, which was the left one, so the
+        // dialogue in a 5.1 film's centre channel went unheard.
+        channelCount: 1,
+        channelCountMode: 'explicit',
+        channelInterpretation: 'speakers',
+      });
+    } catch (e) {
+      model.release().catch(() => {});
+      throw e;
+    }
+    this.entryNode = vadNode;
     this.model = model;
     this.frameProcessor = new FrameProcessor(model.process, model.reset_state, {
       frameSamples: this.options.frameSamples,

chrome/player/ui/audio/AudioChannelMixer.mjs

@@ -10,6 +10,20 @@ import {AudioEqualizer} from './AudioEqualizer.mjs';
 import {MAX_AUDIO_CHANNELS, CHANNEL_NAMES} from './config/AudioProfile.mjs';
 import {VirtualAudioNode} from './VirtualAudioNode.mjs';
 
+/**
+ * Sizes a meter's canvas when its size has changed. Setting a canvas's size, even to the
+ * size it has, clears it and resets its context: that was done every frame on all seven
+ * meters, which are cleared before they are drawn anyway.
+ * @param {HTMLCanvasElement} canvas
+ * @param {number} width
+ * @param {number} height
+ */
+function sizeCanvas(canvas, width, height) {
+  // The size is a whole number: a fraction is dropped when it is set.
+  if (canvas.width !== Math.trunc(width)) canvas.width = width;
+  if (canvas.height !== Math.trunc(height)) canvas.height = height;
+}
+
 export class AudioChannelMixer extends AbstractAudioModule {
   constructor(configManager) {
     super('AudioChannelMixer');
@@ -125,8 +139,7 @@ export class AudioChannelMixer extends AbstractAudioModule {
     const height = canvas.clientHeight * window.devicePixelRatio;
     if (width === 0 || height === 0) return;
 
-    canvas.width = width;
-    canvas.height = height;
+    sizeCanvas(canvas, width, height);
 
     ctx.clearRect(0, 0, width, height);
 
@@ -202,8 +215,7 @@ export class AudioChannelMixer extends AbstractAudioModule {
     const height = canvas.clientHeight * window.devicePixelRatio;
     if (width === 0 || height === 0) return;
 
-    canvas.width = width;
-    canvas.height = height;
+    sizeCanvas(canvas, width, height);
 
     ctx.clearRect(0, 0, width, height);
 
@@ -434,7 +446,7 @@ export class AudioChannelMixer extends AbstractAudioModule {
         els.muteButton.classList.toggle('active', channel.mono);
       } else {
         channel.muted = !channel.muted;
-        els.muteButton.classList.toggle('active', channel.mute);
+        els.muteButton.classList.toggle('active', channel.muted);
       }
       this.updateNodes();
     };
@@ -764,6 +776,13 @@ export class AudioChannelMixer extends AbstractAudioModule {
       return nodes.equalizer.hasNodes() || nodes.compressor.isEnabled();
     });
 
+    // A compressor delays what it compresses by its look-ahead: while any channel has one
+    // on, the channels without wait as long, or they reached the speakers 6 ms early.
+    const anyCompressed = activeChannels.some((i) => this.channelNodes[i].compressor.isEnabled());
+    this.channelNodes.forEach((nodes, i) => {
+      nodes.compressor.setLatencyMatch(anyCompressed && activeChannels.includes(i));
+    });
+
     const needsMerger = hasNonUnityChannelGains || hasActiveNodes || needsAnalyzer;
     const needsSplitter = needsMerger; // numberOfChannels > 1 && needsMerger;
     if (needsMasterGain) {