fix: enhance lyrics processing and synchronization in Player class

Author: 1Lucas1apk

src/lyrics/youtube.js

@@ -101,13 +101,24 @@ export default class YouTubeLyrics {
     const lines = lyrics.events
       .map((event) => {
         const text = event.segs?.map((seg) => seg.utf8).join('') || ''
+        const words =
+          event.segs?.map((seg) => ({
+            text: seg.utf8
+              .replace(/'/g, "'")
+              .replace(/"/g, '"')
+              .replace(/&/g, '&'),
+            timestamp: event.tStartMs + (seg.tOffsetMs || 0),
+            duration: 0
+          })) || []
+
         return {
           text: text
             .replace(/'/g, "'")
             .replace(/"/g, '"')
             .replace(/&/g, '&'),
           time: event.tStartMs,
-          duration: event.dDurationMs || 0
+          duration: event.dDurationMs || 0,
+          words
         }
       })
       .filter((line) => line.text.trim().length > 0)

src/playback/player.js

@@ -320,8 +320,6 @@ export class Player {
       this.isPaused = false
 
       if (!wasResuming && !this._isRestoring) {
-        this._lyricsBasePackets =
-          this.connection?.statistics?.packetsExpected ?? 0
         this._fading('trackStart')
         this._emitTrackStart()
       }
@@ -771,6 +769,8 @@ export class Player {
       resource.setVolume(this.volumePercent / 100)
     }
     this._lyricsBasePosition = startTime
+    this._lyricsBasePackets =
+      this.connection?.statistics?.packetsExpected ?? 0
     this._fading('trackStartArm', { resource })
     this._fading('trackEndSchedule', { startPosition: startTime || 0 })
 
@@ -957,7 +957,7 @@ export class Player {
           clearTimeout(this._lyricsMarkerTimer)
           this._lyricsMarkerTimer = null
         }
-        if (this.isLyricsSubscribed) this._recalculateLyricsIndex()
+        if (this.isLyricsSubscribed) this._recalculateLyricsIndex(undefined, undefined, true)
         this._fading('seek')
         this._fading('trackEndSchedule', { startPosition: this.position })
       }
@@ -1047,6 +1047,9 @@ export class Player {
       'Player',
       `Playing resource for guild ${this.guildId} after source seek`
     )
+    this._lyricsBasePosition = position
+    this._lyricsBasePackets =
+      this.connection?.statistics?.packetsExpected ?? 0
     this.connection.play(resource)
     await this.waitEvent('playerStateChange', (s) => s.status === 'playing')
 
@@ -1099,6 +1102,10 @@ export class Player {
       this._fading('seekPrepare', { resource })
       resource.setFilters(this.filters)
 
+      this._lyricsBasePosition = position
+      this._lyricsBasePackets =
+        this.connection?.statistics?.packetsExpected ?? 0
+
       const oldStream = this.connection.play(resource)
       await this.waitEvent('playerStateChange', (s) => s.status === 'playing')
       if (oldStream) {
@@ -1214,6 +1221,9 @@ export class Player {
       'Player',
       `Playing resource for guild ${this.guildId} after legacy seek`
     )
+    this._lyricsBasePosition = position
+    this._lyricsBasePackets =
+      this.connection?.statistics?.packetsExpected ?? 0
     this.connection.play(resource)
     await this.waitEvent('playerStateChange', (s) => s.status === 'playing')
 
@@ -1603,16 +1613,25 @@ export class Player {
     )
 
     if (lyricsData && lyricsData.loadType === 'lyrics') {
+      const lines = lyricsData.data.lines.map((line) => ({
+        timestamp: line.time,
+        duration: line.duration || 0,
+        line: line.text,
+        words: line.words || [],
+        plugin: {}
+      }))
+
+      for (let i = 0; i < lines.length - 1; i++) {
+        if (lines[i].duration === 0) {
+          lines[i].duration = lines[i + 1].timestamp - lines[i].timestamp
+        }
+      }
+
       const payload = {
         sourceName: this.track.info.sourceName,
         provider: lyricsData.data.provider,
         text: lyricsData.data.lines.map((l) => l.text).join('\n'),
-        lines: lyricsData.data.lines.map((line) => ({
-          timestamp: line.time,
-          duration: line.duration,
-          line: line.text,
-          plugin: {}
-        })),
+        lines,
         plugin: {}
       }
 
@@ -1623,7 +1642,7 @@ export class Player {
         clearTimeout(this._lyricsMarkerTimer)
         this._lyricsMarkerTimer = null
       }
-      this._recalculateLyricsIndex()
+      this._recalculateLyricsIndex(undefined, undefined, true)
       this._syncLyrics(true)
     } else {
       this.currentLyrics = null
@@ -1635,16 +1654,12 @@ export class Player {
     if (!this.isLyricsSubscribed || !this.currentLyrics || !this.currentLyrics.lines) return
     if (this._lyricsMarkerTimer && !force) return
 
-    const stats = this.connection?.statistics
-    if (!stats) return
     const timescale = this.filters.filters?.timescale || {
       speed: 1.0,
       rate: 1.0
     }
     const playbackSpeed = (timescale.speed || 1.0) * (timescale.rate || 1.0)
-    const packets = stats.packetsExpected ?? this._lyricsBasePackets
-    const deltaPackets = Math.max(0, packets - this._lyricsBasePackets)
-    const position = this._lyricsBasePosition + deltaPackets * 20 * playbackSpeed
+    const position = this._getLyricsPosition(playbackSpeed)
     const lines = this.currentLyrics.lines
     this._recalculateLyricsIndex(position, lines)
 
@@ -1657,19 +1672,18 @@ export class Player {
     this._lyricsMarkerTimer = setTimeout(() => {
       this._lyricsMarkerTimer = null
       if (!this.isLyricsSubscribed || !this.currentLyrics || !this.currentLyrics.lines) return
-      const nowStats = this.connection?.statistics
-      if (!nowStats) return
-      const nowPackets = nowStats.packetsExpected ?? this._lyricsBasePackets
-      const nowDeltaPackets = Math.max(0, nowPackets - this._lyricsBasePackets)
-      const nowPosition =
-        this._lyricsBasePosition + nowDeltaPackets * 20 * playbackSpeed
+      const nowPosition = this._getLyricsPosition(playbackSpeed)
       const drift = nowPosition - nextTimestamp
 
       if (drift < -15) {
         this._syncLyrics(true)
         return
       }
 
+      if (Math.abs(drift) > 100) {
+        this._lyricsBasePosition -= drift * 0.25
+      }
+
       this.lyricsLineIndex = nextIndex
       this.emitEvent('LyricsLineEvent', {
         lineIndex: nextIndex,
@@ -1680,23 +1694,26 @@ export class Player {
     }, delayMs)
   }
 
-  _recalculateLyricsIndex(positionOverride, linesOverride) {
+  _getLyricsPosition(playbackSpeed) {
+    const stats = this.connection?.statistics
+    const packets = stats?.packetsExpected ?? this._lyricsBasePackets
+    const deltaPackets = Math.max(0, packets - this._lyricsBasePackets)
+
+    return this._lyricsBasePosition + deltaPackets * 20 * playbackSpeed
+  }
+  _recalculateLyricsIndex(positionOverride, linesOverride, allowBackward = false) {
     if (!this.currentLyrics || !this.currentLyrics.lines) return
 
     const lines = linesOverride || this.currentLyrics.lines
     let position = positionOverride
 
     if (position === undefined) {
-      const stats = this.connection?.statistics
-      if (!stats) return
       const timescale = this.filters.filters?.timescale || {
         speed: 1.0,
         rate: 1.0
       }
       const playbackSpeed = (timescale.speed || 1.0) * (timescale.rate || 1.0)
-      const packets = stats.packetsExpected ?? this._lyricsBasePackets
-      const deltaPackets = Math.max(0, packets - this._lyricsBasePackets)
-      position = this._lyricsBasePosition + deltaPackets * 20 * playbackSpeed
+      position = this._getLyricsPosition(playbackSpeed)
     }
 
     let foundIndex = -1
@@ -1709,6 +1726,10 @@ export class Player {
       }
     }
 
+    if (!allowBackward && foundIndex < this.lyricsLineIndex) {
+      return
+    }
+
     if (foundIndex !== this.lyricsLineIndex) {
       const skipped = foundIndex > this.lyricsLineIndex + 1
       this.lyricsLineIndex = foundIndex