From d44db6ffc0769313d954cfd982efd23a620e12db Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Balusu Date: Wed, 25 Mar 2026 16:58:45 -0400 Subject: [PATCH] fix: prevent IndexError in Whisper timestamp decode on trailing replacement char When the decoded token stream ends with a dangling Unicode replacement character (U+FFFD), the computed index in _split_tokens_on_unicode() could equal len(decoded_full), causing an IndexError. Add a bounds check so that an out-of-range index is treated as a word boundary instead of crashing. Fixes #44869 --- src/transformers/models/whisper/tokenization_whisper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/whisper/tokenization_whisper.py b/src/transformers/models/whisper/tokenization_whisper.py index 755018210f69..6cb0a2e25c72 100644 --- a/src/transformers/models/whisper/tokenization_whisper.py +++ b/src/transformers/models/whisper/tokenization_whisper.py @@ -1329,9 +1329,11 @@ def _split_tokens_on_unicode(tokenizer, tokens: list[int]): current_indices.append(token_idx) decoded = tokenizer.decode(current_tokens, decode_with_timestamps=True) + target_index = unicode_offset + decoded.index(replacement_char) if replacement_char in decoded else None if ( replacement_char not in decoded - or decoded_full[unicode_offset + decoded.index(replacement_char)] == replacement_char + or target_index >= len(decoded_full) + or decoded_full[target_index] == replacement_char ): words.append(decoded) word_tokens.append(current_tokens)