diff --git a/src/transformers/models/whisper/tokenization_whisper.py b/src/transformers/models/whisper/tokenization_whisper.py
index 755018210f69..6cb0a2e25c72 100644
--- a/src/transformers/models/whisper/tokenization_whisper.py
+++ b/src/transformers/models/whisper/tokenization_whisper.py
@@ -1329,9 +1329,11 @@ def _split_tokens_on_unicode(tokenizer, tokens: list[int]):
         current_indices.append(token_idx)
         decoded = tokenizer.decode(current_tokens, decode_with_timestamps=True)
 
+        target_index = unicode_offset + decoded.index(replacement_char) if replacement_char in decoded else None
         if (
             replacement_char not in decoded
-            or decoded_full[unicode_offset + decoded.index(replacement_char)] == replacement_char
+            or target_index >= len(decoded_full)
+            or decoded_full[target_index] == replacement_char
         ):
             words.append(decoded)
             word_tokens.append(current_tokens)