Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion packages/backend/embedding_atlas/projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,30 @@ def _ftyp_brands(data: bytes) -> set[bytes]:
_HEIC_BRANDS = {b"heic", b"heix", b"hevc", b"hevx", b"heim", b"heis", b"hevm", b"hevs"}


# MPEG audio and AAC ADTS frames have no magic bytes, only a sync word at the
# start of each frame. 0xFF padding and other binary data can start with the
# same bits, so the rest of the header must also be valid.
def _is_adts_header(data: bytes) -> bool:
# 12-bit sync word, layer 00, and a sampling frequency index below 13.
return (
len(data) >= 3
and data[0] == 0xFF
and (data[1] & 0xF6) == 0xF0
and (data[2] >> 2) & 0x0F < 13
)


def _is_mpeg_audio_header(data: bytes) -> bool:
# 11-bit sync word, then no reserved version, layer, bitrate or sample rate.
if len(data) < 3 or data[0] != 0xFF or (data[1] & 0xE0) != 0xE0:
return False
version = (data[1] >> 3) & 0x03
layer = (data[1] >> 1) & 0x03
bitrate = data[2] >> 4
sample_rate = (data[2] >> 2) & 0x03
return version != 0b01 and layer != 0b00 and bitrate != 0x0F and sample_rate != 0b11


def _detect_mime_type(data: bytes) -> str | None:
"""Return the MIME type of image or audio bytes based on magic bytes, or None if unknown."""
# Image formats
Expand Down Expand Up @@ -298,7 +322,11 @@ def _detect_mime_type(data: bytes) -> str | None:
return "audio/flac"
if data[:4] == b"OggS": # OGG (Vorbis/Opus)
return "audio/ogg"
if data[:3] == b"ID3" or data[:2] == b"\xff\xfb": # MP3 (ID3 tag or sync frame)
if data[:3] == b"ID3": # MP3 with an ID3v2 tag
return "audio/mpeg"
if _is_adts_header(data): # AAC without a container
return "audio/aac"
if _is_mpeg_audio_header(data): # MP3/MP2/MP1 without an ID3 tag
return "audio/mpeg"
if len(data) >= 12 and data[4:8] == b"ftyp": # MP4/M4A container (non-image brands)
return "audio/mp4"
Expand Down
26 changes: 26 additions & 0 deletions packages/backend/tests/test_modality_detection.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,32 @@ def test_mp3_sync_frame(self):
data = b"\xff\xfb\x90\x00" + b"\x00" * 20
assert _detect_binary_modality(data) == "audio"

@pytest.mark.parametrize(
"header",
[
b"\xff\xfa", # MPEG-1 Layer III with CRC
b"\xff\xf3", # MPEG-2 Layer III (16/22.05/24 kHz)
b"\xff\xf2", # MPEG-2 Layer III with CRC
b"\xff\xe3", # MPEG-2.5 Layer III (8/11.025/12 kHz)
b"\xff\xf1", # AAC ADTS
],
)
def test_mpeg_and_adts_sync_frames(self, header):
data = header + b"\x00" * 20
assert _detect_binary_modality(data) == "audio"

@pytest.mark.parametrize(
"data",
[
b"\xff\xd8\xff\xe0" + b"\x00" * 20, # JPEG must win over the sync check
b"\xff\xff\xff" + b"\x00" * 20, # 0xFF padding
b"\xff\xe1\x00" + b"\x00" * 20, # not valid MPEG or ADTS
b"\xff", # too short
],
)
def test_sync_like_bytes_that_are_not_audio(self, data):
assert _detect_binary_modality(data) == "image"

def test_mp4_m4a(self):
data = b"\x00\x00\x00\x1cftyp" + b"M4A " + b"\x00" * 20
assert _detect_binary_modality(data) == "audio"
Expand Down
Loading