diff --git a/src/basic_memory/markdown/plugins.py b/src/basic_memory/markdown/plugins.py index d0890e0a4..b96a733e8 100644 --- a/src/basic_memory/markdown/plugins.py +++ b/src/basic_memory/markdown/plugins.py @@ -1,16 +1,46 @@ """Markdown-it plugins for Basic Memory markdown parsing.""" +import re from typing import List, Any, Dict from basic_memory.utils import normalize_project_reference from markdown_it import MarkdownIt from markdown_it.token import Token +# Transcript timecodes like [00:00:11] or [1:02:03.500] share the bracket shape of +# observation categories, so indexed transcripts would mint one junk observation per +# spoken line. A category that is purely a clock value is never a semantic category; +# those bracket prefixes stay ordinary content (issue #1219). +_TIMESTAMP_CATEGORY = re.compile(r"^\d{1,3}:\d{2}(:\d{2})?([.,]\d{1,3})?$") + + +def _is_task_marker_category(category: str) -> bool: + """Recognize checkbox-marker shapes the GFM/Obsidian task family uses. + + `[ ]`, `[x]`, and `[-]` are excluded upstream, but the extended vocabulary + (`[/]` in progress, `[>]` deferred, `[?]` question, uppercase `[X]`) shares the + bracket shape and would otherwise mint junk one-character categories (#1241). + Single-character alphanumeric categories other than x/X keep parsing as today. + """ + if len(category) != 1: + return False + return category in {"x", "X"} or not category.isalnum() + + +def _observation_category_match(content: str) -> re.Match[str] | None: + """Match ``[category] content``, rejecting timestamp and task-marker shapes.""" + match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content) + if not match: + return None + category = match.group(1).strip() + if _TIMESTAMP_CATEGORY.match(category) or _is_task_marker_category(category): + return None + return match + # Observation handling functions def is_observation(token: Token) -> bool: """Check if token looks like our observation format.""" - import re if token.type != "inline": # pragma: no cover return False @@ -31,7 +61,7 @@ def is_observation(token: Token) -> bool: return False # Check for proper observation format: [category] content - match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content) + match = _observation_category_match(content) # Check for standalone hashtags (words starting with #) # This excludes # in HTML attributes like color="#4285F4" has_tags = any(part.startswith("#") for part in content.split()) @@ -40,13 +70,13 @@ def is_observation(token: Token) -> bool: def parse_observation(token: Token) -> Dict[str, Any]: """Extract observation parts from token.""" - import re # Use token.tag which contains the actual content for test tokens, fallback to content content = (token.tag or token.content).strip() - # Parse [category] with regex - match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content) + # Parse [category] with regex; a timestamp-shaped prefix is not a category, so a + # hashtag-promoted transcript line keeps its timecode inside the content instead. + match = _observation_category_match(content) category = None if match: category = match.group(1).strip() diff --git a/tests/markdown/test_observation_edge_cases.py b/tests/markdown/test_observation_edge_cases.py index 5e1bf3619..8cc03a2bb 100644 --- a/tests/markdown/test_observation_edge_cases.py +++ b/tests/markdown/test_observation_edge_cases.py @@ -127,3 +127,93 @@ def test_unicode_content(): observation = Observation.model_validate(obs) assert "δΈ­ζ–‡" in observation.content assert "πŸ‘" in observation.content + + +def test_timestamp_prefixes_are_not_observation_categories(): + """Transcript timecodes must not mint observations (issue #1219).""" + md = MarkdownIt().use(observation_plugin) + + # The issue's repro: list-item and bare transcript lines plus one real observation. + tokens = md.parse( + "[00:00:11] Speaker: We chose the safer option.\n" + "- [00:01:42] Speaker: Follow up next week.\n" + "- [decision] Use the safer option.\n" + ) + observations = [t.meta["observation"] for t in tokens if t.meta and "observation" in t.meta] + assert len(observations) == 1 + assert observations[0]["category"] == "decision" + assert observations[0]["content"] == "Use the safer option." + + +def test_timestamp_shapes_rejected_across_formats(): + """MM:SS, HH:MM:SS, and fractional-second timecodes all stay ordinary content.""" + md = MarkdownIt().use(observation_plugin) + + for line in ( + "- [1:02] short timecode", + "- [00:00:11] plain timecode", + "- [1:02:03.500] fractional seconds", + "- [100:02:11] long recording hours", + "- [12:03,250] comma milliseconds", + ): + tokens = md.parse(line) + assert not any(t.meta and "observation" in t.meta for t in tokens), line + + +def test_hashtag_promoted_timestamp_line_keeps_timecode_in_content(): + """A tagged transcript line is an observation via its hashtag, never via the timecode.""" + md = MarkdownIt().use(observation_plugin) + + tokens = md.parse("- [00:00:11] Speaker: decision recorded #meeting") + token = next(t for t in tokens if t.type == "inline") + obs = parse_observation(token) + assert obs["category"] is None + assert obs["content"].startswith("[00:00:11] Speaker:") + assert obs["tags"] == ["meeting"] + + +def test_numeric_but_non_timestamp_categories_still_parse(): + """Only pure clock values are rejected; other numeric categories keep working.""" + md = MarkdownIt().use(observation_plugin) + + for line, category in ( + ("- [2024] year in review", "2024"), + ("- [v1:2] odd but not a clock", "v1:2"), + ("- [10:30am] time-of-day words", "10:30am"), + ): + tokens = md.parse(line) + token = next(t for t in tokens if t.type == "inline") + obs = token.meta.get("observation") if token.meta else None + assert obs is not None, line + assert obs["category"] == category + + +def test_extended_checkbox_markers_are_not_observation_categories(): + """Obsidian's extended task markers must not mint observations (issue #1241).""" + md = MarkdownIt().use(observation_plugin) + + for line in ( + "- [/] in progress task", + "- [>] deferred task", + "- [?] maybe task", + "- [!] important task", + "- [X] uppercase done task", + ): + tokens = md.parse(line) + assert not any(t.meta and "observation" in t.meta for t in tokens), line + + +def test_single_character_alphanumeric_categories_still_parse(): + """Only marker shapes are rejected; short real categories keep working.""" + md = MarkdownIt().use(observation_plugin) + + for line, category in ( + ("- [a] annotation shorthand", "a"), + ("- [1] first point", "1"), + ("- [q] question shorthand", "q"), + ): + tokens = md.parse(line) + token = next(t for t in tokens if t.type == "inline") + obs = token.meta.get("observation") if token.meta else None + assert obs is not None, line + assert obs["category"] == category