From 73fa52059aaf0a88e870ce1f080ee091b0689dad Mon Sep 17 00:00:00 2001 From: phernandez Date: Fri, 17 Jul 2026 23:34:52 -0500 Subject: [PATCH] feat(core): honor note timestamps from frontmatter Signed-off-by: phernandez --- docs/DOMAIN_MODEL.md | 12 +- docs/NOTE-FORMAT.md | 21 ++++ src/basic_memory/api/v2/utils.py | 1 + src/basic_memory/index/local_dependencies.py | 2 - .../indexing/accepted_note_mutation_runner.py | 5 - .../indexing/accepted_note_write_runner.py | 24 ++-- src/basic_memory/indexing/batch_indexer.py | 33 ++++-- .../indexing/note_materialization_runner.py | 1 - src/basic_memory/markdown/entity_parser.py | 50 +++++++- src/basic_memory/models/knowledge.py | 9 +- .../repository/search_index_row.py | 10 ++ src/basic_memory/schemas/search.py | 1 + .../services/directory_service.py | 16 +-- src/basic_memory/services/entity_service.py | 107 ++++++++--------- src/basic_memory/services/search_service.py | 19 +-- tests/index/test_local_project_index.py | 32 ++++- .../test_accepted_note_mutation_runner.py | 12 ++ .../test_accepted_note_write_runner.py | 44 +++---- tests/indexing/test_batch_indexer.py | 85 ++++++++++++++ .../test_note_materialization_runner.py | 6 +- tests/markdown/test_entity_parser.py | 83 ++++++++++++- tests/services/test_entity_service_prepare.py | 111 +++++++++++++++++- tests/services/test_prepared_entity_fields.py | 54 --------- .../test_semantic_timestamp_surfaces.py | 78 ++++++++++++ 24 files changed, 604 insertions(+), 212 deletions(-) delete mode 100644 tests/services/test_prepared_entity_fields.py create mode 100644 tests/services/test_semantic_timestamp_surfaces.py diff --git a/docs/DOMAIN_MODEL.md b/docs/DOMAIN_MODEL.md index 2900f9b2f..5f4b88304 100644 --- a/docs/DOMAIN_MODEL.md +++ b/docs/DOMAIN_MODEL.md @@ -53,8 +53,14 @@ which graph and search projections attach. project-scoped, may be absent when permalinks are disabled or inapplicable, and changes only according to the configured move and permalink policies. - `title` is mutable display metadata, not identity. -- `checksum`, timestamps, size, parsed metadata, and indexed text describe synchronized state; - they are not independent knowledge sources. +- For Markdown notes, `created_at` and `updated_at` project the canonical `created` and `modified` + frontmatter values. Missing values fall back independently to file ctime and mtime for legacy + notes; invalid canonical values fail indexing. +- `checksum`, `mtime`, size, and path describe the physical file used for synchronization. They + are bookkeeping, not note semantics, and moves or materialization must not overwrite semantic + timestamps. +- Parsed metadata and indexed text describe synchronized state; they are not independent + knowledge sources. ### Observation @@ -142,7 +148,7 @@ or leaf helpers guess which registry wins. 2. Derive final frontmatter, permalink behavior, path, and Markdown bytes without mutating the caller's input. 3. Parse the accepted Markdown once into its entity, observations, and relations. -4. Persist through the runtime's file-first or DB-first path. +4. Persist the same parsed semantic timestamps through the runtime's file-first or DB-first path. 5. Reconcile the entity and its owned graph projections. 6. Update search projections from the same accepted or materialized content. 7. Resolve pending relation targets after the target entities exist. diff --git a/docs/NOTE-FORMAT.md b/docs/NOTE-FORMAT.md index d760f7671..7b3a05222 100644 --- a/docs/NOTE-FORMAT.md +++ b/docs/NOTE-FORMAT.md @@ -39,6 +39,8 @@ YAML metadata between `---` fences at the top of the file. | `type` | No | `note` | Entity type. Used for schema resolution and filtering. | | `tags` | No | `[]` | List or comma-separated string. Used for organization and search. | | `permalink` | No | generated from title | Stable identifier. Persists even if the file moves. | +| `created` | No | file ctime | Canonical semantic creation timestamp. Accepts ISO 8601 dates or datetimes. | +| `modified` | No | file mtime | Canonical semantic modification timestamp. Accepts ISO 8601 dates or datetimes. | | `schema` | No | none | Schema attachment — dict (inline), string (reference), or omitted (implicit). | Custom fields are allowed. Any key not in the standard set is stored as `entity_metadata` and indexed for search and filtering. @@ -67,6 +69,25 @@ YAML automatically converts some values to native types. Basic Memory normalizes This prevents errors when downstream code expects string values. +### Canonical Note Timestamps + +`created` and `modified` describe the note, not the current file object. Basic Memory parses these +canonical fields once when it accepts or indexes Markdown and carries the resulting typed values +through entity, search, and directory projections. + +- Date-only values use midnight in the machine's local timezone. +- Datetimes without an offset use the machine's local timezone. +- Explicit UTC or numeric offsets remain unchanged. +- A missing or null field falls back independently to the file's ctime or mtime. When file stats + are unavailable, both missing values use one timestamp from the current operation. +- An invalid canonical value is an indexing error for that field; Basic Memory does not silently + replace it with a file timestamp. + +Filesystem mtime, checksum, size, and path remain physical synchronization bookkeeping. A move, +materialization, or incidental bookkeeping update does not change a note's semantic timestamps. +Passive indexing does not add or rewrite `created` or `modified`; notes without them remain +compatible through the filesystem fallback. Timestamp aliases are not canonical fields. + ## Observations An observation is a categorized fact about the entity. Written as a Markdown list item. diff --git a/src/basic_memory/api/v2/utils.py b/src/basic_memory/api/v2/utils.py index 5f4f1f802..a988ae389 100644 --- a/src/basic_memory/api/v2/utils.py +++ b/src/basic_memory/api/v2/utils.py @@ -274,6 +274,7 @@ async def to_search_results( content=result.content, matched_chunk=result.matched_chunk_text, file_path=_required_str(result.file_path, "file_path"), + updated_at=result.updated_at, metadata=result.metadata, entity_id=entity_id, observation_id=observation_id, diff --git a/src/basic_memory/index/local_dependencies.py b/src/basic_memory/index/local_dependencies.py index 69dac98bf..fe31d0c11 100644 --- a/src/basic_memory/index/local_dependencies.py +++ b/src/basic_memory/index/local_dependencies.py @@ -517,8 +517,6 @@ async def index_changed_markdown_file( refreshed_entities[0].id, { "checksum": indexed.checksum, - "created_at": file_metadata.created_at, - "updated_at": file_metadata.modified_at, "mtime": file_metadata.modified_at.timestamp(), "size": file_metadata.size, }, diff --git a/src/basic_memory/indexing/accepted_note_mutation_runner.py b/src/basic_memory/indexing/accepted_note_mutation_runner.py index 86cfe57ac..348e592fe 100644 --- a/src/basic_memory/indexing/accepted_note_mutation_runner.py +++ b/src/basic_memory/indexing/accepted_note_mutation_runner.py @@ -483,7 +483,6 @@ async def _run_accepted_note_create( session, prepared=prepared, project_id=project.id, - now=now, user_profile_value=user_profile_value, repositories=dependencies.write_repositories, ) @@ -573,7 +572,6 @@ async def _run_accepted_note_update( session, prepared=prepared_write.prepared, project_id=project.id, - now=now, user_profile_value=user_profile_value, external_id=request.entity_external_id, repositories=dependencies.write_repositories, @@ -627,7 +625,6 @@ async def _run_accepted_note_update( entity=entity, data=request.data, current_note_content=current_note_content, - now=now, user_profile_value=user_profile_value, ) except (ParseError, ValueError) as error: @@ -698,7 +695,6 @@ async def _run_accepted_note_edit( find_text=request.data.find_text, expected_replacements=request.data.expected_replacements, replace_subsections=request.data.replace_subsections, - now=now, user_profile_value=user_profile_value, ) except (ParseError, ValueError) as error: @@ -798,7 +794,6 @@ async def _run_accepted_note_move( current_note_content=current_note_content, accepted_file_path=accepted_file_path, should_update_permalink=should_update_permalink, - now=now, user_profile_value=user_profile_value, ) except (ParseError, ValueError) as error: diff --git a/src/basic_memory/indexing/accepted_note_write_runner.py b/src/basic_memory/indexing/accepted_note_write_runner.py index 7cd511715..076e610c3 100644 --- a/src/basic_memory/indexing/accepted_note_write_runner.py +++ b/src/basic_memory/indexing/accepted_note_write_runner.py @@ -67,6 +67,12 @@ def permalink(self) -> str | None: ... @property def file_path(self) -> RuntimeFilePath: ... + @property + def created_at(self) -> datetime: ... + + @property + def updated_at(self) -> datetime: ... + class AcceptedPreparedEntityWriteSource(Protocol): """Prepared markdown/entity state produced by Basic Memory note semantics.""" @@ -100,6 +106,7 @@ class AcceptedPreparedEntityTarget(Protocol): content_type: str permalink: str | None file_path: RuntimeFilePath + created_at: datetime updated_at: datetime last_updated_by: str | None @@ -389,7 +396,6 @@ async def prepare_accepted_note_replace( entity: Entity, data: EntitySchema, current_note_content: AcceptedNoteContentSource, - now: datetime, user_profile_value: str | None, ) -> AcceptedPreparedNoteWrite: """Prepare a full accepted replacement and apply its entity fields.""" @@ -406,7 +412,6 @@ async def prepare_accepted_note_replace( apply_accepted_prepared_entity_fields( entity, prepared.entity_fields, - updated_at=now, user_profile_value=user_profile_value, ) await session.flush() @@ -425,7 +430,6 @@ async def prepare_accepted_note_edit( find_text: str | None, expected_replacements: int, replace_subsections: bool, - now: datetime, user_profile_value: str | None, ) -> AcceptedPreparedNoteWrite: """Prepare a partial accepted edit and apply its entity fields.""" @@ -447,7 +451,6 @@ async def prepare_accepted_note_edit( apply_accepted_prepared_entity_fields( entity, prepared.entity_fields, - updated_at=now, user_profile_value=user_profile_value, ) await session.flush() @@ -462,7 +465,6 @@ async def prepare_accepted_note_move( current_note_content: AcceptedNoteContentSource, accepted_file_path: RuntimeFilePath, should_update_permalink: bool, - now: datetime, user_profile_value: str | None, ) -> AcceptedPreparedNoteMove: """Prepare a DB-first move and apply the accepted path/permalink fields.""" @@ -495,7 +497,6 @@ async def prepare_accepted_note_move( ) entity.file_path = result.file_path entity.permalink = result.permalink - entity.updated_at = now entity.last_updated_by = user_profile_value await session.flush() return result @@ -505,7 +506,6 @@ def apply_accepted_prepared_entity_fields( entity: AcceptedPreparedEntityTarget, entity_fields: AcceptedPreparedEntityFields, *, - updated_at: datetime, user_profile_value: str | None, ) -> None: """Copy prepared accepted markdown fields onto an entity row.""" @@ -515,14 +515,14 @@ def apply_accepted_prepared_entity_fields( entity.content_type = entity_fields.content_type entity.permalink = entity_fields.permalink entity.file_path = entity_fields.file_path - entity.updated_at = updated_at + entity.created_at = entity_fields.created_at + entity.updated_at = entity_fields.updated_at entity.last_updated_by = user_profile_value def accepted_pending_entity_write_from_prepared( prepared: AcceptedPreparedEntityWriteSource, *, - now: datetime, user_profile_value: str | None, external_id: str | None = None, ) -> AcceptedPendingEntityWrite: @@ -535,8 +535,8 @@ def accepted_pending_entity_write_from_prepared( content_type=fields.content_type, permalink=fields.permalink, file_path=fields.file_path, - created_at=now, - updated_at=now, + created_at=fields.created_at, + updated_at=fields.updated_at, created_by=user_profile_value, last_updated_by=user_profile_value, external_id=external_id, @@ -548,7 +548,6 @@ async def create_accepted_pending_entity( *, prepared: AcceptedPreparedEntityWriteSource, project_id: ProjectId, - now: datetime, user_profile_value: str | None, external_id: str | None = None, repositories: AcceptedNoteWriteRepositories, @@ -559,7 +558,6 @@ async def create_accepted_pending_entity( session, accepted_pending_entity_write_from_prepared( prepared, - now=now, user_profile_value=user_profile_value, external_id=external_id, ), diff --git a/src/basic_memory/indexing/batch_indexer.py b/src/basic_memory/indexing/batch_indexer.py index 99e5c384f..c2d0fc43f 100644 --- a/src/basic_memory/indexing/batch_indexer.py +++ b/src/basic_memory/indexing/batch_indexer.py @@ -485,7 +485,11 @@ async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity: updated = await self.entity_repository.update( session, entity_id, - self._entity_metadata_updates(file, checksum, include_created_at=is_new_entity), + self._resource_metadata_updates( + file, + checksum, + include_created_at=is_new_entity, + ), ) if updated is None: raise ValueError(f"Failed to update file entity metadata for {file.path}") @@ -631,7 +635,10 @@ async def _persist_markdown_file( session=session, ) prepared = await self._reconcile_persisted_permalink(prepared, entity) - metadata_updates = self._entity_metadata_updates(prepared.file, prepared.final_checksum) + metadata_updates = self._file_bookkeeping_updates( + prepared.file, + prepared.final_checksum, + ) updated = await self.entity_repository.update_fields( session, entity.id, @@ -711,27 +718,37 @@ async def _resolve_checksum(self, file: IndexInputFile) -> str: raise ValueError(f"Missing checksum and content for file: {file.path}") return await compute_checksum(file.content) - def _entity_metadata_updates( + def _file_bookkeeping_updates( self, file: IndexInputFile, checksum: str, - *, - include_created_at: bool = True, ) -> dict[str, object]: + """Return physical file state without changing note semantics.""" updates: dict[str, object] = { "file_path": file.path, "checksum": checksum, "size": file.size, } - if include_created_at and file.created_at is not None: - updates["created_at"] = file.created_at if file.last_modified is not None: - updates["updated_at"] = file.last_modified updates["mtime"] = file.last_modified.timestamp() if file.content_type is not None: updates["content_type"] = file.content_type return updates + def _resource_metadata_updates( + self, + file: IndexInputFile, + checksum: str, + *, + include_created_at: bool = True, + ) -> dict[str, object]: + updates = self._file_bookkeeping_updates(file, checksum) + if include_created_at and file.created_at is not None: + updates["created_at"] = file.created_at + if file.last_modified is not None: + updates["updated_at"] = file.last_modified + return updates + def _apply_entity_metadata_updates(self, entity: Entity, updates: dict[str, object]) -> None: """Keep the returned entity aligned with metadata written without reload.""" for key, value in updates.items(): diff --git a/src/basic_memory/indexing/note_materialization_runner.py b/src/basic_memory/indexing/note_materialization_runner.py index a757c474e..5dd9fd0ec 100644 --- a/src/basic_memory/indexing/note_materialization_runner.py +++ b/src/basic_memory/indexing/note_materialization_runner.py @@ -502,7 +502,6 @@ async def publish_written_file_state( file_path=written_file.file_path, file_checksum=written_file.file_checksum, ) - entity.updated_at = written_file.file_updated_at entity.mtime = written_file.file_updated_at.timestamp() entity.size = len(prepared_write.markdown_content.encode("utf-8")) await session.flush() diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index f8676dd91..47487b205 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -4,7 +4,7 @@ """ from dataclasses import dataclass, field -from datetime import date, datetime +from datetime import UTC, date, datetime from pathlib import Path from typing import Any, Optional @@ -119,6 +119,33 @@ def normalize_frontmatter_metadata(metadata: dict) -> dict: return {key: normalize_frontmatter_value(value) for key, value in metadata.items()} +def _parse_frontmatter_timestamp( + metadata: dict[str, Any], + field_name: str, + *, + fallback: datetime, +) -> datetime: + """Parse one canonical semantic timestamp after YAML value normalization.""" + value = metadata.get(field_name) + if value is None: + return fallback + if not isinstance(value, str): + raise ValueError(f"Invalid ISO 8601 value for frontmatter field '{field_name}': {value!r}") + + try: + timestamp = datetime.fromisoformat(value) + except ValueError as exc: + raise ValueError( + f"Invalid ISO 8601 value for frontmatter field '{field_name}': {value!r}" + ) from exc + + # ISO date-only and naive datetime values describe local wall-clock time. + # Explicit offsets are already unambiguous and must remain unchanged. + if timestamp.utcoffset() is None: + return timestamp.astimezone() + return timestamp + + @dataclass class EntityContent: content: str @@ -299,10 +326,25 @@ async def parse_markdown_content( entity_frontmatter = EntityFrontmatter(metadata=metadata) entity_content = parse(post.content) - # Use provided timestamps or current time as fallback + # Canonical frontmatter timestamps describe note semantics. File times are + # only compatibility fallbacks for notes that do not declare them. now = datetime.now().astimezone() - created = datetime.fromtimestamp(ctime).astimezone() if ctime else now - modified = datetime.fromtimestamp(mtime).astimezone() if mtime else now + created_fallback = ( + datetime.fromtimestamp(ctime, tz=UTC).astimezone() if ctime is not None else now + ) + modified_fallback = ( + datetime.fromtimestamp(mtime, tz=UTC).astimezone() if mtime is not None else now + ) + created = _parse_frontmatter_timestamp( + metadata, + "created", + fallback=created_fallback, + ) + modified = _parse_frontmatter_timestamp( + metadata, + "modified", + fallback=modified_fallback, + ) return EntityMarkdown( frontmatter=entity_frontmatter, diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index cb31ea74d..b6b6c6a2c 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -20,7 +20,7 @@ Float, text, ) -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from basic_memory.models.base import Base from basic_memory.runtime.storage import RUNTIME_MARKDOWN_CONTENT_TYPE @@ -95,7 +95,6 @@ class Entity(Base): updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=lambda: datetime.now().astimezone(), - onupdate=lambda: datetime.now().astimezone(), ) # Who created this entity (cloud user_profile_id UUID, null for local/CLI usage) @@ -127,6 +126,12 @@ class Entity(Base): uselist=False, ) + @validates("created_at", "updated_at") + def _normalize_semantic_timestamp(self, attribute_name: str, value: datetime) -> datetime: + """Keep SQLite's timezone-naive storage faithful to the represented instant.""" + del attribute_name + return ensure_timezone_aware(value).astimezone() + @property def relations(self): """Get all relations (incoming and outgoing) for this entity.""" diff --git a/src/basic_memory/repository/search_index_row.py b/src/basic_memory/repository/search_index_row.py index 9ffe6a738..bfdc17f6e 100644 --- a/src/basic_memory/repository/search_index_row.py +++ b/src/basic_memory/repository/search_index_row.py @@ -7,6 +7,7 @@ from pathlib import Path from basic_memory.schemas.search import SearchItemType +from basic_memory.utils import ensure_timezone_aware @dataclass @@ -44,6 +45,15 @@ class SearchIndexRow: CONTENT_DISPLAY_LIMIT = 4000 + def __post_init__(self) -> None: + """Restore typed, timezone-aware datetimes from raw search query results.""" + if isinstance(self.created_at, str): + self.created_at = datetime.fromisoformat(self.created_at) + if isinstance(self.updated_at, str): + self.updated_at = datetime.fromisoformat(self.updated_at) + self.created_at = ensure_timezone_aware(self.created_at) + self.updated_at = ensure_timezone_aware(self.updated_at) + @property def content(self): """Return truncated content for display. Full content in content_snippet.""" diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index 3c10477c1..21684409b 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -125,6 +125,7 @@ class SearchResult(BaseModel): content: Optional[str] = None matched_chunk: Optional[str] = None file_path: str + updated_at: Optional[datetime] = None metadata: Optional[dict] = None diff --git a/src/basic_memory/services/directory_service.py b/src/basic_memory/services/directory_service.py index 58cad7065..859cb9317 100644 --- a/src/basic_memory/services/directory_service.py +++ b/src/basic_memory/services/directory_service.py @@ -3,7 +3,6 @@ import fnmatch import logging import os -from datetime import datetime from typing import Dict, List, Optional, Sequence @@ -22,17 +21,6 @@ logger = logging.getLogger(__name__) -def _mtime_to_datetime(entity: Entity) -> datetime: - """Convert entity mtime (file modification time) to datetime. - - Returns the file's actual modification time, falling back to updated_at - if mtime is not available. - """ - if entity.mtime: # pragma: no cover - return datetime.fromtimestamp(entity.mtime).astimezone() # pragma: no cover - return entity.updated_at - - class DirectoryService: """Service for working with directory trees.""" @@ -105,7 +93,7 @@ async def get_directory_tree(self) -> DirectoryNode: entity_id=file.id, note_type=file.note_type, content_type=file.content_type, - updated_at=_mtime_to_datetime(file), + updated_at=file.updated_at, ) # Add to parent directory's children @@ -312,7 +300,7 @@ def _build_directory_tree_from_entities( entity_id=file.id, note_type=file.note_type, content_type=file.content_type, - updated_at=_mtime_to_datetime(file), + updated_at=file.updated_at, ) # Add to parent directory's children diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 93261efd5..3fb709c58 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -15,6 +15,7 @@ from basic_memory import db from basic_memory.config import ProjectConfig, BasicMemoryConfig from basic_memory.file_utils import ( + ParseError, has_frontmatter, parse_frontmatter, remove_frontmatter, @@ -78,24 +79,8 @@ class PreparedEntityFields: content_type: str permalink: str | None file_path: str - - -def apply_prepared_entity_fields( - entity: EntityModel, - entity_fields: PreparedEntityFields, - *, - updated_at: datetime, - user_profile_value: str | None, -) -> None: - """Copy prepared accepted markdown fields onto an entity row.""" - entity.title = entity_fields.title - entity.note_type = entity_fields.note_type - entity.entity_metadata = entity_fields.entity_metadata - entity.content_type = entity_fields.content_type - entity.permalink = entity_fields.permalink - entity.file_path = entity_fields.file_path - entity.updated_at = updated_at - entity.last_updated_by = user_profile_value + created_at: datetime + updated_at: datetime @dataclass(frozen=True) @@ -601,22 +586,27 @@ def _build_entity_fields( self, *, file_path: Path, - title: str, - note_type: str, content_type: str, - metadata: dict[str, Any] | None, permalink: str | None, + entity_markdown: EntityMarkdown, ) -> PreparedEntityFields: """Build the entity row data that mirrors accepted markdown state.""" - normalized_metadata = normalize_frontmatter_metadata(metadata or {}) + if entity_markdown.created is None or entity_markdown.modified is None: # pragma: no cover + raise ValueError("Prepared markdown requires created and modified timestamps") + + normalized_metadata = normalize_frontmatter_metadata( + entity_markdown.frontmatter.metadata or {} + ) entity_metadata = {k: v for k, v in normalized_metadata.items() if v is not None} return PreparedEntityFields( - title=title, - note_type=note_type, + title=entity_markdown.frontmatter.title, + note_type=entity_markdown.frontmatter.type, file_path=file_path.as_posix(), content_type=content_type, entity_metadata=entity_metadata or None, permalink=permalink, + created_at=entity_markdown.created, + updated_at=entity_markdown.modified, ) async def _build_prepared_write( @@ -624,7 +614,9 @@ async def _build_prepared_write( *, file_path: Path, markdown_content: str, - entity_fields: PreparedEntityFields, + content_type: str, + permalink: str | None, + preserved_created_at: datetime | None = None, ) -> PreparedEntityWrite: """Parse accepted markdown once so all persistence paths share the same state.""" # Trigger: both local and cloud-style callers need the exact same accepted markdown. @@ -634,6 +626,15 @@ async def _build_prepared_write( entity_markdown = await self.entity_parser.parse_markdown_content( file_path=file_path, content=markdown_content, + # DB-first updates have no file ctime. Reuse the existing semantic creation + # time as that field's fallback so editing a legacy note cannot make it "new". + ctime=(preserved_created_at.timestamp() if preserved_created_at is not None else None), + ) + entity_fields = self._build_entity_fields( + file_path=file_path, + content_type=content_type, + permalink=permalink, + entity_markdown=entity_markdown, ) return PreparedEntityWrite( file_path=file_path, @@ -710,18 +711,11 @@ async def prepare_create_entity_content( # store it in note_content first and materialize later without re-deriving anything. post = await schema_to_markdown(schema) markdown_content = dump_frontmatter(post) - entity_fields = self._build_entity_fields( - file_path=file_path, - title=schema.title, - note_type=schema.note_type, - content_type=schema.content_type, - metadata=post.metadata, - permalink=permalink, - ) return await self._build_prepared_write( file_path=file_path, markdown_content=markdown_content, - entity_fields=entity_fields, + content_type=schema.content_type, + permalink=permalink, ) async def prepare_update_entity_content( @@ -743,10 +737,17 @@ async def prepare_update_entity_content( schema = schema.model_copy(deep=True) file_path = Path(schema.file_path) current_file_path = Path(entity.file_path) - existing_markdown = await self.entity_parser.parse_markdown_content( - file_path=current_file_path, - content=existing_content, - ) + existing_metadata: dict[str, Any] = {} + if has_frontmatter(existing_content): + try: + existing_metadata = parse_frontmatter(existing_content) + except ParseError: + # Trigger: the old note has frontmatter fences but malformed YAML. + # Why: a full replacement must be able to repair that note, and malformed + # metadata cannot be merged safely into the replacement. + # Outcome: discard only the invalid merge input; the final accepted markdown + # is still parsed and validated below. + pass content_markdown = self._apply_schema_frontmatter_overrides(schema) # Trigger: a full replacement may also rename the note by changing title or directory. @@ -777,7 +778,9 @@ async def prepare_update_entity_content( # Full updates preserve unrecognized frontmatter keys from the existing note. # That keeps Basic Memory's write semantics stable for hand-authored metadata while still # letting the incoming schema replace the fields it explicitly owns. - merged_metadata = deepcopy(existing_markdown.frontmatter.metadata) + # Existing frontmatter is a merge input, not accepted state. Semantic validation happens + # after the incoming metadata has had a chance to repair invalid canonical values. + merged_metadata = deepcopy(existing_metadata) merged_metadata.update(post.metadata) merged_metadata["permalink"] = resolved_permalink @@ -785,18 +788,12 @@ async def prepare_update_entity_content( merged_post.metadata.update(merged_metadata) markdown_content = dump_frontmatter(merged_post) - entity_fields = self._build_entity_fields( - file_path=file_path, - title=schema.title, - note_type=schema.note_type, - content_type=schema.content_type, - metadata=merged_post.metadata, - permalink=resolved_permalink, - ) return await self._build_prepared_write( file_path=file_path, markdown_content=markdown_content, - entity_fields=entity_fields, + content_type=schema.content_type, + permalink=resolved_permalink, + preserved_created_at=entity.created_at, ) async def prepare_edit_entity_content( @@ -875,21 +872,13 @@ async def prepare_edit_entity_content( metadata=metadata, ) markdown_content = title_reconciliation.markdown_content - title = title_reconciliation.title - metadata = title_reconciliation.metadata - entity_fields = self._build_entity_fields( - file_path=file_path, - title=title, - note_type=note_type, - content_type=entity.content_type, - metadata=metadata, - permalink=permalink, - ) return await self._build_prepared_write( file_path=file_path, markdown_content=markdown_content, - entity_fields=entity_fields, + content_type=entity.content_type, + permalink=permalink, + preserved_created_at=entity.created_at, ) async def prepare_move_entity_content( diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index e8efabce1..be5257ec8 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -58,17 +58,6 @@ def _strip_nul(value: str) -> str: return value.replace("\x00", "") -def _mtime_to_datetime(entity: Entity) -> datetime: - """Convert entity mtime (file modification time) to datetime. - - Returns the file's actual modification time, falling back to updated_at - if mtime is not available. - """ - if entity.mtime: - return datetime.fromtimestamp(entity.mtime).astimezone() - return entity.updated_at - - class SearchService: """Service for search operations. @@ -716,7 +705,7 @@ async def index_entity_file( "note_type": entity.note_type, }, created_at=entity.created_at, - updated_at=_mtime_to_datetime(entity), + updated_at=entity.updated_at, project_id=entity.project_id, ) ) @@ -798,7 +787,7 @@ async def index_entity_markdown( "note_type": entity.note_type, }, created_at=entity.created_at, - updated_at=_mtime_to_datetime(entity), + updated_at=entity.updated_at, project_id=entity.project_id, ) ) @@ -833,7 +822,7 @@ async def index_entity_markdown( "tags": obs.tags, }, created_at=entity.created_at, - updated_at=_mtime_to_datetime(entity), + updated_at=entity.updated_at, project_id=entity.project_id, ) ) @@ -861,7 +850,7 @@ async def index_entity_markdown( to_id=rel.to_id, relation_type=rel.relation_type, created_at=entity.created_at, - updated_at=_mtime_to_datetime(entity), + updated_at=entity.updated_at, project_id=entity.project_id, ) ) diff --git a/tests/index/test_local_project_index.py b/tests/index/test_local_project_index.py index 0a90688a3..08b55dae3 100644 --- a/tests/index/test_local_project_index.py +++ b/tests/index/test_local_project_index.py @@ -337,7 +337,7 @@ async def test_local_project_index_uses_file_mtime_for_new_markdown_entities( assert abs(entity.mtime - expected_mtime) < 2 -async def test_local_project_index_updates_entity_mtime_on_file_modification( +async def test_local_project_index_preserves_semantic_timestamp_on_file_modification( test_project: Project, project_config, entity_repository, @@ -345,12 +345,22 @@ async def test_local_project_index_updates_entity_mtime_on_file_modification( config_manager, monkeypatch, ) -> None: - """Modified markdown entities use the current file modification timestamp.""" + """Reindex updates physical bookkeeping without changing note semantics.""" del config_manager note_path = project_config.home / "notes" / "timestamp-update.md" note_path.parent.mkdir(parents=True, exist_ok=True) - note_path.write_text("# Timestamp Update\n\nInitial content.\n", encoding="utf-8") + note_path.write_text( + """--- +created: 2023-12-01T08:00:00Z +modified: 2023-12-02T09:30:00Z +--- +# Timestamp Update + +Initial content. +""", + encoding="utf-8", + ) initial_mtime = datetime(2024, 1, 2, 3, 4, 5, tzinfo=timezone.utc).timestamp() os.utime(note_path, (initial_mtime, initial_mtime)) @@ -361,7 +371,17 @@ async def test_local_project_index_updates_entity_mtime_on_file_modification( assert first.enqueued_files == 1 note_path.write_text( - "# Timestamp Update\n\nModified content.\n\n## Observations\n- [test] Timestamp moved.\n", + """--- +created: 2023-12-01T08:00:00Z +modified: 2023-12-02T09:30:00Z +--- +# Timestamp Update + +Modified content. + +## Observations +- [test] Timestamp moved. +""", encoding="utf-8", ) modified_mtime = datetime(2024, 1, 2, 4, 5, 6, tzinfo=timezone.utc).timestamp() @@ -378,8 +398,8 @@ async def test_local_project_index_updates_entity_mtime_on_file_modification( entity = await entity_repository.get_by_file_path(session, "notes/timestamp-update.md") assert entity is not None - assert entity.updated_at.timestamp() != initial_mtime - assert abs(entity.updated_at.timestamp() - modified_mtime) < 2 + assert entity.created_at == datetime(2023, 12, 1, 8, 0, tzinfo=timezone.utc) + assert entity.updated_at == datetime(2023, 12, 2, 9, 30, tzinfo=timezone.utc) assert entity.mtime is not None assert abs(entity.mtime - modified_mtime) < 2 assert len(entity.observations) == 1 diff --git a/tests/indexing/test_accepted_note_mutation_runner.py b/tests/indexing/test_accepted_note_mutation_runner.py index 8e77c34ee..c6f0115c0 100644 --- a/tests/indexing/test_accepted_note_mutation_runner.py +++ b/tests/indexing/test_accepted_note_mutation_runner.py @@ -46,6 +46,8 @@ _NOW = datetime(2026, 6, 20, 14, 30, tzinfo=UTC) +_PREPARED_CREATED_AT = datetime(2024, 1, 15, 10, 30, tzinfo=UTC) +_PREPARED_UPDATED_AT = datetime(2024, 1, 16, 11, 45, tzinfo=UTC) _ACTOR_ID = UUID("11111111-1111-4111-8111-111111111111") @@ -57,6 +59,8 @@ class _PreparedFields: content_type: str permalink: str | None file_path: str + created_at: datetime = _PREPARED_CREATED_AT + updated_at: datetime = _PREPARED_UPDATED_AT @dataclass(frozen=True, slots=True) @@ -612,6 +616,8 @@ async def test_run_accepted_note_create_persists_prepared_markdown() -> None: assert preparer.calls == [(schema, False, session)] assert preparer.skip_conflict_checks == [True] assert pending_entity_repository.calls[0][1].created_by == str(_ACTOR_ID) + assert entity.created_at == _PREPARED_CREATED_AT + assert entity.updated_at == _PREPARED_UPDATED_AT assert note_content_accept_repository.calls[0][1].markdown_content == "# Accepted\n" assert note_content_accept_repository.calls[0][1].db_version == 1 assert search_repository.calls[0][1].content_snippet == "Accepted" @@ -746,6 +752,8 @@ async def test_run_accepted_note_update_replaces_existing_note_content() -> None assert session.flush_count == 1 assert note_content_accept_repository.calls[0][1].db_version == 2 assert note_content_accept_repository.calls[0][1].markdown_content == "# Replacement\n" + assert entity.created_at == _PREPARED_CREATED_AT + assert entity.updated_at == _PREPARED_UPDATED_AT assert change.status_code == 200 assert isinstance(change.payload, RuntimeAcceptedNoteResponse) assert change.payload.title == "Replacement" @@ -1113,6 +1121,8 @@ async def test_run_accepted_note_edit_applies_patch_against_db_content() -> None ) ] assert note_content_accept_repository.calls[0][1].last_source == "mcp" + assert entity.created_at == _PREPARED_CREATED_AT + assert entity.updated_at == _PREPARED_UPDATED_AT assert change.status_code == 200 assert change.materialization is not None assert change.materialization.source == "mcp" @@ -1172,6 +1182,8 @@ async def test_run_accepted_note_move_carries_previous_path_and_materialized_cle ] assert entity.file_path == "archive/accepted.md" assert entity.permalink == "archive/accepted" + assert entity.created_at == _NOW + assert entity.updated_at == _NOW assert change.status_code == 200 assert change.materialization is not None assert change.materialization.previous_file_path == "notes/accepted.md" diff --git a/tests/indexing/test_accepted_note_write_runner.py b/tests/indexing/test_accepted_note_write_runner.py index bdcf18c28..53d9ed60d 100644 --- a/tests/indexing/test_accepted_note_write_runner.py +++ b/tests/indexing/test_accepted_note_write_runner.py @@ -42,6 +42,10 @@ from basic_memory.schemas.base import Entity as EntitySchema +_PREPARED_CREATED_AT = datetime(2024, 1, 15, 10, 30, tzinfo=UTC) +_PREPARED_UPDATED_AT = datetime(2024, 1, 16, 11, 45, tzinfo=UTC) + + @dataclass(frozen=True, slots=True) class _PreparedFields: title: str @@ -50,6 +54,8 @@ class _PreparedFields: content_type: str permalink: str | None file_path: str + created_at: datetime = _PREPARED_CREATED_AT + updated_at: datetime = _PREPARED_UPDATED_AT @dataclass(frozen=True, slots=True) @@ -478,7 +484,6 @@ async def test_prepare_accepted_note_replace_applies_entity_fields() -> None: session = _FlushSession() entity = _entity() schema = _schema() - now = datetime(2026, 6, 19, 12, 30, tzinfo=UTC) fields = _PreparedFields( title="Replacement", note_type="decision", @@ -496,7 +501,6 @@ async def test_prepare_accepted_note_replace_applies_entity_fields() -> None: entity=entity, data=schema, current_note_content=_note_content(), - now=now, user_profile_value="user-2", ) @@ -509,7 +513,8 @@ async def test_prepare_accepted_note_replace_applies_entity_fields() -> None: assert entity.note_type == "decision" assert entity.entity_metadata == {"status": "accepted"} assert entity.file_path == "notes/replacement.md" - assert entity.updated_at == now + assert entity.created_at == _PREPARED_CREATED_AT + assert entity.updated_at == _PREPARED_UPDATED_AT assert entity.last_updated_by == "user-2" assert session.flush_count == 1 @@ -518,7 +523,6 @@ async def test_prepare_accepted_note_replace_applies_entity_fields() -> None: async def test_prepare_accepted_note_edit_applies_entity_fields() -> None: session = _FlushSession() entity = _entity() - now = datetime(2026, 6, 19, 12, 45, tzinfo=UTC) fields = _PreparedFields( title="Edited", note_type="note", @@ -541,7 +545,6 @@ async def test_prepare_accepted_note_edit_applies_entity_fields() -> None: find_text="# Accepted", expected_replacements=1, replace_subsections=True, - now=now, user_profile_value=None, ) @@ -563,13 +566,14 @@ async def test_prepare_accepted_note_edit_applies_entity_fields() -> None: assert entity.title == "Edited" assert entity.permalink == "edited" assert entity.file_path == "notes/edited.md" + assert entity.created_at == _PREPARED_CREATED_AT + assert entity.updated_at == _PREPARED_UPDATED_AT assert entity.last_updated_by is None assert session.flush_count == 1 def test_apply_accepted_prepared_entity_fields_updates_mutable_entity() -> None: entity = _entity() - now = datetime(2026, 6, 19, 13, 0, tzinfo=UTC) apply_accepted_prepared_entity_fields( entity, @@ -581,7 +585,6 @@ def test_apply_accepted_prepared_entity_fields_updates_mutable_entity() -> None: permalink="applied", file_path="schemas/applied.md", ), - updated_at=now, user_profile_value="user-3", ) @@ -591,7 +594,8 @@ def test_apply_accepted_prepared_entity_fields_updates_mutable_entity() -> None: assert entity.content_type == "text/markdown" assert entity.permalink == "applied" assert entity.file_path == "schemas/applied.md" - assert entity.updated_at == now + assert entity.created_at == _PREPARED_CREATED_AT + assert entity.updated_at == _PREPARED_UPDATED_AT assert entity.last_updated_by == "user-3" @@ -599,7 +603,8 @@ def test_apply_accepted_prepared_entity_fields_updates_mutable_entity() -> None: async def test_prepare_accepted_note_move_without_permalink_update_keeps_current_markdown() -> None: session = _FlushSession() entity = _entity() - now = datetime(2026, 6, 19, 13, 15, tzinfo=UTC) + original_created_at = entity.created_at + original_updated_at = entity.updated_at current = _note_content() current.markdown_content = "---\ntitle: legacy\n\n# Body still matters\n" @@ -610,7 +615,6 @@ async def test_prepare_accepted_note_move_without_permalink_update_keeps_current current_note_content=current, accepted_file_path="archive/accepted.md", should_update_permalink=False, - now=now, user_profile_value="user-4", ) @@ -621,7 +625,8 @@ async def test_prepare_accepted_note_move_without_permalink_update_keeps_current assert result.db_checksum == sha256(str(current.markdown_content).encode()).hexdigest() assert entity.file_path == "archive/accepted.md" assert entity.permalink == "accepted" - assert entity.updated_at == now + assert entity.created_at == original_created_at + assert entity.updated_at == original_updated_at assert entity.last_updated_by == "user-4" assert session.flush_count == 1 @@ -630,7 +635,8 @@ async def test_prepare_accepted_note_move_without_permalink_update_keeps_current async def test_prepare_accepted_note_move_with_permalink_update_uses_preparer() -> None: session = _FlushSession() entity = _entity() - now = datetime(2026, 6, 19, 13, 30, tzinfo=UTC) + original_created_at = entity.created_at + original_updated_at = entity.updated_at prepared = _PreparedMove( file_path=Path("archive/prepared.md"), markdown_content="# Prepared\n", @@ -646,7 +652,6 @@ async def test_prepare_accepted_note_move_with_permalink_update_uses_preparer() current_note_content=_note_content(), accepted_file_path="archive/accepted.md", should_update_permalink=True, - now=now, user_profile_value=None, ) @@ -660,17 +665,15 @@ async def test_prepare_accepted_note_move_with_permalink_update_uses_preparer() assert result.db_checksum == sha256(b"# Prepared\n").hexdigest() assert entity.file_path == "archive/prepared.md" assert entity.permalink == "archive/prepared" - assert entity.updated_at == now + assert entity.created_at == original_created_at + assert entity.updated_at == original_updated_at assert entity.last_updated_by is None assert session.flush_count == 1 def test_accepted_pending_entity_write_from_prepared_maps_core_fields() -> None: - now = datetime(2026, 6, 19, 12, 0, tzinfo=UTC) - write = accepted_pending_entity_write_from_prepared( _prepared(), - now=now, user_profile_value="user-1", external_id="note-1", ) @@ -682,8 +685,8 @@ def test_accepted_pending_entity_write_from_prepared_maps_core_fields() -> None: content_type="text/markdown", permalink="accepted", file_path="notes/accepted.md", - created_at=now, - updated_at=now, + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, created_by="user-1", last_updated_by="user-1", external_id="note-1", @@ -695,13 +698,10 @@ async def test_create_accepted_pending_entity_uses_repository_protocol() -> None session = cast(AsyncSession, object()) entity = _entity() repository = _PendingEntityRepository(entity) - now = datetime(2026, 6, 19, 12, 0, tzinfo=UTC) - result = await create_accepted_pending_entity( session, prepared=_prepared(), project_id=7, - now=now, user_profile_value=None, repositories=_repository_provider(pending_entity_repository=repository), ) diff --git a/tests/indexing/test_batch_indexer.py b/tests/indexing/test_batch_indexer.py index 3766b41e3..8bdee2ff3 100644 --- a/tests/indexing/test_batch_indexer.py +++ b/tests/indexing/test_batch_indexer.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +from datetime import UTC, datetime +import os from pathlib import Path from textwrap import dedent from unittest.mock import AsyncMock @@ -16,6 +18,7 @@ from basic_memory.indexing.models import IndexInputFile, StorageIndexFileWriter from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.schemas import Entity as EntitySchema +from basic_memory.schemas.search import SearchItemType, SearchQuery from basic_memory.services.exceptions import SyncFatalError @@ -202,6 +205,88 @@ async def test_batch_indexer_creates_entities_with_real_db_session( assert beta.title == "Beta" +@pytest.mark.asyncio +async def test_batch_indexer_preserves_markdown_semantic_timestamps_on_reindex( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + project_config, +): + path = "notes/canonical-timestamps.md" + absolute_path = project_config.home / path + canonical_created = datetime(2024, 1, 15, 10, 30, tzinfo=UTC) + canonical_modified = datetime.fromisoformat("2024-01-16T11:45:00+05:30") + frontmatter = dedent( + """ + --- + title: Canonical Timestamps + type: note + created: 2024-01-15T10:30:00Z + modified: 2024-01-16T11:45:00+05:30 + --- + """ + ).lstrip() + await _create_file(absolute_path, f"{frontmatter}First body\n") + os.utime(absolute_path, (1_730_000_000, 1_730_000_000)) + + batch_indexer = _make_batch_indexer( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + ) + first_input = await _load_input(file_service, path) + first_result = await batch_indexer.index_files({path: first_input}, max_concurrent=1) + + assert first_result.errors == [] + async with db.scoped_session(search_service.session_maker) as session: + entity = await entity_repository.get_by_file_path(session, path) + + assert entity is not None + assert first_input.last_modified is not None + assert entity.created_at == canonical_created + assert entity.updated_at == canonical_modified + assert entity.mtime == first_input.last_modified.timestamp() + + await _create_file(absolute_path, f"{frontmatter}Second body\n") + os.utime(absolute_path, (1_740_000_000, 1_740_000_000)) + second_input = await _load_input(file_service, path) + second_result = await batch_indexer.index_files({path: second_input}, max_concurrent=1) + + assert second_result.errors == [] + async with db.scoped_session(search_service.session_maker) as session: + reindexed = await entity_repository.get_by_file_path(session, path) + + assert reindexed is not None + assert second_input.last_modified is not None + assert reindexed.created_at == canonical_created + assert reindexed.updated_at == canonical_modified + assert reindexed.mtime == second_input.last_modified.timestamp() + assert reindexed.mtime != entity.mtime + + included = await search_service.search( + SearchQuery( + after_date="2024-01-16T06:00:00Z", + entity_types=[SearchItemType.ENTITY], + ) + ) + excluded = await search_service.search( + SearchQuery( + after_date="2024-01-16T06:30:00Z", + entity_types=[SearchItemType.ENTITY], + ) + ) + + assert [row.file_path for row in included] == [path] + assert included[0].updated_at == canonical_modified + assert excluded == [] + + @pytest.mark.asyncio async def test_batch_indexer_returns_original_markdown_content_when_no_frontmatter_rewrite( app_config, diff --git a/tests/indexing/test_note_materialization_runner.py b/tests/indexing/test_note_materialization_runner.py index 34109ae6e..75f572d36 100644 --- a/tests/indexing/test_note_materialization_runner.py +++ b/tests/indexing/test_note_materialization_runner.py @@ -303,6 +303,8 @@ def materialization_entity(*, file_path: str = "notes/a.md") -> Entity: content_type="text/markdown", file_path=file_path, checksum="old-file-sum", + created_at=datetime(2024, 1, 15, 10, 30, tzinfo=UTC), + updated_at=datetime(2024, 1, 16, 11, 45, tzinfo=UTC), ) @@ -515,6 +517,7 @@ async def test_repository_note_materialization_publisher_updates_current_written prepared = prepared_write(request) written = written_file() entity = materialization_entity() + semantic_updated_at = entity.updated_at note_content = materialization_note_content() session = FakeRepositorySession(entity=entity, note_content=note_content) session_lock = FakeSessionLock() @@ -558,7 +561,8 @@ async def test_repository_note_materialization_publisher_updates_current_written }, ) ] - assert entity.updated_at == written.file_updated_at + assert entity.updated_at == semantic_updated_at + assert entity.mtime == written.file_updated_at.timestamp() assert entity.size == len(b"# A note\n") assert session.flush_count == 1 diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index bfcf46c19..f6159da42 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -1,6 +1,6 @@ """Tests for entity markdown parsing.""" -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from textwrap import dedent @@ -58,6 +58,8 @@ async def test_parse_complete_file(project_config, entity_parser, valid_entity_c assert entity.frontmatter.type == "component" assert entity.frontmatter.permalink == "auth_service" assert set(entity.frontmatter.tags) == {"authentication", "security", "core"} + assert entity.created == datetime(2024, 12, 21, 14, 0, tzinfo=UTC) + assert entity.modified == datetime(2024, 12, 21, 14, 0, tzinfo=UTC) # Check content assert "Core authentication service that handles user authentication." in entity.content @@ -251,6 +253,85 @@ async def test_parse_file_with_absolute_path(project_config, entity_parser): assert entity.modified is not None +@pytest.mark.asyncio +async def test_parse_canonical_timestamp_formats(entity_parser): + date_only = await entity_parser.parse_markdown_content( + Path("date-only.md"), + "---\ncreated: 2024-01-15\nmodified: 2024-01-16\n---\nBody", + ) + naive = await entity_parser.parse_markdown_content( + Path("naive.md"), + "---\ncreated: 2024-01-15T10:30:00\nmodified: 2024-01-16T11:45:00\n---\nBody", + ) + offset = await entity_parser.parse_markdown_content( + Path("offset.md"), + "---\ncreated: 2024-01-15T10:30:00+05:30\nmodified: 2024-01-16T11:45:00Z\n---\nBody", + ) + + assert date_only.created == datetime(2024, 1, 15).astimezone() + assert date_only.modified == datetime(2024, 1, 16).astimezone() + assert naive.created == datetime(2024, 1, 15, 10, 30).astimezone() + assert naive.modified == datetime(2024, 1, 16, 11, 45).astimezone() + assert offset.created == datetime.fromisoformat("2024-01-15T10:30:00+05:30") + assert offset.modified == datetime(2024, 1, 16, 11, 45, tzinfo=UTC) + + +@pytest.mark.asyncio +async def test_parse_missing_and_null_timestamps_fall_back_individually(entity_parser): + entity = await entity_parser.parse_markdown_content( + Path("fallback.md"), + "---\ncreated: null\nmodified: 2024-01-16T11:45:00Z\n---\nBody", + ctime=0, + mtime=1, + ) + + assert entity.created == datetime(1970, 1, 1, tzinfo=UTC).astimezone() + assert entity.modified == datetime(2024, 1, 16, 11, 45, tzinfo=UTC) + + missing_modified = await entity_parser.parse_markdown_content( + Path("missing-modified.md"), + "---\ncreated: 2024-01-15T10:30:00Z\n---\nBody", + ctime=0, + mtime=1, + ) + + assert missing_modified.created == datetime(2024, 1, 15, 10, 30, tzinfo=UTC) + assert missing_modified.modified == datetime(1970, 1, 1, 0, 0, 1, tzinfo=UTC).astimezone() + + +@pytest.mark.asyncio +async def test_parse_without_file_stats_uses_one_operation_timestamp(entity_parser): + before = datetime.now().astimezone() + entity = await entity_parser.parse_markdown_content(Path("unstored.md"), "Body") + after = datetime.now().astimezone() + + assert before <= entity.created <= after + assert entity.modified == entity.created + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("created", "not-a-timestamp"), + ("modified", "Jan 15, 2024"), + ("created", "[2024-01-15]"), + ], +) +async def test_parse_invalid_canonical_timestamp_fails_for_field( + entity_parser, + field_name: str, + value: str, +): + with pytest.raises(ValueError, match=rf"frontmatter field '{field_name}'"): + await entity_parser.parse_markdown_content( + Path("invalid.md"), + f"---\n{field_name}: {value}\n---\nBody", + ctime=0, + mtime=1, + ) + + # @pytest.mark.asyncio # async def test_parse_file_invalid_yaml(test_config, entity_parser): # """Test parsing file with invalid YAML frontmatter.""" diff --git a/tests/services/test_entity_service_prepare.py b/tests/services/test_entity_service_prepare.py index dcf3bcab0..900f7d9b5 100644 --- a/tests/services/test_entity_service_prepare.py +++ b/tests/services/test_entity_service_prepare.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import FrozenInstanceError +from datetime import UTC, datetime import pytest @@ -21,7 +22,15 @@ async def test_prepare_create_entity_content_matches_create_entity_with_content( title="Prepared Create", directory="notes", note_type="note", - content="---\nstatus: draft\npermalink: prepared/create\n---\nCreate body", + content=( + "---\n" + "status: draft\n" + "permalink: prepared/create\n" + "created: 2024-01-15T10:30:00Z\n" + "modified: 2024-01-16T11:45:00Z\n" + "---\n" + "Create body" + ), ) prepared = await entity_service.prepare_create_entity_content(schema) @@ -34,6 +43,8 @@ async def test_prepare_create_entity_content_matches_create_entity_with_content( assert prepared.entity_fields.note_type == result.entity.note_type assert prepared.entity_fields.permalink == result.entity.permalink assert prepared.entity_fields.entity_metadata == result.entity.entity_metadata + assert prepared.entity_fields.created_at == result.entity.created_at + assert prepared.entity_fields.updated_at == result.entity.updated_at @pytest.mark.asyncio @@ -43,7 +54,14 @@ async def test_prepare_create_entity_content_returns_typed_entity_fields(entity_ title="Typed Fields", directory="notes", note_type="decision", - content="---\nstatus: accepted\n---\nBody", + content=( + "---\n" + "status: accepted\n" + "created: 2024-01-15T10:30:00Z\n" + "modified: 2024-01-16T11:45:00+05:30\n" + "---\n" + "Body" + ), ) ) @@ -54,11 +72,15 @@ async def test_prepare_create_entity_content_returns_typed_entity_fields(entity_ "title": "Typed Fields", "type": "decision", "status": "accepted", + "created": "2024-01-15T10:30:00+00:00", + "modified": "2024-01-16T11:45:00+05:30", "permalink": "test-project/notes/typed-fields", }, content_type="text/markdown", permalink="test-project/notes/typed-fields", file_path="notes/Typed Fields.md", + created_at=datetime(2024, 1, 15, 10, 30, tzinfo=UTC), + updated_at=datetime.fromisoformat("2024-01-16T11:45:00+05:30"), ) with pytest.raises(FrozenInstanceError): setattr(prepared.entity_fields, "title", "Changed") @@ -163,6 +185,7 @@ async def test_prepare_update_entity_content_matches_update_entity_with_content( update_schema, existing_content, ) + original_created_at = created.created_at result = await entity_service.update_entity_with_content(created, update_schema) prepared_frontmatter = parse_frontmatter(prepared.markdown_content) @@ -171,6 +194,8 @@ async def test_prepare_update_entity_content_matches_update_entity_with_content( assert prepared.entity_fields.title == result.entity.title assert prepared.entity_fields.note_type == result.entity.note_type assert prepared.entity_fields.permalink == result.entity.permalink + assert prepared.entity_fields.created_at == original_created_at + assert result.entity.created_at == original_created_at assert prepared_frontmatter["owner"] == "alice" assert prepared_frontmatter["status"] == "published" assert prepared_frontmatter["reviewed_by"] == "bob" @@ -215,6 +240,85 @@ async def test_prepare_update_entity_content_can_change_file_path( assert await file_service.exists("journal/Renamed Note.md") +@pytest.mark.asyncio +async def test_prepare_update_entity_content_can_repair_invalid_canonical_timestamps( + entity_service, + file_service, +) -> None: + created = await entity_service.create_entity( + EntitySchema( + title="Repair Timestamps", + directory="notes", + note_type="note", + content="Original body", + ) + ) + invalid_content = ( + "---\ncreated: yesterday\nmodified: last week\nowner: alice\n---\nOriginal body" + ) + await file_service.write_file(created.file_path, invalid_content) + update_schema = EntitySchema( + title="Repair Timestamps", + directory="notes", + note_type="note", + content=( + "---\ncreated: 2024-01-15T10:30:00Z\nmodified: 2024-01-16T11:45:00Z\n---\nRepaired body" + ), + ) + + prepared = await entity_service.prepare_update_entity_content( + created, + update_schema, + invalid_content, + ) + result = await entity_service.update_entity_with_content(created, update_schema) + + assert prepared.entity_fields.created_at == datetime(2024, 1, 15, 10, 30, tzinfo=UTC) + assert prepared.entity_fields.updated_at == datetime(2024, 1, 16, 11, 45, tzinfo=UTC) + assert result.entity.created_at == prepared.entity_fields.created_at + assert result.entity.updated_at == prepared.entity_fields.updated_at + assert parse_frontmatter(result.content)["owner"] == "alice" + + +@pytest.mark.asyncio +async def test_prepare_update_entity_content_can_repair_malformed_frontmatter( + entity_service, + file_service, +) -> None: + created = await entity_service.create_entity( + EntitySchema( + title="Repair Malformed Frontmatter", + directory="notes", + note_type="note", + content="Original body", + ) + ) + malformed_content = "---\nstatus: [draft\n---\nOriginal body" + await file_service.write_file(created.file_path, malformed_content) + update_schema = EntitySchema( + title="Repair Malformed Frontmatter", + directory="notes", + note_type="note", + content=( + "---\ncreated: 2024-01-15T10:30:00Z\nmodified: 2024-01-16T11:45:00Z\n" + "status: repaired\n---\nRepaired body" + ), + ) + + prepared = await entity_service.prepare_update_entity_content( + created, + update_schema, + malformed_content, + ) + result = await entity_service.update_entity_with_content(created, update_schema) + + assert prepared.entity_fields.created_at == datetime(2024, 1, 15, 10, 30, tzinfo=UTC) + assert prepared.entity_fields.updated_at == datetime(2024, 1, 16, 11, 45, tzinfo=UTC) + assert result.content == prepared.markdown_content + assert parse_frontmatter(result.content)["status"] == "repaired" + assert remove_frontmatter(result.content) == "Repaired body" + + @pytest.mark.asyncio async def test_prepare_update_entity_content_preserves_permalink_when_move_updates_disabled( entity_service, @@ -351,6 +455,7 @@ async def test_prepare_edit_entity_content_matches_edit_entity_with_content( content="After edit", find_text="Before edit", ) + original_created_at = created.created_at result = await entity_service.edit_entity_with_content( identifier=created.permalink, operation="find_replace", @@ -363,6 +468,8 @@ async def test_prepare_edit_entity_content_matches_edit_entity_with_content( assert prepared.entity_fields.title == result.entity.title assert prepared.entity_fields.note_type == result.entity.note_type assert prepared.entity_fields.permalink == result.entity.permalink + assert prepared.entity_fields.created_at == original_created_at + assert result.entity.created_at == original_created_at @pytest.mark.asyncio diff --git a/tests/services/test_prepared_entity_fields.py b/tests/services/test_prepared_entity_fields.py deleted file mode 100644 index bc4b3ac14..000000000 --- a/tests/services/test_prepared_entity_fields.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Tests for prepared entity field helpers.""" - -from datetime import UTC, datetime -from uuid import uuid4 - -from basic_memory.models import Entity -from basic_memory.services.entity_service import ( - PreparedEntityFields, - apply_prepared_entity_fields, -) - - -def test_apply_prepared_entity_fields_updates_accepted_entity_state() -> None: - now = datetime(2026, 4, 13, 12, 0, tzinfo=UTC) - updated_at = datetime(2026, 4, 13, 13, 0, tzinfo=UTC) - entity = Entity( - id=42, - external_id=str(uuid4()), - title="Original", - note_type="note", - entity_metadata={"topic": "tests"}, - content_type="text/markdown", - project_id=7, - permalink="main/notes/original", - file_path="notes/original.md", - checksum=None, - created_at=now, - updated_at=now, - created_by="creator", - last_updated_by="creator", - ) - - apply_prepared_entity_fields( - entity, - PreparedEntityFields( - title="Renamed", - note_type="decision", - entity_metadata={"status": "accepted"}, - content_type="text/markdown", - permalink="main/journal/renamed", - file_path="journal/renamed.md", - ), - updated_at=updated_at, - user_profile_value="editor-123", - ) - - assert entity.title == "Renamed" - assert entity.note_type == "decision" - assert entity.entity_metadata == {"status": "accepted"} - assert entity.content_type == "text/markdown" - assert entity.permalink == "main/journal/renamed" - assert entity.file_path == "journal/renamed.md" - assert entity.updated_at == updated_at - assert entity.last_updated_by == "editor-123" diff --git a/tests/services/test_semantic_timestamp_surfaces.py b/tests/services/test_semantic_timestamp_surfaces.py new file mode 100644 index 000000000..b013ad014 --- /dev/null +++ b/tests/services/test_semantic_timestamp_surfaces.py @@ -0,0 +1,78 @@ +"""Tests for semantic note timestamps on read-facing surfaces.""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import UTC, datetime +from typing import Any, cast + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory.api.v2.utils import to_search_results +from basic_memory.models import Entity +from basic_memory.repository import EntityRepository +from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.services.directory_service import DirectoryService + + +class _EmptyEntityService: + async def get_entities_by_id(self, ids: list[int]) -> Sequence[Any]: + return () + + +async def test_search_result_returns_indexed_semantic_timestamp() -> None: + semantic_updated_at = datetime(2024, 1, 16, 11, 45, tzinfo=UTC) + row = SearchIndexRow( + project_id=7, + id=42, + entity_id=42, + type="entity", + title="Timestamped", + permalink="notes/timestamped", + file_path="notes/timestamped.md", + created_at=datetime(2024, 1, 15, 10, 30, tzinfo=UTC), + updated_at=semantic_updated_at, + ) + + results = await to_search_results(_EmptyEntityService(), [row]) + + assert results[0].updated_at == semantic_updated_at + + +def test_directory_result_returns_entity_semantic_timestamp() -> None: + semantic_updated_at = datetime(2024, 1, 16, 11, 45, tzinfo=UTC) + entity = Entity( + id=42, + external_id="note-42", + project_id=7, + title="Timestamped", + note_type="note", + content_type="text/markdown", + permalink="notes/timestamped", + file_path="notes/timestamped.md", + created_at=datetime(2024, 1, 15, 10, 30, tzinfo=UTC), + updated_at=semantic_updated_at, + mtime=1_800_000_000, + ) + service = DirectoryService( + cast(EntityRepository, object()), + cast(async_sessionmaker[AsyncSession], object()), + ) + + tree = service._build_directory_tree_from_entities([entity], "/") + + assert tree.children[0].children[0].updated_at == semantic_updated_at + + +def test_search_index_row_normalizes_raw_naive_datetimes() -> None: + row = SearchIndexRow( + project_id=7, + id=42, + type="entity", + file_path="notes/timestamped.md", + created_at=datetime(2024, 1, 15, 10, 30), + updated_at=datetime(2024, 1, 16, 11, 45), + ) + + assert row.created_at.utcoffset() is not None + assert row.updated_at.utcoffset() is not None