diff --git a/src/basic_memory/alembic/versions/r1m2n3o4p5q6_add_relation_generation.py b/src/basic_memory/alembic/versions/r1m2n3o4p5q6_add_relation_generation.py new file mode 100644 index 000000000..b4e47d3d7 --- /dev/null +++ b/src/basic_memory/alembic/versions/r1m2n3o4p5q6_add_relation_generation.py @@ -0,0 +1,95 @@ +"""Add generation ownership to relation projections. + +Revision ID: r1m2n3o4p5q6 +Revises: q0l1m2n3o4p5 +Create Date: 2026-08-09 17:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +revision: str = "r1m2n3o4p5q6" +down_revision: Union[str, None] = "q0l1m2n3o4p5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Backfill relation rows with their source note's accepted generation.""" + op.add_column( + "relation_search_refresh", + sa.Column("publication_generation", sa.BigInteger(), nullable=True), + ) + op.create_index( + "ix_relation_search_refresh_project_publication_generation", + "relation_search_refresh", + ["project_id", "publication_generation"], + unique=False, + ) + op.add_column( + "relation", + sa.Column( + "generation", + sa.BigInteger(), + nullable=False, + server_default=sa.text("0"), + ), + ) + + connection = op.get_bind() + if connection.dialect.name == "postgresql": + op.execute( + """ + UPDATE relation + SET generation = note_content.db_version + FROM note_content + WHERE relation.from_id = note_content.entity_id + AND relation.project_id = note_content.project_id + """ + ) + else: + op.execute( + """ + UPDATE relation + SET generation = COALESCE( + ( + SELECT note_content.db_version + FROM note_content + WHERE note_content.entity_id = relation.from_id + AND note_content.project_id = relation.project_id + ), + 0 + ) + """ + ) + + op.create_index( + "ix_relation_project_from_generation", + "relation", + ["project_id", "from_id", "generation"], + unique=False, + ) + + +def downgrade() -> None: + """Remove relation generation ownership.""" + op.drop_index("ix_relation_project_from_generation", table_name="relation") + if op.get_bind().dialect.name == "postgresql": + op.drop_column("relation", "generation") + else: + with op.batch_alter_table("relation") as batch_op: + batch_op.drop_column("generation") + + op.drop_index( + "ix_relation_search_refresh_project_publication_generation", + table_name="relation_search_refresh", + ) + if op.get_bind().dialect.name == "postgresql": + op.drop_column("relation_search_refresh", "publication_generation") + else: + with op.batch_alter_table("relation_search_refresh") as batch_op: + batch_op.drop_column("publication_generation") diff --git a/src/basic_memory/deps/services.py b/src/basic_memory/deps/services.py index b4d8d125c..51482d37c 100644 --- a/src/basic_memory/deps/services.py +++ b/src/basic_memory/deps/services.py @@ -283,6 +283,7 @@ async def get_index_file_executor_v2_external( raise RuntimeError("Index file executor requires a project-scoped entity repository") batch_indexer = BatchIndexer( + project_id=project_id, app_config=app_config, entity_service=entity_service, entity_repository=entity_repository, diff --git a/src/basic_memory/index/local_dependencies.py b/src/basic_memory/index/local_dependencies.py index 62fe162dd..7314ddce7 100644 --- a/src/basic_memory/index/local_dependencies.py +++ b/src/basic_memory/index/local_dependencies.py @@ -39,6 +39,7 @@ FileIndexOperation, FileIndexResult, IndexEntitySearchWriter, + IndexedEntity, IndexInputFile, IndexingBatchResult, StorageIndexFileWriter, @@ -321,21 +322,33 @@ async def index_markdown_file( resolve_relations=True, refresh_unchanged_derived_state=existing is not None, ) - outcome = await self.note_content_reconciler.reconcile( + reconciliation = await self.note_content_reconciler.reconcile( entity=synced.entity, markdown_content=synced.markdown_content, observed_at=synced.updated_at, source=source, anchor=anchor, ) - if outcome == "current": + # Trigger: the indexed file is older than the accepted DB generation. + # Why: retrying identical bytes cannot make that file lineage current. + # Outcome: preserve accepted relations and finish without publishing this pass. + if reconciliation.status == "deferred": break + if reconciliation.generation is not None: + generation_is_current = await self.publish_relation_generation( + synced, + generation=reconciliation.generation, + ) + if generation_is_current: + break + logger.info( - "Retrying markdown index after concurrent accepted note write: {}", + "Retrying markdown index without a current relation generation: {}", file_path, entity_id=synced.entity.id, attempt=attempt + 1, + reconciliation_status=reconciliation.status, ) else: raise NoteContentChangedDuringIndexError( @@ -428,6 +441,39 @@ async def index_regular_file( operation=operation, ) + async def publish_relation_generation( + self, + synced: SyncedMarkdownFile, + *, + generation: int, + ) -> bool: + """Publish one claimed relation set, then preserve legacy inline resolution.""" + indexed = IndexedEntity( + path=synced.file_path, + entity_id=synced.entity.id, + permalink=synced.entity.permalink, + checksum=synced.checksum, + content_type=synced.content_type, + markdown_content=synced.markdown_content, + relations=synced.relations, + resolve_relations=synced.resolve_relations, + ) + generation_is_current = await self.batch_indexer.publish_relation_generation( + indexed, + generation=generation, + ) + if generation_is_current and synced.resolve_relations: + await self.batch_indexer.resolve_relation_targets( + [synced.entity.id], + max_concurrent=1, + ) + if generation_is_current: + await self.batch_indexer.refresh_indexed_entity_search( + indexed, + generation=generation, + ) + return generation_is_current + async def index_current_markdown_file( self, path: RuntimeFilePath, @@ -488,6 +534,8 @@ async def index_current_markdown_file( content_type=self.file_service.content_type(path), updated_at=file_metadata.modified_at, size=file_metadata.size, + relations=(), + resolve_relations=resolve_relations, ) return await self.index_changed_markdown_file( @@ -536,6 +584,8 @@ async def refresh_unchanged_markdown_file( content_type=self.file_service.content_type(input_file.path), updated_at=file_metadata.modified_at, size=file_metadata.size, + relations=indexed.relations, + resolve_relations=resolve_relations, ) async def index_changed_markdown_file( @@ -602,6 +652,8 @@ async def index_changed_markdown_file( content_type=self.file_service.content_type(input_file.path), updated_at=file_metadata.modified_at, size=file_metadata.size, + relations=indexed.relations, + resolve_relations=resolve_relations, ) @@ -671,6 +723,7 @@ async def build_local_index_project_dependencies( app_config=app_config, ) batch_indexer = BatchIndexer( + project_id=project.id, app_config=app_config, entity_service=entity_service, entity_repository=entity_repository, diff --git a/src/basic_memory/indexing/accepted_note_mutation_runner.py b/src/basic_memory/indexing/accepted_note_mutation_runner.py index 190146e92..458b5c7d9 100644 --- a/src/basic_memory/indexing/accepted_note_mutation_runner.py +++ b/src/basic_memory/indexing/accepted_note_mutation_runner.py @@ -16,12 +16,13 @@ AcceptedNoteCreatePreparer, AcceptedNoteEditPreparer, AcceptedNoteMovePreparer, - AcceptedNoteSelfRelationResolver, AcceptedPreparedNoteWrite, AcceptedNoteReplacePreparer, + AcceptedNoteSelfRelationResolver, AcceptedNoteWriteRepositories, create_accepted_pending_entity, delete_accepted_note, + lock_accepted_note_content_for_entity_mutation, persist_accepted_note_move, persist_accepted_note_snapshot, prepare_accepted_note_create, @@ -29,6 +30,7 @@ prepare_accepted_note_move, prepare_accepted_note_replace, ) +from basic_memory.indexing.relation_persistence import RelationGenerationPublication from basic_memory.models import Entity, NoteContent, Project from basic_memory.repository import NoteContentVersionConflict from basic_memory.repository.note_file_vacate_repository import NoteFileVacateRepository @@ -305,6 +307,14 @@ class AcceptedNoteMutationDependencies: verify_storage_absent_on_create: bool = False +@dataclass(frozen=True, slots=True) +class AcceptedNoteMutationResult: + """Accepted response plus relation work that must run after commit.""" + + change: AcceptedNoteMutationChange + relation_publication: RelationGenerationPublication | None = None + + def accepted_note_integrity_rejection(error: IntegrityError) -> AcceptedNoteMutationRejection: """Map repository integrity errors into portable accepted-note rejections.""" conflict_kind = classify_accepted_note_write_conflict(str(error.orig or error)) @@ -374,7 +384,7 @@ async def run_accepted_note_create( *, request: AcceptedNoteCreateMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: """Accept a new markdown note into DB state without materializing its file.""" try: return await _run_accepted_note_create(session, request=request, dependencies=dependencies) @@ -387,7 +397,7 @@ async def run_accepted_note_update( *, request: AcceptedNoteUpdateMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: """Accept a PUT create-or-replace into DB state without materializing its file.""" try: return await _run_accepted_note_update(session, request=request, dependencies=dependencies) @@ -402,7 +412,7 @@ async def run_accepted_note_edit( *, request: AcceptedNoteEditMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: """Accept a partial note edit into DB state without materializing its file.""" try: return await _run_accepted_note_edit(session, request=request, dependencies=dependencies) @@ -417,7 +427,7 @@ async def run_accepted_note_move( *, request: AcceptedNoteMoveMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: """Accept a note move into DB state without materializing its file.""" try: return await _run_accepted_note_move(session, request=request, dependencies=dependencies) @@ -432,7 +442,7 @@ async def run_accepted_note_delete( *, request: AcceptedNoteDeleteMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: """Delete one accepted note and return any materialized-file cleanup.""" project = await load_accepted_note_mutation_project( session, @@ -446,11 +456,13 @@ async def run_accepted_note_delete( load_relations=False, ) if entity is None: - return await delete_accepted_note( - session, - project_id=project.id, - entity=None, - repositories=dependencies.write_repositories, + return AcceptedNoteMutationResult( + change=await delete_accepted_note( + session, + project_id=project.id, + entity=None, + repositories=dependencies.write_repositories, + ) ) note_content = await load_accepted_note_content( @@ -460,12 +472,14 @@ async def run_accepted_note_delete( dependencies=dependencies, missing_kind=None, ) - return await delete_accepted_note( - session, - project_id=project.id, - entity=entity, - note_content=note_content, - repositories=dependencies.write_repositories, + return AcceptedNoteMutationResult( + change=await delete_accepted_note( + session, + project_id=project.id, + entity=entity, + note_content=note_content, + repositories=dependencies.write_repositories, + ) ) @@ -474,7 +488,7 @@ async def _run_accepted_note_create( *, request: AcceptedNoteCreateMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: ensure_accepted_note_markdown_entity(request.data) now = accepted_note_mutation_utc_now() @@ -519,19 +533,22 @@ async def _run_accepted_note_create( entity=entity, prepared=prepared, db_checksum=prepared_write.db_checksum, - self_relation_resolver=preparer, last_source=request.source, updated_at=now, + self_relation_resolver=preparer, repositories=dependencies.write_repositories, ) - return plan_accepted_note_write_change( - status_code=201, - entity=entity, - note_content=persisted.note_content, - actor_user_profile_id=request.actor.user_profile_id, - actor_kind=request.actor.kind, - actor_name=request.actor.name, - fallback_source=request.source, + return AcceptedNoteMutationResult( + change=plan_accepted_note_write_change( + status_code=201, + entity=entity, + note_content=persisted.note_content, + actor_user_profile_id=request.actor.user_profile_id, + actor_kind=request.actor.kind, + actor_name=request.actor.name, + fallback_source=request.source, + ), + relation_publication=persisted.relation_publication, ) @@ -540,7 +557,7 @@ async def _run_accepted_note_update( *, request: AcceptedNoteUpdateMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: ensure_accepted_note_markdown_entity(request.data) now = accepted_note_mutation_utc_now() @@ -640,11 +657,9 @@ async def _run_accepted_note_update( # Optimistic-concurrency precondition: the caller sent the db_checksum it # last synced; if the accepted row has advanced to a different write, # reject with the current checksum so the client rebases instead of - # clobbering the newer write (issue #1445). Cloud main checked this under - # SELECT ... FOR UPDATE; core needs no row lock because accept_write's - # compare-and-set on db_version already guarantees a write planned against - # this read cannot land stale — a write slipping in between this check and - # the CAS trips the CAS and surfaces the concurrent-write 409 instead. + # clobbering the newer write (issue #1445). The lock order is defined by + # current_relation_generation_statement; accept_write's compare-and-set + # remains the portable stale-write guard. if ( request.base_checksum is not None and current_note_content.db_checksum != request.base_checksum @@ -691,13 +706,13 @@ async def _run_accepted_note_update( entity=entity, prepared=prepared, db_checksum=prepared_write.db_checksum, - self_relation_resolver=preparer, last_source=request.source, updated_at=now, current_note_content=current_note_content, existing_file_path=existing_file_path, accepted_file_path=entity.file_path, source_file_checksum=vacated_source[1] if vacated_source is not None else None, + self_relation_resolver=preparer, repositories=dependencies.write_repositories, ) if ( @@ -711,15 +726,18 @@ async def _run_accepted_note_update( file_path=vacated_source[0], file_checksum=vacated_source[1], ) - return plan_accepted_note_write_change( - status_code=201 if created else 200, - entity=entity, - note_content=persisted.note_content, - actor_user_profile_id=request.actor.user_profile_id, - actor_kind=request.actor.kind, - actor_name=request.actor.name, - cleanup_after_write=persisted.previous_file_delete, - fallback_source=request.source, + return AcceptedNoteMutationResult( + change=plan_accepted_note_write_change( + status_code=201 if created else 200, + entity=entity, + note_content=persisted.note_content, + actor_user_profile_id=request.actor.user_profile_id, + actor_kind=request.actor.kind, + actor_name=request.actor.name, + cleanup_after_write=persisted.previous_file_delete, + fallback_source=request.source, + ), + relation_publication=persisted.relation_publication, ) @@ -728,7 +746,7 @@ async def _run_accepted_note_edit( *, request: AcceptedNoteEditMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: now = accepted_note_mutation_utc_now() user_profile_value = ( str(request.actor.user_profile_id) if request.actor.user_profile_id is not None else None @@ -764,21 +782,24 @@ async def _run_accepted_note_edit( entity=entity, prepared=prepared, db_checksum=prepared_write.db_checksum, - self_relation_resolver=preparer, last_source=request.source, updated_at=now, current_note_content=current_note_content, accepted_file_path=entity.file_path, + self_relation_resolver=preparer, repositories=dependencies.write_repositories, ) - return plan_accepted_note_write_change( - status_code=200, - entity=entity, - note_content=persisted.note_content, - actor_user_profile_id=request.actor.user_profile_id, - actor_kind=request.actor.kind, - actor_name=request.actor.name, - fallback_source=request.source, + return AcceptedNoteMutationResult( + change=plan_accepted_note_write_change( + status_code=200, + entity=entity, + note_content=persisted.note_content, + actor_user_profile_id=request.actor.user_profile_id, + actor_kind=request.actor.kind, + actor_name=request.actor.name, + fallback_source=request.source, + ), + relation_publication=persisted.relation_publication, ) @@ -787,7 +808,7 @@ async def _run_accepted_note_move( *, request: AcceptedNoteMoveMutation, dependencies: AcceptedNoteMutationDependencies, -) -> AcceptedNoteMutationChange: +) -> AcceptedNoteMutationResult: try: accepted_file_path = normalize_note_move_destination_path(request.destination_path) except ValueError as error: @@ -847,7 +868,7 @@ async def _run_accepted_note_move( reject_accepted_note_mutation(AcceptedNoteMutationRejectKind.conflict, str(error)) try: prepared_move = await prepare_accepted_note_move( - preparer if should_update_permalink else None, + preparer, session, entity=entity, current_note_content=current_note_content, @@ -867,6 +888,7 @@ async def _run_accepted_note_move( current_note_content=current_note_content, existing_file_path=existing_file_path, source_file_checksum=vacated_source_checksum, + self_relation_resolver=preparer, repositories=dependencies.write_repositories, ) # Trigger: storage confirms which source bytes this move vacated. @@ -880,16 +902,19 @@ async def _run_accepted_note_move( file_path=existing_file_path, file_checksum=vacated_source_checksum, ) - return plan_accepted_note_write_change( - status_code=200, - entity=entity, - note_content=persisted.note_content, - actor_user_profile_id=request.actor.user_profile_id, - actor_kind=request.actor.kind, - actor_name=request.actor.name, - previous_file_path=existing_file_path, - cleanup_after_write=persisted.previous_file_delete, - fallback_source=request.source, + return AcceptedNoteMutationResult( + change=plan_accepted_note_write_change( + status_code=200, + entity=entity, + note_content=persisted.note_content, + actor_user_profile_id=request.actor.user_profile_id, + actor_kind=request.actor.kind, + actor_name=request.actor.name, + previous_file_path=existing_file_path, + cleanup_after_write=persisted.previous_file_delete, + fallback_source=request.source, + ), + relation_publication=persisted.relation_publication, ) @@ -960,6 +985,13 @@ async def load_required_accepted_note_content( missing_kind: AcceptedNoteMutationRejectKind, ) -> NoteContent: """Load required accepted DB note content or reject the mutation.""" + # Claim the source before preparation; current_relation_generation_statement + # is the canonical authority for the cross-table lock order. + await lock_accepted_note_content_for_entity_mutation( + session, + project_id=project_id, + entity_id=entity_id, + ) note_content = await load_accepted_note_content( session, project_id=project_id, diff --git a/src/basic_memory/indexing/accepted_note_write_runner.py b/src/basic_memory/indexing/accepted_note_write_runner.py index d0b2853b9..bb6d614e6 100644 --- a/src/basic_memory/indexing/accepted_note_write_runner.py +++ b/src/basic_memory/indexing/accepted_note_write_runner.py @@ -7,12 +7,15 @@ from datetime import datetime from typing import Any, Protocol +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from basic_memory import file_utils -from basic_memory.indexing.accepted_note_search import ( - accepted_search_content_from_markdown, - build_accepted_note_search_row, +from basic_memory.indexing.accepted_note_search import build_accepted_note_search_row +from basic_memory.indexing.models import IndexedRelation +from basic_memory.indexing.relation_persistence import ( + RelationGenerationPublication, + RelationGenerationStore, ) from basic_memory.models import Entity, NoteContent from basic_memory.repository import ( @@ -94,17 +97,6 @@ async def prepare_edit_entity_content( ) -> PreparedEntityWrite: ... -class AcceptedNoteSelfRelationResolver(Protocol): - """Capability for resolving ambiguity-safe self-links during acceptance.""" - - async def resolve_deferred_self_relation( - self, - target: str, - entity: Entity, - session: AsyncSession | None = ..., - ) -> Entity | None: ... - - class AcceptedNoteMovePreparer(Protocol): """Capability that derives accepted markdown for a note move.""" @@ -114,6 +106,7 @@ async def prepare_move_entity_content( current_content: str, destination_path: str, *, + should_update_permalink: bool, session: AsyncSession | None = ..., ) -> PreparedEntityMove: ... @@ -125,6 +118,17 @@ async def verify_move_destination_absent( ) -> None: ... +class AcceptedNoteSelfRelationResolver(Protocol): + """Resolve only ambiguity-safe self-links while accepted bytes are in hand.""" + + async def resolve_deferred_self_relation( + self, + target: str, + entity: Entity, + session: AsyncSession | None = ..., + ) -> Entity | None: ... + + class AcceptedNoteDeleteEntitySource(RuntimeDeletedNoteFileDeleteEntitySource, Protocol): """Entity identity required to delete one accepted note row.""" @@ -182,15 +186,8 @@ async def replace_accepted_observations( ) -> None: ... -class AcceptedNoteRelationRepository(Protocol): - """Repository capability for replacing one accepted note's outgoing relations.""" - - async def replace_accepted_outgoing_relations( - self, - session: AsyncSession, - entity_id: RuntimeEntityId, - relations: Sequence[AcceptedRelationWrite], - ) -> None: ... +class AcceptedNoteRelationRepository(RelationGenerationStore, Protocol): + """Generation-fenced relation persistence for accepted note writes.""" class AcceptedNoteWriteRepositories(Protocol): @@ -239,6 +236,7 @@ class AcceptedPreparedNoteMove: search_content: str permalink: str | None db_checksum: RuntimeNoteContentChecksum + relations: tuple[AcceptedRelationWrite, ...] @dataclass(frozen=True, slots=True) @@ -247,6 +245,7 @@ class AcceptedPersistedNoteWrite: note_content: NoteContent previous_file_delete: RuntimePendingNoteFileDelete | None = None + relation_publication: RelationGenerationPublication | None = None async def prepare_accepted_note_create( @@ -341,7 +340,7 @@ async def prepare_accepted_note_edit( async def prepare_accepted_note_move( - preparer: AcceptedNoteMovePreparer | None, + preparer: AcceptedNoteMovePreparer, session: AsyncSession, *, entity: Entity, @@ -352,31 +351,21 @@ async def prepare_accepted_note_move( ) -> AcceptedPreparedNoteMove: """Prepare a DB-first move and apply the accepted path/permalink fields.""" current_content = str(current_note_content.markdown_content) - file_path = accepted_file_path - permalink = entity.permalink - markdown_content = current_content - search_content = accepted_search_content_from_markdown(markdown_content) - - if should_update_permalink: - if preparer is None: - raise ValueError("Accepted note move requires a preparer to update the permalink") - prepared = await preparer.prepare_move_entity_content( - entity, - current_content, - accepted_file_path, - session=session, - ) - file_path = prepared.file_path.as_posix() - permalink = prepared.permalink - markdown_content = prepared.markdown_content - search_content = prepared.search_content + prepared = await preparer.prepare_move_entity_content( + entity, + current_content, + accepted_file_path, + should_update_permalink=should_update_permalink, + session=session, + ) result = AcceptedPreparedNoteMove( - file_path=file_path, - markdown_content=markdown_content, - search_content=search_content, - permalink=permalink, - db_checksum=await file_utils.compute_checksum(markdown_content), + file_path=prepared.file_path.as_posix(), + markdown_content=prepared.markdown_content, + search_content=prepared.search_content, + permalink=prepared.permalink, + db_checksum=await file_utils.compute_checksum(prepared.markdown_content), + relations=prepared.relations, ) entity.file_path = result.file_path entity.permalink = result.permalink @@ -594,22 +583,18 @@ async def _persist_accepted_note_content_and_search( ) -async def _replace_accepted_note_graph( +async def _replace_accepted_note_observations( session: AsyncSession, *, entity: Entity, prepared: PreparedEntityWrite, - self_relation_resolver: AcceptedNoteSelfRelationResolver, repositories: AcceptedNoteWriteRepositories, ) -> None: - """Persist the accepted note's observations and relations in one transaction. + """Persist observations alongside the accepted note-content generation. The accepted markdown was already parsed during prepare, so the graph rows - are committed alongside note_content and search instead of waiting for a - later ``index_file`` pass to reparse the materialized file. Without this the - observation/relation tables stay empty after a successful DB-first write, so - schema inference and relation traversal are nondeterministic until an - unrelated storage notification happens to fire (issue #1076). + are committed alongside note_content and search. Relations use a separate + generation-fenced publication after this transaction commits. """ observation_repository = repositories.observation_repository(entity.project_id) await observation_repository.replace_accepted_observations( @@ -618,37 +603,42 @@ async def _replace_accepted_note_graph( prepared.observations, ) - # General deferred resolution skips target_id == from_id to avoid binding an - # ambiguous title to the wrong note. Reuse the indexing path's narrow, - # ambiguity-safe self resolver here so filepath/permalink self-links do not - # remain unresolved forever after a DB-first write. - relations: list[AcceptedRelationWrite] = [] - for relation in prepared.relations: - if relation.target_id is not None: - relations.append(relation) - continue - target_entity = await self_relation_resolver.resolve_deferred_self_relation( - relation.target_name, - entity, - session=session, - ) - if target_entity is None: - relations.append(relation) - continue - relations.append( - AcceptedRelationWrite( + +async def accepted_relation_generation_publication( + session: AsyncSession, + *, + entity: Entity, + note_content: NoteContent, + relations: Sequence[AcceptedRelationWrite], + self_relation_resolver: AcceptedNoteSelfRelationResolver, +) -> RelationGenerationPublication: + """Carry original target names plus ambiguity-safe self targets into publication.""" + indexed_relations: list[IndexedRelation] = [] + for relation in relations: + target_id = relation.target_id + if target_id is None: + target = await self_relation_resolver.resolve_deferred_self_relation( + relation.target_name, + entity, + session=session, + ) + target_id = target.id if target is not None else None + if target_id is not None and target_id != entity.id: + raise ValueError("Accepted relation pre-resolution is restricted to self-links") + indexed_relations.append( + IndexedRelation( relation_type=relation.relation_type, - target_name=target_entity.title, + target_name=relation.target_name, context=relation.context, - target_id=target_entity.id, + target_id=target_id, ) ) - relation_repository = repositories.relation_repository(entity.project_id) - await relation_repository.replace_accepted_outgoing_relations( - session, - entity.id, - relations, + return RelationGenerationPublication( + project_id=entity.project_id, + entity_id=entity.id, + generation=note_content.db_version, + relations=tuple(indexed_relations), ) @@ -658,13 +648,13 @@ async def persist_accepted_note_snapshot( entity: Entity, prepared: PreparedEntityWrite, db_checksum: RuntimeNoteContentChecksum, - self_relation_resolver: AcceptedNoteSelfRelationResolver, last_source: RuntimeNoteChangeSource | None, updated_at: datetime, current_note_content: RuntimeAcceptedNoteContentWriteSource | None = None, existing_file_path: RuntimeFilePath | None = None, accepted_file_path: RuntimeFilePath | None = None, source_file_checksum: RuntimeFileChecksum | None = None, + self_relation_resolver: AcceptedNoteSelfRelationResolver, repositories: AcceptedNoteWriteRepositories, ) -> AcceptedPersistedNoteWrite: """Persist one complete accepted Markdown snapshot in the caller's transaction.""" @@ -682,14 +672,23 @@ async def persist_accepted_note_snapshot( source_file_checksum=source_file_checksum, repositories=repositories, ) - await _replace_accepted_note_graph( + await _replace_accepted_note_observations( session, entity=entity, prepared=prepared, - self_relation_resolver=self_relation_resolver, repositories=repositories, ) - return persisted + return AcceptedPersistedNoteWrite( + note_content=persisted.note_content, + previous_file_delete=persisted.previous_file_delete, + relation_publication=await accepted_relation_generation_publication( + session, + entity=entity, + note_content=persisted.note_content, + relations=prepared.relations, + self_relation_resolver=self_relation_resolver, + ), + ) async def persist_accepted_note_move( @@ -702,10 +701,11 @@ async def persist_accepted_note_move( current_note_content: RuntimeAcceptedNoteContentWriteSource, existing_file_path: RuntimeFilePath, source_file_checksum: RuntimeFileChecksum | None = None, + self_relation_resolver: AcceptedNoteSelfRelationResolver, repositories: AcceptedNoteWriteRepositories, ) -> AcceptedPersistedNoteWrite: """Persist the explicitly narrower content/search state for an accepted move.""" - return await _persist_accepted_note_content_and_search( + persisted = await _persist_accepted_note_content_and_search( session, entity=entity, markdown_content=prepared.markdown_content, @@ -719,6 +719,17 @@ async def persist_accepted_note_move( source_file_checksum=source_file_checksum, repositories=repositories, ) + return AcceptedPersistedNoteWrite( + note_content=persisted.note_content, + previous_file_delete=persisted.previous_file_delete, + relation_publication=await accepted_relation_generation_publication( + session, + entity=entity, + note_content=persisted.note_content, + relations=prepared.relations, + self_relation_resolver=self_relation_resolver, + ), + ) async def delete_accepted_note_entity( @@ -730,6 +741,37 @@ async def delete_accepted_note_entity( await session.delete(entity) +async def lock_accepted_note_content_for_entity_mutation( + session: AsyncSession, + *, + project_id: ProjectId, + entity_id: RuntimeEntityId, +) -> None: + """Lock the source before entity mutation; see current_relation_generation_statement.""" + await session.scalar( + select(NoteContent.entity_id) + .where( + NoteContent.project_id == project_id, + NoteContent.entity_id == entity_id, + ) + .with_for_update() + ) + + +async def lock_accepted_note_content_for_delete( + session: AsyncSession, + *, + project_id: ProjectId, + entity_id: RuntimeEntityId, +) -> None: + """Compatibility name for the accepted-delete lock-order boundary.""" + await lock_accepted_note_content_for_entity_mutation( + session, + project_id=project_id, + entity_id=entity_id, + ) + + async def delete_accepted_note( session: AsyncSession, *, @@ -745,6 +787,14 @@ async def delete_accepted_note( note_content=note_content, ) if entity is not None: + if note_content is not None: + # Relation publication locks NoteContent before its Entity foreign-key check. + # Taking the same order here prevents delete/publication from holding opposite locks. + await lock_accepted_note_content_for_delete( + session, + project_id=project_id, + entity_id=entity.id, + ) await delete_accepted_note_search_index( session, project_id=project_id, diff --git a/src/basic_memory/indexing/batch_indexer.py b/src/basic_memory/indexing/batch_indexer.py index 20f42fa90..ee8ca1f7a 100644 --- a/src/basic_memory/indexing/batch_indexer.py +++ b/src/basic_memory/indexing/batch_indexer.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Awaitable, Callable, Mapping, TypeVar +from typing import Awaitable, Callable, Mapping, Sequence, TypeVar from loguru import logger from sqlalchemy.exc import IntegrityError @@ -15,25 +15,37 @@ import logfire from basic_memory import db from basic_memory.config import BasicMemoryConfig -from basic_memory.file_utils import compute_checksum, has_frontmatter, remove_frontmatter +from basic_memory.file_utils import ( + ParseError, + compute_checksum, + has_frontmatter, + remove_frontmatter, +) from basic_memory.markdown.schemas import EntityMarkdown from basic_memory.indexing.models import ( IndexEntitySearchWriter, IndexedEntity, + IndexedRelation, IndexFileWriter, IndexFrontmatterUpdate, IndexingBatchResult, IndexInputFile, + RelationGenerationBatchResult, ) -from basic_memory.models import Entity, Relation +from basic_memory.indexing.relation_resolution import RepositoryRelationResolutionRuntime +from basic_memory.indexing.relation_persistence import RelationGenerationPublisher +from basic_memory.models import Entity +from basic_memory.repository import EntityRepository, RelationRepository +from basic_memory.repository.note_content_repository import NoteContentRepository from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.runtime.storage import ( + ProjectId, RUNTIME_MARKDOWN_CONTENT_TYPE, runtime_file_path_is_markdown_note, ) from basic_memory.services import EntityService +from basic_memory.services.bulk_link_resolver import BulkLinkResolver from basic_memory.services.exceptions import SyncFatalError -from basic_memory.repository import EntityRepository, RelationRepository T = TypeVar("T") @@ -51,6 +63,25 @@ async def index_entity_data(self, entity: Entity, content: str | None = None) -> await self.search_writer.index_entity_data(entity, content=content) +@dataclass(frozen=True, slots=True) +class RelationResolutionSearchWriter: + """Adapt the portable single-entity writer to resolver batch refreshes.""" + + search_writer: IndexEntitySearchWriter + + async def index_entities( + self, + entities: Sequence[Entity], + *, + content_by_entity_id: Mapping[int, str], + ) -> None: + for entity in sorted(entities, key=lambda item: item.id): + await self.search_writer.index_entity_data( + entity, + content=content_by_entity_id.get(entity.id), + ) + + @dataclass(slots=True) class _PreparedMarkdownFile: file: IndexInputFile @@ -69,6 +100,8 @@ class _PreparedEntity: content_type: str | None search_content: str | None markdown_content: str | None = None + relations: tuple[IndexedRelation, ...] = () + resolve_relations: bool = True @dataclass(slots=True) @@ -83,6 +116,7 @@ class BatchIndexer: def __init__( self, *, + project_id: ProjectId, app_config: BasicMemoryConfig, entity_service: EntityService, entity_repository: EntityRepository, @@ -98,6 +132,21 @@ def __init__( self.search_service = search_service self.file_writer = file_writer self.session_maker = session_maker + self.relation_generation_publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + session_maker=session_maker, + ) + self.relation_resolution = RepositoryRelationResolutionRuntime( + session_maker=session_maker, + relation_repository=relation_repository, + entity_repository=entity_repository, + note_content_repository=NoteContentRepository(project_id=project_id), + target_resolver=BulkLinkResolver( + entity_repository=entity_repository, + app_config=app_config, + ), + entity_indexer=RelationResolutionSearchWriter(search_service), + ) async def index_files( self, @@ -135,8 +184,6 @@ async def index_files( error_by_path.update(normalization_errors) indexed_entities: list[IndexedEntity] = [] - resolved_count = 0 - unresolved_count = 0 search_indexed = 0 prepared_entities: dict[str, _PreparedEntity] = {} @@ -160,17 +207,6 @@ async def index_files( error_by_path.update(regular_errors) prepared_entities.update(regular_upserts) - markdown_entity_ids = [ - prepared_entities[path].entity_id - for path in markdown_paths - if path in prepared_entities - ] - if markdown_entity_ids: - resolved_count, unresolved_count = await self._resolve_batch_relations( - markdown_entity_ids, - max_concurrent=max_concurrent, - ) - async with db.scoped_session(self.session_maker) as session: refreshed_entities = await self.entity_repository.find_by_ids( session, [prepared.entity_id for prepared in prepared_entities.values()] @@ -197,8 +233,6 @@ async def index_files( return IndexingBatchResult( indexed=indexed_entities, errors=[(path, error_by_path[path]) for path in ordered_paths if path in error_by_path], - relations_resolved=resolved_count, - relations_unresolved=unresolved_count, search_indexed=search_indexed, ) @@ -234,8 +268,6 @@ async def index_markdown_file( persisted = await self._persist_markdown_file( prepared, is_new=new, - resolve_relations=resolve_relations, - reload_entity=False, ) existing_permalink_by_path[file.path] = persisted.entity.permalink @@ -249,7 +281,11 @@ async def index_markdown_file( if len(refreshed) != 1: # pragma: no cover raise ValueError(f"Failed to reload indexed entity for {file.path}") entity = refreshed[0] - prepared_entity = self._build_prepared_entity(persisted.prepared, entity) + prepared_entity = await self._build_prepared_entity( + persisted.prepared, + entity, + resolve_relations=resolve_relations, + ) if index_search: with logfire.span( @@ -266,6 +302,8 @@ async def index_markdown_file( checksum=prepared_entity.checksum, content_type=prepared_entity.content_type, markdown_content=prepared_entity.markdown_content, + relations=prepared_entity.relations, + resolve_relations=prepared_entity.resolve_relations, ) async def _get_file_path_to_permalink_map(self) -> dict[str, str | None]: @@ -427,7 +465,7 @@ def _reserve_batch_permalink( async def _upsert_markdown_file(self, prepared: _PreparedMarkdownFile) -> _PreparedEntity: persisted = await self._persist_markdown_file(prepared) - return self._build_prepared_entity(persisted.prepared, persisted.entity) + return await self._build_prepared_entity(persisted.prepared, persisted.entity) async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity: checksum = await self._resolve_checksum(file) @@ -503,78 +541,170 @@ async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity: content_type=file.content_type, search_content=None, markdown_content=None, + relations=(), + resolve_relations=False, ) # --- Relations --- - async def _resolve_batch_relations( + async def publish_relation_generation( self, - entity_ids: list[int], + indexed: IndexedEntity, + *, + generation: int, + ) -> bool: + """Publish one indexed entity's parsed relations after generation claim.""" + return await self.relation_generation_publisher.publish( + entity_id=indexed.entity_id, + generation=generation, + relations=indexed.relations, + ) + + async def publish_relation_generations( + self, + indexed_entities: list[IndexedEntity], *, + generation_by_entity_id: Mapping[int, int], max_concurrent: int, - ) -> tuple[int, int]: - unresolved_relation_lists = await asyncio.gather( - *(self._find_unresolved_relations_for_entity(entity_id) for entity_id in entity_ids) + ) -> RelationGenerationBatchResult: + """Publish claimed batch generations before resolving forward references.""" + indexed_by_path = { + indexed.path: indexed + for indexed in indexed_entities + if indexed.markdown_content is not None and indexed.entity_id in generation_by_entity_id + } + published, errors = await self._run_bounded( + sorted(indexed_by_path), + limit=max_concurrent, + worker=lambda path: self.publish_relation_generation( + indexed_by_path[path], + generation=generation_by_entity_id[indexed_by_path[path].entity_id], + ), ) - unresolved_relations = [ - relation for relation_list in unresolved_relation_lists for relation in relation_list + resolvable_entity_ids = [ + indexed_by_path[path].entity_id + for path in sorted(published) + if published[path] and indexed_by_path[path].resolve_relations ] + resolved_count = 0 + unresolved_count = 0 + if resolvable_entity_ids: + resolved_count, unresolved_count = await self._resolve_batch_relations( + resolvable_entity_ids, + max_concurrent=max_concurrent, + ) - if not unresolved_relations: - return 0, 0 + _, refresh_errors = await self._run_bounded( + [path for path in sorted(published) if published[path]], + limit=max_concurrent, + worker=lambda path: self.refresh_indexed_entity_search( + indexed_by_path[path], + generation=generation_by_entity_id[indexed_by_path[path].entity_id], + ), + ) + errors.update(refresh_errors) - semaphore = asyncio.Semaphore(max_concurrent) + return RelationGenerationBatchResult( + errors=tuple((path, errors[path]) for path in sorted(errors)), + relations_resolved=resolved_count, + relations_unresolved=unresolved_count, + ) - async def resolve_relation(relation: Relation) -> int: - async with semaphore: - try: - # strict=True for deferred resolution: only fill in to_id on an - # exact permalink/title/file_path match. Fuzzy fallback would silently - # resolve ambiguous links to whichever entity shares tokens with the - # link text, mismatching this with the sync_service forward-reference - # path and producing confidently-wrong graph edges. See - # sync_service.resolve_forward_references for the same change. - async with db.scoped_session(self.session_maker) as session: - resolved_entity = await self.entity_service.link_resolver.resolve_link( - relation.to_name, strict=True, session=session - ) - if resolved_entity is None or resolved_entity.id == relation.from_id: - return 0 - - try: - async with db.scoped_session(self.session_maker) as session: - await self.relation_repository.update( - session, - relation.id, - { - "to_id": resolved_entity.id, - "to_name": resolved_entity.title, - }, - ) - except IntegrityError: - async with db.scoped_session(self.session_maker) as session: - await self.relation_repository.delete(session, relation.id) - return 1 - except Exception as exc: # pragma: no cover - defensive logging - logger.warning( - "Batch relation resolution failed", - relation_id=relation.id, - from_id=relation.from_id, - to_name=relation.to_name, - error=str(exc), - ) - return 0 + async def resolve_relation_targets( + self, + entity_ids: list[int], + *, + max_concurrent: int, + ) -> tuple[int, int]: + """Resolve newly published relations through the shared guarded resolver.""" + return await self._resolve_batch_relations(entity_ids, max_concurrent=max_concurrent) + + async def refresh_indexed_entity_search( + self, + indexed: IndexedEntity, + *, + generation: int, + ) -> IndexedEntity: + """Refresh search only while this publication generation remains accepted.""" + async with db.scoped_session(self.session_maker) as session: + refresh = await self.relation_repository.load_search_refresh_for_generation( + session, + entity_id=indexed.entity_id, + generation=generation, + ) + # Trigger: a newer accepted note generation won after this publication. + # Why: combining generation-N parsed markdown with N+1 entity state would + # produce a search row that never represented one coherent note version. + # Outcome: terminal-wins; N+1 owns its refresh and this pass leaves durable + # marker IDs untouched for a later retry. + if refresh is None: + return indexed - resolved_counts = await asyncio.gather( - *(resolve_relation(relation) for relation in unresolved_relations) + try: + search_content = ( + remove_frontmatter(indexed.markdown_content) + if indexed.markdown_content is not None + else None + ) + except ParseError: + search_content = indexed.markdown_content + + prepared = _PreparedEntity( + path=indexed.path, + entity_id=indexed.entity_id, + permalink=indexed.permalink, + checksum=indexed.checksum, + content_type=indexed.content_type, + search_content=search_content, + markdown_content=indexed.markdown_content, + relations=indexed.relations, + resolve_relations=indexed.resolve_relations, ) + refreshed = await self._refresh_search_index(prepared, refresh.entity) + async with db.scoped_session(self.session_maker) as session: + # Trigger: N+1 can be accepted after N loaded its coherent snapshot but + # before N finishes the external search write. + # Why: N must not consume the last repair marker after rendering stale + # bytes; N+1 owns convergence and may already have completed its pass. + # Outcome: the guarded completion either retires N's observed markers or + # leaves fresh durable work that repairs a late stale write. + await self.relation_repository.complete_search_refresh_for_generation( + session, + entity_id=indexed.entity_id, + generation=generation, + refresh_ids=refresh.refresh_ids, + ) + return refreshed + + async def _resolve_batch_relations( + self, + entity_ids: list[int], + *, + max_concurrent: int, + ) -> tuple[int, int]: + if max_concurrent < 1: + raise ValueError("max_concurrent must be greater than zero") + + ordered_entity_ids = sorted(set(entity_ids)) + unresolved_relation_lists = await asyncio.gather( + *( + self._find_unresolved_relations_for_entity(entity_id) + for entity_id in ordered_entity_ids + ) + ) + unresolved_before = sum(len(relations) for relations in unresolved_relation_lists) + + for entity_id in ordered_entity_ids: + await self.relation_resolution.resolve_relations(entity_id=entity_id) remaining_relation_lists = await asyncio.gather( - *(self._find_unresolved_relations_for_entity(entity_id) for entity_id in entity_ids) + *( + self._find_unresolved_relations_for_entity(entity_id) + for entity_id in ordered_entity_ids + ) ) remaining_unresolved = sum(len(relations) for relations in remaining_relation_lists) - - return sum(resolved_counts), remaining_unresolved + return max(0, unresolved_before - remaining_unresolved), remaining_unresolved async def _find_unresolved_relations_for_entity(self, entity_id: int): """Load unresolved relations for one entity in a service-owned session.""" @@ -606,6 +736,8 @@ async def _refresh_search_index( checksum=prepared.checksum, content_type=prepared.content_type, markdown_content=prepared.markdown_content, + relations=prepared.relations, + resolve_relations=prepared.resolve_relations, ) # --- Helpers --- @@ -615,8 +747,6 @@ async def _persist_markdown_file( prepared: _PreparedMarkdownFile, *, is_new: bool | None = None, - resolve_relations: bool = True, - reload_entity: bool = True, ) -> _PersistedMarkdownFile: async with db.scoped_session(self.session_maker) as session: existing = await self.entity_repository.get_by_file_path( @@ -626,15 +756,19 @@ async def _persist_markdown_file( ) if is_new is None: is_new = existing is None - entity = await self.entity_service.upsert_entity_from_markdown( - Path(prepared.file.path), - prepared.markdown, - is_new=is_new, - existing_entity=existing, - resolve_relations=resolve_relations, - reload_entity=reload_entity, - session=session, - ) + if is_new: + entity = await self.entity_service.create_entity_from_markdown( + Path(prepared.file.path), + prepared.markdown, + session=session, + ) + else: + entity = await self.entity_service.update_entity_and_observations( + Path(prepared.file.path), + prepared.markdown, + existing_entity=existing, + session=session, + ) prepared = await self._reconcile_persisted_permalink(prepared, entity) metadata_updates = self._file_bookkeeping_updates( prepared.file, @@ -693,11 +827,28 @@ async def _reconcile_persisted_permalink( file_contains_frontmatter=prepared.file_contains_frontmatter, ) - def _build_prepared_entity( + async def _build_prepared_entity( self, prepared: _PreparedMarkdownFile, entity: Entity, + *, + resolve_relations: bool = True, ) -> _PreparedEntity: + indexed_relations: list[IndexedRelation] = [] + for relation in prepared.markdown.relations: + resolved = await self.entity_service.resolve_deferred_self_relation( + relation.target, + entity, + ) + indexed_relations.append( + IndexedRelation( + relation_type=relation.type, + target_name=relation.target, + context=relation.context, + target_id=resolved.id if resolved else None, + ) + ) + return _PreparedEntity( path=prepared.file.path, entity_id=entity.id, @@ -710,6 +861,8 @@ def _build_prepared_entity( else remove_frontmatter(prepared.content) ), markdown_content=prepared.content, + relations=tuple(indexed_relations), + resolve_relations=resolve_relations, ) async def _resolve_checksum(self, file: IndexInputFile) -> str: diff --git a/src/basic_memory/indexing/directory_delete_runner.py b/src/basic_memory/indexing/directory_delete_runner.py index bec6c1f52..377dbad57 100644 --- a/src/basic_memory/indexing/directory_delete_runner.py +++ b/src/basic_memory/indexing/directory_delete_runner.py @@ -7,7 +7,7 @@ from enum import StrEnum from typing import Literal, NotRequired, Protocol, TypedDict -from sqlalchemy import bindparam, delete, select, text +from sqlalchemy import bindparam, delete, exists, select, text from sqlalchemy.ext.asyncio import AsyncSession from basic_memory.models import Entity, NoteContent, Project, Relation @@ -15,6 +15,9 @@ ProjectIndexExternalVectorCleaner, delete_project_index_vector_rows, ) +from basic_memory.repository.relation_repository import ( + lock_note_content_before_entity_mutation, +) from basic_memory.runtime.cleanup import ( RuntimeDeleteStatus, RuntimeDirectoryFileSnapshot, @@ -89,6 +92,14 @@ def deleted_files(self) -> tuple[RuntimeFilePath, ...]: return tuple(file_snapshot.file_path for file_snapshot in self.files) +@dataclass(frozen=True, slots=True) +class DirectoryEntityDeleteResult: + """Rows accepted by the guarded directory delete and their repair work.""" + + deleted_entity_ids: frozenset[int] = frozenset() + relation_cleanup_entity_ids: frozenset[int] = frozenset() + + class DirectoryDeleteAcceptanceStore(Protocol): """Repository capability for accepting directory deletes into DB state.""" @@ -113,12 +124,13 @@ async def delete_directory_entities( session: AsyncSession, *, project_id: ProjectId, + directory: RuntimeFilePath, entity_ids: Sequence[int], - ) -> frozenset[int]: - """Delete the accepted entity rows and return surviving relation sources. + ) -> DirectoryEntityDeleteResult: + """Delete current directory members and return accepted rows plus repair work. - The returned ids are entities OUTSIDE the deleted set whose relations pointed - into it; their search rows need reindexing to drop now-dangling relations. + Relation cleanup ids are entities OUTSIDE the deleted set whose relations + pointed into it; their search rows need reindexing to drop dangling relations. """ @@ -185,9 +197,7 @@ async def load_directory_file_snapshots( escaped_directory = directory_delete_like_prefix(directory) query = query.where(Entity.file_path.like(f"{escaped_directory}/%", escape="\\")) - result = await session.execute( - query.order_by(Entity.file_path.asc()).with_for_update(of=Entity) - ) + result = await session.execute(query.order_by(Entity.file_path.asc())) return [ plan_directory_file_snapshot( entity_id=int(row.id), @@ -208,27 +218,98 @@ async def delete_directory_entities( session: AsyncSession, *, project_id: ProjectId, + directory: RuntimeFilePath, entity_ids: Sequence[int], - ) -> frozenset[int]: + ) -> DirectoryEntityDeleteResult: if not entity_ids: - return frozenset() - deleted_entity_ids = tuple(entity_ids) + return DirectoryEntityDeleteResult() + snapshotted_entity_ids = tuple(sorted(set(entity_ids))) + + # The directory snapshot is intentionally a plain read. Claim every accepted + # source here, immediately before the first operation that can lock Entity or + # Relation rows. See current_relation_generation_statement for the canonical + # NoteContent-first invariant shared with relation publication. + await lock_note_content_before_entity_mutation( + session, + project_id=project_id, + entity_ids=snapshotted_entity_ids, + ) + + membership_predicates = [ + Entity.id.in_(snapshotted_entity_ids), + Entity.project_id == project_id, + ] + if directory not in {"", "/"}: + escaped_directory = directory_delete_like_prefix(directory) + membership_predicates.append( + Entity.file_path.like(f"{escaped_directory}/%", escape="\\") + ) + + # The NoteContent fence is this flow's only lock construct. Re-read current + # membership without locking so projection cleanup follows only rows still in + # the directory, then repeat the predicate in DELETE so the stale snapshot + # itself never authorizes mutation. + current_directory_entities = await session.execute( + select(Entity.id).where(*membership_predicates).order_by(Entity.id) + ) + current_directory_entity_ids = tuple( + int(entity_id) for entity_id in current_directory_entities.scalars() + ) + if not current_directory_entity_ids: + return DirectoryEntityDeleteResult() # Capture surviving sources before the delete: Relation.to_id CASCADE will drop # the relation table rows for incoming links from entities outside the directory, # but those sources own the matching search_index relation rows and are never in # the deleted set, so the caller must reindex them to clear the stale rows. - surviving_relation_sources = await session.execute( - select(Relation.from_id) - .where( + incoming_relations = await session.execute( + select(Relation.id, Relation.to_id, Relation.from_id).where( Relation.project_id == project_id, - Relation.to_id.in_(deleted_entity_ids), - Relation.from_id.not_in(deleted_entity_ids), + Relation.to_id.in_(current_directory_entity_ids), + Relation.from_id.not_in(current_directory_entity_ids), + ) + ) + incoming_relation_snapshots_list: list[tuple[int, int, int]] = [] + for relation_id, target_id, source_id in incoming_relations.tuples().all(): + if target_id is None: # pragma: no cover + raise RuntimeError("Resolved incoming relation is missing its target id") + incoming_relation_snapshots_list.append( + (int(relation_id), int(target_id), int(source_id)) + ) + incoming_relation_snapshots = tuple(incoming_relation_snapshots_list) + incoming_relation_ids = tuple( + relation_id for relation_id, _, _ in incoming_relation_snapshots + ) + + uncaptured_incoming_relation = select(Relation.id).where( + Relation.project_id == project_id, + Relation.to_id == Entity.id, + Relation.from_id.not_in(current_directory_entity_ids), + ) + if incoming_relation_ids: + uncaptured_incoming_relation = uncaptured_incoming_relation.where( + Relation.id.not_in(incoming_relation_ids) ) - .distinct() + + # Trigger: a resolver can publish an incoming edge after the unlocked capture. + # Why: deleting its target would cascade that edge without scheduling its source + # for search repair. The lock budget deliberately excludes target Entity locks. + # Outcome: optimistic deletion rejects only targets with uncaptured incoming work; + # a later directory pass can retry from a fresh snapshot. + guarded_entity_delete = delete(Entity).where( + *membership_predicates, + ~exists(uncaptured_incoming_relation.correlate(Entity)), ) + deleted_entities = await session.execute(guarded_entity_delete.returning(Entity.id)) + deleted_entity_ids = frozenset(int(entity_id) for entity_id in deleted_entities.scalars()) + if not deleted_entity_ids: + return DirectoryEntityDeleteResult() + ordered_deleted_entity_ids = tuple(sorted(deleted_entity_ids)) + relation_cleanup_entity_ids = frozenset( - int(source_id) for source_id in surviving_relation_sources.scalars() + source_id + for _, target_id, source_id in incoming_relation_snapshots + if target_id in deleted_entity_ids ) await session.execute( @@ -238,23 +319,25 @@ async def delete_directory_entities( WHERE project_id = :project_id AND entity_id IN :entity_ids """ ).bindparams(bindparam("entity_ids", expanding=True)), - {"project_id": project_id, "entity_ids": deleted_entity_ids}, + {"project_id": project_id, "entity_ids": ordered_deleted_entity_ids}, ) if self.external_vector_cleaner is None: await delete_project_index_vector_rows( session, project_id=project_id, - entity_ids=deleted_entity_ids, + entity_ids=ordered_deleted_entity_ids, ) else: await delete_project_index_vector_rows( session, project_id=project_id, - entity_ids=deleted_entity_ids, + entity_ids=ordered_deleted_entity_ids, external_vector_cleaner=self.external_vector_cleaner, ) - await session.execute(delete(Entity).where(Entity.id.in_(deleted_entity_ids))) - return relation_cleanup_entity_ids + return DirectoryEntityDeleteResult( + deleted_entity_ids=deleted_entity_ids, + relation_cleanup_entity_ids=relation_cleanup_entity_ids, + ) def normalize_directory_delete_path(directory: str) -> RuntimeFilePath: @@ -493,15 +576,21 @@ async def accept_directory_delete( if not file_snapshots: return DirectoryDeleteAcceptance(project_id=project_id, files=()) - relation_cleanup_entity_ids = await store.delete_directory_entities( + delete_result = await store.delete_directory_entities( session, project_id=project_id, + directory=directory, entity_ids=[snapshot.entity_id for snapshot in file_snapshots], ) + deleted_file_snapshots = tuple( + snapshot + for snapshot in file_snapshots + if snapshot.entity_id in delete_result.deleted_entity_ids + ) return DirectoryDeleteAcceptance( project_id=project_id, - files=file_snapshots, - relation_cleanup_entity_ids=relation_cleanup_entity_ids, + files=deleted_file_snapshots, + relation_cleanup_entity_ids=delete_result.relation_cleanup_entity_ids, ) diff --git a/src/basic_memory/indexing/external_file_delete_runner.py b/src/basic_memory/indexing/external_file_delete_runner.py index 4e37a97bd..da38275c6 100644 --- a/src/basic_memory/indexing/external_file_delete_runner.py +++ b/src/basic_memory/indexing/external_file_delete_runner.py @@ -11,6 +11,9 @@ from basic_memory import db from basic_memory.models import Relation from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache +from basic_memory.repository.relation_repository import ( + lock_note_content_before_entity_mutation, +) from basic_memory.runtime.cleanup import RuntimeExternalFileDeletePlan from basic_memory.runtime.note_content import ( RuntimeDeletedNoteEntityDeleteSource, @@ -109,6 +112,13 @@ async def delete_entity_if_file_path_matches( raise RuntimeError("External file delete requires a project-scoped entity repository") async with db.scoped_session(self.session_maker) as session: + # See current_relation_generation_statement: a delete may cascade through + # Entity and Relation, so claim its accepted NoteContent source first. + await lock_note_content_before_entity_mutation( + session, + project_id=self.entity_repository.project_id, + entity_ids=(entity_id,), + ) relation_cleanup_entity_ids = await relation_cleanup_sources_for_deleted_entity( session, project_id=self.entity_repository.project_id, diff --git a/src/basic_memory/indexing/file_indexer.py b/src/basic_memory/indexing/file_indexer.py index befbd2d24..7a3a470e1 100644 --- a/src/basic_memory/indexing/file_indexer.py +++ b/src/basic_memory/indexing/file_indexer.py @@ -14,7 +14,7 @@ from basic_memory import db from basic_memory.indexing.note_content_reconciliation import ( NoteContentReconciliationAnchor, - NoteContentReconciliationOutcome, + NoteContentReconciliationResult, ) from basic_memory.indexing.models import ( FileIndexOperation, @@ -102,6 +102,15 @@ async def index_file( source: str, ) -> FileIndexResult: ... + async def publish_relation_generation( + self, + synced: SyncedMarkdownFile, + *, + generation: int, + ) -> bool: + """Publish parsed relations only while ``generation`` remains current.""" + ... + class IndexMarkdownNoteContentReconciler(Protocol): """Note-content capability needed after canonical markdown sync succeeds.""" @@ -117,7 +126,7 @@ async def reconcile( observed_at: datetime | None, source: str, anchor: NoteContentReconciliationAnchor | None = None, - ) -> NoteContentReconciliationOutcome: ... + ) -> NoteContentReconciliationResult: ... class NoteContentChangedDuringIndexError(RuntimeError): @@ -197,21 +206,33 @@ async def index_markdown_file( refresh_unchanged_derived_state=existing is not None, ) - outcome = await self.note_content_reconciler.reconcile( + reconciliation = await self.note_content_reconciler.reconcile( entity=synced.entity, markdown_content=synced.markdown_content, observed_at=synced.updated_at, source=source, anchor=anchor, ) - if outcome == "current": + # Trigger: the indexed file is older than the accepted DB generation. + # Why: retrying identical bytes cannot make that file lineage current. + # Outcome: preserve accepted relations and finish without publishing this pass. + if reconciliation.status == "deferred": break + if reconciliation.generation is not None: + generation_is_current = await self.markdown_indexer.publish_relation_generation( + synced, + generation=reconciliation.generation, + ) + if generation_is_current: + break + log.info( - "Retrying markdown index after concurrent accepted note write: {}", + "Retrying markdown index without a current relation generation: {}", file_path, entity_id=synced.entity.id, attempt=attempt + 1, + reconciliation_status=reconciliation.status, ) else: raise NoteContentChangedDuringIndexError( diff --git a/src/basic_memory/indexing/forward_reference_resolution.py b/src/basic_memory/indexing/forward_reference_resolution.py index 7ac378e5a..f13caf898 100644 --- a/src/basic_memory/indexing/forward_reference_resolution.py +++ b/src/basic_memory/indexing/forward_reference_resolution.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Protocol -from sqlalchemy import case, select, update +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory import db @@ -17,7 +17,11 @@ RelationResolutionEntityRepository, UnresolvedRelation, ) -from basic_memory.models import Relation +from basic_memory.models import Entity +from basic_memory.repository.relation_repository import ( + RelationRepository, + ResolvedRelationWrite, +) class ForwardReferenceRelationSource(Protocol): @@ -47,6 +51,8 @@ class ForwardReferenceUpdate: source_entity_id: EntityId target_entity_id: EntityId link_text: LinkText + source_generation: int + relation_type: str @dataclass(frozen=True, slots=True) @@ -101,13 +107,11 @@ async def list_unresolved_forward_references( self, ) -> tuple[UnresolvedRelation, ...]: async with db.scoped_session(self.session_maker) as session: - result = await session.execute( - select(Relation).where( - Relation.project_id == self.project_id, - Relation.to_id.is_(None), + return tuple( + await RelationRepository(project_id=self.project_id).find_unresolved_relations( + session ) ) - return tuple(result.scalars().all()) @dataclass(frozen=True, slots=True) @@ -134,18 +138,31 @@ async def apply_forward_reference_updates( if not updates: return - relation_ids = [update.relation_id for update in updates] - target_entity_ids_by_relation_id = { - update.relation_id: update.target_entity_id for update in updates - } - async with db.scoped_session(self.session_maker) as session: - stmt = ( - update(Relation) - .where(Relation.id.in_(relation_ids)) - .values(to_id=case(target_entity_ids_by_relation_id, value=Relation.id)) + target_entities = await session.execute( + select(Entity.id, Entity.external_id).where( + Entity.project_id == self.project_id, + Entity.id.in_(sorted({update.target_entity_id for update in updates})), + ) + ) + target_external_id_by_id = dict(target_entities.tuples().all()) + writes = tuple( + ResolvedRelationWrite( + relation_id=update.relation_id, + from_id=update.source_entity_id, + generation=update.source_generation, + original_target_name=update.link_text, + target_id=update.target_entity_id, + target_external_id=target_external_id_by_id[update.target_entity_id], + relation_type=update.relation_type, + ) + for update in updates + if update.target_entity_id in target_external_id_by_id + ) + await RelationRepository(project_id=self.project_id).apply_resolved_targets( + session, + writes, ) - await session.execute(stmt) @dataclass(frozen=True, slots=True) @@ -257,6 +274,8 @@ def plan_forward_reference_resolution( source_entity_id=relation.from_id, target_entity_id=target_entity_id, link_text=link_text, + source_generation=relation.generation, + relation_type=relation.relation_type, ) ) entity_ids_to_refresh.add(target_entity_id) diff --git a/src/basic_memory/indexing/index_batch_runtime.py b/src/basic_memory/indexing/index_batch_runtime.py index 282a5d17d..cb4d28ac9 100644 --- a/src/basic_memory/indexing/index_batch_runtime.py +++ b/src/basic_memory/indexing/index_batch_runtime.py @@ -21,6 +21,7 @@ IndexFrontmatterStorage, IndexingBatchResult, IndexInputFile, + RelationGenerationBatchResult, StorageIndexFileWriter, ) from basic_memory.indexing.note_content_batch_reconciliation import ( @@ -53,6 +54,16 @@ async def index_files( ) -> IndexingBatchResult: """Index one batch of storage-neutral input files.""" + async def publish_relation_generations( + self, + indexed_entities: list[IndexedEntity], + *, + generation_by_entity_id: Mapping[int, int], + max_concurrent: int, + ) -> RelationGenerationBatchResult: + """Publish relation projections only for claimed note generations.""" + ... + @dataclass(frozen=True, slots=True) class IndexBatchRuntime[EntityT: IndexedNoteContentEntity, FileInfoT: LoadedIndexFile]: @@ -90,7 +101,7 @@ async def index_loaded_files( max_concurrent=max_concurrent, parse_max_concurrent=parse_max_concurrent, ) - note_content_errors = await reconcile_indexed_note_content_batch( + reconciliation = await reconcile_indexed_note_content_batch( result.indexed, file_infos=files, entity_repository=self.entity_repository, @@ -101,7 +112,17 @@ async def index_loaded_files( source=self.note_content_source, file_reader=self.file_reader, ) - result.errors.extend(error.as_tuple() for error in note_content_errors) + relation_result = await self.batch_indexer.publish_relation_generations( + result.indexed, + generation_by_entity_id={ + claim.entity_id: claim.generation for claim in reconciliation.generations + }, + max_concurrent=metadata_update_max_concurrent or max_concurrent, + ) + result.errors.extend(error.as_tuple() for error in reconciliation.errors) + result.errors.extend(relation_result.errors) + result.relations_resolved = relation_result.relations_resolved + result.relations_unresolved = relation_result.relations_unresolved result.search_indexed = count_search_indexed_entities(result.indexed) return result @@ -141,6 +162,7 @@ def build_default_index_batch_runtime[FileInfoT: LoadedIndexFile]( # Wrapping it in a markdown-only filter here regressed full/startup scans: # non-markdown files were persisted but never added to the search index. batch_indexer = BatchIndexer( + project_id=project_id, app_config=app_config, entity_service=entity_service, entity_repository=entity_repository, diff --git a/src/basic_memory/indexing/models.py b/src/basic_memory/indexing/models.py index d31610ba4..76f980da8 100644 --- a/src/basic_memory/indexing/models.py +++ b/src/basic_memory/indexing/models.py @@ -104,6 +104,16 @@ class IndexFrontmatterWriteResult: content: str +@dataclass(frozen=True, slots=True) +class IndexedRelation: + """One parsed outgoing relation waiting for generation-owned publication.""" + + relation_type: str + target_name: str + context: str | None + target_id: int | None = None + + @dataclass(slots=True) class IndexedEntity: """Stable output describing one file that finished indexing successfully.""" @@ -114,6 +124,17 @@ class IndexedEntity: checksum: str content_type: str | None = None markdown_content: str | None = None + relations: tuple[IndexedRelation, ...] = () + resolve_relations: bool = True + + +@dataclass(frozen=True, slots=True) +class RelationGenerationBatchResult: + """Relation publication and legacy resolution outcome for one indexed batch.""" + + errors: tuple[tuple[str, str], ...] = () + relations_resolved: int = 0 + relations_unresolved: int = 0 class FileIndexOperation(StrEnum): @@ -801,6 +822,8 @@ class SyncedMarkdownFile: content_type: str updated_at: datetime size: int + relations: tuple[IndexedRelation, ...] = () + resolve_relations: bool = True @dataclass(slots=True) diff --git a/src/basic_memory/indexing/note_content_batch_reconciliation.py b/src/basic_memory/indexing/note_content_batch_reconciliation.py index a24511148..cab5823aa 100644 --- a/src/basic_memory/indexing/note_content_batch_reconciliation.py +++ b/src/basic_memory/indexing/note_content_batch_reconciliation.py @@ -15,7 +15,7 @@ from basic_memory import db from basic_memory.indexing.file_index_planning import FileIndexPath from basic_memory.indexing.models import IndexedEntity -from basic_memory.indexing.note_content_reconciliation import NoteContentReconciliationOutcome +from basic_memory.indexing.note_content_reconciliation import NoteContentReconciliationResult from basic_memory.indexing.note_content_reconciler import NoteContentReconcileFileReader @@ -73,7 +73,7 @@ async def reconcile( markdown_content: str, observed_at: datetime | None, source: str, - ) -> NoteContentReconciliationOutcome | None: + ) -> NoteContentReconciliationResult: """Apply the note_content state change for one markdown entity.""" @@ -89,6 +89,23 @@ def as_tuple(self) -> tuple[FileIndexPath, str]: return self.path, self.message +@dataclass(frozen=True, slots=True) +class IndexedNoteContentGeneration: + """Exact note-content generation claimed for one indexed markdown entity.""" + + path: FileIndexPath + entity_id: int + generation: int + + +@dataclass(frozen=True, slots=True) +class IndexedNoteContentBatchReconciliation: + """Generation claims and per-file failures produced after one batch index.""" + + generations: tuple[IndexedNoteContentGeneration, ...] + errors: tuple[IndexedNoteContentReconciliationError, ...] + + def indexed_note_content_utc_now() -> datetime: """Return the current UTC time used to stamp rewritten indexed observations.""" return datetime.now(tz=UTC) @@ -164,7 +181,9 @@ class IndexedNoteContentReconciliationTask[ source: str file_reader: NoteContentReconcileFileReader | None = None - async def run(self) -> IndexedNoteContentReconciliationError | None: + async def run( + self, + ) -> IndexedNoteContentGeneration | IndexedNoteContentReconciliationError | None: if self.indexed.markdown_content is None: return None @@ -192,21 +211,34 @@ async def run(self) -> IndexedNoteContentReconciliationError | None: return None markdown_content = fresh.content.decode("utf-8") observed_at = fresh.last_modified + publication_matches_indexed_snapshot = markdown_content == self.indexed.markdown_content else: markdown_content = self.indexed.markdown_content observed_at = self.timestamp_provider( self.indexed, self.file_infos.get(self.indexed.path), ) + publication_matches_indexed_snapshot = True try: - await self.note_content_reconciler.reconcile( + result = await self.note_content_reconciler.reconcile( entity=entity, markdown_content=markdown_content, observed_at=observed_at, source=self.source, ) - return None + if result.generation is None: + return None + # Trigger: local reconciliation observed bytes newer than the parsed scan payload. + # Why: a generation must identify the exact bytes that produced its relations. + # Outcome: keep the fresh content claim, but withhold stale relation publication. + if not publication_matches_indexed_snapshot: + return None + return IndexedNoteContentGeneration( + path=self.indexed.path, + entity_id=self.indexed.entity_id, + generation=result.generation, + ) except Exception as exc: # pragma: no cover - defensive logging # The entity/search writes are already durable by this point. Report # the note_content follow-up failure as a per-file indexing error. @@ -228,13 +260,13 @@ async def reconcile_indexed_note_content_batch[ timestamp_provider: IndexedNoteContentObservedAt[FileInfoT] = indexed_note_content_observed_at, source: str = "index", file_reader: NoteContentReconcileFileReader | None = None, -) -> tuple[IndexedNoteContentReconciliationError, ...]: +) -> IndexedNoteContentBatchReconciliation: """Hydrate note_content rows for indexed markdown entities after batch indexing.""" markdown_entities = tuple( indexed for indexed in indexed_entities if indexed.markdown_content is not None ) if not markdown_entities: - return () + return IndexedNoteContentBatchReconciliation(generations=(), errors=()) async with db.scoped_session(session_maker) as session: stored_entities = await entity_repository.find_by_ids( @@ -259,10 +291,16 @@ async def reconcile_indexed_note_content_batch[ max_concurrent=max_concurrent, ) + generations: list[IndexedNoteContentGeneration] = [] errors: list[IndexedNoteContentReconciliationError] = [] for indexed, result in zip(markdown_entities, results, strict=True): if isinstance(result, BaseException): errors.append(IndexedNoteContentReconciliationError(indexed.path, str(result))) - elif result is not None: + elif isinstance(result, IndexedNoteContentReconciliationError): errors.append(result) - return tuple(errors) + elif result is not None: + generations.append(result) + return IndexedNoteContentBatchReconciliation( + generations=tuple(generations), + errors=tuple(errors), + ) diff --git a/src/basic_memory/indexing/note_content_reconciler.py b/src/basic_memory/indexing/note_content_reconciler.py index 9fcabad46..50618d5d0 100644 --- a/src/basic_memory/indexing/note_content_reconciler.py +++ b/src/basic_memory/indexing/note_content_reconciler.py @@ -22,7 +22,7 @@ NoteContentPromoted, NoteContentReconciliationAnchor, NoteContentReconciliationDeferred, - NoteContentReconciliationOutcome, + NoteContentReconciliationResult, NoteContentState, NoteContentSource, NoteContentWriteStatus, @@ -317,7 +317,7 @@ async def reconcile( observed_at: datetime | None, source: NoteContentSource, anchor: NoteContentReconciliationAnchor | None = None, - ) -> NoteContentReconciliationOutcome: + ) -> NoteContentReconciliationResult: """Apply the shared file-vs-DB rule for one markdown entity.""" observed_checksum = await file_utils.compute_checksum(markdown_content) observed_timestamp = observed_at or datetime.now(tz=UTC) @@ -347,7 +347,7 @@ async def reconcile( "accepted state changed during indexing", entity.id, ) - return "stale" + return NoteContentReconciliationResult.stale() if note_content is None: plan = plan_note_content_reconciliation(None, observed) @@ -359,7 +359,7 @@ async def reconcile( session, note_content_from_bootstrap(entity.id, plan), ) - return "current" + return NoteContentReconciliationResult.current(plan.db_version) except IntegrityError: # Concurrent repair/index workers can both observe a missing row before # one wins the insert. Reload the winner and let normal reconciliation @@ -379,7 +379,7 @@ async def reconcile( "accepted state was created during indexing", entity.id, ) - return "stale" + return NoteContentReconciliationResult.stale() # The plan is computed from the row read above; guard the write on # that db_version so a concurrent accepted API mutation that advanced @@ -401,7 +401,7 @@ async def reconcile( current_state.file_version, current_state.file_write_status, ) - return "current" + return NoteContentReconciliationResult.deferred() applied = await apply_note_content_update_plan( self._note_content_repository, @@ -417,9 +417,18 @@ async def reconcile( expected_db_version, entity.id, ) - return "stale" + return NoteContentReconciliationResult.stale() - return "current" + if isinstance(plan, NoteContentFileObserved): + # The observed bytes describe an older materialized file while DB content is + # already ahead. They may update file bookkeeping, but they cannot publish + # semantic relations under the newer accepted generation. + return NoteContentReconciliationResult.deferred() + + claimed_generation = ( + plan.db_version if isinstance(plan, NoteContentPromoted) else expected_db_version + ) + return NoteContentReconciliationResult.current(claimed_generation) async def reconcile_note_content_for_entity( diff --git a/src/basic_memory/indexing/note_content_reconciliation.py b/src/basic_memory/indexing/note_content_reconciliation.py index ab30297c3..7e62dff89 100644 --- a/src/basic_memory/indexing/note_content_reconciliation.py +++ b/src/basic_memory/indexing/note_content_reconciliation.py @@ -4,11 +4,11 @@ from dataclasses import dataclass from datetime import datetime -from typing import Literal +from typing import Literal, Self type NoteContentChecksum = str type NoteContentSource = str -type NoteContentReconciliationOutcome = Literal["current", "stale"] +type NoteContentReconciliationStatus = Literal["current", "stale", "deferred"] type NoteContentWriteStatus = Literal[ "pending", "writing", @@ -18,6 +18,36 @@ ] +@dataclass(frozen=True, slots=True) +class NoteContentReconciliationResult: + """Whether observed bytes claimed an exact accepted note generation.""" + + status: NoteContentReconciliationStatus + generation: int | None = None + + def __post_init__(self) -> None: + generation_is_claimed = self.generation is not None + if generation_is_claimed != (self.status == "current"): + raise ValueError("Only current note-content reconciliation can claim a generation") + + @classmethod + def current(cls, generation: int) -> Self: + """Return a successful claim for one authoritative database generation.""" + if generation < 1: + raise ValueError("Claimed note-content generation must be positive") + return cls(status="current", generation=generation) + + @classmethod + def stale(cls) -> Self: + """Return an observation superseded by a concurrent accepted write.""" + return cls(status="stale") + + @classmethod + def deferred(cls) -> Self: + """Return an observation whose DB/file lineage is not safe to promote.""" + return cls(status="deferred") + + @dataclass(frozen=True, slots=True) class ObservedNoteContent: """One observed markdown file version ready to compare against note_content.""" diff --git a/src/basic_memory/indexing/note_materialization_runner.py b/src/basic_memory/indexing/note_materialization_runner.py index d6efb1b9c..efa923174 100644 --- a/src/basic_memory/indexing/note_materialization_runner.py +++ b/src/basic_memory/indexing/note_materialization_runner.py @@ -428,8 +428,17 @@ async def publish_written_file_state( ) # Materialization publishes NoteContent and Entity in one transaction. - # Lock Entity before reading the publish state so an Entity-first move - # cannot change the destination path while this publisher waits. + # The canonical order lives in current_relation_generation_statement: + # claim NoteContent before any Entity lock so publication, accepted + # mutation, deletion, and materialization cannot form a lock cycle. + note_content = await session.scalar( + select(NoteContent) + .where( + NoteContent.entity_id == request.entity_id, + NoteContent.project_id == request.project_id, + ) + .with_for_update() + ) entity = await session.scalar( select(Entity) .where( @@ -438,7 +447,6 @@ async def publish_written_file_state( ) .with_for_update() ) - note_content = await session.get(NoteContent, request.entity_id) publish_plan = plan_written_note_materialization_publish( request=request, prepared_write=prepared_write, diff --git a/src/basic_memory/indexing/project_delete_runner.py b/src/basic_memory/indexing/project_delete_runner.py index af49a4c57..f3f18196e 100644 --- a/src/basic_memory/indexing/project_delete_runner.py +++ b/src/basic_memory/indexing/project_delete_runner.py @@ -17,6 +17,9 @@ from basic_memory.repository.accepted_note_vector_cleanup import ( project_external_vector_index_names, ) +from basic_memory.repository.relation_repository import ( + lock_project_note_content_before_project_mutation, +) from basic_memory.repository.semantic_errors import SemanticVectorIndexExtensionError from basic_memory.runtime.cleanup import ( RuntimeDeleteStatus, @@ -217,6 +220,13 @@ async def hard_delete_project( request: RuntimeProjectDeleteJobRequest, ) -> ProjectHardDeleteOutcome: async with db.scoped_session(self.session_maker) as session: + # A project delete cascades through Entity and Relation. Claim all accepted + # sources first; current_relation_generation_statement documents the + # canonical NoteContent-first lock-order invariant. + await lock_project_note_content_before_project_mutation( + session, + project_id=request.project_id, + ) # Trigger: the project was reactivated while the per-file cleanup loop ran. # Why: preflight checked is_active once, potentially long before this # transaction; hard-deleting a reactivated project destroys live data. diff --git a/src/basic_memory/indexing/project_index_maintenance.py b/src/basic_memory/indexing/project_index_maintenance.py index e5c811cdb..a7ae0d820 100644 --- a/src/basic_memory/indexing/project_index_maintenance.py +++ b/src/basic_memory/indexing/project_index_maintenance.py @@ -16,6 +16,9 @@ ProjectIndexExternalVectorCleaner, delete_project_index_vector_rows, ) +from basic_memory.repository.relation_repository import ( + lock_note_content_before_entity_mutation, +) from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache from basic_memory.runtime.storage import ProjectExternalId, ProjectId @@ -361,10 +364,19 @@ async def delete_project_index_entities( external_vector_cleaner: ProjectIndexExternalVectorCleaner | None = None, ) -> frozenset[int]: """Delete indexed entities and return surviving relation sources needing repair.""" - deleted_entity_ids = tuple(entity_ids) + deleted_entity_ids = tuple(sorted(set(entity_ids))) if not deleted_entity_ids: return frozenset() + # See current_relation_generation_statement for the canonical lock order. + # Entity deletion can cascade into Relation, so accepted sources are claimed + # in sorted order before either table can be locked by this transaction. + await lock_note_content_before_entity_mutation( + session, + project_id=project_id, + entity_ids=deleted_entity_ids, + ) + surviving_relation_sources = await session.execute( select(Relation.from_id) .where( @@ -632,6 +644,15 @@ async def apply_project_index_move_batch( int(row["id"]): target_paths_by_old_path[str(row["file_path"])] for row in target_rows } + replaced_entity_ids = frozenset(int(row["id"]) for row in replacement_rows) + # A move updates NoteContent and Entity while also deleting any replaced + # destination entities. Claim the complete union once, in canonical order, + # before content planning or any mutation can acquire a later lock. + await lock_note_content_before_entity_mutation( + session, + project_id=self.project_id, + entity_ids=(*target_paths_by_entity_id, *replaced_entity_ids), + ) content_plan = await self._plan_move_content_updates( session, target_rows=target_rows, @@ -639,10 +660,8 @@ async def apply_project_index_move_batch( ) # --- Apply the batched replacement deletes and path/content updates --- - replaced_entity_ids: frozenset[int] = frozenset() relation_cleanup_entity_ids: frozenset[int] = frozenset() if updated_old_paths: - replaced_entity_ids = frozenset(int(row["id"]) for row in replacement_rows) relation_cleanup_entity_ids = await delete_project_index_entities( session, project_id=self.project_id, diff --git a/src/basic_memory/indexing/relation_persistence.py b/src/basic_memory/indexing/relation_persistence.py new file mode 100644 index 000000000..680f3b96f --- /dev/null +++ b/src/basic_memory/indexing/relation_persistence.py @@ -0,0 +1,136 @@ +"""Publish parsed relations under one accepted note-content generation.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from itertools import batched +from typing import Protocol + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.indexing.models import IndexedRelation +from basic_memory.repository.relation_repository import ( + RELATION_GENERATION_WRITE_STATEMENT_SIZE, + AcceptedRelationWrite, + RelationGenerationWriteResult, +) +from basic_memory.runtime.storage import ProjectId, RuntimeEntityId, RuntimeNoteContentVersion + + +class RelationGenerationStore(Protocol): + """Repository operations needed to publish one relation generation.""" + + async def begin_relation_generation_publication( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: ... + + async def upsert_relation_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + relations: Sequence[AcceptedRelationWrite], + ) -> RelationGenerationWriteResult: ... + + async def cleanup_relation_generations( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: ... + + +@dataclass(frozen=True, slots=True) +class RelationGenerationPublication: + """Relation intent authorized by one accepted note-content generation.""" + + project_id: ProjectId + entity_id: RuntimeEntityId + generation: RuntimeNoteContentVersion + relations: tuple[IndexedRelation, ...] + + +@dataclass(frozen=True, slots=True) +class RelationGenerationPublisher: + """Commit sorted relation chunks, then cleanup, under repeated source fences.""" + + relation_repository: RelationGenerationStore + session_maker: async_sessionmaker[AsyncSession] + + async def publish( + self, + *, + entity_id: int, + generation: int, + relations: Sequence[IndexedRelation], + ) -> bool: + """Return whether every statement retained ownership of ``generation``.""" + relations_by_identity: dict[tuple[str, str], IndexedRelation] = {} + for relation in relations: + if relation.target_id is not None and relation.target_id != entity_id: + raise ValueError("Only the source entity may be pre-resolved during publication") + identity = relation.relation_type, relation.target_name + relations_by_identity.setdefault(identity, relation) + + ordered_relations: list[AcceptedRelationWrite] = [] + resolved_identities: set[tuple[str, int]] = set() + for _, relation in sorted(relations_by_identity.items()): + if relation.target_id is not None: + resolved_identity = relation.relation_type, relation.target_id + # Constraint: safe aliases have distinct authored-name identities but share the + # resolved relation uniqueness domain. Keep the lexical first alias so input order + # cannot decide which valid source representation is published. + if resolved_identity in resolved_identities: + continue + resolved_identities.add(resolved_identity) + ordered_relations.append( + AcceptedRelationWrite( + relation_type=relation.relation_type, + target_name=relation.target_name, + context=relation.context, + target_id=relation.target_id, + ) + ) + + # Commit retry intent before the first independently committed chunk. If + # any later statement fails, change detection will re-drive this source. + async with db.scoped_session(self.session_maker) as session: + publication = await self.relation_repository.begin_relation_generation_publication( + session, + entity_id=entity_id, + generation=generation, + ) + if not publication.generation_is_current: + return False + + for relation_chunk in batched( + ordered_relations, + RELATION_GENERATION_WRITE_STATEMENT_SIZE, + ): + async with db.scoped_session(self.session_maker) as session: + result = await self.relation_repository.upsert_relation_generation( + session, + entity_id=entity_id, + generation=generation, + relations=relation_chunk, + ) + if not result.generation_is_current: + return False + + # Cleanup is intentionally its own transaction. An empty desired set still + # reaches this statement and removes every row older than the claimed generation. + async with db.scoped_session(self.session_maker) as session: + cleanup = await self.relation_repository.cleanup_relation_generations( + session, + entity_id=entity_id, + generation=generation, + ) + return cleanup.generation_is_current diff --git a/src/basic_memory/indexing/relation_resolution.py b/src/basic_memory/indexing/relation_resolution.py index b3b593b01..17ed52ba1 100644 --- a/src/basic_memory/indexing/relation_resolution.py +++ b/src/basic_memory/indexing/relation_resolution.py @@ -39,12 +39,11 @@ def plan_resolved_relation_write_batches( *, target_size: int = RELATION_RESOLUTION_WRITE_BATCH_SIZE, ) -> tuple[ResolvedRelationWriteBatch, ...]: - """Pack writes without splitting a source/relation-type collision domain. + """Pack writes without splitting a source/relation-type target collision domain. - Both relation uniqueness constraints include ``from_id`` and ``relation_type``. Writes in - that shared domain must be planned together so aliases can exchange their canonical names - without a later batch appearing to occupy the destination. A single domain may exceed the - target size; correctness takes precedence over the preferred commit size. + Resolved uniqueness includes ``from_id`` and ``relation_type``. Aliases that + resolve to the same target must be planned together so one deterministic + winner is chosen before any batch commits. """ if target_size < 1: raise ValueError("Relation write batch target size must be positive") @@ -86,6 +85,9 @@ def id(self) -> int: ... @property def from_id(self) -> int: ... + @property + def generation(self) -> int: ... + @property def to_name(self) -> str: ... @@ -304,10 +306,10 @@ async def resolve_relations( ResolvedRelationWrite( relation_id=relation.id, from_id=relation.from_id, + generation=relation.generation, original_target_name=relation.to_name, target_id=resolved_entity.id, target_external_id=resolved_entity.external_id, - target_name=resolved_entity.title, relation_type=relation.relation_type, ) ) diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index 9349e5d48..fa9e88ac4 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -343,6 +343,12 @@ class Relation(Base): Index("ix_relation_type", "relation_type"), Index("ix_relation_from_id", "from_id"), # Add FK indexes Index("ix_relation_to_id", "to_id"), + Index( + "ix_relation_project_from_generation", + "project_id", + "from_id", + "generation", + ), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride] @@ -354,6 +360,14 @@ class Relation(Base): to_name: Mapped[str] = mapped_column(String) relation_type: Mapped[str] = mapped_column(String) context: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + # Relation rows are a projection of one accepted note-content generation. + # Zero is reserved for rows written by pre-generation binaries during a rolling deploy. + generation: Mapped[int] = mapped_column( + BigInteger, + nullable=False, + default=0, + server_default=text("0"), + ) # Relationships from_entity = relationship( diff --git a/src/basic_memory/models/relation_search_refresh.py b/src/basic_memory/models/relation_search_refresh.py index 497a4ee28..5e3dfe447 100644 --- a/src/basic_memory/models/relation_search_refresh.py +++ b/src/basic_memory/models/relation_search_refresh.py @@ -2,7 +2,7 @@ from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, Index, Integer +from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer from sqlalchemy.orm import Mapped, mapped_column from basic_memory.models.base import Base @@ -20,6 +20,11 @@ class RelationSearchRefresh(Base): __table_args__ = ( Index("ix_relation_search_refresh_project_id", "project_id"), Index("ix_relation_search_refresh_entity_id", "entity_id"), + Index( + "ix_relation_search_refresh_project_publication_generation", + "project_id", + "publication_generation", + ), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) @@ -33,6 +38,7 @@ class RelationSearchRefresh(Base): ForeignKey("entity.id", ondelete="CASCADE"), nullable=False, ) + publication_generation: Mapped[int | None] = mapped_column(BigInteger, nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=lambda: datetime.now().astimezone(), diff --git a/src/basic_memory/repository/entity_repository.py b/src/basic_memory/repository/entity_repository.py index 1898e5ea2..5d117300b 100644 --- a/src/basic_memory/repository/entity_repository.py +++ b/src/basic_memory/repository/entity_repository.py @@ -7,7 +7,7 @@ from loguru import logger -from sqlalchemy import exists, func, select +from sqlalchemy import case, exists, func, or_, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import load_only, selectinload @@ -15,6 +15,7 @@ from sqlalchemy.engine import Row from basic_memory.models.knowledge import Entity, Observation, Relation +from basic_memory.models.relation_search_refresh import RelationSearchRefresh from basic_memory.repository.repository import Repository type EntityMetadata = dict[str, Any] | None @@ -340,8 +341,27 @@ async def get_by_file_paths( # Convert all paths to POSIX strings for consistent comparison posix_paths = [Path(fp).as_posix() for fp in file_paths] # pragma: no cover - # Query ONLY file_path and checksum columns (not full Entity objects) - query = select(Entity.file_path, Entity.checksum).where( # pragma: no cover + # A pending relation publication means the file projection is incomplete even + # when its content checksum already matches. Mask it as unknown so the normal + # change detector re-drives the exact canonical bytes on the next scan. + publication_pending = exists().where( + RelationSearchRefresh.project_id == Entity.project_id, + RelationSearchRefresh.entity_id == Entity.id, + RelationSearchRefresh.publication_generation.is_not(None), + ) + # Generation zero is the server default for old binaries during the + # drain window. Force a new-code scan to replace those rows from the + # canonical file instead of letting them evade the generation fence. + legacy_relation_pending = exists().where( + Relation.project_id == Entity.project_id, + Relation.from_id == Entity.id, + Relation.generation == 0, + ) + indexed_checksum = case( + (or_(publication_pending, legacy_relation_pending), None), + else_=Entity.checksum, + ).label("checksum") + query = select(Entity.file_path, indexed_checksum).where( # pragma: no cover Entity.file_path.in_(posix_paths) ) query = self._add_project_filter(query) # pragma: no cover diff --git a/src/basic_memory/repository/relation_repository.py b/src/basic_memory/repository/relation_repository.py index a4851f196..6772eff50 100644 --- a/src/basic_memory/repository/relation_repository.py +++ b/src/basic_memory/repository/relation_repository.py @@ -2,32 +2,47 @@ from dataclasses import dataclass from itertools import batched -from typing import override, Sequence, List, Optional, Any, cast +from typing import override, Sequence, List, Optional -from sqlalchemy import and_, case, delete, exists, or_, select, update -from sqlalchemy.engine import CursorResult +from sqlalchemy import ( + Integer, + String, + Text, + and_, + case, + delete, + exists, + literal, + or_, + select, + tuple_, + union_all, + update, +) from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload, aliased +from sqlalchemy.orm import joinedload, selectinload, aliased +from sqlalchemy.orm.attributes import InstrumentedAttribute from sqlalchemy.orm.interfaces import LoaderOption +from sqlalchemy.sql import Select from sqlalchemy.sql.elements import ColumnElement -from basic_memory.models import Entity, Relation, RelationSearchRefresh +from basic_memory.models import Entity, NoteContent, Observation, Relation, RelationSearchRefresh from basic_memory.repository.repository import Repository RESOLVED_RELATION_WRITE_STATEMENT_SIZE = 250 +RELATION_GENERATION_WRITE_STATEMENT_SIZE = 250 +LEGACY_RELATION_GENERATION = 0 @dataclass(frozen=True, slots=True) class AcceptedRelationWrite: - """One outgoing relation parsed from accepted markdown, ready to persist. + """One outgoing relation parsed from accepted markdown. - Most targets are carried by name and left for forward-reference resolution. - Safe self-relations can carry ``target_id`` because the general resolver - deliberately skips them; persisting that ID in the accepted transaction - keeps DB-first writes consistent with the normal indexing path (issue #1076). + ``target_id`` is reserved for an ambiguity-safe self-link. Other targets + stay unresolved so the generation-fenced resolver owns their backfill. """ relation_type: str @@ -36,20 +51,136 @@ class AcceptedRelationWrite: target_id: int | None = None +@dataclass(frozen=True, slots=True) +class RelationGenerationWriteResult: + """Whether one guarded relation-generation statement still owned its source.""" + + generation_is_current: bool + + +def current_relation_generation_statement( + *, + project_id: int, + entity_id: int, + generation: int, +) -> Select[tuple[int]]: + """Build the source-generation fence acquired before a projection write. + + Lock-order invariant: every transaction that can reach a note's + ``NoteContent`` and then lock its ``Entity`` or ``Relation`` rows acquires + all relevant ``NoteContent`` rows first, sorted by entity ID for bulk work. + Relation publication may lock ``Entity`` through foreign-key enforcement; + accepted mutations and cascading deletes must claim the same sources before + preparing or flushing later-table changes. + + PostgreSQL renders ``FOR UPDATE`` and holds the note-content row through the + relation statement. SQLite intentionally omits the clause; its first guarded + relation mutation then enters SQLite's single-writer serialization. + """ + return ( + select(NoteContent.entity_id) + .where( + NoteContent.project_id == project_id, + NoteContent.entity_id == entity_id, + NoteContent.db_version == generation, + ) + .with_for_update() + ) + + +async def lock_note_content_before_entity_mutation( + session: AsyncSession, + *, + project_id: int, + entity_ids: Sequence[int], +) -> None: + """Lock accepted note rows in canonical order before mutating their entities. + + See ``current_relation_generation_statement`` for the authoritative + NoteContent-before-Entity lock-order invariant. Sorting the complete set + before the first entity mutation also keeps overlapping bulk operations + from acquiring source fences in opposite orders. + """ + ordered_entity_ids = tuple(sorted(set(entity_ids))) + if not ordered_entity_ids: + return + + await session.execute( + select(NoteContent.entity_id) + .where( + NoteContent.project_id == project_id, + NoteContent.entity_id.in_(ordered_entity_ids), + ) + .order_by(NoteContent.entity_id) + .with_for_update() + ) + + +async def lock_project_note_content_before_project_mutation( + session: AsyncSession, + *, + project_id: int, +) -> None: + """Lock a project's accepted notes before a cascading project mutation. + + See ``current_relation_generation_statement`` for the canonical lock-order + invariant. A project hard delete can cascade through Entity and Relation, + so it claims every NoteContent source in sorted order before locking Project. + """ + await session.execute( + select(NoteContent.entity_id) + .where(NoteContent.project_id == project_id) + .order_by(NoteContent.entity_id) + .with_for_update() + ) + + +def current_relation_generation_predicate( + *, + project_id: int, + entity_id: int | InstrumentedAttribute[int], + generation: int | InstrumentedAttribute[int], +) -> ColumnElement[bool]: + """Require an accepted generation or an unbootstrapped legacy source. + + The note-content migration did not backfill existing entities. Its relation + rows remain at generation zero until canonical markdown is reconciled. Keep + those rows usable only while the source still has no ``NoteContent``; once + bootstrapped, the ordinary generation fence owns every subsequent mutation. + """ + source_has_note_content = exists().where( + NoteContent.project_id == project_id, + NoteContent.entity_id == entity_id, + ) + generation_is_legacy = ( + literal(generation) == LEGACY_RELATION_GENERATION + if isinstance(generation, int) + else generation == LEGACY_RELATION_GENERATION + ) + return or_( + exists().where( + NoteContent.project_id == project_id, + NoteContent.entity_id == entity_id, + NoteContent.db_version == generation, + ), + and_(generation_is_legacy, ~source_has_note_content), + ) + + @dataclass(frozen=True, slots=True) class ResolvedRelationWrite: """Compare-and-set command for one unresolved relation target.""" relation_id: int from_id: int + generation: int original_target_name: str target_id: int target_external_id: str - target_name: str relation_type: str @property - def unresolved_identity(self) -> tuple[int, int, None, str, str]: + def unresolved_identity(self) -> tuple[int, int, None, str, str, int]: """Return the row identity that must still exist before mutation.""" return ( self.relation_id, @@ -57,6 +188,7 @@ def unresolved_identity(self) -> tuple[int, int, None, str, str]: None, self.original_target_name, self.relation_type, + self.generation, ) @property @@ -76,17 +208,72 @@ class ResolvedRelationWriteResult: def current_resolved_relation_write_predicate( write: ResolvedRelationWrite, + *, + project_id: int, ) -> ColumnElement[bool]: - """Require both the unresolved edge and its snapshotted target to remain current.""" + """Require source generation, unresolved edge, and target identity to remain current.""" return and_( + Relation.project_id == project_id, Relation.id == write.relation_id, Relation.from_id == write.from_id, Relation.to_id.is_(None), Relation.to_name == write.original_target_name, Relation.relation_type == write.relation_type, + Relation.generation == write.generation, + current_relation_generation_predicate( + project_id=project_id, + entity_id=write.from_id, + generation=write.generation, + ), exists().where( Entity.id == write.target_id, Entity.external_id == write.target_external_id, + Entity.project_id == project_id, + ), + ) + + +@dataclass(frozen=True, slots=True) +class ExistingRelationSnapshot: + """Locked relation identity used for guarded resolver cleanup.""" + + relation_id: int + from_id: int + to_id: int | None + to_name: str + relation_type: str + generation: int + + @property + def target_key(self) -> tuple[int, int, str] | None: + """Return the resolved uniqueness identity when this row has a target.""" + if self.to_id is None: + return None + return self.from_id, self.to_id, self.relation_type + + +def current_relation_snapshot_predicate( + snapshot: ExistingRelationSnapshot, + *, + project_id: int, + expected_generation: int, +) -> ColumnElement[bool]: + """Delete a snapshotted row only while its source still owns the planned generation.""" + target_predicate = ( + Relation.to_id.is_(None) if snapshot.to_id is None else Relation.to_id == snapshot.to_id + ) + return and_( + Relation.project_id == project_id, + Relation.id == snapshot.relation_id, + Relation.from_id == snapshot.from_id, + target_predicate, + Relation.to_name == snapshot.to_name, + Relation.relation_type == snapshot.relation_type, + Relation.generation == snapshot.generation, + current_relation_generation_predicate( + project_id=project_id, + entity_id=snapshot.from_id, + generation=expected_generation, ), ) @@ -99,9 +286,19 @@ class PendingRelationSearchRefresh: entity_id: int +@dataclass(frozen=True, slots=True) +class CurrentRelationGenerationSearchRefresh: + """One generation-coherent entity snapshot and its visible refresh work.""" + + entity: Entity + refresh_ids: tuple[int, ...] + + class RelationRepository(Repository[Relation]): """Repository for Relation model with memory-specific operations.""" + project_id: int + def __init__(self, project_id: int): """Initialize with project_id filter. @@ -150,21 +347,228 @@ async def find_by_type(self, session: AsyncSession, relation_type: str) -> Seque result = await self.execute_query(session, query) return result.scalars().all() - async def delete_outgoing_relations_from_entity( - self, session: AsyncSession, entity_id: int - ) -> None: - """Delete outgoing relations for an entity. + async def upsert_relation_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + relations: Sequence[AcceptedRelationWrite], + ) -> RelationGenerationWriteResult: + """Publish one bounded relation chunk only while its source generation is current.""" + relations_by_identity: dict[tuple[str, str], AcceptedRelationWrite] = {} + for relation in relations: + if relation.target_id is not None and relation.target_id != entity_id: + raise ValueError("Only the source entity may be pre-resolved during publication") + identity = relation.relation_type, relation.target_name + relations_by_identity.setdefault(identity, relation) - Only deletes relations where this entity is the source (from_id), - as these are the ones owned by this entity's markdown file. - """ - query = delete(Relation).where(Relation.from_id == entity_id) - query = query.where(Relation.project_id == self.project_id) - await session.execute(query) + ordered_relations: list[AcceptedRelationWrite] = [] + resolved_identities: set[tuple[str, int]] = set() + for identity in sorted(relations_by_identity): + relation = relations_by_identity[identity] + if relation.target_id is not None: + resolved_identity = relation.relation_type, relation.target_id + if resolved_identity in resolved_identities: + continue + resolved_identities.add(resolved_identity) + ordered_relations.append(relation) + + if not ordered_relations: + raise ValueError("Relation generation upsert requires at least one relation") + if len(ordered_relations) > RELATION_GENERATION_WRITE_STATEMENT_SIZE: + raise ValueError( + "Relation generation upsert exceeds the bounded statement size " + f"of {RELATION_GENERATION_WRITE_STATEMENT_SIZE}" + ) + + current_generation = await session.scalar( + current_relation_generation_statement( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ) + ) + if current_generation is None: + return RelationGenerationWriteResult(generation_is_current=False) + + pre_resolved_relations = [ + relation for relation in ordered_relations if relation.target_id is not None + ] + if pre_resolved_relations: + # Constraint: name identity is the upsert arbiter, while safe self-links also occupy + # the resolved identity domain. Replace an older alias in this same transaction so + # changing the authored self-link cannot collide before generation cleanup runs. + await session.execute( + delete(Relation).where( + Relation.project_id == self.project_id, + Relation.from_id == entity_id, + or_( + *( + and_( + Relation.to_id == relation.target_id, + Relation.relation_type == relation.relation_type, + Relation.to_name != relation.target_name, + ) + for relation in pre_resolved_relations + ) + ), + current_relation_generation_predicate( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ), + ) + ) + + desired_relations = union_all( + *( + select( + literal(relation.target_id, type_=Integer).label("to_id"), + literal(relation.target_name, type_=String).label("to_name"), + literal(relation.relation_type, type_=String).label("relation_type"), + literal(relation.context, type_=Text).label("context"), + ) + for relation in ordered_relations + ) + ).subquery() + generation_is_current = current_relation_generation_predicate( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ) + rows = ( + select( + literal(self.project_id).label("project_id"), + literal(entity_id).label("from_id"), + desired_relations.c.to_id, + desired_relations.c.to_name, + desired_relations.c.relation_type, + desired_relations.c.context, + literal(generation).label("generation"), + ) + .select_from(desired_relations) + .where(generation_is_current) + ) + + dialect_name = session.bind.dialect.name if session.bind else "sqlite" + insert_statement = ( + pg_insert(Relation) if dialect_name == "postgresql" else sqlite_insert(Relation) + ).from_select( + [ + Relation.project_id, + Relation.from_id, + Relation.to_id, + Relation.to_name, + Relation.relation_type, + Relation.context, + Relation.generation, + ], + rows, + ) + statement = insert_statement.on_conflict_do_update( + index_elements=[Relation.from_id, Relation.to_name, Relation.relation_type], + set_={ + "generation": insert_statement.excluded.generation, + "context": insert_statement.excluded.context, + "to_id": insert_statement.excluded.to_id, + }, + where=Relation.generation < insert_statement.excluded.generation, + ) + await session.execute(statement) + return RelationGenerationWriteResult(generation_is_current=True) + + async def begin_relation_generation_publication( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + """Persist retry work before publishing any relation chunk.""" + current_generation = await session.scalar( + current_relation_generation_statement( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ) + ) + if current_generation is None: + return RelationGenerationWriteResult(generation_is_current=False) + + session.add( + RelationSearchRefresh( + project_id=self.project_id, + entity_id=entity_id, + publication_generation=generation, + ) + ) + return RelationGenerationWriteResult(generation_is_current=True) + + async def cleanup_relation_generations( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + """Delete superseded rows only while the cleanup generation remains current.""" + current_generation = await session.scalar( + current_relation_generation_statement( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ) + ) + if current_generation is None: + return RelationGenerationWriteResult(generation_is_current=False) + + await session.execute( + delete(Relation).where( + Relation.project_id == self.project_id, + Relation.from_id == entity_id, + Relation.generation < generation, + current_relation_generation_predicate( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ), + ) + ) + + # Successful cleanup converts every retry for this or an older generation + # into search-refresh work in the same transaction. A newer generation's + # marker remains pending for its own publisher. + converted = await session.execute( + update(RelationSearchRefresh) + .where( + RelationSearchRefresh.project_id == self.project_id, + RelationSearchRefresh.entity_id == entity_id, + RelationSearchRefresh.publication_generation.is_not(None), + RelationSearchRefresh.publication_generation <= generation, + ) + .values(publication_generation=None) + .returning(RelationSearchRefresh.id) + ) + if converted.scalars().first() is None: + session.add( + RelationSearchRefresh( + project_id=self.project_id, + entity_id=entity_id, + ) + ) + return RelationGenerationWriteResult(generation_is_current=True) async def find_unresolved_relations(self, session: AsyncSession) -> Sequence[Relation]: - """Find all unresolved relations, where to_id is null.""" - query = self.select().filter(Relation.to_id.is_(None)) + """Find unresolved relations owned by their source's current generation.""" + query = self.select().filter( + Relation.to_id.is_(None), + current_relation_generation_predicate( + project_id=self.project_id, + entity_id=Relation.from_id, + generation=Relation.generation, + ), + ) result = await self.execute_query(session, query) return result.scalars().all() @@ -179,7 +583,15 @@ async def find_unresolved_relations_for_entity( Returns: List of unresolved relations where this entity is the source. """ - query = self.select().filter(Relation.from_id == entity_id, Relation.to_id.is_(None)) + query = self.select().filter( + Relation.from_id == entity_id, + Relation.to_id.is_(None), + current_relation_generation_predicate( + project_id=self.project_id, + entity_id=Relation.from_id, + generation=Relation.generation, + ), + ) result = await self.execute_query(session, query) return result.scalars().all() @@ -192,7 +604,10 @@ async def list_pending_search_refreshes( """Return committed relation changes whose source projection needs refresh.""" query = ( select(RelationSearchRefresh.id, RelationSearchRefresh.entity_id) - .where(RelationSearchRefresh.project_id == self.project_id) + .where( + RelationSearchRefresh.project_id == self.project_id, + RelationSearchRefresh.publication_generation.is_(None), + ) .order_by(RelationSearchRefresh.id) ) if entity_id is not None: @@ -203,6 +618,58 @@ async def list_pending_search_refreshes( for refresh_id, refresh_entity_id in result.tuples().all() ] + async def load_search_refresh_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> CurrentRelationGenerationSearchRefresh | None: + """Load refresh inputs only while the published generation is still accepted. + + Entity state, the generation fence, and the exact pending marker IDs are + observed by one statement. If a newer accepted generation already won, + the old publisher performs no search write and leaves all markers for the + newer flow or the durable resolver retry. + """ + result = await session.execute( + select(Entity, RelationSearchRefresh.id) + .join( + NoteContent, + and_( + NoteContent.project_id == Entity.project_id, + NoteContent.entity_id == Entity.id, + ), + ) + .outerjoin( + RelationSearchRefresh, + and_( + RelationSearchRefresh.project_id == Entity.project_id, + RelationSearchRefresh.entity_id == Entity.id, + RelationSearchRefresh.publication_generation.is_(None), + ), + ) + .where( + Entity.project_id == self.project_id, + Entity.id == entity_id, + NoteContent.db_version == generation, + ) + .options( + joinedload(Entity.observations).joinedload(Observation.entity), + joinedload(Entity.outgoing_relations).joinedload(Relation.from_entity), + joinedload(Entity.outgoing_relations).joinedload(Relation.to_entity), + ) + .order_by(RelationSearchRefresh.id) + ) + rows = result.unique().tuples().all() + if not rows: + return None + + return CurrentRelationGenerationSearchRefresh( + entity=rows[0][0], + refresh_ids=tuple(refresh_id for _, refresh_id in rows if refresh_id is not None), + ) + async def clear_pending_search_refreshes( self, session: AsyncSession, @@ -216,41 +683,125 @@ async def clear_pending_search_refreshes( ) ) + async def complete_search_refresh_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + refresh_ids: Sequence[int], + ) -> bool: + """Retire observed work only if the rendered generation is still current. + + Search storage is intentionally outside this repository transaction. A + newer accepted generation can therefore win while the caller is rendering + the older snapshot. Repeating the generation predicate in the marker + mutation makes that race visible: stale writers leave fresh durable work + for the winning generation instead of consuming the final repair signal. + """ + generation_is_current = current_relation_generation_predicate( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ) + if refresh_ids: + await session.execute( + delete(RelationSearchRefresh).where( + RelationSearchRefresh.project_id == self.project_id, + RelationSearchRefresh.id.in_(refresh_ids), + generation_is_current, + ) + ) + + is_current = bool(await session.scalar(select(generation_is_current))) + if not is_current: + session.add( + RelationSearchRefresh( + project_id=self.project_id, + entity_id=entity_id, + ) + ) + return is_current + async def apply_resolved_targets( self, session: AsyncSession, writes: Sequence[ResolvedRelationWrite], ) -> ResolvedRelationWriteResult: - """Resolve relation targets in one transaction without duplicate edges. - - Both relation uniqueness constraints can collide when aliases resolve to - the same canonical entity. The old row-at-a-time path handled that by - deleting the later unresolved row after an ``IntegrityError``. Planning - the accepted and redundant rows up front preserves that behavior while - allowing the mutations to run as set-based statements. - """ + """Backfill targets only while each source relation generation remains current.""" if not writes: return ResolvedRelationWriteResult(frozenset(), ()) ordered_writes = sorted(writes, key=lambda write: write.relation_id) - # PostgreSQL row locks preserve snapshotted targets until commit. Bound - # the locking reads as well as writes so a large collision domain never - # exceeds backend parameter limits. SQLite ignores FOR UPDATE, so each - # first mutation also checks the external ID before its write lock. + + # Publishers and resolvers both acquire the source NoteContent lock before + # relation rows. Targets remain an identity snapshot: explicitly locking a + # mutual target Entity here would invert with the source-Entity key-share + # lock required by the durable search-refresh marker. + requested_source_entity_ids = sorted({write.from_id for write in ordered_writes}) + source_result = await session.execute( + select(NoteContent.entity_id, NoteContent.db_version) + .where( + NoteContent.project_id == self.project_id, + NoteContent.entity_id.in_(requested_source_entity_ids), + ) + .order_by(NoteContent.entity_id) + .with_for_update() + ) + current_generation_by_source = dict(source_result.tuples().all()) + + # The statement guards below repeat the no-NoteContent condition, so a + # concurrent bootstrap closes this compatibility path before mutation. + legacy_source_entity_ids = { + write.from_id + for write in ordered_writes + if write.generation == LEGACY_RELATION_GENERATION + and write.from_id not in current_generation_by_source + } + current_generation_by_source.update( + (source_entity_id, LEGACY_RELATION_GENERATION) + for source_entity_id in legacy_source_entity_ids + ) + + stale_relation_ids = [ + write.relation_id + for write in ordered_writes + if current_generation_by_source.get(write.from_id) != write.generation + ] + generation_current_writes = [ + write + for write in ordered_writes + if current_generation_by_source.get(write.from_id) == write.generation + ] + if not generation_current_writes: + return ResolvedRelationWriteResult( + affected_entity_ids=frozenset(), + duplicate_relation_ids=(), + stale_relation_ids=tuple(sorted(stale_relation_ids)), + ) + + # The guarded UPDATE repeats the target external-ID predicate. PostgreSQL's + # FK check then acquires the target key-share lock only for the accepted + # mutation; SQLite's first mutation enters single-writer serialization. current_target_identities: set[tuple[int, str]] = set() - target_ids = sorted({write.target_id for write in ordered_writes}) + target_ids = sorted({write.target_id for write in generation_current_writes}) for target_id_batch in batched( target_ids, RESOLVED_RELATION_WRITE_STATEMENT_SIZE, ): target_result = await session.execute( select(Entity.id, Entity.external_id) - .where(Entity.id.in_(target_id_batch)) + .where( + Entity.project_id == self.project_id, + Entity.id.in_(target_id_batch), + ) .order_by(Entity.id) - .with_for_update() ) current_target_identities.update(target_result.tuples().all()) - requested_source_entity_ids = {write.from_id for write in ordered_writes} + + collision_domains = sorted( + {(write.from_id, write.relation_type) for write in generation_current_writes} + ) result = await session.execute( select( Relation.id, @@ -258,57 +809,135 @@ async def apply_resolved_targets( Relation.to_id, Relation.to_name, Relation.relation_type, - ).where( + Relation.generation, + ) + .where( Relation.project_id == self.project_id, - Relation.from_id.in_(requested_source_entity_ids), + tuple_(Relation.from_id, Relation.relation_type).in_(collision_domains), ) + .order_by(Relation.id) + .with_for_update() ) - existing_relations = result.tuples().all() + existing_relations = [ + ExistingRelationSnapshot( + relation_id=relation_id, + from_id=from_id, + to_id=to_id, + to_name=to_name, + relation_type=relation_type, + generation=generation, + ) + for relation_id, from_id, to_id, to_name, relation_type, generation in result.tuples() + ] existing_relations_by_id = { - relation_id: (relation_id, from_id, to_id, to_name, relation_type) - for relation_id, from_id, to_id, to_name, relation_type in existing_relations + snapshot.relation_id: snapshot for snapshot in existing_relations } current_writes: list[ResolvedRelationWrite] = [] - stale_relation_ids: list[int] = [] - for write in ordered_writes: + for write in generation_current_writes: + snapshot = existing_relations_by_id.get(write.relation_id) if ( - existing_relations_by_id.get(write.relation_id) != write.unresolved_identity + snapshot is None + or ( + snapshot.relation_id, + snapshot.from_id, + snapshot.to_id, + snapshot.to_name, + snapshot.relation_type, + snapshot.generation, + ) + != write.unresolved_identity or write.target_identity not in current_target_identities ): stale_relation_ids.append(write.relation_id) continue current_writes.append(write) - relation_ids = {write.relation_id for write in current_writes} + resolved_snapshots_by_target: dict[ + tuple[int, int, str], list[ExistingRelationSnapshot] + ] = {} + for snapshot in existing_relations: + current_source_generation = current_generation_by_source[snapshot.from_id] + if snapshot.generation > current_source_generation: + raise RuntimeError( + "Relation generation cannot be newer than its source note_content: " + f"relation_id={snapshot.relation_id}, " + f"relation_generation={snapshot.generation}, " + f"source_generation={current_source_generation}" + ) + if snapshot.target_key is not None: + resolved_snapshots_by_target.setdefault(snapshot.target_key, []).append(snapshot) - occupied_target_keys: set[tuple[int, int, str]] = set() - occupied_name_keys: set[tuple[int, str, str]] = set() - all_name_keys: set[tuple[int, str, str]] = set() - for relation_id, from_id, to_id, to_name, relation_type in existing_relations: - name_key = (from_id, to_name, relation_type) - all_name_keys.add(name_key) - if relation_id in relation_ids: - continue - occupied_name_keys.add(name_key) - if to_id is not None: - occupied_target_keys.add((from_id, to_id, relation_type)) + writes_by_target: dict[tuple[int, int, str], list[ResolvedRelationWrite]] = {} + for write in current_writes: + key = (write.from_id, write.target_id, write.relation_type) + writes_by_target.setdefault(key, []).append(write) accepted_writes: list[ResolvedRelationWrite] = [] - planned_duplicate_writes: list[ResolvedRelationWrite] = [] - for write in current_writes: - target_key = (write.from_id, write.target_id, write.relation_type) - name_key = (write.from_id, write.target_name, write.relation_type) - if target_key in occupied_target_keys or name_key in occupied_name_keys: - planned_duplicate_writes.append(write) - continue - accepted_writes.append(write) - occupied_target_keys.add(target_key) - occupied_name_keys.add(name_key) + duplicate_writes: list[ResolvedRelationWrite] = [] + duplicate_snapshot_deletes: list[tuple[ExistingRelationSnapshot, int]] = [] + superseded_snapshot_deletes: list[tuple[ExistingRelationSnapshot, int]] = [] + for target_key, target_writes in sorted(writes_by_target.items()): + source_id = target_key[0] + expected_generation = current_generation_by_source[source_id] + target_snapshots = resolved_snapshots_by_target.get(target_key, []) + current_snapshots: list[ExistingRelationSnapshot] = [] + for snapshot in target_snapshots: + if snapshot.generation < expected_generation: + superseded_snapshot_deletes.append((snapshot, expected_generation)) + else: + current_snapshots.append(snapshot) + + winner_id = min( + [write.relation_id for write in target_writes] + + [snapshot.relation_id for snapshot in current_snapshots] + ) + for write in target_writes: + if write.relation_id == winner_id: + accepted_writes.append(write) + else: + duplicate_writes.append(write) + duplicate_snapshot_deletes.extend( + (snapshot, expected_generation) + for snapshot in current_snapshots + if snapshot.relation_id != winner_id + ) + + deleted_snapshot_ids: set[int] = set() + snapshot_delete_commands = [ + *superseded_snapshot_deletes, + *duplicate_snapshot_deletes, + ] + for delete_batch in batched( + snapshot_delete_commands, + RESOLVED_RELATION_WRITE_STATEMENT_SIZE, + ): + delete_result = await session.execute( + delete(Relation) + .where( + or_( + *( + current_relation_snapshot_predicate( + snapshot, + project_id=self.project_id, + expected_generation=expected_generation, + ) + for snapshot, expected_generation in delete_batch + ) + ) + ) + .returning(Relation.id) + ) + deleted_snapshot_ids.update(delete_result.scalars().all()) - duplicate_relation_ids: list[int] = [] + duplicate_relation_ids = { + snapshot.relation_id + for snapshot, _ in duplicate_snapshot_deletes + if snapshot.relation_id in deleted_snapshot_ids + } + deleted_duplicate_write_ids: set[int] = set() for write_batch in batched( - planned_duplicate_writes, + duplicate_writes, RESOLVED_RELATION_WRITE_STATEMENT_SIZE, ): delete_result = await session.execute( @@ -316,13 +945,20 @@ async def apply_resolved_targets( .where( Relation.project_id == self.project_id, or_( - *(current_resolved_relation_write_predicate(write) for write in write_batch) + *( + current_resolved_relation_write_predicate( + write, + project_id=self.project_id, + ) + for write in write_batch + ) ), ) .returning(Relation.id) ) deleted_relation_ids = set(delete_result.scalars().all()) - duplicate_relation_ids.extend( + deleted_duplicate_write_ids.update(deleted_relation_ids) + duplicate_relation_ids.update( write.relation_id for write in write_batch if write.relation_id in deleted_relation_ids @@ -333,90 +969,52 @@ async def apply_resolved_targets( if write.relation_id not in deleted_relation_ids ) - staged_writes: list[ResolvedRelationWrite] = [] - if accepted_writes: - temporary_names_by_relation_id: dict[int, str] = {} - for write in accepted_writes: - temporary_name = f"__basic_memory_resolving_relation_{write.relation_id}__" - while (write.from_id, temporary_name, write.relation_type) in all_name_keys: - temporary_name += "_" - temporary_names_by_relation_id[write.relation_id] = temporary_name - all_name_keys.add((write.from_id, temporary_name, write.relation_type)) - - # Clear both unique keys before assigning canonical targets. This - # makes alias swaps safe on databases that check uniqueness row by - # row inside a multi-row UPDATE. The identity predicates make this - # first mutation a compare-and-set, so a reused relation ID cannot - # redirect a stale resolution command onto a replacement edge. - staged_relation_ids: set[int] = set() - for write_batch in batched( - accepted_writes, - RESOLVED_RELATION_WRITE_STATEMENT_SIZE, - ): - batch_temporary_names = { - write.relation_id: temporary_names_by_relation_id[write.relation_id] - for write in write_batch - } - stage_result = await session.execute( - update(Relation) - .where( - Relation.project_id == self.project_id, - or_( - *( - current_resolved_relation_write_predicate(write) - for write in write_batch + updated_relation_ids: set[int] = set() + for write_batch in batched( + accepted_writes, + RESOLVED_RELATION_WRITE_STATEMENT_SIZE, + ): + update_result = await session.execute( + update(Relation) + .where( + or_( + *( + current_resolved_relation_write_predicate( + write, + project_id=self.project_id, ) - ), + for write in write_batch + ) ) - .values( - to_id=None, - to_name=case(batch_temporary_names, value=Relation.id), + ) + .values( + to_id=case( + {write.relation_id: write.target_id for write in write_batch}, + value=Relation.id, ) - .returning(Relation.id) - .execution_options(synchronize_session=False) ) - staged_relation_ids.update(stage_result.scalars().all()) - staged_writes = [ - write for write in accepted_writes if write.relation_id in staged_relation_ids - ] + .returning(Relation.id) + .execution_options(synchronize_session=False) + ) + batch_updated_relation_ids = set(update_result.scalars().all()) + updated_relation_ids.update(batch_updated_relation_ids) stale_relation_ids.extend( write.relation_id - for write in accepted_writes - if write.relation_id not in staged_relation_ids + for write in write_batch + if write.relation_id not in batch_updated_relation_ids ) - if staged_writes: - # Keep the whole collision domain in this transaction, but bound - # each SQL expression below SQLite's expression-depth limit. - for write_batch in batched( - staged_writes, - RESOLVED_RELATION_WRITE_STATEMENT_SIZE, - ): - await session.execute( - update(Relation) - .where( - Relation.project_id == self.project_id, - Relation.id.in_(write.relation_id for write in write_batch), - ) - .values( - to_id=case( - {write.relation_id: write.target_id for write in write_batch}, - value=Relation.id, - ), - to_name=case( - {write.relation_id: write.target_name for write in write_batch}, - value=Relation.id, - ), - ) - .execution_options(synchronize_session=False) - ) - writes_by_id = {write.relation_id: write for write in current_writes} - mutated_relation_ids = set(duplicate_relation_ids) - mutated_relation_ids.update(write.relation_id for write in staged_writes) source_entity_ids = { - writes_by_id[relation_id].from_id for relation_id in mutated_relation_ids + writes_by_id[relation_id].from_id + for relation_id in deleted_duplicate_write_ids | updated_relation_ids } + deleted_snapshots_by_id = { + snapshot.relation_id: snapshot for snapshot, _ in snapshot_delete_commands + } + source_entity_ids.update( + deleted_snapshots_by_id[relation_id].from_id for relation_id in deleted_snapshot_ids + ) # Trigger: relation targets changed or duplicate edges were removed. # Why: a later storage/search failure must not consume the only evidence @@ -433,96 +1031,10 @@ async def apply_resolved_targets( return ResolvedRelationWriteResult( affected_entity_ids=frozenset(source_entity_ids), - duplicate_relation_ids=tuple(duplicate_relation_ids), + duplicate_relation_ids=tuple(sorted(duplicate_relation_ids)), stale_relation_ids=tuple(sorted(stale_relation_ids)), ) - async def add_all_ignore_duplicates( - self, session: AsyncSession, relations: List[Relation] - ) -> int: - """Bulk insert relations, ignoring duplicates. - - Uses ON CONFLICT DO NOTHING to skip relations that would violate the - unique constraint on (from_id, to_name, relation_type). This is useful - for bulk operations where the same link may appear multiple times in - a document. - - Works with both SQLite and PostgreSQL dialects. - - Args: - relations: List of Relation objects to insert - - Returns: - Number of relations actually inserted (excludes duplicates) - """ - if not relations: - return 0 - - # Convert Relation objects to dicts for insert - values = [ - { - "project_id": r.project_id if r.project_id else self.project_id, - "from_id": r.from_id, - "to_id": r.to_id, - "to_name": r.to_name, - "relation_type": r.relation_type, - "context": r.context, - } - for r in relations - ] - - # Check dialect to use appropriate insert - dialect_name = session.bind.dialect.name if session.bind else "sqlite" - - if dialect_name == "postgresql": # pragma: no cover - # PostgreSQL: use RETURNING to count inserted rows - # (rowcount is 0 for ON CONFLICT DO NOTHING) - stmt = ( # pragma: no cover - pg_insert(Relation).values(values).on_conflict_do_nothing().returning(Relation.id) - ) - result = await session.execute(stmt) # pragma: no cover - return len(result.fetchall()) # pragma: no cover - else: - # SQLite: rowcount works correctly - stmt = sqlite_insert(Relation).values(values) - stmt = stmt.on_conflict_do_nothing() - result = cast(CursorResult[Any], await session.execute(stmt)) - return result.rowcount if result.rowcount > 0 else 0 - - async def replace_accepted_outgoing_relations( - self, - session: AsyncSession, - entity_id: int, - relations: Sequence[AcceptedRelationWrite], - ) -> None: - """Replace an entity's outgoing relations with the accepted markdown set. - - Delete-then-insert mirrors ``EntityService.update_entity_relations``: - the markdown file owns its outgoing links, so an accepted write replaces - the prior set. Ordinary targets are written unresolved and linked by the - forward-reference job. Safe self-relations already carry their resolved - ID because that job intentionally skips self targets. Runs inside the - caller's transaction so the graph commits atomically with - note_content/search (issue #1076). - """ - await self.delete_outgoing_relations_from_entity(session, entity_id) - if not relations: - return - rows = [ - Relation( - project_id=self.project_id, - from_id=entity_id, - to_id=rel.target_id, - to_name=rel.target_name, - relation_type=rel.relation_type, - context=rel.context, - ) - for rel in relations - ] - # A single markdown file can repeat the same link; ignore-duplicates keeps - # the unique (from_id, to_name, relation_type) constraint from aborting. - await self.add_all_ignore_duplicates(session, rows) - @override def get_load_options(self) -> List[LoaderOption]: return [selectinload(Relation.from_entity), selectinload(Relation.to_entity)] diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 8e7669743..be9ea32fb 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -12,6 +12,10 @@ from basic_memory import db from basic_memory.config import ProjectConfig, BasicMemoryConfig from basic_memory.file_utils import remove_frontmatter +from basic_memory.indexing.models import IndexedRelation +from basic_memory.indexing.note_content_reconciliation import NoteContentReconciliationAnchor +from basic_memory.indexing.note_content_reconciler import NoteContentReconciler +from basic_memory.indexing.relation_persistence import RelationGenerationPublisher from basic_memory.markdown import EntityMarkdown from basic_memory.markdown.entity_parser import ( EntityParser, @@ -19,10 +23,10 @@ ) from basic_memory.markdown.utils import entity_model_from_markdown from basic_memory.models import Entity as EntityModel -from basic_memory.models import Observation, Relation -from basic_memory.models.knowledge import Entity +from basic_memory.models import Observation from basic_memory.repository import ObservationRepository, RelationRepository from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.note_content_repository import NoteContentRepository from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.runtime.note_move import normalize_note_move_destination_path from basic_memory.schemas import Entity as EntitySchema @@ -111,34 +115,6 @@ def __init__( # Default returns None for local/CLI usage. Cloud overrides this to read from UserContext. self.get_user_id: Callable[[], Optional[str]] = lambda: None - async def detect_file_path_conflicts( - self, - file_path: str, - skip_check: bool = False, - session: AsyncSession | None = None, - ) -> List[str]: - """Delegate file-path conflict detection to the shared preparation capability.""" - return await self._note_preparation.detect_file_path_conflicts( - file_path, - skip_check=skip_check, - session=session, - ) - - async def resolve_permalink( - self, - file_path: Permalink | Path, - markdown: Optional[EntityMarkdown] = None, - skip_conflict_check: bool = False, - session: AsyncSession | None = None, - ) -> str: - """Delegate permalink resolution to the shared preparation capability.""" - return await self._note_preparation.resolve_permalink( - file_path, - markdown, - skip_conflict_check=skip_conflict_check, - session=session, - ) - def _coerce_schema_input(self, schema: EntitySchema | EntityModel) -> EntitySchema: """Normalize legacy Entity-like inputs into the schema shape prepare methods expect.""" if isinstance(schema, EntitySchema): @@ -181,15 +157,22 @@ def _sync_prepared_schema_state( else: source_schema._permalink = prepared.entity_fields.permalink - async def _read_persisted_write_content(self, file_path: Path) -> tuple[str, str]: - """Read the stored markdown after write-time formatting has finished.""" + async def _read_persisted_write_snapshot( + self, + file_path: Path, + ) -> tuple[EntityMarkdown, str, str]: + """Read and parse the stored markdown after write-time formatting has finished.""" # Trigger: format-on-save or platform-specific text writes can change the stored markdown # after prepare accepted the request. - # Why: API responses and inline search indexing should describe the note that actually - # landed on disk, not the pre-write snapshot. - # Outcome: write helpers return persisted markdown plus search content derived from it. + # Why: responses, search, and relation generations must all describe the same bytes that + # landed on disk, not the pre-write parse paired with a later file snapshot. + # Outcome: callers publish relations reparsed from the exact persisted markdown. persisted_content = await self.file_service.read_file_content(file_path) - return persisted_content, remove_frontmatter(persisted_content) + persisted_markdown = await self.entity_parser.parse_markdown_content( + file_path=file_path, + content=persisted_content, + ) + return persisted_markdown, persisted_content, remove_frontmatter(persisted_content) def _paths_share_storage_target(self, left: Path, right: Path) -> bool: """Return whether two relative project paths point at the same stored file.""" @@ -202,6 +185,119 @@ def _paths_share_storage_target(self, left: Path, right: Path) -> bool: except OSError: return False + async def resolve_deferred_self_relation( + self, + target: str, + entity: EntityModel, + session: AsyncSession | None = None, + ) -> EntityModel | None: + """Resolve only the source aliases that the background resolver intentionally skips.""" + return await self._note_preparation.resolve_deferred_self_relation( + target, + entity, + session=session, + ) + + async def _publish_markdown_relations( + self, + *, + entity: EntityModel, + markdown: EntityMarkdown, + markdown_content: str, + anchor: NoteContentReconciliationAnchor, + ) -> EntityModel: + """Claim the stored note generation, then publish its unresolved relations.""" + # Constraint: this public service surface and normal indexers can touch the same project DB. + # There is no single-writer exemption: every path must claim note_content before publishing. + reconciler = NoteContentReconciler( + note_content_repository=NoteContentRepository(project_id=self.repository.project_id), + session_maker=self.session_maker, + ) + reconciliation = await reconciler.reconcile( + entity=entity, + markdown_content=markdown_content, + observed_at=entity.updated_at, + source="entity_service", + anchor=anchor, + ) + if reconciliation.generation is None: + return entity + + indexed_relations: list[IndexedRelation] = [] + for relation in markdown.relations: + resolved = await self.resolve_deferred_self_relation(relation.target, entity) + indexed_relations.append( + IndexedRelation( + relation_type=relation.type, + target_name=relation.target, + context=relation.context, + target_id=resolved.id if resolved else None, + ) + ) + + publisher = RelationGenerationPublisher( + relation_repository=self.relation_repository, + session_maker=self.session_maker, + ) + published = await publisher.publish( + entity_id=entity.id, + generation=reconciliation.generation, + relations=indexed_relations, + ) + if not published: + return entity + + async with db.scoped_session(self.session_maker) as session: + reloaded = await self.repository.find_by_ids(session, [entity.id]) + reloaded_entity = reloaded[0] + relation_order = { + (relation.type, relation.target): index + for index, relation in enumerate(markdown.relations) + } + reloaded_entity.outgoing_relations.sort( + key=lambda relation: relation_order[(relation.relation_type, relation.to_name)] + ) + return reloaded_entity + + async def _capture_note_content_anchor( + self, + entity_id: int | None, + ) -> NoteContentReconciliationAnchor: + """Capture accepted content state before the compatibility writer touches storage.""" + reconciler = NoteContentReconciler( + note_content_repository=NoteContentRepository(project_id=self.repository.project_id), + session_maker=self.session_maker, + ) + return await reconciler.capture_anchor(entity_id) + + async def detect_file_path_conflicts( + self, + file_path: str, + skip_check: bool = False, + session: AsyncSession | None = None, + ) -> List[str]: + """Delegate file-path conflict detection to the shared preparation capability.""" + return await self._note_preparation.detect_file_path_conflicts( + file_path, + skip_check=skip_check, + session=session, + ) + + async def resolve_permalink( + self, + file_path: Permalink | Path, + markdown: Optional[EntityMarkdown] = None, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, + ) -> str: + """Delegate permalink resolution to the shared preparation capability.""" + return await self._note_preparation.resolve_permalink( + file_path, + markdown, + skip_conflict_check=skip_conflict_check, + session=session, + ) + async def prepare_create_entity_content( self, schema: EntitySchema, @@ -335,15 +431,11 @@ async def verify_move_destination_absent( ) async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]: - """Create new entity or update existing one. - Returns: (entity, is_new) where is_new is True if a new entity was created - """ + """Create a new entity or update the exact existing file/permalink match.""" logger.debug( f"Creating or updating entity: {schema.file_path}, permalink: {schema.permalink}" ) - # Try to find existing entity using strict resolution (no fuzzy search) - # This prevents incorrectly matching similar file paths like "Node A.md" and "Node C.md" existing = await self.link_resolver.resolve_link( schema.file_path, strict=True, @@ -359,45 +451,48 @@ async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityMod if existing: logger.debug(f"Found existing entity: {existing.file_path}") return await self.update_entity(existing, self._coerce_schema_input(schema)), False - else: - # Create new entity - return await self.create_entity(self._coerce_schema_input(schema)), True + return await self.create_entity(self._coerce_schema_input(schema)), True async def create_entity(self, schema: EntitySchema) -> EntityModel: - """Create a new entity and write to filesystem.""" + """Create a new entity and write it to the filesystem.""" return (await self.create_entity_with_content(schema)).entity async def create_entity_with_content(self, schema: EntitySchema) -> EntityWriteResult: - """Create a new entity and return both the entity row and written markdown.""" + """Create a new entity, then publish relations after the DB commit.""" logger.debug(f"Creating entity: {schema.title}") + relation_anchor = await self._capture_note_content_anchor(None) async with db.scoped_session(self.session_maker) as session: - # --- Prepare Accepted State --- - # Derive the canonical markdown/entity fields before touching the filesystem. prepared = await self.prepare_create_entity_content(schema, session=session) self._sync_prepared_schema_state(schema, prepared) - # --- Persist File, Then Indexable DB State --- - # Local mode still writes the file immediately; the prepare object keeps semantics separate - # from that persistence step. checksum = await self.file_service.write_file( - prepared.file_path, prepared.markdown_content + prepared.file_path, + prepared.markdown_content, ) - entity = await self.upsert_entity_from_markdown( + entity = await self.create_entity_from_markdown( prepared.file_path, prepared.entity_markdown, - is_new=True, session=session, ) updated = await self.repository.update(session, entity.id, {"checksum": checksum}) if not updated: # pragma: no cover raise ValueError(f"Failed to update entity checksum after create: {entity.id}") - persisted_content, search_content = await self._read_persisted_write_content( - prepared.file_path - ) - return EntityWriteResult( - entity=updated, - content=persisted_content, - search_content=search_content, - ) + + ( + persisted_markdown, + persisted_content, + search_content, + ) = await self._read_persisted_write_snapshot(prepared.file_path) + updated = await self._publish_markdown_relations( + entity=updated, + markdown=persisted_markdown, + markdown_content=persisted_content, + anchor=relation_anchor, + ) + return EntityWriteResult( + entity=updated, + content=persisted_content, + search_content=search_content, + ) async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel: """Update an entity's content and metadata.""" @@ -406,18 +501,18 @@ async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> Enti ).entity async def update_entity_with_content( - self, entity: EntityModel, schema: EntitySchema + self, + entity: EntityModel, + schema: EntitySchema, ) -> EntityWriteResult: - """Update an entity and return both the entity row and written markdown.""" + """Update an entity, then publish relations after the DB commit.""" schema = self._coerce_schema_input(schema) logger.debug( f"Updating entity with permalink: {entity.permalink} content-type: {schema.content_type}" ) + relation_anchor = await self._capture_note_content_anchor(entity.id) async with db.scoped_session(self.session_maker) as session: - # --- Read Current File State --- - # Full replacements merge with existing frontmatter, so local mode still needs the current - # file contents as input to the prepare step. existing_content = await self.file_service.read_file_content(entity.file_path) prepared = await self.prepare_update_entity_content( entity, @@ -427,10 +522,6 @@ async def update_entity_with_content( ) self._sync_prepared_schema_state(schema, prepared) previous_file_path = Path(entity.file_path) - # Trigger: a full replacement also renames the note to a different canonical path. - # Why: Path.replace() overwrites existing files, so the destination must be conflict-free - # before we write or we can clobber another note and only fail later at the DB layer. - # Outcome: conflicting rename attempts fail before touching either file on disk. if ( prepared.file_path.as_posix() != previous_file_path.as_posix() and await self.file_service.exists(prepared.file_path) @@ -439,38 +530,42 @@ async def update_entity_with_content( raise EntityAlreadyExistsError( f"file already exists at destination path: {prepared.file_path.as_posix()}" ) - # --- Persist Prepared State --- + checksum = await self.file_service.write_file( prepared.file_path, prepared.markdown_content, ) - entity = await self.upsert_entity_from_markdown( + entity = await self.update_entity_and_observations( prepared.file_path, prepared.entity_markdown, - is_new=False, existing_entity=entity, session=session, ) if prepared.file_path.as_posix() != previous_file_path.as_posix(): - # Trigger: a full replacement changed the canonical note path. - # Why: the new file has already been written and the entity now points at it. - # Outcome: remove the stale old file so local Basic Memory mirrors cloud's queued cleanup. if not self._paths_share_storage_target(previous_file_path, prepared.file_path): await self.file_service.delete_file(previous_file_path) - entity = await self.repository.update(session, entity.id, {"checksum": checksum}) - if not entity: # pragma: no cover + updated = await self.repository.update(session, entity.id, {"checksum": checksum}) + if not updated: # pragma: no cover raise ValueError( f"Failed to update entity checksum after update: {prepared.file_path}" ) - persisted_content, search_content = await self._read_persisted_write_content( - prepared.file_path - ) - return EntityWriteResult( - entity=entity, - content=persisted_content, - search_content=search_content, - ) + ( + persisted_markdown, + persisted_content, + search_content, + ) = await self._read_persisted_write_snapshot(prepared.file_path) + updated = await self._publish_markdown_relations( + entity=updated, + markdown=persisted_markdown, + markdown_content=persisted_content, + anchor=relation_anchor, + ) + return EntityWriteResult( + entity=updated, + content=persisted_content, + search_content=search_content, + ) async def delete_entity(self, permalink_or_id: str | int) -> bool: """Delete entity and its file.""" @@ -677,151 +772,6 @@ def _apply_markdown_entity_fields( key: value for key, value in normalized_metadata.items() if value is not None } - async def upsert_entity_from_markdown( - self, - file_path: Path, - markdown: EntityMarkdown, - *, - is_new: bool, - existing_entity: EntityModel | None = None, - resolve_relations: bool = True, - reload_entity: bool = True, - session: AsyncSession | None = None, - ) -> EntityModel: - """Create/update entity and relations from parsed markdown.""" - async with db.scoped_session(self.session_maker, session) as active_session: - if is_new: - created = await self.create_entity_from_markdown( - file_path, markdown, session=active_session - ) - else: - created = await self.update_entity_and_observations( - file_path, - markdown, - existing_entity=existing_entity, - session=active_session, - ) - # Pass the entity through so relation work does not have to rediscover the source row. - return await self.update_entity_relations( - created, - markdown, - resolve_targets=resolve_relations, - reload_entity=reload_entity, - session=active_session, - ) - - async def update_entity_relations( - self, - entity: EntityModel, - markdown: EntityMarkdown, - *, - resolve_targets: bool = True, - reload_entity: bool = True, - session: AsyncSession | None = None, - ) -> EntityModel: - """Update relations for entity. - - Accepts the entity object directly to avoid a redundant DB fetch. - Only entity.id and entity.permalink are used from the passed-in object. - """ - entity_id = entity.id - logger.debug(f"Updating relations for entity: {entity.file_path}") - - async with db.scoped_session(self.session_maker, session) as active_session: - # Clear existing relations first - await self.relation_repository.delete_outgoing_relations_from_entity( - active_session, entity_id - ) - - if markdown.relations: - if resolve_targets: - # Exact target resolution is useful for local sync, but expensive for cloud - # one-file jobs. Cloud can write unresolved rows and let a relation repair pass - # fill in to_id later. - resolved_entities: list[Entity | Exception | None] = [] - for rel in markdown.relations: - try: - # Savepoint: a DB-level lookup failure would otherwise - # poison the caller's write transaction — on Postgres - # every later statement raises PendingRollbackError with - # the root cause hidden. Scoping the lookup keeps the - # relation writes below alive. - async with active_session.begin_nested(): - resolved = await self.link_resolver.resolve_link( - rel.target, - strict=True, - load_relations=False, - session=active_session, - ) - except Exception as exc: - # The failure intentionally degrades to a forward - # reference below, but losing the error silently hides - # real defects — log it with the link context. - logger.warning( - f"Relation target resolution failed for '{rel.target}' " - f"from entity {entity.file_path}; keeping forward reference", - entity_id=entity_id, - error=str(exc), - ) - resolved = exc - resolved_entities.append(resolved) - else: - resolved_entities = [None] * len(markdown.relations) - - # Process results and create relation records - relations_to_add = [] - for rel, resolved in zip(markdown.relations, resolved_entities): - # Handle exceptions from gather and None results - target_entity: Optional[Entity] = None - if not isinstance(resolved, Exception): - # Relation target resolution keeps exceptions as values so a failed lookup - # becomes an unresolved forward reference instead of aborting the write. - target_entity = resolved - - if target_entity is None and not resolve_targets: - target_entity = await self.resolve_deferred_self_relation( - rel.target, entity, session=active_session - ) - - # if the target is found, store the id - target_id = target_entity.id if target_entity else None - # if the target is found, store the title, otherwise add the target for a "forward link" - target_name = target_entity.title if target_entity else rel.target - - # Create the relation - relation = Relation( - project_id=self.relation_repository.project_id, - from_id=entity_id, - to_id=target_id, - to_name=target_name, - relation_type=rel.type, - context=rel.context, - ) - relations_to_add.append(relation) - - # Batch insert all relations - if relations_to_add: - await self.relation_repository.add_all_ignore_duplicates( - active_session, relations_to_add - ) - - if not reload_entity: - return entity - - # Reload entity with relations via PK lookup (faster than get_by_file_path string match). - reloaded = await self.repository.find_by_ids(active_session, [entity_id]) - return reloaded[0] - - async def resolve_deferred_self_relation( - self, target: str, entity: EntityModel, session: AsyncSession | None = None - ) -> EntityModel | None: - """Resolve only self-relations that are safe to identify in deferred mode.""" - return await self._note_preparation.resolve_deferred_self_relation( - target, - entity, - session=session, - ) - async def edit_entity( self, identifier: str, @@ -832,26 +782,7 @@ async def edit_entity( expected_replacements: int = 1, replace_subsections: bool = True, ) -> EntityModel: - """Edit an existing entity's content using various operations. - - Args: - identifier: Entity identifier (permalink, title, etc.) - operation: The editing operation (append, prepend, find_replace, replace_section) - content: The content to add or use for replacement - section: For replace_section operation - the markdown header - find_text: For find_replace operation - the text to find and replace - expected_replacements: For find_replace operation - expected number of replacements (default: 1) - replace_subsections: For replace_section operation - replace nested - subsections along with the section body (default True); False stops - at the first heading of any level, preserving them - - Returns: - The updated entity model - - Raises: - EntityNotFoundError: If the entity cannot be found - ValueError: If required parameters are missing for the operation or replacement count doesn't match expected - """ + """Edit an existing entity's content.""" return ( await self.edit_entity_with_content( identifier=identifier, @@ -874,7 +805,7 @@ async def edit_entity_with_content( expected_replacements: int = 1, replace_subsections: bool = True, ) -> EntityWriteResult: - """Edit an entity and return both the entity row and written markdown.""" + """Edit an entity, then publish relations after the DB commit.""" logger.debug(f"Editing entity: {identifier}, operation: {operation}") entity = await self.link_resolver.resolve_link( @@ -885,12 +816,10 @@ async def edit_entity_with_content( if not entity: raise EntityNotFoundError(f"Entity not found: {identifier}") + relation_anchor = await self._capture_note_content_anchor(entity.id) file_path = Path(entity.file_path) current_content, _ = await self.file_service.read_file(file_path) async with db.scoped_session(self.session_maker) as session: - # --- Prepare Against Explicit Base Content --- - # The edit operation is the semantic step; file/DB writes below are just persistence of that - # accepted result. prepared = await self.prepare_edit_entity_content( entity, current_content, @@ -902,32 +831,36 @@ async def edit_entity_with_content( replace_subsections=replace_subsections, session=session, ) - checksum = await self.file_service.write_file( file_path, prepared.markdown_content, ) - - # --- Rebuild Structured Knowledge State --- - # Non-fast edits remain fully synchronous locally: once the file write succeeds, we refresh - # observations, relations, and checksum in the same request. - entity = await self.upsert_entity_from_markdown( + entity = await self.update_entity_and_observations( file_path, prepared.entity_markdown, - is_new=False, + existing_entity=entity, session=session, ) - - entity = await self.repository.update(session, entity.id, {"checksum": checksum}) - if not entity: # pragma: no cover + updated = await self.repository.update(session, entity.id, {"checksum": checksum}) + if not updated: # pragma: no cover raise ValueError(f"Failed to update entity checksum after edit: {file_path}") - persisted_content, search_content = await self._read_persisted_write_content(file_path) - return EntityWriteResult( - entity=entity, - content=persisted_content, - search_content=search_content, - ) + ( + persisted_markdown, + persisted_content, + search_content, + ) = await self._read_persisted_write_snapshot(file_path) + updated = await self._publish_markdown_relations( + entity=updated, + markdown=persisted_markdown, + markdown_content=persisted_content, + anchor=relation_anchor, + ) + return EntityWriteResult( + entity=updated, + content=persisted_content, + search_content=search_content, + ) def apply_edit_operation( self, diff --git a/src/basic_memory/services/file_service.py b/src/basic_memory/services/file_service.py index b539a45b0..f301cbb00 100644 --- a/src/basic_memory/services/file_service.py +++ b/src/basic_memory/services/file_service.py @@ -29,7 +29,7 @@ @dataclass(frozen=True, slots=True) class FrontmatterUpdateResult: - """Final content emitted by a frontmatter rewrite without a follow-up reread.""" + """Exact persisted UTF-8 content and checksum from a frontmatter rewrite.""" checksum: str content: str @@ -504,21 +504,21 @@ async def update_frontmatter_with_result( await file_utils.write_file_atomic(full_path, final_content) # Format file if configured - content_for_checksum = final_content if self.app_config: - formatted_content = await file_utils.format_file( + await file_utils.format_file( full_path, self.app_config, is_markdown=self.is_markdown(path) ) - if formatted_content is not None: - content_for_checksum = formatted_content # pragma: no cover # Trigger: frontmatter normalization may persist bytes that differ from the # in-memory string because of formatter output or platform newline handling. - # Why: follow-up scans and checksum-based move detection read raw bytes from disk. - # Outcome: the returned checksum always matches the file that was just written. + # Why: generation publication must parse the same bytes whose checksum and + # note_content generation are claimed; an LF payload paired with CRLF disk + # bytes is not one coherent snapshot. + # Outcome: one binary read supplies both the exact decoded payload and checksum. + persisted_bytes = await self.read_file_bytes(full_path) return FrontmatterUpdateResult( - checksum=await self.compute_checksum(full_path), - content=content_for_checksum, + checksum=await file_utils.compute_checksum(persisted_bytes), + content=persisted_bytes.decode("utf-8"), ) except FileOperationError: diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index ba5822e89..a02bf8242 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -8,6 +8,7 @@ from typing import Literal, Protocol from uuid import UUID +from loguru import logger from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory.indexing.accepted_note_mutation_runner import ( @@ -19,6 +20,7 @@ AcceptedNoteMutationDependencies, AcceptedNoteMutationRejected, AcceptedNoteMutationRejection, + AcceptedNoteMutationResult, AcceptedNoteUpdateMutation, run_accepted_note_create, run_accepted_note_delete, @@ -26,6 +28,10 @@ run_accepted_note_move, run_accepted_note_update, ) +from basic_memory.indexing.relation_persistence import ( + RelationGenerationPublication, + RelationGenerationPublisher, +) from basic_memory.runtime.note_content import ( RuntimeAcceptedNoteChange, RuntimeNoteContentResponsePayload, @@ -150,6 +156,48 @@ def __init__( self.actor_resolver = actor_resolver self.read_cache = read_cache + async def _publish_relation_generation( + self, + publication: RelationGenerationPublication | None, + ) -> None: + """Publish accepted relation intent only after note-content commit.""" + if publication is None: + return + + repository = self.mutation_dependencies.write_repositories.relation_repository( + publication.project_id + ) + publisher = RelationGenerationPublisher( + relation_repository=repository, + session_maker=self.session_maker, + ) + await publisher.publish( + entity_id=publication.entity_id, + generation=publication.generation, + relations=publication.relations, + ) + + async def _finish_mutation( + self, + result: AcceptedNoteMutationResult, + ) -> AcceptedNoteChange: + """Run post-commit relation publication and expose the accepted response.""" + try: + await self._publish_relation_generation(result.relation_publication) + except Exception: + publication = result.relation_publication + # Trigger: derived relation publication fails after accepted content committed. + # Why: failing the response would strand file materialization even though the + # canonical DB write succeeded; a later index pass can republish the same generation. + # Outcome: preserve the accepted change and surface the repairable failure in logs. + logger.exception( + "Relation publication failed after accepted note commit; continuing " + "materialization: entity_id={} generation={}", + publication.entity_id if publication is not None else None, + publication.generation if publication is not None else None, + ) + return result.change + @asynccontextmanager async def _mutation_cache_scope( self, @@ -257,7 +305,7 @@ async def create_note( try: async with self._mutation_cache_scope(project_external_id): async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_create( + result = await run_accepted_note_create( session, request=AcceptedNoteCreateMutation( project_external_id=project_external_id, @@ -271,6 +319,7 @@ async def create_note( ), dependencies=self.mutation_dependencies, ) + accepted = await self._finish_mutation(result) return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error @@ -314,7 +363,7 @@ async def update_note( invalidate_on_rejection=freshening_may_have_published, ): async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_update( + result = await run_accepted_note_update( session, request=AcceptedNoteUpdateMutation( project_external_id=project_external_id, @@ -330,6 +379,7 @@ async def update_note( ), dependencies=self.mutation_dependencies, ) + accepted = await self._finish_mutation(result) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error return accepted @@ -364,7 +414,7 @@ async def edit_note( invalidate_on_rejection=freshening_may_have_published, ): async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_edit( + result = await run_accepted_note_edit( session, request=AcceptedNoteEditMutation( project_external_id=project_external_id, @@ -379,6 +429,7 @@ async def edit_note( ), dependencies=self.mutation_dependencies, ) + accepted = await self._finish_mutation(result) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error return accepted @@ -413,7 +464,7 @@ async def move_note( invalidate_on_rejection=freshening_may_have_published, ): async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_move( + result = await run_accepted_note_move( session, request=AcceptedNoteMoveMutation( project_external_id=project_external_id, @@ -428,6 +479,7 @@ async def move_note( ), dependencies=self.mutation_dependencies, ) + accepted = await self._finish_mutation(result) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error return accepted @@ -450,7 +502,7 @@ async def delete_note( invalidate_on_rejection=freshening_may_have_published, ): async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_delete( + result = await run_accepted_note_delete( session, request=AcceptedNoteDeleteMutation( project_external_id=project_external_id, @@ -458,6 +510,7 @@ async def delete_note( ), dependencies=self.mutation_dependencies, ) + accepted = await self._finish_mutation(result) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error return accepted diff --git a/src/basic_memory/services/note_preparation.py b/src/basic_memory/services/note_preparation.py index d5616b504..0d5c70330 100644 --- a/src/basic_memory/services/note_preparation.py +++ b/src/basic_memory/services/note_preparation.py @@ -98,6 +98,7 @@ class PreparedEntityMove: markdown_content: str search_content: str permalink: str | None + relations: tuple[AcceptedRelationWrite, ...] = () @dataclass(frozen=True, slots=True) @@ -793,6 +794,7 @@ async def prepare_move_entity_content( current_content: str, destination_path: str, *, + should_update_permalink: bool | None = None, session: AsyncSession | None = None, ) -> PreparedEntityMove: from basic_memory.indexing.accepted_note_search import accepted_search_content_from_markdown @@ -800,22 +802,40 @@ async def prepare_move_entity_content( file_path = Path(normalize_note_move_destination_path(destination_path)) markdown_content = current_content permalink = entity.permalink - disable_permalinks = bool( - dependencies.app_config and dependencies.app_config.disable_permalinks - ) - update_permalinks_on_move = bool( - dependencies.app_config and dependencies.app_config.update_permalinks_on_move - ) - if not disable_permalinks and (update_permalinks_on_move or entity.permalink is None): + update_permalink = should_update_permalink + if update_permalink is None: + disable_permalinks = bool( + dependencies.app_config and dependencies.app_config.disable_permalinks + ) + update_permalinks_on_move = bool( + dependencies.app_config and dependencies.app_config.update_permalinks_on_move + ) + update_permalink = not disable_permalinks and ( + update_permalinks_on_move or entity.permalink is None + ) + if update_permalink: permalink = await resolve_permalink(dependencies, file_path, session=session) post = frontmatter.loads(markdown_content) post.metadata["permalink"] = permalink markdown_content = dump_frontmatter(post) + entity_markdown = await dependencies.entity_parser.parse_markdown_content( + file_path=file_path, + content=markdown_content, + ctime=entity.created_at.timestamp() if entity.created_at is not None else None, + ) return PreparedEntityMove( file_path=file_path, markdown_content=markdown_content, search_content=accepted_search_content_from_markdown(markdown_content), permalink=permalink, + relations=tuple( + AcceptedRelationWrite( + relation_type=relation.type, + target_name=relation.target, + context=relation.context, + ) + for relation in entity_markdown.relations + ), ) @@ -973,6 +993,7 @@ async def prepare_move_entity_content( current_content: str, destination_path: str, *, + should_update_permalink: bool | None = None, session: AsyncSession | None = None, ) -> PreparedEntityMove: return await prepare_move_entity_content( @@ -980,6 +1001,7 @@ async def prepare_move_entity_content( entity, current_content, destination_path, + should_update_permalink=should_update_permalink, session=session, ) diff --git a/test-int/test_note_materialization_lock_order.py b/test-int/test_note_materialization_lock_order.py index 63e149dcb..c27abaee4 100644 --- a/test-int/test_note_materialization_lock_order.py +++ b/test-int/test_note_materialization_lock_order.py @@ -1,4 +1,4 @@ -"""Postgres regression coverage for Entity-NoteContent materialization lock order.""" +"""Postgres regression coverage for NoteContent-Entity materialization lock order.""" from __future__ import annotations @@ -16,6 +16,9 @@ NoteMaterializationSessionLock, RepositoryNoteMaterializationPublisher, ) +from basic_memory.indexing.accepted_note_write_runner import ( + lock_accepted_note_content_for_entity_mutation, +) from basic_memory.models import Entity, NoteContent, Project from basic_memory.repository.note_content_repository import NoteContentRepository from basic_memory.runtime.note_content import ( @@ -45,48 +48,14 @@ async def lock_note_materialization( self.started.set() -class PausingNoteContentRepository(NoteContentRepository): - """Hold the materializer after NoteContent is updated but before Entity flush.""" - - def __init__(self, project_id: int) -> None: - super().__init__(project_id) - self.update_applied = asyncio.Event() - self.release_update = asyncio.Event() - - @override - async def update_state_fields( - self, - session: AsyncSession, - entity_id: int, - *, - expected_db_version: int | None = None, - **updates: Any, - ) -> NoteContent | None: - note_content = await super().update_state_fields( - session, - entity_id, - expected_db_version=expected_db_version, - **updates, - ) - self.update_applied.set() - await self.release_update.wait() - return note_content - - @pytest.mark.asyncio @pytest.mark.parametrize("move_during_wait", [False, True], ids=["stable-path", "moved-path"]) -async def test_materialization_waits_for_entity_before_updating_note_content( +async def test_materialization_and_accepted_mutation_share_note_content_first_order( engine_factory, test_project: Project, move_during_wait: bool, ) -> None: - """An Entity-first reconciliation transaction must not deadlock materialization. - - Before the fix, the publisher reaches ``update_applied`` while this test owns - the Entity row. An Entity-first reconciliation update would then wait on the - publisher's NoteContent lock while the publisher waits on Entity: the exact - deadlock cycle reported in #1187. - """ + """An accepted mutation can take Entity while materialization waits on NoteContent.""" engine, session_maker = engine_factory if engine.dialect.name != "postgresql": pytest.skip("row-lock ordering requires PostgreSQL") @@ -140,21 +109,20 @@ async def test_materialization_waits_for_entity_before_updating_note_content( file_updated_at=datetime(2026, 8, 5, 1, 1, tzinfo=UTC), ) session_lock = StartedMaterializationLock() - note_content_repository = PausingNoteContentRepository(test_project.id) publisher = RepositoryNoteMaterializationPublisher( session_maker=session_maker, session_lock=session_lock, - note_content_store=lambda _project_id: note_content_repository, ) publish_task: asyncio.Task[Any] | None = None - async with session_maker() as reconciliation_session: - await reconciliation_session.begin() + async with session_maker() as mutation_session: + await mutation_session.begin() try: - locked_entity = await reconciliation_session.scalar( - select(Entity).where(Entity.id == entity_id).with_for_update() + await lock_accepted_note_content_for_entity_mutation( + mutation_session, + project_id=test_project.id, + entity_id=entity_id, ) - assert locked_entity is not None publish_task = asyncio.create_task( publisher.publish_written_file_state( @@ -164,34 +132,38 @@ async def test_materialization_waits_for_entity_before_updating_note_content( ) ) await asyncio.wait_for(session_lock.started.wait(), timeout=2) - - with pytest.raises(TimeoutError): - await asyncio.wait_for( - note_content_repository.update_applied.wait(), - timeout=0.5, - ) + # Let PostgreSQL enqueue the materializer's NoteContent lock. With + # the old Entity-first order, the next lock request closes a cycle. + await asyncio.sleep(0.1) + + locked_entity = await asyncio.wait_for( + mutation_session.scalar( + select(Entity).where(Entity.id == entity_id).with_for_update() + ), + timeout=2, + ) + assert locked_entity is not None if move_during_wait: locked_entity.file_path = "notes/moved-during-materialization.md" - await reconciliation_session.execute( + await mutation_session.execute( update(NoteContent) .where(NoteContent.entity_id == entity_id) .values(file_path=locked_entity.file_path) ) else: - await reconciliation_session.execute( + locked_entity.title = "Accepted mutation completed" + await mutation_session.execute( update(NoteContent) .where(NoteContent.entity_id == entity_id) - .values(last_materialization_error="entity-first reconciliation") + .values(last_materialization_error="accepted mutation overlap") ) - await reconciliation_session.commit() + await mutation_session.commit() - note_content_repository.release_update.set() result = await asyncio.wait_for(publish_task, timeout=2) finally: - note_content_repository.release_update.set() - if reconciliation_session.in_transaction(): - await reconciliation_session.rollback() + if mutation_session.in_transaction(): + await mutation_session.rollback() if publish_task is not None and not publish_task.done(): publish_task.cancel() with suppress(asyncio.CancelledError): @@ -213,6 +185,7 @@ async def test_materialization_waits_for_entity_before_updating_note_content( assert entity.mtime != written_file.file_updated_at.timestamp() else: assert result.status is RuntimeNoteMaterializationStatus.written + assert entity.title == "Accepted mutation completed" assert note_content.file_version == 1 assert note_content.file_checksum == "materialized-checksum" assert note_content.file_write_status == "synced" diff --git a/test-int/test_relation_generation_concurrency.py b/test-int/test_relation_generation_concurrency.py new file mode 100644 index 000000000..9a4465024 --- /dev/null +++ b/test-int/test_relation_generation_concurrency.py @@ -0,0 +1,563 @@ +"""PostgreSQL concurrency coverage for generation-versioned relation persistence.""" + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from pathlib import Path +from typing import cast + +import pytest + +from basic_memory import db +from basic_memory.index.local_dependencies import build_local_markdown_file_indexer +from basic_memory.indexing.accepted_note_write_runner import ( + lock_accepted_note_content_for_delete, + lock_accepted_note_content_for_entity_mutation, +) +from basic_memory.indexing.batch_indexer import BatchIndexer +from basic_memory.indexing.directory_delete_runner import ( + RepositoryDirectoryDeleteAcceptanceStore, +) +from basic_memory.indexing.index_batch_runtime import build_default_index_batch_runtime +from basic_memory.indexing.models import IndexInputFile +from basic_memory.markdown import EntityParser, MarkdownProcessor +from basic_memory.models import Entity, NoteContent +from basic_memory.repository import ( + AcceptedNoteContentWrite, + EntityRepository, + NoteContentRepository, + ObservationRepository, + RelationRepository, +) +from basic_memory.repository.relation_repository import ( + AcceptedRelationWrite, + current_relation_generation_statement, +) +from basic_memory.services import EntityService, FileService +from basic_memory.services.link_resolver import LinkResolver + + +async def write_note(path: Path, *, title: str, target: str | None, version: str) -> None: + """Write one deterministic note generation for the concurrency exercise.""" + relation = f"\n- links_to [[{target}]]\n" if target is not None else "" + content = f"---\ntitle: {title}\ntype: note\n---\n\n# {title}\n\n{version}\n{relation}" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +async def load_input(file_service, path: str) -> IndexInputFile: + """Load the current file bytes through the production storage adapter.""" + metadata = await file_service.get_file_metadata(path) + return IndexInputFile( + path=path, + size=metadata.size, + checksum=await file_service.compute_checksum(path), + content_type=file_service.content_type(path), + last_modified=metadata.modified_at, + created_at=metadata.created_at, + content=await file_service.read_file_bytes(path), + ) + + +class EntityUpdateRendezvous: + """Release two source-entity transactions only after both hold their row lock.""" + + def __init__(self) -> None: + self.arrivals = 0 + self.ready = asyncio.Event() + + async def wait(self) -> None: + self.arrivals += 1 + if self.arrivals == 2: + self.ready.set() + await asyncio.wait_for(self.ready.wait(), timeout=5) + + +@pytest.mark.asyncio +async def test_relation_publication_and_directory_delete_share_note_content_first_lock_order( + engine_factory, + test_project, +) -> None: + """A bulk directory delete never holds Entity while waiting on publication's source.""" + engine, session_maker = engine_factory + if engine.dialect.name != "postgresql": + pytest.skip("row-lock deadlock regression requires PostgreSQL") + + now = datetime.now(tz=UTC) + async with db.scoped_session(session_maker) as session: + entities: list[Entity] = [] + for title in ("Directory sibling", "Directory source"): + entity = Entity( + project_id=test_project.id, + title=title, + note_type="note", + permalink=f"directory-race/{title.lower().replace(' ', '-')}", + file_path=f"directory-race/{title.lower().replace(' ', '-')}.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + session.add(entity) + await session.flush() + session.add( + NoteContent( + entity_id=entity.id, + project_id=test_project.id, + external_id=entity.external_id, + file_path=entity.file_path, + markdown_content=f"# {title}\n", + db_version=1, + db_checksum=f"generation-{entity.id}", + file_write_status="synced", + ) + ) + entities.append(entity) + sibling_id, source_id = (entity.id for entity in entities) + + source_locked = asyncio.Event() + directory_snapshot_loaded = asyncio.Event() + relation_repository = RelationRepository(project_id=test_project.id) + store = RepositoryDirectoryDeleteAcceptanceStore() + + async def publish_relation() -> None: + async with session_maker() as session: + async with session.begin(): + locked_entity_id = await session.scalar( + current_relation_generation_statement( + project_id=test_project.id, + entity_id=source_id, + generation=1, + ) + ) + assert locked_entity_id == source_id + source_locked.set() + await asyncio.wait_for(directory_snapshot_loaded.wait(), timeout=5) + result = await relation_repository.upsert_relation_generation( + session, + entity_id=source_id, + generation=1, + relations=[AcceptedRelationWrite("links_to", "Target", None)], + ) + assert result.generation_is_current + + async def delete_directory() -> None: + await asyncio.wait_for(source_locked.wait(), timeout=5) + async with session_maker() as session: + async with session.begin(): + snapshots = await store.load_directory_file_snapshots( + session, + project_id=test_project.id, + directory="directory-race", + ) + assert {snapshot.entity_id for snapshot in snapshots} == {sibling_id, source_id} + directory_snapshot_loaded.set() + await store.delete_directory_entities( + session, + project_id=test_project.id, + directory="directory-race", + entity_ids=[source_id, sibling_id], + ) + + await asyncio.wait_for( + asyncio.gather(publish_relation(), delete_directory()), + timeout=10, + ) + + async with db.scoped_session(session_maker) as session: + assert await session.get(Entity, sibling_id) is None + assert await session.get(Entity, source_id) is None + assert await relation_repository.find_all(session) == [] + + +@pytest.mark.asyncio +async def test_relation_publication_and_delete_share_note_content_first_lock_order( + engine_factory, + test_project, +) -> None: + """A same-note publisher and delete complete without a NoteContent/Entity lock cycle.""" + engine, session_maker = engine_factory + if engine.dialect.name != "postgresql": + pytest.skip("row-lock deadlock regression requires PostgreSQL") + + now = datetime.now(tz=UTC) + async with db.scoped_session(session_maker) as session: + source_entity = Entity( + project_id=test_project.id, + title="Delete race source", + note_type="note", + permalink="delete-race/source", + file_path="delete-race/source.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + session.add(source_entity) + await session.flush() + source_entity_id = source_entity.id + session.add( + NoteContent( + entity_id=source_entity_id, + project_id=test_project.id, + external_id=source_entity.external_id, + file_path=source_entity.file_path, + markdown_content="# Source\n", + db_version=1, + db_checksum="generation-1", + file_write_status="synced", + ) + ) + + source_locked = asyncio.Event() + delete_waiting = asyncio.Event() + relation_repository = RelationRepository(project_id=test_project.id) + + async def publish_relation() -> None: + async with session_maker() as session: + async with session.begin(): + locked_entity_id = await session.scalar( + current_relation_generation_statement( + project_id=test_project.id, + entity_id=source_entity_id, + generation=1, + ) + ) + assert locked_entity_id == source_entity_id + source_locked.set() + await asyncio.wait_for(delete_waiting.wait(), timeout=5) + result = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity_id, + generation=1, + relations=[AcceptedRelationWrite("links_to", "Target", None)], + ) + assert result.generation_is_current + + async def delete_source() -> None: + await asyncio.wait_for(source_locked.wait(), timeout=5) + async with session_maker() as session: + async with session.begin(): + entity = await session.get(Entity, source_entity_id) + assert entity is not None + lock_task = asyncio.create_task( + lock_accepted_note_content_for_delete( + session, + project_id=test_project.id, + entity_id=source_entity_id, + ) + ) + # Let PostgreSQL enqueue the NoteContent lock before the publisher + # requests its Entity foreign-key lock, proving the shared order. + await asyncio.sleep(0) + delete_waiting.set() + await lock_task + await session.delete(entity) + + await asyncio.wait_for( + asyncio.gather(publish_relation(), delete_source()), + timeout=10, + ) + + async with db.scoped_session(session_maker) as session: + assert await session.get(Entity, source_entity_id) is None + assert await session.get(NoteContent, source_entity_id) is None + assert await relation_repository.find_all(session) == [] + + +@pytest.mark.asyncio +async def test_relation_publication_and_update_share_note_content_first_lock_order( + engine_factory, + test_project, +) -> None: + """A same-note publisher and update complete without a NoteContent/Entity lock cycle.""" + engine, session_maker = engine_factory + if engine.dialect.name != "postgresql": + pytest.skip("row-lock deadlock regression requires PostgreSQL") + + now = datetime.now(tz=UTC) + async with db.scoped_session(session_maker) as session: + source_entity = Entity( + project_id=test_project.id, + title="Update race source", + note_type="note", + permalink="update-race/source", + file_path="update-race/source.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + session.add(source_entity) + await session.flush() + source_entity_id = source_entity.id + session.add( + NoteContent( + entity_id=source_entity_id, + project_id=test_project.id, + external_id=source_entity.external_id, + file_path=source_entity.file_path, + markdown_content="# Generation 1\n", + db_version=1, + db_checksum="generation-1", + file_write_status="synced", + ) + ) + + source_locked = asyncio.Event() + update_waiting = asyncio.Event() + relation_repository = RelationRepository(project_id=test_project.id) + note_content_repository = NoteContentRepository(project_id=test_project.id) + + async def publish_relation() -> None: + async with session_maker() as session: + async with session.begin(): + locked_entity_id = await session.scalar( + current_relation_generation_statement( + project_id=test_project.id, + entity_id=source_entity_id, + generation=1, + ) + ) + assert locked_entity_id == source_entity_id + source_locked.set() + await asyncio.wait_for(update_waiting.wait(), timeout=5) + result = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity_id, + generation=1, + relations=[AcceptedRelationWrite("links_to", "Target", None)], + ) + assert result.generation_is_current + + async def update_source() -> None: + await asyncio.wait_for(source_locked.wait(), timeout=5) + async with session_maker() as session: + async with session.begin(): + entity = await session.get(Entity, source_entity_id) + assert entity is not None + lock_task = asyncio.create_task( + lock_accepted_note_content_for_entity_mutation( + session, + project_id=test_project.id, + entity_id=source_entity_id, + ) + ) + # The accepted writer queues on NoteContent before changing Entity, + # so the publisher can take its source foreign-key lock and commit. + await asyncio.sleep(0) + update_waiting.set() + await lock_task + entity.title = "Updated without deadlock" + await session.flush() + await note_content_repository.accept_write( + session, + AcceptedNoteContentWrite( + entity_id=source_entity_id, + markdown_content="# Generation 2\n", + db_version=2, + db_checksum="generation-2", + last_source="test", + updated_at=datetime.now(tz=UTC), + ), + ) + + await asyncio.wait_for( + asyncio.gather(publish_relation(), update_source()), + timeout=10, + ) + + async with db.scoped_session(session_maker) as session: + entity = await session.get(Entity, source_entity_id) + note_content = await session.get(NoteContent, source_entity_id) + assert entity is not None + assert entity.title == "Updated without deadlock" + assert note_content is not None + assert note_content.db_version == 2 + + +@pytest.mark.asyncio +async def test_mutual_relation_generations_complete_under_concurrent_persistence( + engine_factory, + app_config, + search_service, + project_config, + test_project, + monkeypatch, +) -> None: + """Mutual batch and single-file updates finish without the old lock cycle.""" + engine, session_maker = engine_factory + if engine.dialect.name != "postgresql": + pytest.skip("row-lock deadlock regression requires PostgreSQL") + + app_config.disable_permalinks = True + entity_repository = EntityRepository(project_id=test_project.id) + observation_repository = ObservationRepository(project_id=test_project.id) + relation_repository = RelationRepository(project_id=test_project.id) + entity_parser = EntityParser(project_config.home) + file_service = FileService( + project_config.home, + MarkdownProcessor(entity_parser, app_config=app_config), + app_config=app_config, + ) + entity_service = EntityService( + entity_parser=entity_parser, + entity_repository=entity_repository, + observation_repository=observation_repository, + relation_repository=relation_repository, + file_service=file_service, + link_resolver=LinkResolver( + entity_repository, + search_service, + session_maker=session_maker, + app_config=app_config, + ), + session_maker=session_maker, + search_service=search_service, + app_config=app_config, + ) + alpha_path = "notes/alpha.md" + beta_path = "notes/beta.md" + absolute_alpha_path = project_config.home / alpha_path + absolute_beta_path = project_config.home / beta_path + + runtime = build_default_index_batch_runtime( + project_id=test_project.id, + app_config=app_config, + entity_service=entity_service, + entity_repository=entity_repository, + relation_repository=relation_repository, + search_writer=search_service, + frontmatter_storage=file_service, + content_type_provider=file_service, + session_maker=session_maker, + ) + batch_indexer = cast(BatchIndexer, runtime.batch_indexer) + local_indexer = build_local_markdown_file_indexer( + project_id=test_project.id, + file_service=file_service, + session_maker=session_maker, + entity_repository=entity_repository, + batch_indexer=batch_indexer, + search_service=search_service, + ) + + # Seed committed targets so the historical delete/insert path would resolve + # both mutual links while each transaction still held its own Entity row lock. + await write_note(absolute_alpha_path, title="Alpha", target=None, version="Seed alpha") + await write_note(absolute_beta_path, title="Beta", target=None, version="Seed beta") + seed_result = await runtime.index_loaded_files( + { + alpha_path: await load_input(file_service, alpha_path), + beta_path: await load_input(file_service, beta_path), + }, + max_concurrent=2, + ) + assert seed_result.errors == [] + + original_update = entity_service.update_entity_and_observations + + # Batch direction: both persistence transactions hold their source Entity lock + # before continuing. The old path then inserted A -> B and B -> A in those same + # transactions, creating the production FK lock cycle. Relation publication now + # begins only after both short entity transactions commit. + batch_rendezvous = EntityUpdateRendezvous() + + async def synchronized_batch_update(*args, **kwargs): + entity = await original_update(*args, **kwargs) + await batch_rendezvous.wait() + return entity + + monkeypatch.setattr( + entity_service, + "update_entity_and_observations", + synchronized_batch_update, + ) + await write_note( + absolute_alpha_path, + title="Alpha", + target="Beta", + version="Batch generation", + ) + await write_note( + absolute_beta_path, + title="Beta", + target="Alpha", + version="Batch generation", + ) + batch_result = await asyncio.wait_for( + runtime.index_loaded_files( + { + alpha_path: await load_input(file_service, alpha_path), + beta_path: await load_input(file_service, beta_path), + }, + max_concurrent=2, + ), + timeout=15, + ) + assert batch_rendezvous.arrivals == 2 + assert batch_result.errors == [] + assert batch_result.relations_unresolved == 0 + + # Single-file direction: two independent retry loops take the same source-lock + # rendezvous, then claim and publish their own next generations concurrently. + single_file_rendezvous = EntityUpdateRendezvous() + + async def synchronized_single_file_update(*args, **kwargs): + entity = await original_update(*args, **kwargs) + await single_file_rendezvous.wait() + return entity + + monkeypatch.setattr( + entity_service, + "update_entity_and_observations", + synchronized_single_file_update, + ) + await write_note( + absolute_alpha_path, + title="Alpha", + target="Beta", + version="Single-file generation", + ) + await write_note( + absolute_beta_path, + title="Beta", + target="Alpha", + version="Single-file generation", + ) + single_file_results = await asyncio.wait_for( + asyncio.gather( + local_indexer.index_markdown_file(alpha_path, source="test"), + local_indexer.index_markdown_file(beta_path, source="test"), + ), + timeout=15, + ) + assert single_file_rendezvous.arrivals == 2 + assert {result.file_path for result in single_file_results} == {alpha_path, beta_path} + + async with db.scoped_session(session_maker) as session: + alpha = await entity_repository.get_by_file_path(session, alpha_path) + beta = await entity_repository.get_by_file_path(session, beta_path) + assert alpha is not None + assert beta is not None + note_contents = await NoteContentRepository(project_id=test_project.id).find_by_ids( + session, + [alpha.id, beta.id], + ) + relations = await relation_repository.find_all(session) + + generation_by_entity_id = { + note_content.entity_id: note_content.db_version for note_content in note_contents + } + assert generation_by_entity_id == {alpha.id: 3, beta.id: 3} + assert { + ( + relation.from_id, + relation.to_id, + relation.to_name, + relation.relation_type, + relation.generation, + ) + for relation in relations + } == { + (alpha.id, beta.id, "Beta", "links_to", generation_by_entity_id[alpha.id]), + (beta.id, alpha.id, "Alpha", "links_to", generation_by_entity_id[beta.id]), + } diff --git a/tests/cloud/test_cloud_services.py b/tests/cloud/test_cloud_services.py index 4c08d8707..6dfebdfc2 100644 --- a/tests/cloud/test_cloud_services.py +++ b/tests/cloud/test_cloud_services.py @@ -17,9 +17,12 @@ AcceptedNoteMutationRejectKind, AcceptedNoteMutationRejected, AcceptedNoteMutationRejection, + AcceptedNoteMutationResult, AcceptedNoteUpdateMutation, ) +from basic_memory.indexing.relation_persistence import RelationGenerationPublication from basic_memory.indexing.directory_delete_runner import ( + DirectoryEntityDeleteResult, DirectoryDeleteRejectKind, DirectoryDeleteRuntime, ) @@ -107,13 +110,15 @@ async def delete_directory_entities( session: AsyncSession, *, project_id: int, + directory: str, entity_ids, - ) -> frozenset[int]: + ) -> DirectoryEntityDeleteResult: assert session is not None assert project_id == 3 + assert directory == "notes" assert list(entity_ids) == [7] # No surviving relation sources point into this directory in the fixture. - return frozenset() + return DirectoryEntityDeleteResult(deleted_entity_ids=frozenset({7})) class FakeDirectoryFileDeleteEnqueuer: @@ -442,6 +447,7 @@ async def test_note_content_mutation_service_delegates_create_to_core_runner(mon user_profile_id = uuid4() data = EntitySchema(title="Created", directory="notes", content="# Created") returned = SimpleNamespace(status_code=201, payload={"ok": True}) + mutation_result = AcceptedNoteMutationResult(change=cast(Any, returned)) calls: list[tuple[AsyncSession, AcceptedNoteCreateMutation, object]] = [] async def fake_runner( @@ -451,7 +457,7 @@ async def fake_runner( dependencies: AcceptedNoteMutationDependencies, ): calls.append((repository_session, request, dependencies)) - return returned + return mutation_result monkeypatch.setattr(note_content_writes, "run_accepted_note_create", fake_runner) @@ -481,6 +487,167 @@ async def fake_runner( assert received_dependencies is dependencies +@pytest.mark.asyncio +async def test_note_content_mutation_service_publishes_relations_after_commit(monkeypatch) -> None: + events: list[str] = [] + returned = cast(Any, SimpleNamespace(status_code=201, payload={"ok": True})) + publication = RelationGenerationPublication( + project_id=7, + entity_id=42, + generation=3, + relations=(), + ) + + class RecordingTransaction: + async def __aenter__(self): + events.append("transaction_enter") + return self + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> None: + events.append("transaction_exit") + + class RecordingSession: + async def __aenter__(self): + events.append("session_enter") + return self + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> None: + events.append("session_exit") + + def begin(self) -> RecordingTransaction: + return RecordingTransaction() + + class RecordingSessionMaker: + def __call__(self) -> RecordingSession: + return RecordingSession() + + relation_repository = object() + + class WriteRepositories: + def relation_repository(self, project_id: int) -> object: + assert project_id == publication.project_id + events.append("repository") + return relation_repository + + dependencies = cast( + AcceptedNoteMutationDependencies, + SimpleNamespace(write_repositories=WriteRepositories()), + ) + + async def fake_runner( + repository_session: AsyncSession, + *, + request: AcceptedNoteCreateMutation, + dependencies: AcceptedNoteMutationDependencies, + ) -> AcceptedNoteMutationResult: + _ = repository_session, request, dependencies + events.append("runner") + return AcceptedNoteMutationResult( + change=returned, + relation_publication=publication, + ) + + class RecordingPublisher: + def __init__(self, *, relation_repository: object, session_maker: object) -> None: + assert relation_repository is not None + assert session_maker is not None + + async def publish( + self, + *, + entity_id: int, + generation: int, + relations: object, + ) -> bool: + assert entity_id == publication.entity_id + assert generation == publication.generation + assert relations == publication.relations + events.append("publish") + return True + + monkeypatch.setattr(note_content_writes, "run_accepted_note_create", fake_runner) + monkeypatch.setattr(note_content_writes, "RelationGenerationPublisher", RecordingPublisher) + + service = NoteContentMutationService( + session_maker=cast(async_sessionmaker[AsyncSession], RecordingSessionMaker()), + mutation_dependencies=dependencies, + ) + + accepted = await service.create_note( + project_external_id="project-123", + data=EntitySchema(title="Created", directory="notes", content="# Created"), + user_profile_id=None, + source="api", + ) + + assert accepted is returned + assert events == [ + "session_enter", + "transaction_enter", + "runner", + "transaction_exit", + "session_exit", + "repository", + "publish", + ] + + +@pytest.mark.asyncio +async def test_note_content_mutation_service_continues_after_relation_publication_failure( + monkeypatch, +) -> None: + """Derived graph failure cannot suppress the committed change's materialization.""" + returned = cast(Any, SimpleNamespace(status_code=201, payload={"ok": True})) + publication = RelationGenerationPublication( + project_id=7, + entity_id=42, + generation=3, + relations=(), + ) + + class WriteRepositories: + def relation_repository(self, project_id: int) -> object: + assert project_id == publication.project_id + return object() + + dependencies = cast( + AcceptedNoteMutationDependencies, + SimpleNamespace(write_repositories=WriteRepositories()), + ) + + async def fake_runner(*args, **kwargs) -> AcceptedNoteMutationResult: + _ = args, kwargs + return AcceptedNoteMutationResult( + change=returned, + relation_publication=publication, + ) + + class FailingPublisher: + def __init__(self, *, relation_repository: object, session_maker: object) -> None: + assert relation_repository is not None + assert session_maker is not None + + async def publish(self, **kwargs: object) -> bool: + assert kwargs["entity_id"] == publication.entity_id + raise RuntimeError("relation publication unavailable") + + monkeypatch.setattr(note_content_writes, "run_accepted_note_create", fake_runner) + monkeypatch.setattr(note_content_writes, "RelationGenerationPublisher", FailingPublisher) + service = NoteContentMutationService( + session_maker=cast(async_sessionmaker[AsyncSession], FakeSessionMaker()), + mutation_dependencies=dependencies, + ) + + accepted = await service.create_note( + project_external_id="project-123", + data=EntitySchema(title="Created", directory="notes", content="# Created"), + user_profile_id=None, + source="api", + ) + + assert accepted is returned + + @pytest.mark.asyncio async def test_note_content_mutation_service_uses_injected_actor_resolver(monkeypatch) -> None: """A runtime adapter can replace route-passed actor values with its own @@ -499,7 +666,9 @@ async def fake_runner( dependencies: AcceptedNoteMutationDependencies, ): calls.append(request) - return SimpleNamespace(status_code=201, payload={"ok": True}) + return AcceptedNoteMutationResult( + change=cast(Any, SimpleNamespace(status_code=201, payload={"ok": True})) + ) monkeypatch.setattr(note_content_writes, "run_accepted_note_create", fake_runner) @@ -564,6 +733,7 @@ async def test_note_content_mutation_service_delegates_remaining_methods_to_core expected_replacements=1, ) returned = SimpleNamespace(status_code=200, payload={"ok": True}) + mutation_result = AcceptedNoteMutationResult(change=cast(Any, returned)) calls: list[tuple[str, AsyncSession, object, object]] = [] def runner(name: str): @@ -574,7 +744,7 @@ async def fake_runner( dependencies: object, ): calls.append((name, repository_session, request, dependencies)) - return returned + return mutation_result return fake_runner @@ -802,14 +972,19 @@ async def delete_directory_entities( session: AsyncSession, *, project_id: int, + directory: str, entity_ids, - ) -> frozenset[int]: + ) -> DirectoryEntityDeleteResult: await super().delete_directory_entities( session, project_id=project_id, + directory=directory, entity_ids=entity_ids, ) - return frozenset({99, 42}) + return DirectoryEntityDeleteResult( + deleted_entity_ids=frozenset(entity_ids), + relation_cleanup_entity_ids=frozenset({99, 42}), + ) refresher = RecordingRelationCleanupRefresher() service = DirectoryDeleteService( diff --git a/tests/conftest.py b/tests/conftest.py index 75f5b31f5..7d92cd778 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,12 +32,14 @@ from basic_memory.db import DatabaseType from basic_memory.index.local_project import LocalProjectIndexRunner from basic_memory.index.watch_service import WatchService +from basic_memory.indexing.relation_resolution import RepositoryRelationResolutionRuntime from basic_memory.markdown import EntityParser from basic_memory.markdown.markdown_processor import MarkdownProcessor from basic_memory.models import Base from basic_memory.models.knowledge import Entity from basic_memory.models.project import Project from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.note_content_repository import NoteContentRepository from basic_memory.repository.observation_repository import ObservationRepository from basic_memory.repository.project_repository import ProjectRepository from basic_memory.repository.relation_repository import RelationRepository @@ -47,6 +49,7 @@ ProjectService, ) from basic_memory.services.directory_service import DirectoryService +from basic_memory.services.bulk_link_resolver import BulkLinkResolver from basic_memory.services.file_service import FileService from basic_memory.services.link_resolver import LinkResolver from basic_memory.services.search_service import SearchService @@ -660,6 +663,9 @@ async def test_graph( file_service, entity_service, session_maker, + app_config, + project_repository, + test_project, ): """Create a test knowledge graph with entities, relations and observations.""" @@ -726,6 +732,20 @@ async def test_graph( ) ) + relation_resolution = RepositoryRelationResolutionRuntime( + session_maker=session_maker, + relation_repository=relation_repository, + entity_repository=entity_repository, + note_content_repository=NoteContentRepository(project_id=test_project.id), + target_resolver=BulkLinkResolver( + entity_repository=entity_repository, + app_config=app_config, + project_repository=project_repository, + ), + entity_indexer=search_service, + ) + await relation_resolution.resolve_relations() + # get latest async with db.scoped_session(session_maker) as session: entities = await entity_repository.find_all(session) diff --git a/tests/db/test_memory_db_session_isolation.py b/tests/db/test_memory_db_session_isolation.py index 54afe099b..0a7f56d29 100644 --- a/tests/db/test_memory_db_session_isolation.py +++ b/tests/db/test_memory_db_session_isolation.py @@ -59,7 +59,7 @@ async def test_concurrent_session_rollback_does_not_destroy_uncommitted_writes() write_in_flight = asyncio.Event() async def writer() -> None: - # Mirrors RelationRepository.add_all_ignore_duplicates: INSERT executed, + # Mirrors relation-generation publication: INSERT executed, # commit only happens at scoped_session exit several awaits later. async with db.scoped_session(session_maker) as session: await session.execute( diff --git a/tests/index/test_local_markdown_file_indexer.py b/tests/index/test_local_markdown_file_indexer.py index 238758614..8c32129de 100644 --- a/tests/index/test_local_markdown_file_indexer.py +++ b/tests/index/test_local_markdown_file_indexer.py @@ -17,6 +17,7 @@ from basic_memory.indexing.batch_indexer import BatchIndexer from basic_memory.indexing.file_indexer import IndexMarkdownNoteContentReconciler from basic_memory.indexing.models import IndexEntitySearchWriter, SyncedMarkdownFile +from basic_memory.indexing.note_content_reconciliation import NoteContentReconciliationResult from basic_memory.services import FileService @@ -61,12 +62,21 @@ async def test_local_file_indexer_logs_brace_path_through_retry( entity_repository.get_by_file_path = AsyncMock(return_value=entity) note_content_reconciler = Mock() note_content_reconciler.capture_anchor = AsyncMock(return_value=None) - note_content_reconciler.reconcile = AsyncMock(side_effect=["stale", "current"]) + note_content_reconciler.reconcile = AsyncMock( + side_effect=[ + NoteContentReconciliationResult.stale(), + NoteContentReconciliationResult.current(3), + ] + ) + batch_indexer = Mock() + batch_indexer.publish_relation_generation = AsyncMock(return_value=True) + batch_indexer.resolve_relation_targets = AsyncMock() + batch_indexer.refresh_indexed_entity_search = AsyncMock() indexer = LocalMarkdownFileIndexer( file_service=cast(FileService, Mock()), session_maker=cast(async_sessionmaker[AsyncSession], _FakeSession), entity_repository=cast(LocalIndexEntityRepository, entity_repository), - batch_indexer=cast(BatchIndexer, Mock()), + batch_indexer=cast(BatchIndexer, batch_indexer), search_service=cast(IndexEntitySearchWriter, Mock()), note_content_reconciler=cast( IndexMarkdownNoteContentReconciler, @@ -91,7 +101,7 @@ async def test_local_file_indexer_logs_brace_path_through_retry( logger.remove(sink_id) assert note_content_reconciler.reconcile.await_count == 2 - assert f"Retrying markdown index after concurrent accepted note write: {file_path}" in ( + assert f"Retrying markdown index without a current relation generation: {file_path}" in ( rendered_messages ) assert f"Indexed markdown file: {file_path}" in rendered_messages diff --git a/tests/index/test_local_project_index.py b/tests/index/test_local_project_index.py index fae34ed59..d7f2aec6a 100644 --- a/tests/index/test_local_project_index.py +++ b/tests/index/test_local_project_index.py @@ -42,6 +42,7 @@ IndexFileJobStatus, IndexInputFile, IndexedEntity, + IndexedRelation, IndexingBatchResult, ) from basic_memory.indexing.project_index_coordinator import ( @@ -58,6 +59,7 @@ ResolvedRelationTarget, UnresolvedRelation, ) +from basic_memory.indexing.relation_persistence import RelationGenerationPublisher from basic_memory.models import Entity, Project, Relation from basic_memory.repository import EntityRepository from basic_memory.repository.note_content_repository import ( @@ -1722,6 +1724,7 @@ async def test_local_relation_resolution_refreshes_pending_source_without_markdo test_project: Project, project_config, entity_repository, + relation_repository, session_maker: async_sessionmaker[AsyncSession], search_service, config_manager, @@ -1780,12 +1783,13 @@ async def test_local_relation_resolution_refreshes_pending_source_without_markdo "updated_at": accepted_at, }, ) + accepted_generation = current_note_content.db_version + 1 await note_content_repository.accept_write( session, AcceptedNoteContentWrite( entity_id=source.id, markdown_content=accepted_markdown, - db_version=current_note_content.db_version + 1, + db_version=accepted_generation, db_checksum=accepted_checksum, last_source="test", updated_at=accepted_at, @@ -1794,6 +1798,22 @@ async def test_local_relation_resolution_refreshes_pending_source_without_markdo source_id = source.id target_id = target.id + generation_is_current = await RelationGenerationPublisher( + relation_repository=relation_repository, + session_maker=session_maker, + ).publish( + entity_id=source_id, + generation=accepted_generation, + relations=( + IndexedRelation( + relation_type="relates_to", + target_name="Pending Target", + context=None, + ), + ), + ) + assert generation_is_current + # The accepted database state is durable, but its Markdown projection is # intentionally absent when relation resolution refreshes the source. source_path.unlink() diff --git a/tests/indexing/test_accepted_note_mutation_runner.py b/tests/indexing/test_accepted_note_mutation_runner.py index 1320e6c51..157fe3767 100644 --- a/tests/indexing/test_accepted_note_mutation_runner.py +++ b/tests/indexing/test_accepted_note_mutation_runner.py @@ -46,6 +46,7 @@ AcceptedObservationWrite, AcceptedRelationWrite, ) +from basic_memory.repository.relation_repository import RelationGenerationWriteResult from basic_memory.repository.entity_repository import AcceptedPendingEntityWrite from basic_memory.runtime.note_content import RuntimeAcceptedNoteResponse from basic_memory.schemas.base import Entity as EntitySchema @@ -143,6 +144,7 @@ def __init__(self) -> None: self.deleted: list[object] = [] self.added: list[object] = [] self.flush_count = 0 + self.scalar_count = 0 self.bind = SimpleNamespace(dialect=SimpleNamespace(name="sqlite")) async def delete(self, value: object) -> None: @@ -155,6 +157,11 @@ async def execute(self, query: object) -> _EmptyResult: # No existing note_file_vacate marker for this test; the move records a fresh one via add(). return _EmptyResult() + async def scalar(self, statement: object) -> int: + assert statement is not None + self.scalar_count += 1 + return 42 + async def flush(self) -> None: self.flush_count += 1 @@ -195,7 +202,7 @@ def __init__( AsyncSession | None, ] ] = [] - self.move_calls: list[tuple[Entity, str, str, AsyncSession | None]] = [] + self.move_calls: list[tuple[Entity, str, str, bool, AsyncSession | None]] = [] self.self_relation_calls: list[tuple[str, Entity, AsyncSession | None]] = [] async def prepare_create_entity_content( @@ -266,9 +273,12 @@ async def prepare_move_entity_content( current_content: str, destination_path: str, *, + should_update_permalink: bool, session: AsyncSession | None = None, ) -> PreparedEntityMove: - self.move_calls.append((entity, current_content, destination_path, session)) + self.move_calls.append( + (entity, current_content, destination_path, should_update_permalink, session) + ) return self.prepared_move async def verify_move_destination_absent( @@ -481,14 +491,37 @@ class _RelationRepository: def __init__(self) -> None: self.calls: list[tuple[int, Sequence[AcceptedRelationWrite]]] = [] - async def replace_accepted_outgoing_relations( + async def begin_relation_generation_publication( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + raise AssertionError( + "relation publication was not expected inside the accepted transaction" + ) + + async def upsert_relation_generation( self, session: AsyncSession, + *, entity_id: int, + generation: int, relations: Sequence[AcceptedRelationWrite], - ) -> None: - _ = session - self.calls.append((entity_id, list(relations))) + ) -> RelationGenerationWriteResult: + raise AssertionError( + "relation publication was not expected inside the accepted transaction" + ) + + async def cleanup_relation_generations( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + raise AssertionError("relation cleanup was not expected inside the accepted transaction") @dataclass(frozen=True, slots=True) @@ -678,7 +711,7 @@ async def test_run_accepted_note_create_persists_prepared_markdown( note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_create( + result = await run_accepted_note_create( session, request=AcceptedNoteCreateMutation( project_external_id="project-123", @@ -701,6 +734,7 @@ async def test_run_accepted_note_create_persists_prepared_markdown( ), ) + change = result.change assert project_repository.calls == [(session, "project-123")] assert entity_lookup_repository.file_path_calls == [(session, "notes/Accepted.md", False)] assert preparer_factory.projects == [project] @@ -723,6 +757,8 @@ async def test_run_accepted_note_create_persists_prepared_markdown( assert change.materialization.actor_kind == "user" assert change.materialization.actor_name == "Ada" assert change.materialization.previous_file_path is None + assert result.relation_publication is not None + assert result.relation_publication.generation == 1 assert persistence_calls[0].await_count == 1 assert persistence_calls[1].await_count == 0 @@ -777,7 +813,7 @@ async def test_run_accepted_note_create_allows_equivalent_non_markdown_resource_ filename_conflicts=["notes/accepted.png"], ) - change = await run_accepted_note_create( + result = await run_accepted_note_create( session, request=AcceptedNoteCreateMutation( project_external_id="project-123", @@ -796,6 +832,7 @@ async def test_run_accepted_note_create_allows_equivalent_non_markdown_resource_ ), ) + change = result.change assert change.status_code == 201 assert preparer.conflict_calls == [("notes/Accepted.md", False, session)] assert preparer.skip_conflict_checks == [True] @@ -820,7 +857,7 @@ async def test_run_accepted_note_update_replaces_existing_note_content( note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_update( + result = await run_accepted_note_update( cast(AsyncSession, session), request=AcceptedNoteUpdateMutation( project_external_id="project-123", @@ -840,11 +877,13 @@ async def test_run_accepted_note_update_replaces_existing_note_content( ), ) + change = result.change assert entity_lookup_repository.external_id_calls == [ (cast(AsyncSession, session), "note-123", False) ] assert note_content_lookup_repository.calls == [(cast(AsyncSession, session), entity.id)] assert preparer.replace_calls == [(entity, schema, "# Old\n", cast(AsyncSession, session))] + assert session.scalar_count == 1 assert session.flush_count == 1 assert note_content_accept_repository.calls[0][1].db_version == 2 assert note_content_accept_repository.calls[0][1].markdown_content == "# Replacement\n" @@ -856,6 +895,8 @@ async def test_run_accepted_note_update_replaces_existing_note_content( assert change.materialization is not None assert change.materialization.db_version == 2 assert change.materialization.previous_file_path is None + assert result.relation_publication is not None + assert result.relation_publication.generation == 2 assert persistence_calls[0].await_count == 1 assert persistence_calls[1].await_count == 0 @@ -879,7 +920,7 @@ async def test_run_accepted_note_update_accepts_matching_base_checksum() -> None note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_update( + result = await run_accepted_note_update( cast(AsyncSession, session), request=AcceptedNoteUpdateMutation( project_external_id="project-123", @@ -900,6 +941,7 @@ async def test_run_accepted_note_update_accepts_matching_base_checksum() -> None ), ) + change = result.change assert change.status_code == 200 assert note_content_accept_repository.calls[0][1].db_version == 2 assert note_content_accept_repository.calls[0][1].markdown_content == "# Replacement\n" @@ -985,7 +1027,7 @@ async def test_run_accepted_note_update_accepts_relay_self_supersede_on_stale_ba note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_update( + result = await run_accepted_note_update( cast(AsyncSession, session), request=AcceptedNoteUpdateMutation( project_external_id="project-123", @@ -1006,6 +1048,7 @@ async def test_run_accepted_note_update_accepts_relay_self_supersede_on_stale_ba ), ) + change = result.change assert change.status_code == 200 assert note_content_accept_repository.calls[0][1].db_version == 2 assert note_content_accept_repository.calls[0][1].markdown_content == "# Replacement\n" @@ -1034,7 +1077,7 @@ async def test_run_accepted_note_update_relay_supersedes_foreign_head() -> None: note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_update( + result = await run_accepted_note_update( cast(AsyncSession, session), request=AcceptedNoteUpdateMutation( project_external_id="project-123", @@ -1055,6 +1098,7 @@ async def test_run_accepted_note_update_relay_supersedes_foreign_head() -> None: ), ) + change = result.change assert change.status_code == 200 assert note_content_accept_repository.calls[0][1].db_version == 2 @@ -1236,7 +1280,7 @@ async def test_run_accepted_note_update_creates_missing_entity_without_base_chec note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_update( + result = await run_accepted_note_update( cast(AsyncSession, session), request=AcceptedNoteUpdateMutation( project_external_id="project-123", @@ -1256,6 +1300,7 @@ async def test_run_accepted_note_update_creates_missing_entity_without_base_chec ), ) + change = result.change assert change.status_code == 201 assert len(pending_entity_repository.calls) == 1 assert note_content_accept_repository.calls[0][1].db_version == 1 @@ -1380,7 +1425,7 @@ async def test_run_accepted_note_edit_applies_patch_against_db_content( note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_edit( + result = await run_accepted_note_edit( cast(AsyncSession, session), request=AcceptedNoteEditMutation( project_external_id="project-123", @@ -1405,6 +1450,7 @@ async def test_run_accepted_note_edit_applies_patch_against_db_content( ), ) + change = result.change assert preparer.edit_calls == [ ( entity, @@ -1546,7 +1592,7 @@ async def test_run_accepted_note_move_carries_previous_path_and_materialized_cle note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_move( + result = await run_accepted_note_move( cast(AsyncSession, session), request=AcceptedNoteMoveMutation( project_external_id="project-123", @@ -1574,8 +1620,9 @@ async def test_run_accepted_note_move_carries_previous_path_and_materialized_cle ), ) + change = result.change assert preparer.move_calls == [ - (entity, "# Old\n", "archive/accepted.md", cast(AsyncSession, session)) + (entity, "# Old\n", "archive/accepted.md", True, cast(AsyncSession, session)) ] assert entity.file_path == "archive/accepted.md" assert entity.permalink == "archive/accepted" @@ -1594,6 +1641,8 @@ async def test_run_accepted_note_move_carries_previous_path_and_materialized_cle assert preparer_factory.checksum_calls == [(project, "notes/accepted.md")] assert persistence_calls[0].await_count == 0 assert persistence_calls[1].await_count == 1 + assert result.relation_publication is not None + assert result.relation_publication.generation == note_content.db_version @pytest.mark.asyncio @@ -1652,7 +1701,7 @@ async def test_run_accepted_note_delete_removes_entity_and_returns_cleanup() -> note_content_accept_repository = _NoteContentAcceptRepository(note_content) search_repository = _SearchRepository() - change = await run_accepted_note_delete( + result = await run_accepted_note_delete( cast(AsyncSession, session), request=AcceptedNoteDeleteMutation( project_external_id="project-123", @@ -1669,13 +1718,16 @@ async def test_run_accepted_note_delete_removes_entity_and_returns_cleanup() -> ), ) + change = result.change assert session.deleted == [entity] assert search_repository.deleted_entity_ids == [entity.id] assert search_repository.deleted_vector_entity_ids == [entity.id] + assert session.scalar_count == 1 assert change.status_code == 200 assert change.file_delete is not None assert change.file_delete.file_path == "notes/accepted.md" assert change.file_delete.file_checksum == "file-checksum" + assert result.relation_publication is None def _prepared_with_graph( @@ -1726,7 +1778,7 @@ async def test_run_accepted_note_create_persists_graph_rows() -> None: observation_repository = _ObservationRepository() relation_repository = _RelationRepository() - change = await run_accepted_note_create( + result = await run_accepted_note_create( session, request=AcceptedNoteCreateMutation( project_external_id="project-123", @@ -1747,28 +1799,40 @@ async def test_run_accepted_note_create_persists_graph_rows() -> None: ), ) + change = result.change assert change.status_code == 201 - # The parsed graph is persisted against the new entity in the same transaction. + # Observations stay with accepted content; relations publish after commit. assert observation_repository.calls == [(entity.id, observations)] - assert relation_repository.calls == [(entity.id, relations)] + assert relation_repository.calls == [] + assert result.relation_publication is not None + assert result.relation_publication.generation == note_content.db_version + assert result.relation_publication.relations[0].target_name == "XSYS Target" @pytest.mark.asyncio -async def test_run_accepted_note_create_resolves_self_relation_in_transaction() -> None: - """Create resolves its own safe permalink before persisting the graph.""" +async def test_run_accepted_note_create_pre_resolves_only_unambiguous_self_links() -> None: + """Safe self aliases resolve inline while ambiguous title aliases stay deferred.""" session = cast(AsyncSession, object()) self_relation = AcceptedRelationWrite( relation_type="documents", target_name="accepted", context=None, ) - prepared = _prepared_with_graph(observations=[], relations=[self_relation]) + ambiguous_relation = AcceptedRelationWrite( + relation_type="mentions", + target_name="Accepted", + context=None, + ) + prepared = _prepared_with_graph( + observations=[], + relations=[self_relation, ambiguous_relation], + ) entity = _entity() note_content = _note_content(entity) preparer = _CreatePreparer(prepared) relation_repository = _RelationRepository() - change = await run_accepted_note_create( + result = await run_accepted_note_create( session, request=AcceptedNoteCreateMutation( project_external_id="project-123", @@ -1788,21 +1852,19 @@ async def test_run_accepted_note_create_resolves_self_relation_in_transaction() ), ) + change = result.change assert change.status_code == 201 - assert [call[0] for call in preparer.self_relation_calls] == ["accepted"] - assert relation_repository.calls == [ - ( - entity.id, - [ - AcceptedRelationWrite( - relation_type="documents", - target_name=entity.title, - context=None, - target_id=entity.id, - ) - ], - ) + assert preparer.self_relation_calls == [ + ("accepted", entity, session), + ("Accepted", entity, session), ] + assert relation_repository.calls == [] + assert result.relation_publication is not None + relations_by_name = { + relation.target_name: relation for relation in result.relation_publication.relations + } + assert relations_by_name["accepted"].target_id == entity.id + assert relations_by_name["Accepted"].target_id is None @pytest.mark.asyncio @@ -1821,7 +1883,7 @@ async def test_run_accepted_note_update_replaces_graph_rows() -> None: observation_repository = _ObservationRepository() relation_repository = _RelationRepository() - change = await run_accepted_note_update( + result = await run_accepted_note_update( cast(AsyncSession, session), request=AcceptedNoteUpdateMutation( project_external_id="project-123", @@ -1843,9 +1905,12 @@ async def test_run_accepted_note_update_replaces_graph_rows() -> None: ), ) + change = result.change assert change.status_code == 200 assert observation_repository.calls == [(entity.id, observations)] - assert relation_repository.calls == [(entity.id, relations)] + assert relation_repository.calls == [] + assert result.relation_publication is not None + assert result.relation_publication.relations[0].target_name == "Other" @pytest.mark.asyncio @@ -1858,7 +1923,7 @@ async def test_run_accepted_note_edit_clears_graph_when_markdown_drops_it() -> N observation_repository = _ObservationRepository() relation_repository = _RelationRepository() - change = await run_accepted_note_edit( + result = await run_accepted_note_edit( cast(AsyncSession, session), request=AcceptedNoteEditMutation( project_external_id="project-123", @@ -1885,7 +1950,10 @@ async def test_run_accepted_note_edit_clears_graph_when_markdown_drops_it() -> N ), ) + change = result.change assert change.status_code == 200 - # An empty parsed set still hits the repos so stale rows are cleared, not left behind. + # Empty observations clear in the transaction; empty relations still emit cleanup work. assert observation_repository.calls == [(entity.id, [])] - assert relation_repository.calls == [(entity.id, [])] + assert relation_repository.calls == [] + assert result.relation_publication is not None + assert result.relation_publication.relations == () diff --git a/tests/indexing/test_accepted_note_write_runner.py b/tests/indexing/test_accepted_note_write_runner.py index fbaadb2bb..4f38544bd 100644 --- a/tests/indexing/test_accepted_note_write_runner.py +++ b/tests/indexing/test_accepted_note_write_runner.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime from hashlib import sha256 from pathlib import Path @@ -44,6 +44,7 @@ AcceptedObservationWrite, AcceptedRelationWrite, ) +from basic_memory.repository.relation_repository import RelationGenerationWriteResult from basic_memory.repository.entity_repository import AcceptedPendingEntityWrite from basic_memory.schemas.base import Entity as EntitySchema from basic_memory.services.note_preparation import ( @@ -147,28 +148,37 @@ class _RelationRepository: def __init__(self) -> None: self.calls: list[tuple[int, Sequence[AcceptedRelationWrite]]] = [] - async def replace_accepted_outgoing_relations( + async def begin_relation_generation_publication( self, session: AsyncSession, + *, entity_id: int, - relations: Sequence[AcceptedRelationWrite], - ) -> None: - self.calls.append((entity_id, relations)) - + generation: int, + ) -> RelationGenerationWriteResult: + raise AssertionError( + "relation publication was not expected inside the accepted transaction" + ) -class _SelfRelationResolver: - def __init__(self, result: Entity | None = None) -> None: - self.result = result - self.calls: list[tuple[str, Entity, AsyncSession | None]] = [] + async def upsert_relation_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + relations: Sequence[AcceptedRelationWrite], + ) -> RelationGenerationWriteResult: + raise AssertionError( + "relation publication was not expected inside the accepted transaction" + ) - async def resolve_deferred_self_relation( + async def cleanup_relation_generations( self, - target: str, - entity: Entity, - session: AsyncSession | None = None, - ) -> Entity | None: - self.calls.append((target, entity, session)) - return self.result + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + raise AssertionError("relation cleanup was not expected inside the accepted transaction") def test_accepted_note_write_repositories_name_persistence_behavior() -> None: @@ -207,8 +217,14 @@ def relation_repository(self, project_id: int) -> _RelationRepository: class _DeleteSession: def __init__(self, events: list[tuple[str, int]] | None = None) -> None: self.deleted: list[object] = [] + self.scalar_count = 0 self.events = events + async def scalar(self, statement: object) -> int: + assert statement is not None + self.scalar_count += 1 + return 42 + async def delete(self, entity: object) -> None: self.deleted.append(entity) if self.events is not None: @@ -303,7 +319,7 @@ async def prepare_edit_entity_content( class _MovePreparer: def __init__(self, prepared: PreparedEntityMove) -> None: self.prepared = prepared - self.calls: list[tuple[Entity, str, str, AsyncSession | None]] = [] + self.calls: list[tuple[Entity, str, str, bool, AsyncSession | None]] = [] async def prepare_move_entity_content( self, @@ -311,9 +327,12 @@ async def prepare_move_entity_content( current_content: str, destination_path: str, *, + should_update_permalink: bool, session: AsyncSession | None = None, ) -> PreparedEntityMove: - self.calls.append((entity, current_content, destination_path, session)) + self.calls.append( + (entity, current_content, destination_path, should_update_permalink, session) + ) return self.prepared async def verify_move_destination_absent( @@ -325,6 +344,23 @@ async def verify_move_destination_absent( return None +@dataclass(slots=True) +class _SelfRelationResolver: + """Resolve the exact self-link names selected by one focused test.""" + + resolved_names: set[str] = field(default_factory=set) + calls: list[tuple[str, Entity, AsyncSession | None]] = field(default_factory=list) + + async def resolve_deferred_self_relation( + self, + target: str, + entity: Entity, + session: AsyncSession | None = None, + ) -> Entity | None: + self.calls.append((target, entity, session)) + return entity if target in self.resolved_names else None + + def _unexpected_pending_entity_repository(_project_id: int) -> _PendingEntityRepository: raise AssertionError("pending entity repository was not expected") @@ -695,8 +731,16 @@ async def test_prepare_accepted_note_move_without_permalink_update_keeps_current current = _note_content() current.markdown_content = "---\ntitle: legacy\n\n# Body still matters\n" + prepared = _PreparedMove( + file_path=Path("archive/accepted.md"), + markdown_content=str(current.markdown_content), + search_content=str(current.markdown_content), + permalink="accepted", + ) + preparer = _MovePreparer(prepared) + result = await prepare_accepted_note_move( - None, + preparer, cast(AsyncSession, session), entity=entity, current_note_content=current, @@ -716,6 +760,15 @@ async def test_prepare_accepted_note_move_without_permalink_update_keeps_current assert entity.updated_at == original_updated_at assert entity.last_updated_by == "user-4" assert session.flush_count == 1 + assert preparer.calls == [ + ( + entity, + str(current.markdown_content), + "archive/accepted.md", + False, + cast(AsyncSession, session), + ) + ] @pytest.mark.asyncio @@ -743,7 +796,7 @@ async def test_prepare_accepted_note_move_with_permalink_update_uses_preparer() ) assert preparer.calls == [ - (entity, "# Accepted\n", "archive/accepted.md", cast(AsyncSession, session)), + (entity, "# Accepted\n", "archive/accepted.md", True, cast(AsyncSession, session)), ] assert result.file_path == "archive/prepared.md" assert result.markdown_content == "# Prepared\n" @@ -907,7 +960,7 @@ async def test_delete_accepted_note_search_index_uses_repository_protocol() -> N @pytest.mark.asyncio -async def test_persist_accepted_note_snapshot_persists_content_search_and_graph() -> None: +async def test_persist_accepted_note_snapshot_emits_relation_generation() -> None: session = cast(AsyncSession, object()) entity = _entity() entity.file_path = "notes/new.md" @@ -918,6 +971,7 @@ async def test_persist_accepted_note_snapshot_persists_content_search_and_graph( current_note_content.file_version = 3 current_note_content.file_checksum = "old-file-checksum" persisted_note_content = _note_content() + persisted_note_content.db_version = 5 content_repository = _NoteContentRepository(persisted_note_content) search_repository = _SearchRepository() observation_repository = _ObservationRepository() @@ -939,24 +993,24 @@ async def test_persist_accepted_note_snapshot_persists_content_search_and_graph( observations=(observation,), relations=(relation,), ) + self_relation_resolver = _SelfRelationResolver() result = await persist_accepted_note_snapshot( session, entity=entity, prepared=prepared, db_checksum="new-db-checksum", - self_relation_resolver=_SelfRelationResolver(), last_source="api", updated_at=updated_at, current_note_content=current_note_content, existing_file_path="notes/old.md", accepted_file_path="notes/new.md", source_file_checksum="db-checksum", + self_relation_resolver=self_relation_resolver, repositories=_repository_provider( note_content_repository=content_repository, search_repository=search_repository, observation_repository=observation_repository, - relation_repository=relation_repository, ), ) @@ -985,11 +1039,19 @@ async def test_persist_accepted_note_snapshot_persists_content_search_and_graph( assert search_repository.calls[0].entity_id == entity.id assert search_repository.calls[0].content_snippet == "New body" assert observation_repository.calls == [(entity.id, prepared.observations)] - assert relation_repository.calls == [(entity.id, prepared.relations)] + assert relation_repository.calls == [] + assert result.relation_publication is not None + assert result.relation_publication.project_id == entity.project_id + assert result.relation_publication.entity_id == entity.id + assert result.relation_publication.generation == 5 + assert result.relation_publication.relations[0].relation_type == "documents" + assert result.relation_publication.relations[0].target_name == "Another Note" + assert result.relation_publication.relations[0].target_id is None + assert self_relation_resolver.calls == [("Another Note", entity, session)] @pytest.mark.asyncio -async def test_persist_accepted_note_move_is_explicitly_content_and_search_only() -> None: +async def test_persist_accepted_note_move_emits_relation_generation() -> None: session = cast(AsyncSession, _FlushSession()) entity = _entity() entity.file_path = "notes/new.md" @@ -997,8 +1059,16 @@ async def test_persist_accepted_note_move_is_explicitly_content_and_search_only( current_note_content.file_path = "notes/old.md" content_repository = _NoteContentRepository(_note_content()) search_repository = _SearchRepository() + move_preparer = _MovePreparer( + _PreparedMove( + file_path=Path("notes/new.md"), + markdown_content=str(current_note_content.markdown_content), + search_content=str(current_note_content.markdown_content), + permalink=entity.permalink, + ) + ) prepared = await prepare_accepted_note_move( - None, + move_preparer, session, entity=entity, current_note_content=current_note_content, @@ -1007,7 +1077,7 @@ async def test_persist_accepted_note_move_is_explicitly_content_and_search_only( user_profile_value=None, ) - await persist_accepted_note_move( + result = await persist_accepted_note_move( session, entity=entity, prepared=prepared, @@ -1015,6 +1085,7 @@ async def test_persist_accepted_note_move_is_explicitly_content_and_search_only( updated_at=datetime(2026, 6, 19, 14, 0, tzinfo=UTC), current_note_content=current_note_content, existing_file_path="notes/old.md", + self_relation_resolver=_SelfRelationResolver(), repositories=_repository_provider( note_content_repository=content_repository, search_repository=search_repository, @@ -1023,6 +1094,9 @@ async def test_persist_accepted_note_move_is_explicitly_content_and_search_only( assert len(content_repository.calls) == 1 assert len(search_repository.calls) == 1 + assert result.relation_publication is not None + assert result.relation_publication.generation == result.note_content.db_version + assert result.relation_publication.relations == () @pytest.mark.asyncio @@ -1074,6 +1148,7 @@ async def test_delete_accepted_note_plans_cleanup_and_deletes_entity() -> None: assert search_repository.deleted_entity_ids == [entity.id] assert search_repository.deleted_vector_entity_ids == [entity.id] + assert session.scalar_count == 1 assert session.deleted == [entity] assert events == [ ("search", entity.id), @@ -1097,9 +1172,8 @@ async def test_delete_accepted_note_plans_cleanup_and_deletes_entity() -> None: @pytest.mark.asyncio -async def test_persist_accepted_note_snapshot_resolves_safe_self_relation() -> None: - """A safe self-link carries its ID because deferred resolution skips self targets.""" - relation_repository = _RelationRepository() +async def test_persist_accepted_note_snapshot_pre_resolves_unambiguous_self_relation() -> None: + """Accepted publication keeps the authored alias and safe self target together.""" entity = _entity() prepared = _prepared( markdown_content="# Accepted\n", @@ -1122,63 +1196,54 @@ async def test_persist_accepted_note_snapshot_resolves_safe_self_relation() -> N ) ], ) - resolver = _SelfRelationResolver(entity) - - await persist_accepted_note_snapshot( - cast(AsyncSession, object()), + session = cast(AsyncSession, object()) + resolver = _SelfRelationResolver(resolved_names={"notes/accepted"}) + result = await persist_accepted_note_snapshot( + session, entity=entity, prepared=prepared, db_checksum="snapshot-checksum", - self_relation_resolver=resolver, last_source="api", updated_at=entity.updated_at, + self_relation_resolver=resolver, repositories=_repository_provider( note_content_repository=_NoteContentRepository(_note_content()), search_repository=_SearchRepository(), observation_repository=_ObservationRepository(), - relation_repository=relation_repository, ), ) - assert relation_repository.calls == [ - ( - entity.id, - [ - AcceptedRelationWrite( - relation_type="documents", - target_name=entity.title, - context=None, - target_id=entity.id, - ) - ], - ) - ] + assert result.relation_publication is not None + assert result.relation_publication.relations[0].target_name == "notes/accepted" + assert result.relation_publication.relations[0].target_id == entity.id + assert resolver.calls == [("notes/accepted", entity, session)] @pytest.mark.asyncio -async def test_persist_accepted_note_snapshot_forwards_empty_graph_sets() -> None: - """A note with no observations/relations still clears the graph (empty replace).""" +async def test_persist_accepted_note_snapshot_emits_empty_relation_generation() -> None: + """An empty relation set still emits a publication so cleanup can remove stale rows.""" observation_repository = _ObservationRepository() relation_repository = _RelationRepository() repositories = _repository_provider( note_content_repository=_NoteContentRepository(_note_content()), search_repository=_SearchRepository(), observation_repository=observation_repository, - relation_repository=relation_repository, ) prepared = _prepared() entity = _entity() - await persist_accepted_note_snapshot( + result = await persist_accepted_note_snapshot( cast(AsyncSession, object()), entity=entity, prepared=prepared, db_checksum="snapshot-checksum", - self_relation_resolver=_SelfRelationResolver(), last_source="api", updated_at=entity.updated_at, + self_relation_resolver=_SelfRelationResolver(), repositories=repositories, ) assert observation_repository.calls == [(42, [])] - assert relation_repository.calls == [(42, [])] + assert relation_repository.calls == [] + assert result.relation_publication is not None + assert result.relation_publication.relations == () diff --git a/tests/indexing/test_batch_indexer.py b/tests/indexing/test_batch_indexer.py index 00f410ed7..be2fe746f 100644 --- a/tests/indexing/test_batch_indexer.py +++ b/tests/indexing/test_batch_indexer.py @@ -15,7 +15,14 @@ from basic_memory import db from basic_memory.file_utils import remove_frontmatter from basic_memory.indexing.batch_indexer import BatchIndexer -from basic_memory.indexing.models import IndexInputFile, StorageIndexFileWriter +from basic_memory.indexing.models import ( + IndexingBatchResult, + IndexInputFile, + RelationGenerationBatchResult, + StorageIndexFileWriter, +) +from basic_memory.indexing.note_content_reconciler import NoteContentReconciler +from basic_memory.repository import NoteContentRepository from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.schemas import Entity as EntitySchema from basic_memory.schemas.search import SearchItemType, SearchQuery @@ -47,6 +54,7 @@ def _make_batch_indexer( app_config, entity_service, entity_repository, relation_repository, search_service, file_service ) -> BatchIndexer: return BatchIndexer( + project_id=relation_repository.project_id, app_config=app_config, entity_service=entity_service, entity_repository=entity_repository, @@ -57,6 +65,53 @@ def _make_batch_indexer( ) +async def _claim_and_publish_relations( + batch_indexer: BatchIndexer, + result: IndexingBatchResult, + *, + entity_repository, + relation_repository, + session_maker, + max_concurrent: int, +) -> RelationGenerationBatchResult: + """Exercise the post-index generation claim and relation publication boundary.""" + markdown_entities = [ + indexed for indexed in result.indexed if indexed.markdown_content is not None + ] + async with db.scoped_session(session_maker) as session: + entities = await entity_repository.find_by_ids( + session, + [indexed.entity_id for indexed in markdown_entities], + ) + entity_by_id = {entity.id: entity for entity in entities} + reconciler = NoteContentReconciler( + note_content_repository=NoteContentRepository( + project_id=relation_repository.project_id, + ), + session_maker=session_maker, + ) + generation_by_entity_id: dict[int, int] = {} + for indexed in markdown_entities: + reconciliation = await reconciler.reconcile( + entity=entity_by_id[indexed.entity_id], + markdown_content=indexed.markdown_content or "", + observed_at=datetime.now(tz=UTC), + source="test", + ) + assert reconciliation.generation is not None + generation_by_entity_id[indexed.entity_id] = reconciliation.generation + + publication = await batch_indexer.publish_relation_generations( + result.indexed, + generation_by_entity_id=generation_by_entity_id, + max_concurrent=max_concurrent, + ) + result.errors.extend(publication.errors) + result.relations_resolved = publication.relations_resolved + result.relations_unresolved = publication.relations_unresolved + return publication + + @pytest.mark.asyncio async def test_batch_indexer_parses_markdown_with_parallel_path( app_config, @@ -450,6 +505,22 @@ async def test_batch_indexer_resolves_relations_and_refreshes_search( max_concurrent=2, parse_max_concurrent=2, ) + async with db.scoped_session(search_service.session_maker) as session: + source_before_claim = await entity_repository.get_by_file_path(session, source_path) + assert source_before_claim is not None + assert source_before_claim.outgoing_relations == [] + indexed_source = next(indexed for indexed in result.indexed if indexed.path == source_path) + assert [ + (relation.relation_type, relation.target_name) for relation in indexed_source.relations + ] == [("depends_on", "Target")] + await _claim_and_publish_relations( + batch_indexer, + result, + entity_repository=entity_repository, + relation_repository=relation_repository, + session_maker=search_service.session_maker, + max_concurrent=2, + ) async with db.scoped_session(search_service.session_maker) as session: source = await entity_repository.get_by_file_path(session, source_path) @@ -593,12 +664,12 @@ async def test_batch_indexer_assigns_unique_permalinks_for_batch_local_conflicts assert indexed_by_path[path_two].markdown_content is not None assert indexed_by_path[path_one].markdown_content != original_contents[path_one] assert indexed_by_path[path_two].markdown_content != original_contents[path_two] - assert indexed_by_path[path_one].markdown_content == await file_service.read_file_content( - path_one - ) - assert indexed_by_path[path_two].markdown_content == await file_service.read_file_content( - path_two - ) + assert indexed_by_path[path_one].markdown_content == ( + project_config.home / path_one + ).read_bytes().decode("utf-8") + assert indexed_by_path[path_two].markdown_content == ( + project_config.home / path_two + ).read_bytes().decode("utf-8") async with db.scoped_session(search_service.session_maker) as session: entities = await entity_repository.find_all(session) @@ -647,6 +718,14 @@ async def test_batch_indexer_uses_parsed_markdown_body_for_malformed_frontmatter max_concurrent=1, parse_max_concurrent=1, ) + await _claim_and_publish_relations( + batch_indexer, + result, + entity_repository=entity_repository, + relation_repository=relation_repository, + session_maker=search_service.session_maker, + max_concurrent=1, + ) # Trigger: malformed frontmatter should pass through without normalization. # Why: Windows can still surface that unchanged file with CRLF line endings. @@ -801,7 +880,7 @@ async def stale_permalink(*args, **kwargs) -> str: index_search=False, ) - persisted_content = await file_service.read_file_content(path) + persisted_content = (project_config.home / path).read_bytes().decode("utf-8") assert indexed.permalink == f"{conflicting_permalink}-1" assert indexed.markdown_content == persisted_content @@ -852,11 +931,19 @@ async def test_batch_indexer_index_markdown_file_can_defer_relation_resolution( file_service, ) - await batch_indexer.index_markdown_file( + indexed = await batch_indexer.index_markdown_file( await _load_input(file_service, path), index_search=False, resolve_relations=False, ) + await _claim_and_publish_relations( + batch_indexer, + IndexingBatchResult(indexed=[indexed]), + entity_repository=entity_repository, + relation_repository=relation_repository, + session_maker=search_service.session_maker, + max_concurrent=1, + ) resolve_link.assert_not_awaited() async with db.scoped_session(search_service.session_maker) as session: @@ -868,7 +955,356 @@ async def test_batch_indexer_index_markdown_file_can_defer_relation_resolution( @pytest.mark.asyncio -async def test_batch_indexer_uses_strict_link_resolution_for_deferred_relations( +async def test_relation_publication_search_failure_leaves_retry_marker( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + project_config, + monkeypatch, +): + """A failed post-publication search write remains discoverable after this batch exits.""" + path = "notes/retry-relation-search.md" + await _create_file( + project_config.home / path, + "# Retry Relation Search\n\n- [note] Exact observation snapshot\n", + ) + batch_indexer = _make_batch_indexer( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + ) + indexed = await batch_indexer.index_markdown_file( + await _load_input(file_service, path), + index_search=False, + resolve_relations=False, + ) + original_index_entity_data = search_service.index_entity_data + + async def fail_search_refresh(*args, **kwargs): + del args, kwargs + raise OSError("relation search refresh failed") + + monkeypatch.setattr(search_service, "index_entity_data", fail_search_refresh) + publication = await _claim_and_publish_relations( + batch_indexer, + IndexingBatchResult(indexed=[indexed]), + entity_repository=entity_repository, + relation_repository=relation_repository, + session_maker=search_service.session_maker, + max_concurrent=1, + ) + + assert [(error_path, str(error)) for error_path, error in publication.errors] == [ + (path, "relation search refresh failed") + ] + async with db.scoped_session(search_service.session_maker) as session: + pending_after_failure = await relation_repository.list_pending_search_refreshes( + session, + entity_id=indexed.entity_id, + ) + assert [refresh.entity_id for refresh in pending_after_failure] == [indexed.entity_id] + async with db.scoped_session(search_service.session_maker) as session: + note_content = await NoteContentRepository( + project_id=relation_repository.project_id + ).get_by_entity_id(session, indexed.entity_id) + assert note_content is not None + + monkeypatch.setattr(search_service, "index_entity_data", original_index_entity_data) + await batch_indexer.refresh_indexed_entity_search( + indexed, + generation=note_content.db_version, + ) + + async with db.scoped_session(search_service.session_maker) as session: + pending_after_retry = await relation_repository.list_pending_search_refreshes( + session, + entity_id=indexed.entity_id, + ) + assert pending_after_retry == [] + + +@pytest.mark.asyncio +async def test_relation_search_refresh_skips_superseded_publication_generation( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + project_config, + monkeypatch, +): + """Generation N must not refresh search from its payload after N+1 is accepted.""" + path = "notes/superseded-relation-search.md" + await _create_file(project_config.home / path, "# Generation N\n") + batch_indexer = _make_batch_indexer( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + ) + indexed = await batch_indexer.index_markdown_file( + await _load_input(file_service, path), + index_search=False, + resolve_relations=False, + ) + async with db.scoped_session(search_service.session_maker) as session: + entities = await entity_repository.find_by_ids(session, [indexed.entity_id]) + assert len(entities) == 1 + + reconciler = NoteContentReconciler( + note_content_repository=NoteContentRepository( + project_id=relation_repository.project_id, + ), + session_maker=search_service.session_maker, + ) + generation_n = await reconciler.reconcile( + entity=entities[0], + markdown_content=indexed.markdown_content or "", + observed_at=datetime.now(tz=UTC), + source="test", + ) + assert generation_n.generation is not None + generation_n_value = generation_n.generation + assert await batch_indexer.publish_relation_generation( + indexed, + generation=generation_n_value, + ) + + generation_n_plus_one = await reconciler.reconcile( + entity=entities[0], + markdown_content="# Generation N+1\n", + observed_at=datetime.now(tz=UTC), + source="test", + ) + assert generation_n_plus_one.generation == generation_n_value + 1 + + search_write = AsyncMock(side_effect=AssertionError("superseded refresh must be terminal")) + monkeypatch.setattr(search_service, "index_entity_data", search_write) + + refreshed = await batch_indexer.refresh_indexed_entity_search( + indexed, + generation=generation_n_value, + ) + + assert refreshed is indexed + search_write.assert_not_awaited() + async with db.scoped_session(search_service.session_maker) as session: + pending = await relation_repository.list_pending_search_refreshes( + session, + entity_id=indexed.entity_id, + ) + assert [refresh.entity_id for refresh in pending] == [indexed.entity_id] + + +@pytest.mark.asyncio +async def test_relation_search_refresh_requeues_generation_lost_during_write( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + project_config, + monkeypatch, +): + """A late N write cannot consume the final repair signal after N+1 wins.""" + path = "notes/search-generation-race.md" + await _create_file(project_config.home / path, "# Generation N\n") + batch_indexer = _make_batch_indexer( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + ) + indexed = await batch_indexer.index_markdown_file( + await _load_input(file_service, path), + index_search=False, + resolve_relations=False, + ) + async with db.scoped_session(search_service.session_maker) as session: + entities = await entity_repository.find_by_ids(session, [indexed.entity_id]) + assert len(entities) == 1 + + reconciler = NoteContentReconciler( + note_content_repository=NoteContentRepository( + project_id=relation_repository.project_id, + ), + session_maker=search_service.session_maker, + ) + generation_n = await reconciler.reconcile( + entity=entities[0], + markdown_content=indexed.markdown_content or "", + observed_at=datetime.now(tz=UTC), + source="test", + ) + assert generation_n.generation is not None + generation_n_value = generation_n.generation + assert await batch_indexer.publish_relation_generation( + indexed, + generation=generation_n_value, + ) + + async def accept_newer_generation_during_search(*args, **kwargs) -> None: + del args, kwargs + generation_n_plus_one = await reconciler.reconcile( + entity=entities[0], + markdown_content="# Generation N+1\n", + observed_at=datetime.now(tz=UTC), + source="test", + ) + assert generation_n_plus_one.generation == generation_n_value + 1 + + # Model N+1 completing its own refresh before N returns from the external + # search writer. N's post-write guard must create later repair work. + async with db.scoped_session(search_service.session_maker) as session: + observed = await relation_repository.list_pending_search_refreshes( + session, + entity_id=indexed.entity_id, + ) + await relation_repository.clear_pending_search_refreshes( + session, + [refresh.id for refresh in observed], + ) + + monkeypatch.setattr( + search_service, + "index_entity_data", + accept_newer_generation_during_search, + ) + + await batch_indexer.refresh_indexed_entity_search( + indexed, + generation=generation_n_value, + ) + + async with db.scoped_session(search_service.session_maker) as session: + pending = await relation_repository.list_pending_search_refreshes( + session, + entity_id=indexed.entity_id, + ) + assert [refresh.entity_id for refresh in pending] == [indexed.entity_id] + + +@pytest.mark.asyncio +async def test_batch_indexer_publishes_only_ambiguity_safe_self_relations( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + project_config, +): + """File indexing preserves safe self-links without guessing ambiguous title targets.""" + path = "notes/self-links.md" + content = dedent( + """ + --- + title: Self Links + type: note + --- + + # Self Links + + - links_to [[Self Links]] + - links_to [[notes/self-links.md]] + - mentions [[Self Links]] + """ + ).strip() + await _create_file(project_config.home / path, content) + batch_indexer = _make_batch_indexer( + app_config, + entity_service, + entity_repository, + relation_repository, + search_service, + file_service, + ) + + indexed = await batch_indexer.index_markdown_file( + await _load_input(file_service, path), + index_search=False, + resolve_relations=False, + ) + assert {relation.target_id for relation in indexed.relations} == {indexed.entity_id} + + await _claim_and_publish_relations( + batch_indexer, + IndexingBatchResult(indexed=[indexed]), + entity_repository=entity_repository, + relation_repository=relation_repository, + session_maker=search_service.session_maker, + max_concurrent=1, + ) + async with db.scoped_session(search_service.session_maker) as session: + source = await entity_repository.get_by_file_path(session, path) + assert source is not None + assert sorted( + (relation.relation_type, relation.to_name, relation.to_id) + for relation in source.outgoing_relations + ) == [ + ("links_to", "Self Links", source.id), + ("mentions", "Self Links", source.id), + ] + + await entity_service.create_entity_with_content( + EntitySchema( + title="Self Links", + directory="duplicates", + content="# Self Links\n\nA different note with the same title.\n", + ) + ) + await _create_file(project_config.home / path, f"{content}\n\nUpdated source bytes.\n") + + ambiguous = await batch_indexer.index_markdown_file( + await _load_input(file_service, path), + index_search=False, + resolve_relations=False, + ) + target_ids = { + (relation.relation_type, relation.target_name): relation.target_id + for relation in ambiguous.relations + } + assert target_ids == { + ("links_to", "Self Links"): None, + ("links_to", "notes/self-links.md"): source.id, + ("mentions", "Self Links"): None, + } + + await _claim_and_publish_relations( + batch_indexer, + IndexingBatchResult(indexed=[ambiguous]), + entity_repository=entity_repository, + relation_repository=relation_repository, + session_maker=search_service.session_maker, + max_concurrent=1, + ) + async with db.scoped_session(search_service.session_maker) as session: + reloaded = await entity_repository.get_by_file_path(session, path) + assert reloaded is not None + assert sorted( + (relation.relation_type, relation.to_name, relation.to_id) + for relation in reloaded.outgoing_relations + ) == [ + ("links_to", "Self Links", None), + ("links_to", "notes/self-links.md", source.id), + ("mentions", "Self Links", None), + ] + + +@pytest.mark.asyncio +async def test_batch_indexer_uses_exact_bulk_resolution_for_deferred_relations( app_config, entity_service, entity_repository, @@ -878,14 +1314,7 @@ async def test_batch_indexer_uses_strict_link_resolution_for_deferred_relations( project_config, monkeypatch, ): - """Regression: batch indexer's deferred relation resolution must call - resolve_link with strict=True. - - Mirror of sync_service.resolve_forward_references. Fuzzy fallback in the - deferred path silently fills in to_id from BM25/ts_rank results, polluting - the graph with confidently-wrong edges. Entity-creation already uses - strict=True; this is the other deferred path. - """ + """Deferred relations use the shared exact-match bulk resolver.""" path = "notes/source.md" await _create_file( project_config.home / path, @@ -912,25 +1341,31 @@ async def test_batch_indexer_uses_strict_link_resolution_for_deferred_relations( file_service, ) - original_resolve_link = entity_service.link_resolver.resolve_link - seen_strict: list[object] = [] + target_resolver_type = type(batch_indexer.relation_resolution.target_resolver) + original_resolve_targets = target_resolver_type.resolve_relation_targets + seen_target_batches: list[tuple[str, ...]] = [] - async def spy_resolve_link(*args, **kwargs): - seen_strict.append(kwargs.get("strict", False)) - return await original_resolve_link(*args, **kwargs) + async def spy_resolve_targets(self, link_texts, *, session): + seen_target_batches.append(tuple(link_texts)) + return await original_resolve_targets(self, link_texts, session=session) - monkeypatch.setattr(entity_service.link_resolver, "resolve_link", spy_resolve_link) + monkeypatch.setattr(target_resolver_type, "resolve_relation_targets", spy_resolve_targets) - await batch_indexer.index_files( + result = await batch_indexer.index_files( {path: await _load_input(file_service, path)}, max_concurrent=1, ) - - assert seen_strict, "batch indexer did not invoke link_resolver.resolve_link" - assert all(strict is True for strict in seen_strict), ( - f"Deferred resolution must call resolve_link(strict=True). Observed: {seen_strict!r}" + await _claim_and_publish_relations( + batch_indexer, + result, + entity_repository=entity_repository, + relation_repository=relation_repository, + session_maker=search_service.session_maker, + max_concurrent=1, ) + assert seen_target_batches == [("never-resolves-target",)] + # The unresolvable relation stayed unresolved. async with db.scoped_session(search_service.session_maker) as session: source = await entity_repository.get_by_file_path(session, path) diff --git a/tests/indexing/test_directory_delete_runner.py b/tests/indexing/test_directory_delete_runner.py index 06ea8b254..995210b10 100644 --- a/tests/indexing/test_directory_delete_runner.py +++ b/tests/indexing/test_directory_delete_runner.py @@ -1,19 +1,24 @@ """Tests for portable directory-delete cleanup orchestration.""" from collections.abc import Sequence +from datetime import UTC, datetime from types import SimpleNamespace from typing import cast import basic_memory.indexing.directory_delete_runner as directory_delete_runner_module import pytest +from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql.dml import Delete +from basic_memory import db from basic_memory.indexing.directory_delete_runner import ( DirectoryDeleteAcceptanceRequest, DirectoryDeleteAcceptedResult, DirectoryDeleteFileFailure, DirectoryDeleteRejected, DirectoryDeleteRejectKind, + DirectoryEntityDeleteResult, DirectoryFileDeleteEnqueueError, RepositoryDirectoryDeleteAcceptanceStore, DirectoryDeleteRuntime, @@ -21,6 +26,7 @@ normalize_directory_delete_path, run_directory_delete, ) +from basic_memory.models import Entity, NoteContent, Relation from basic_memory.runtime.cleanup import ( RuntimeDirectoryFileSnapshot, RuntimeFileDeleteResult, @@ -52,6 +58,7 @@ def __init__( self.files = files or [] self.relation_cleanup_entity_ids = relation_cleanup_entity_ids self.loaded_directories: list[str] = [] + self.deleted_directories: list[str] = [] self.deleted_entity_ids: list[tuple[int, ...]] = [] async def load_project_id( @@ -76,11 +83,16 @@ async def delete_directory_entities( session: AsyncSession, *, project_id: int, + directory: str, entity_ids: Sequence[int], - ) -> frozenset[int]: + ) -> DirectoryEntityDeleteResult: assert project_id == self.project_id + self.deleted_directories.append(directory) self.deleted_entity_ids.append(tuple(entity_ids)) - return self.relation_cleanup_entity_ids + return DirectoryEntityDeleteResult( + deleted_entity_ids=frozenset(entity_ids), + relation_cleanup_entity_ids=self.relation_cleanup_entity_ids, + ) class FakeScalarResult: @@ -114,6 +126,9 @@ def __init__( def scalars(self) -> FakeScalarResult: return FakeScalarResult(self.scalar_value, self.scalar_values) + def tuples(self) -> "FakeExecuteResult": + return self + def all(self) -> list[object]: return self.rows @@ -307,6 +322,7 @@ async def test_run_directory_delete_accepts_rows_and_queues_cleanup() -> None: deleted_files=("notes/a.md", "notes/b.md") ) assert store.loaded_directories == ["notes"] + assert store.deleted_directories == ["notes"] assert store.deleted_entity_ids == [(7, 8)] # No guarded skips here, so the payload still reports a clean success. assert result.to_response_payload()["failed_deletes"] == 0 @@ -456,22 +472,28 @@ async def test_repository_directory_delete_store_captures_relation_sources() -> AsyncSession, FakeExecuteSession( [ - FakeExecuteResult(scalar_values=[42, 99]), # surviving relation sources + FakeExecuteResult(), # sorted note_content lock fence + FakeExecuteResult(scalar_values=[7, 8]), # current directory members + FakeExecuteResult(rows=[(101, 7, 42), (102, 8, 99)]), + FakeExecuteResult(scalar_values=[7, 8]), # guarded entity delete FakeExecuteResult(), # search_index delete FakeExecuteResult(scalar_values=[]), # vector rows - FakeExecuteResult(), # entity delete ] ), ) store = RepositoryDirectoryDeleteAcceptanceStore() - relation_cleanup_entity_ids = await store.delete_directory_entities( + delete_result = await store.delete_directory_entities( session, project_id=3, + directory="notes", entity_ids=[7, 8], ) - assert relation_cleanup_entity_ids == frozenset({42, 99}) + assert delete_result == DirectoryEntityDeleteResult( + deleted_entity_ids=frozenset({7, 8}), + relation_cleanup_entity_ids=frozenset({42, 99}), + ) @pytest.mark.asyncio @@ -517,10 +539,12 @@ async def test_repository_directory_delete_store_maps_note_content_snapshots() - )() ] ), - FakeExecuteResult(scalar_values=[]), # surviving relation sources + FakeExecuteResult(), # sorted note_content lock fence + FakeExecuteResult(scalar_values=[7]), # current directory members + FakeExecuteResult(rows=[]), # incoming relation snapshot + FakeExecuteResult(scalar_values=[7]), # guarded entity delete FakeExecuteResult(), # search_index delete FakeExecuteResult(scalar_values=[]), # vector rows - FakeExecuteResult(), # entity delete ] ), ) @@ -536,9 +560,10 @@ async def test_repository_directory_delete_store_maps_note_content_snapshots() - project_id=3, directory="notes", ) - relation_cleanup_entity_ids = await store.delete_directory_entities( + delete_result = await store.delete_directory_entities( session, project_id=3, + directory="notes", entity_ids=[7], ) @@ -552,21 +577,33 @@ async def test_repository_directory_delete_store_maps_note_content_snapshots() - size=42, ) ] - assert relation_cleanup_entity_ids == frozenset() - assert len(fake_session.queries) == 6 + assert delete_result == DirectoryEntityDeleteResult( + deleted_entity_ids=frozenset({7}), + ) + assert len(fake_session.queries) == 8 + assert "FOR UPDATE" not in str(fake_session.queries[1][0]) + assert "ORDER BY note_content.entity_id" in str(fake_session.queries[2][0]) + assert "entity.file_path LIKE" in str(fake_session.queries[3][0]) + guarded_entity_delete = str(fake_session.queries[5][0]) + assert "entity.id IN" in guarded_entity_delete + assert "entity.project_id" in guarded_entity_delete + assert "entity.file_path LIKE" in guarded_entity_delete + assert "NOT (EXISTS" in guarded_entity_delete @pytest.mark.asyncio -async def test_repository_directory_delete_store_clears_vectors_before_entities( +async def test_repository_directory_delete_store_clears_vectors_for_deleted_entities( monkeypatch: pytest.MonkeyPatch, ) -> None: session = cast( AsyncSession, FakeExecuteSession( [ - FakeExecuteResult(scalar_values=[]), # surviving relation sources + FakeExecuteResult(), # sorted note_content lock fence + FakeExecuteResult(scalar_values=[7, 8]), # current directory members + FakeExecuteResult(rows=[]), # incoming relation snapshot + FakeExecuteResult(scalar_values=[7, 8]), # guarded entity delete FakeExecuteResult(), # search_index delete - FakeExecuteResult(), # entity delete ] ), ) @@ -600,12 +637,182 @@ async def fake_delete_project_index_vector_rows( await store.delete_directory_entities( session, project_id=3, + directory="notes", entity_ids=[7, 8], ) - # Vector rows are cleared after the relation-source select and search_index delete - # (2 queries so far) but before the entity delete, so CASCADE cannot race the rows. - assert vector_calls == [(session, 3, (7, 8), 2)] + # Projection cleanup uses only ids returned by the guarded Entity mutation. + assert vector_calls == [(session, 3, (7, 8), 5)] statements = [str(query) for query, _ in fake_session.queries] - assert "DELETE FROM search_index" in statements[1] - assert "DELETE FROM entity" in statements[2] + assert "ORDER BY note_content.entity_id" in statements[0] + assert "DELETE FROM entity" in statements[3] + assert "DELETE FROM search_index" in statements[4] + + +@pytest.mark.asyncio +async def test_repository_directory_delete_revalidates_membership_after_move_out( + session_maker, + test_project, +) -> None: + """A note moved after the snapshot is not authorized by its stale entity id.""" + now = datetime.now(tz=UTC) + async with db.scoped_session(session_maker) as session: + moved_note = Entity( + project_id=test_project.id, + title="Move-out survivor", + note_type="note", + permalink="notes/move-out-survivor", + file_path="notes/move-out-survivor.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + session.add(moved_note) + await session.flush() + moved_note_id = moved_note.id + session.add( + NoteContent( + entity_id=moved_note_id, + project_id=test_project.id, + external_id=moved_note.external_id, + file_path=moved_note.file_path, + markdown_content="# Move-out survivor\n", + db_version=1, + db_checksum="generation-1", + file_write_status="synced", + ) + ) + + store = RepositoryDirectoryDeleteAcceptanceStore() + async with db.scoped_session(session_maker) as session: + snapshots = await store.load_directory_file_snapshots( + session, + project_id=test_project.id, + directory="notes", + ) + assert [snapshot.entity_id for snapshot in snapshots] == [moved_note_id] + + async with db.scoped_session(session_maker) as session: + moved_note = await session.get(Entity, moved_note_id) + note_content = await session.get(NoteContent, moved_note_id) + assert moved_note is not None + assert note_content is not None + moved_note.file_path = "archive/move-out-survivor.md" + note_content.file_path = moved_note.file_path + + async with db.scoped_session(session_maker) as session: + delete_result = await store.delete_directory_entities( + session, + project_id=test_project.id, + directory="notes", + entity_ids=[snapshot.entity_id for snapshot in snapshots], + ) + + assert delete_result == DirectoryEntityDeleteResult() + async with db.scoped_session(session_maker) as session: + moved_note = await session.get(Entity, moved_note_id) + note_content = await session.get(NoteContent, moved_note_id) + assert moved_note is not None + assert moved_note.file_path == "archive/move-out-survivor.md" + assert note_content is not None + assert note_content.file_path == "archive/move-out-survivor.md" + + +@pytest.mark.asyncio +async def test_repository_directory_delete_rejects_uncaptured_incoming_relation( + session_maker, + test_project, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A relation resolved after capture keeps its target and repair source intact.""" + now = datetime.now(tz=UTC) + async with db.scoped_session(session_maker) as session: + source = Entity( + project_id=test_project.id, + title="Surviving source", + note_type="note", + file_path="outside/source.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + target = Entity( + project_id=test_project.id, + title="Delete target", + note_type="note", + file_path="notes/target.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + session.add_all([source, target]) + await session.flush() + session.add( + NoteContent( + entity_id=target.id, + project_id=test_project.id, + external_id=target.external_id, + file_path=target.file_path, + markdown_content="# Delete target\n", + db_version=1, + db_checksum="generation-1", + file_write_status="synced", + ) + ) + relation = Relation( + project_id=test_project.id, + from_id=source.id, + to_id=None, + to_name=target.title, + relation_type="related_to", + generation=0, + ) + session.add(relation) + await session.flush() + target_id = target.id + relation_id = relation.id + + store = RepositoryDirectoryDeleteAcceptanceStore() + async with db.scoped_session(session_maker) as session: + snapshots = await store.load_directory_file_snapshots( + session, + project_id=test_project.id, + directory="notes", + ) + assert [snapshot.entity_id for snapshot in snapshots] == [target_id] + + async with db.scoped_session(session_maker) as session: + original_execute = session.execute + relation_resolved = False + + async def execute_with_relation_race(statement, *args, **kwargs): + nonlocal relation_resolved + if ( + not relation_resolved + and isinstance(statement, Delete) + and statement.table.name == Entity.__tablename__ + ): + relation_resolved = True + await original_execute( + update(Relation).where(Relation.id == relation_id).values(to_id=target_id) + ) + return await original_execute(statement, *args, **kwargs) + + monkeypatch.setattr(session, "execute", execute_with_relation_race) + delete_result = await store.delete_directory_entities( + session, + project_id=test_project.id, + directory="notes", + entity_ids=[target_id], + ) + + assert relation_resolved is True + assert delete_result == DirectoryEntityDeleteResult() + async with db.scoped_session(session_maker) as session: + target = await session.get(Entity, target_id) + note_content = await session.get(NoteContent, target_id) + relation = await session.get(Relation, relation_id) + assert target is not None + assert note_content is not None + assert relation is not None + assert relation.to_id == target_id diff --git a/tests/indexing/test_external_file_delete_runner.py b/tests/indexing/test_external_file_delete_runner.py index d8e5a90cc..93e1f6dbd 100644 --- a/tests/indexing/test_external_file_delete_runner.py +++ b/tests/indexing/test_external_file_delete_runner.py @@ -114,6 +114,7 @@ async def test_repository_external_file_delete_entities_use_scoped_sessions( session_maker = cast(async_sessionmaker[AsyncSession], object()) scoped_session_calls: list[async_sessionmaker[AsyncSession]] = [] relation_cleanup_calls: list[tuple[object, int, int]] = [] + note_content_lock_calls: list[tuple[object, int, tuple[int, ...]]] = [] def fake_scoped_session( scoped_session_maker: async_sessionmaker[AsyncSession], @@ -130,6 +131,14 @@ async def fake_relation_cleanup_sources_for_deleted_entity( relation_cleanup_calls.append((cleanup_session, project_id, entity_id)) return frozenset({7}) + async def fake_lock_note_content_before_entity_mutation( + lock_session: AsyncSession, + *, + project_id: int, + entity_ids: tuple[int, ...], + ) -> None: + note_content_lock_calls.append((lock_session, project_id, entity_ids)) + monkeypatch.setattr( external_file_delete_runner.db, "scoped_session", @@ -140,6 +149,11 @@ async def fake_relation_cleanup_sources_for_deleted_entity( "relation_cleanup_sources_for_deleted_entity", fake_relation_cleanup_sources_for_deleted_entity, ) + monkeypatch.setattr( + external_file_delete_runner, + "lock_note_content_before_entity_mutation", + fake_lock_note_content_before_entity_mutation, + ) entity = FakeDeletedEntity( id=42, @@ -165,6 +179,7 @@ async def fake_relation_cleanup_sources_for_deleted_entity( assert scoped_session_calls == [session_maker, session_maker] assert repository.get_calls == [(session, "notes/deleted.md")] assert repository.delete_calls == [(session, {"id": 42, "file_path": "notes/deleted.md"})] + assert note_content_lock_calls == [(session, 1, (42,))] assert relation_cleanup_calls == [(session, 1, 42)] diff --git a/tests/indexing/test_file_indexer.py b/tests/indexing/test_file_indexer.py index f78a730fe..2894a0126 100644 --- a/tests/indexing/test_file_indexer.py +++ b/tests/indexing/test_file_indexer.py @@ -19,6 +19,7 @@ from basic_memory.indexing.models import FileIndexOperation, FileIndexResult, SyncedMarkdownFile from basic_memory.indexing.note_content_reconciliation import ( NoteContentReconciliationAnchor, + NoteContentReconciliationResult, NoteContentState, ) from basic_memory.indexing.note_content_reconciler import NoteContentReconciler @@ -88,6 +89,7 @@ def _file_indexer( markdown_indexer.session_maker = _FakeSession markdown_indexer.entity_repository = entity_repository markdown_indexer.index_current_markdown_file = AsyncMock(return_value=index_result) + markdown_indexer.publish_relation_generation = AsyncMock(return_value=True) markdown_indexer.index_file = AsyncMock( return_value=FileIndexResult( file_path="notes/note.md", @@ -101,7 +103,9 @@ def _file_indexer( ) note_content_reconciler = Mock() - note_content_reconciler.reconcile = AsyncMock(return_value="current") + note_content_reconciler.reconcile = AsyncMock( + return_value=NoteContentReconciliationResult.current(3) + ) note_content_reconciler.capture_anchor = AsyncMock( return_value=NoteContentReconciliationAnchor( entity_id=existing_entity.id if existing_entity is not None else None, @@ -190,6 +194,10 @@ async def test_file_indexer_indexes_new_markdown_file() -> None: source="s3_webhook", anchor=note_content_reconciler.capture_anchor.return_value, ) + markdown_indexer.publish_relation_generation.assert_awaited_once_with( + synced_file, + generation=3, + ) assert result.file_path == "notes/note.md" assert result.entity_id == 42 assert result.checksum == CHECKSUM @@ -253,7 +261,10 @@ async def test_file_indexer_reindexes_current_file_after_anchor_becomes_stale() stale_file, current_file, ] - note_content_reconciler.reconcile.side_effect = ["stale", "current"] + note_content_reconciler.reconcile.side_effect = [ + NoteContentReconciliationResult.stale(), + NoteContentReconciliationResult.current(4), + ] rendered_messages: list[str] = [] sink_id = logger.add( @@ -269,13 +280,66 @@ async def test_file_indexer_reindexes_current_file_after_anchor_becomes_stale() assert markdown_indexer.index_current_markdown_file.await_count == 2 assert note_content_reconciler.capture_anchor.await_args_list[0].args == (7,) assert note_content_reconciler.capture_anchor.await_args_list[1].args == (7,) - assert f"Retrying markdown index after concurrent accepted note write: {file_path}" in ( + assert f"Retrying markdown index without a current relation generation: {file_path}" in ( rendered_messages ) assert f"Indexed markdown file: {file_path}" in rendered_messages assert result.file_path == file_path assert result.title == "{AG} Plan" + markdown_indexer.publish_relation_generation.assert_awaited_once_with( + current_file, + generation=4, + ) + assert result.checksum == "current-checksum" + + +@pytest.mark.asyncio +async def test_file_indexer_preserves_relations_when_file_lineage_is_deferred() -> None: + """An older file cannot publish relations or become current by retrying identical bytes.""" + existing_entity = _entity(entity_id=7) + file_indexer, markdown_indexer, note_content_reconciler = _file_indexer( + existing_entity=existing_entity, + ) + note_content_reconciler.reconcile.return_value = NoteContentReconciliationResult.deferred() + + result = await file_indexer.index_markdown_file("notes/note.md") + + markdown_indexer.index_current_markdown_file.assert_awaited_once() + markdown_indexer.publish_relation_generation.assert_not_awaited() + assert result.checksum == CHECKSUM + + +@pytest.mark.asyncio +async def test_file_indexer_retries_when_generation_changes_before_relation_publish() -> None: + """Losing the repository source fence retries instead of reporting partial publication.""" + existing_entity = _entity(entity_id=7) + stale_file = _synced_file(entity=existing_entity) + current_file = _synced_file(entity=existing_entity, checksum="current-checksum") + file_indexer, markdown_indexer, note_content_reconciler = _file_indexer( + existing_entity=existing_entity, + synced_file=stale_file, + ) + markdown_indexer.entity_repository.get_by_file_path.side_effect = [ + existing_entity, + existing_entity, + ] + markdown_indexer.index_current_markdown_file.side_effect = [stale_file, current_file] + markdown_indexer.publish_relation_generation.side_effect = [False, True] + note_content_reconciler.reconcile.side_effect = [ + NoteContentReconciliationResult.current(3), + NoteContentReconciliationResult.current(4), + ] + + result = await file_indexer.index_markdown_file("notes/note.md") + assert result.checksum == "current-checksum" + assert markdown_indexer.publish_relation_generation.await_count == 2 + assert markdown_indexer.publish_relation_generation.await_args_list[0].kwargs == { + "generation": 3 + } + assert markdown_indexer.publish_relation_generation.await_args_list[1].kwargs == { + "generation": 4 + } @pytest.mark.asyncio @@ -296,7 +360,10 @@ async def test_file_indexer_reloads_entity_after_initial_absence_becomes_stale() stale_file, current_file, ] - note_content_reconciler.reconcile.side_effect = ["stale", "current"] + note_content_reconciler.reconcile.side_effect = [ + NoteContentReconciliationResult.stale(), + NoteContentReconciliationResult.current(2), + ] result = await file_indexer.index_markdown_file("notes/note.md") @@ -317,7 +384,10 @@ async def test_file_indexer_fails_for_retry_after_repeated_stale_indexes() -> No existing_entity, existing_entity, ] - note_content_reconciler.reconcile.side_effect = ["stale", "stale"] + note_content_reconciler.reconcile.side_effect = [ + NoteContentReconciliationResult.stale(), + NoteContentReconciliationResult.stale(), + ] with pytest.raises( NoteContentChangedDuringIndexError, diff --git a/tests/indexing/test_forward_reference_resolution.py b/tests/indexing/test_forward_reference_resolution.py index a2aa44913..99033ff70 100644 --- a/tests/indexing/test_forward_reference_resolution.py +++ b/tests/indexing/test_forward_reference_resolution.py @@ -3,6 +3,7 @@ from collections.abc import AsyncIterator, Sequence from contextlib import asynccontextmanager from dataclasses import FrozenInstanceError, dataclass +from datetime import UTC, datetime from types import SimpleNamespace from typing import cast @@ -10,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker import basic_memory.indexing.forward_reference_resolution as forward_resolution_module +from basic_memory import db from basic_memory.indexing.forward_reference_resolution import ( ForwardReferenceResolutionPlan, ForwardReferenceResolutionRun, @@ -22,7 +24,7 @@ run_forward_reference_entity_refresh, run_forward_reference_resolution, ) -from basic_memory.models import Entity +from basic_memory.models import Entity, NoteContent, Relation @dataclass(frozen=True, slots=True) @@ -31,6 +33,7 @@ class StubUnresolvedRelation: from_id: int to_name: str relation_type: str = "related_to" + generation: int = 1 class RecordingForwardReferenceRuntime: @@ -101,12 +104,21 @@ def all(self) -> list[object]: class FakeForwardReferenceResult: """Minimal SQLAlchemy result stand-in for repository runtime tests.""" - def __init__(self, *, scalar_values: list[object] | None = None) -> None: + def __init__( + self, + *, + scalar_values: list[object] | None = None, + tuple_values: list[tuple[object, ...]] | None = None, + ) -> None: self.scalar_values = scalar_values or [] + self.tuple_values = tuple_values or [] def scalars(self) -> FakeForwardReferenceScalarResult: return FakeForwardReferenceScalarResult(self.scalar_values) + def tuples(self) -> FakeForwardReferenceScalarResult: + return FakeForwardReferenceScalarResult(list(self.tuple_values)) + class FakeForwardReferenceSession: """Record relation update statements issued by the repository runtime.""" @@ -115,7 +127,12 @@ def __init__(self, results: list[FakeForwardReferenceResult] | None = None) -> N self.results = results or [] self.statements: list[object] = [] - async def execute(self, statement: object) -> FakeForwardReferenceResult: + async def execute( + self, + statement: object, + params: object | None = None, + ) -> FakeForwardReferenceResult: + del params self.statements.append(statement) if self.results: return self.results.pop(0) @@ -161,12 +178,16 @@ def test_plan_forward_reference_resolution_filters_only_exact_safe_updates() -> source_entity_id=10, target_entity_id=99, link_text="Target", + source_generation=1, + relation_type="related_to", ), ForwardReferenceUpdate( relation_id=5, source_entity_id=14, target_entity_id=99, link_text="Target", + source_generation=1, + relation_type="related_to", ), ), entity_ids_to_refresh=frozenset({99}), @@ -206,6 +227,8 @@ async def test_run_forward_reference_resolution_applies_updates_once() -> None: source_entity_id=10, target_entity_id=20, link_text="Target", + source_generation=1, + relation_type="related_to", ), ), entity_ids_to_refresh=frozenset({20}), @@ -309,7 +332,24 @@ async def test_repository_forward_reference_runtime_applies_updates( monkeypatch: pytest.MonkeyPatch, ) -> None: session_maker = cast(async_sessionmaker[AsyncSession], object()) - session = FakeForwardReferenceSession() + session = FakeForwardReferenceSession( + results=[FakeForwardReferenceResult(tuple_values=[(20, "target-20"), (21, "target-21")])] + ) + applied_writes: list[tuple[object, tuple[object, ...]]] = [] + + async def fake_apply_resolved_targets( + repository: object, + active_session: object, + writes: Sequence[object], + ) -> None: + del repository + applied_writes.append((active_session, tuple(writes))) + + monkeypatch.setattr( + forward_resolution_module.RelationRepository, + "apply_resolved_targets", + fake_apply_resolved_targets, + ) @asynccontextmanager async def fake_scoped_session( @@ -332,20 +372,47 @@ async def fake_scoped_session( source_entity_id=10, target_entity_id=20, link_text="Target", + source_generation=1, + relation_type="related_to", ), ForwardReferenceUpdate( relation_id=2, source_entity_id=11, target_entity_id=21, link_text="Other", + source_generation=1, + relation_type="related_to", ), ) ) assert len(session.statements) == 1 - statement_text = str(session.statements[0]) - assert "UPDATE relation" in statement_text - assert "relation.id IN" in statement_text + assert "FROM entity" in str(session.statements[0]) + assert applied_writes == [ + ( + session, + ( + forward_resolution_module.ResolvedRelationWrite( + relation_id=1, + from_id=10, + generation=1, + original_target_name="Target", + target_id=20, + target_external_id="target-20", + relation_type="related_to", + ), + forward_resolution_module.ResolvedRelationWrite( + relation_id=2, + from_id=11, + generation=1, + original_target_name="Other", + target_id=21, + target_external_id="target-21", + relation_type="related_to", + ), + ), + ) + ] @pytest.mark.asyncio @@ -369,6 +436,87 @@ async def fake_scoped_session( await runtime.apply_forward_reference_updates(()) +@pytest.mark.asyncio +async def test_repository_forward_reference_runtime_rejects_stale_source_generation( + session_maker, + test_project, +) -> None: + """The project-index compatibility resolver shares the canonical source fence.""" + now = datetime.now(tz=UTC) + async with db.scoped_session(session_maker) as session: + source = Entity( + project_id=test_project.id, + title="Generation source", + note_type="note", + file_path="generation-source.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + target = Entity( + project_id=test_project.id, + title="Generation target", + note_type="note", + file_path="generation-target.md", + content_type="text/markdown", + created_at=now, + updated_at=now, + ) + session.add_all([source, target]) + await session.flush() + session.add( + NoteContent( + entity_id=source.id, + project_id=test_project.id, + external_id=source.external_id, + file_path=source.file_path, + markdown_content="# Generation source\n", + db_version=2, + db_checksum="generation-2", + file_write_status="synced", + ) + ) + stale_relation = Relation( + project_id=test_project.id, + from_id=source.id, + to_id=None, + to_name=target.title, + relation_type="related_to", + generation=1, + ) + session.add(stale_relation) + await session.flush() + source_id = source.id + target_id = target.id + target_external_id = target.external_id + stale_relation_id = stale_relation.id + + runtime = RepositoryForwardReferenceResolutionRuntime( + session_maker=session_maker, + project_id=test_project.id, + ) + await runtime.apply_forward_reference_updates( + ( + ForwardReferenceUpdate( + relation_id=stale_relation_id, + source_entity_id=source_id, + target_entity_id=target_id, + link_text="Generation target", + source_generation=1, + relation_type="related_to", + ), + ) + ) + + async with db.scoped_session(session_maker) as session: + relation = await session.get(Relation, stale_relation_id) + target = await session.get(Entity, target_id) + assert relation is not None + assert relation.to_id is None + assert target is not None + assert target.external_id == target_external_id + + @pytest.mark.asyncio async def test_repository_forward_reference_entity_refresh_indexes_existing_entity( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/indexing/test_index_batch_runtime.py b/tests/indexing/test_index_batch_runtime.py index 6c61ac3a2..3eb3eb2b8 100644 --- a/tests/indexing/test_index_batch_runtime.py +++ b/tests/indexing/test_index_batch_runtime.py @@ -24,8 +24,10 @@ IndexFrontmatterWriteResult, IndexingBatchResult, IndexInputFile, + RelationGenerationBatchResult, StorageIndexFileWriter, ) +from basic_memory.indexing.note_content_reconciliation import NoteContentReconciliationResult from basic_memory.indexing.note_content_reconciler import NoteContentReconciler from basic_memory.models import Entity from basic_memory.repository import EntityRepository, RelationRepository @@ -40,6 +42,22 @@ class FakeFileInfo: content: bytes | None +@dataclass(frozen=True, slots=True) +class FixedReconcileFileReader: + """Expose a rewritten file snapshot at the reconciliation boundary.""" + + content: bytes + last_modified: datetime | None + + async def get_file(self, path: str) -> FakeFileInfo: + return FakeFileInfo( + size=len(self.content), + checksum=f"reread-{path}", + last_modified=self.last_modified, + content=self.content, + ) + + class PathContentTypeProvider: def content_type(self, path: str) -> str: if path.endswith(".md"): @@ -72,6 +90,8 @@ class RecordingBatchIndexer: calls: list[dict[str, IndexInputFile]] = field(default_factory=list) max_concurrent: int | None = None parse_max_concurrent: int | None = None + published_generation_by_entity_id: dict[int, int] | None = None + publish_max_concurrent: int | None = None async def index_files( self, @@ -85,6 +105,21 @@ async def index_files( self.parse_max_concurrent = parse_max_concurrent return self.result + async def publish_relation_generations( + self, + indexed_entities: list[IndexedEntity], + *, + generation_by_entity_id: Mapping[int, int], + max_concurrent: int, + ) -> RelationGenerationBatchResult: + assert indexed_entities is self.result.indexed + self.published_generation_by_entity_id = dict(generation_by_entity_id) + self.publish_max_concurrent = max_concurrent + return RelationGenerationBatchResult( + relations_resolved=2, + relations_unresolved=3, + ) + @dataclass(slots=True) class FakeEntityRepository: @@ -113,10 +148,11 @@ async def reconcile( markdown_content: str, observed_at: datetime | None, source: str, - ) -> None: + ) -> NoteContentReconciliationResult: self.calls.append((entity, markdown_content, observed_at, source)) if entity.id in self.failing_entity_ids: raise RuntimeError(f"note_content failed for {entity.id}") + return NoteContentReconciliationResult.current(5) def recording_indexed_note_content_timestamps( @@ -234,13 +270,79 @@ async def fake_scoped_session( assert batch_indexer.calls[0]["image.png"].content_type == "application/octet-stream" assert repository.loaded_ids == [10, 20] assert reconciler.calls[0] == (FakeEntity(id=10), "# OK\n", observed_at, "index") + assert batch_indexer.published_generation_by_entity_id == {10: 5} + assert batch_indexer.publish_max_concurrent == 1 assert result.errors == [ ("preexisting.md", "parse failed"), ("bad.md", "note_content failed for 20"), ] + assert result.relations_resolved == 2 + assert result.relations_unresolved == 3 assert result.search_indexed == 2 +@pytest.mark.asyncio +async def test_rewrite_between_scan_and_reconcile_publishes_no_stale_relations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fresh content claim never stamps relations parsed from older scan bytes.""" + observed_at = datetime(2026, 6, 19, 12, 0, tzinfo=UTC) + session = cast(AsyncSession, object()) + session_maker = cast(async_sessionmaker[AsyncSession], object()) + repository = FakeEntityRepository(entities=[FakeEntity(id=10)]) + reconciler = RecordingNoteContentReconciler() + batch_indexer = RecordingBatchIndexer( + result=IndexingBatchResult( + indexed=[ + IndexedEntity( + path="rewritten.md", + entity_id=10, + permalink="rewritten", + checksum="scan-checksum", + content_type="text/markdown", + markdown_content="# Scan snapshot\n- links_to [[Old Target]]\n", + ) + ] + ) + ) + + @asynccontextmanager + async def fake_scoped_session( + scoped_session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + assert scoped_session_maker is session_maker + yield session + + monkeypatch.setattr(batch_reconciliation_module.db, "scoped_session", fake_scoped_session) + rewritten_content = b"# Rewritten snapshot\n- links_to [[New Target]]\n" + runtime = IndexBatchRuntime( + batch_indexer=batch_indexer, + content_type_provider=PathContentTypeProvider(), + entity_repository=repository, + session_maker=session_maker, + note_content_reconciler=reconciler, + timestamp_provider=recording_indexed_note_content_timestamps, + file_reader=FixedReconcileFileReader(rewritten_content, observed_at), + ) + + await runtime.index_loaded_files( + { + "rewritten.md": FakeFileInfo( + size=1, + checksum="scan-checksum", + last_modified=observed_at, + content=b"scan bytes are already parsed by the fake indexer", + ) + }, + max_concurrent=1, + ) + + assert reconciler.calls == [ + (FakeEntity(id=10), rewritten_content.decode(), observed_at, "index") + ] + assert batch_indexer.published_generation_by_entity_id == {} + + def test_count_search_indexed_entities_uses_markdown_content_presence() -> None: assert ( count_search_indexed_entities( diff --git a/tests/indexing/test_note_content_batch_reconciliation.py b/tests/indexing/test_note_content_batch_reconciliation.py index 60e03d25a..76cb7509c 100644 --- a/tests/indexing/test_note_content_batch_reconciliation.py +++ b/tests/indexing/test_note_content_batch_reconciliation.py @@ -16,6 +16,7 @@ import basic_memory.indexing.note_content_batch_reconciliation as batch_reconciliation_module from basic_memory import db, file_utils from basic_memory.indexing.models import IndexedEntity +from basic_memory.indexing.note_content_reconciliation import NoteContentReconciliationResult from basic_memory.indexing.note_content_batch_reconciliation import ( indexed_note_content_observed_at, reconcile_indexed_note_content_batch, @@ -126,7 +127,7 @@ async def reconcile_note_content( markdown_content: str, observed_at: datetime | None, source: str, - ) -> None: + ) -> NoteContentReconciliationResult: await reconcile( entity=entity, markdown_content=markdown_content, @@ -135,6 +136,7 @@ async def reconcile_note_content( ) if entity.id == 43: raise RuntimeError("note_content failed") + return NoteContentReconciliationResult.current(7) monkeypatch.setattr( batch_reconciliation_module.db, @@ -142,7 +144,7 @@ async def reconcile_note_content( fake_scoped_session, ) - errors = await reconcile_indexed_note_content_batch( + reconciliation = await reconcile_indexed_note_content_batch( [ IndexedEntity( path="ok.md", @@ -187,10 +189,17 @@ async def reconcile_note_content( ("ok.md", FakeFileInfo(observed_at=observed_at)), ("bad.md", None), ] - assert [error.as_tuple() for error in errors] == [ + assert [error.as_tuple() for error in reconciliation.errors] == [ ("missing.md", "Entity 404 not found after indexing"), ("bad.md", "note_content failed"), ] + assert reconciliation.generations == ( + batch_reconciliation_module.IndexedNoteContentGeneration( + path="ok.md", + entity_id=42, + generation=7, + ), + ) assert reconcile.await_args_list[0].kwargs == { "entity": FakeEntity(id=42), "markdown_content": "# OK\n", @@ -295,7 +304,7 @@ async def fake_scoped_session( monkeypatch.setattr(batch_reconciliation_module.db, "scoped_session", fake_scoped_session) - errors = await reconcile_indexed_note_content_batch( + reconciliation = await reconcile_indexed_note_content_batch( [ IndexedEntity( path="gone.md", @@ -315,7 +324,8 @@ async def fake_scoped_session( file_reader=StubReconcileFileReader(StubReconcileFile(content=None, last_modified=None)), ) - assert errors == () + assert reconciliation.generations == () + assert reconciliation.errors == () reconcile.assert_not_awaited() @@ -371,7 +381,7 @@ async def test_batch_reader_reconciles_fresh_content_not_scan_snapshot( StubReconcileFile(content=accepted_content.encode("utf-8"), last_modified=now) ) - errors = await reconcile_indexed_note_content_batch( + reconciliation = await reconcile_indexed_note_content_batch( [ IndexedEntity( path=sample_entity.file_path, @@ -390,7 +400,10 @@ async def test_batch_reader_reconciles_fresh_content_not_scan_snapshot( file_reader=reader, ) - assert errors == () + # The reread preserved accepted content, but its generation cannot authorize + # relations parsed from the older scan snapshot. + assert reconciliation.generations == () + assert reconciliation.errors == () async with db.scoped_session(session_maker) as session: row = await repository.get_by_entity_id(session, sample_entity.id) assert row is not None @@ -446,7 +459,7 @@ async def test_batch_without_reader_reverts_to_scan_snapshot( session_maker=session_maker, ) - errors = await reconcile_indexed_note_content_batch( + reconciliation = await reconcile_indexed_note_content_batch( [ IndexedEntity( path=sample_entity.file_path, @@ -464,7 +477,8 @@ async def test_batch_without_reader_reverts_to_scan_snapshot( source="index", ) - assert errors == () + assert [claim.generation for claim in reconciliation.generations] == [6] + assert reconciliation.errors == () async with db.scoped_session(session_maker) as session: row = await repository.get_by_entity_id(session, sample_entity.id) assert row is not None diff --git a/tests/indexing/test_note_content_reconciler.py b/tests/indexing/test_note_content_reconciler.py index 4ade61e26..08585e4a3 100644 --- a/tests/indexing/test_note_content_reconciler.py +++ b/tests/indexing/test_note_content_reconciler.py @@ -105,7 +105,8 @@ async def fake_scoped_session(_session_maker: object): ) repository.create.assert_awaited_once() - assert outcome == "current" + assert outcome.status == "current" + assert outcome.generation == 2 assert session.rollback_count == 1 assert repository.get_by_entity_id.await_count == 2 repository.update_state_fields.assert_awaited_once_with( @@ -169,7 +170,8 @@ async def fake_scoped_session(_session_maker: object): ) repository.create.assert_not_awaited() - assert outcome == "stale" + assert outcome.status == "stale" + assert outcome.generation is None assert repository.update_state_fields.await_count == 1 _, kwargs = repository.update_state_fields.await_args assert kwargs["expected_db_version"] == 3 @@ -227,7 +229,8 @@ async def fake_scoped_session(_session_maker: object): repository.create.assert_not_awaited() repository.update_state_fields.assert_not_awaited() - assert outcome == "stale" + assert outcome.status == "stale" + assert outcome.generation is None @pytest.mark.asyncio @@ -269,7 +272,8 @@ async def fake_scoped_session(_session_maker: object): anchor=NoteContentReconciliationAnchor(entity_id=None, state=None), ) - assert outcome == "stale" + assert outcome.status == "stale" + assert outcome.generation is None repository.create.assert_not_awaited() repository.update_state_fields.assert_not_awaited() @@ -315,7 +319,51 @@ async def fake_scoped_session(_session_maker: object): repository.create.assert_not_awaited() repository.update_state_fields.assert_not_awaited() - assert outcome == "current" + assert outcome.status == "deferred" + assert outcome.generation is None + + +@pytest.mark.asyncio +async def test_reconciler_does_not_claim_newer_db_generation_for_older_file() -> None: + """File bookkeeping may advance without lending DB generation 9 to version 8 bytes.""" + older_content = "# Older materialized file\n" + older_checksum = await file_utils.compute_checksum(older_content) + repository = SimpleNamespace( + get_by_entity_id=AsyncMock( + return_value=SimpleNamespace( + db_version=9, + db_checksum="new-db-checksum", + file_version=8, + file_checksum=older_checksum, + file_write_status="pending", + ) + ), + create=AsyncMock(), + update_state_fields=AsyncMock(return_value=SimpleNamespace()), + ) + + @asynccontextmanager + async def fake_scoped_session(_session_maker: object): + yield FakeSession() + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr( + "basic_memory.indexing.note_content_reconciler.db.scoped_session", + fake_scoped_session, + ) + outcome = await NoteContentReconciler( + note_content_repository=cast(Any, repository), + session_maker=cast(Any, object()), + ).reconcile( + entity=cast(Entity, SimpleNamespace(id=42)), + markdown_content=older_content, + observed_at=datetime(2026, 4, 13, 15, 0, tzinfo=UTC), + source="index", + ) + + assert outcome.status == "deferred" + assert outcome.generation is None + repository.update_state_fields.assert_awaited_once() @pytest.mark.asyncio diff --git a/tests/indexing/test_note_content_reconciliation.py b/tests/indexing/test_note_content_reconciliation.py index 1cbe123d9..d28d4f12f 100644 --- a/tests/indexing/test_note_content_reconciliation.py +++ b/tests/indexing/test_note_content_reconciliation.py @@ -2,6 +2,8 @@ from datetime import UTC, datetime +import pytest + from basic_memory.indexing.note_content_reconciliation import ( AcceptedNoteContentVersion, MaterializedNoteContentFile, @@ -13,6 +15,7 @@ NoteContentMaterializationStatusUpdate, NoteContentPromoted, NoteContentReconciliationDeferred, + NoteContentReconciliationResult, NoteContentState, ObservedNoteContent, plan_note_content_materialization_publish, @@ -21,6 +24,17 @@ ) +def test_reconciliation_result_requires_generation_only_for_current_status() -> None: + assert NoteContentReconciliationResult.current(3).generation == 3 + assert NoteContentReconciliationResult.stale().generation is None + assert NoteContentReconciliationResult.deferred().generation is None + + with pytest.raises(ValueError, match="Only current"): + NoteContentReconciliationResult(status="current") + with pytest.raises(ValueError, match="positive"): + NoteContentReconciliationResult.current(0) + + def _observed(checksum: str = "observed-checksum") -> ObservedNoteContent: return ObservedNoteContent( markdown_content="# Observed\n", diff --git a/tests/indexing/test_note_materialization_runner.py b/tests/indexing/test_note_materialization_runner.py index ec3b94e07..9030aedef 100644 --- a/tests/indexing/test_note_materialization_runner.py +++ b/tests/indexing/test_note_materialization_runner.py @@ -145,7 +145,12 @@ def __init__(self, *, entity: Entity | None, note_content: NoteContent | None) - async def scalar(self, statement: object) -> object | None: self.scalar_statements.append(statement) - return self.entity + statement_text = str(statement) + if "FROM note_content" in statement_text: + return self.note_content + if "FROM entity" in statement_text: + return self.entity + raise AssertionError(f"unexpected scalar statement: {statement_text}") async def get(self, model: type[object], identity: int) -> object | None: assert identity == 42 @@ -568,8 +573,10 @@ async def record_clear_vacate_path( file_checksum="new-file-sum", ) assert session_lock.calls == [(cast(AsyncSession, session), 7, 42)] - assert len(session.scalar_statements) == 1 - assert "FOR UPDATE" in str(session.scalar_statements[0]) + assert len(session.scalar_statements) == 2 + assert "FROM note_content" in str(session.scalar_statements[0]) + assert "FROM entity" in str(session.scalar_statements[1]) + assert all("FOR UPDATE" in str(statement) for statement in session.scalar_statements) assert repository.calls == [ ( cast(AsyncSession, session), diff --git a/tests/indexing/test_project_index_maintenance.py b/tests/indexing/test_project_index_maintenance.py index 994364856..9735bc0df 100644 --- a/tests/indexing/test_project_index_maintenance.py +++ b/tests/indexing/test_project_index_maintenance.py @@ -554,12 +554,13 @@ async def fake_scoped_session( moved_entity_ids=frozenset({10}), missing_paths=("notes/b.md",), ) - assert len(session.statements) == 5 + assert len(session.statements) == 6 assert "SELECT entity.id, entity.file_path" in str(session.statements[0]) assert "SELECT entity.id, entity.file_path" in str(session.statements[1]) - assert "UPDATE entity" in str(session.statements[2]) - assert "UPDATE note_content" in str(session.statements[3]) - assert "UPDATE search_index" in str(session.statements[4]) + assert "ORDER BY note_content.entity_id" in str(session.statements[2]) + assert "UPDATE entity" in str(session.statements[3]) + assert "UPDATE note_content" in str(session.statements[4]) + assert "UPDATE search_index" in str(session.statements[5]) @pytest.mark.asyncio @@ -626,6 +627,8 @@ async def test_repository_project_index_maintenance_store_deletes_replaced_move_ {"id": 20, "file_path": "doc.pdf"}, ] ), + FakeProjectIndexResult(), # complete move-set NoteContent fence + FakeProjectIndexResult(), # replacement-delete NoteContent fence FakeProjectIndexResult(scalar_values=[99]), ] ) @@ -661,16 +664,18 @@ async def fake_scoped_session( replaced_entity_ids=frozenset({20}), relation_cleanup_entity_ids=frozenset({99}), ) - assert len(session.statements) == 9 + assert len(session.statements) == 11 assert "SELECT entity.id, entity.file_path" in str(session.statements[0]) assert "SELECT entity.id, entity.file_path" in str(session.statements[1]) - assert "SELECT DISTINCT relation.from_id" in str(session.statements[2]) - assert "DELETE FROM search_index" in str(session.statements[3]) - assert "sqlite_master" in str(session.statements[4]) - assert "DELETE FROM entity" in str(session.statements[5]) - assert "UPDATE entity" in str(session.statements[6]) - assert "UPDATE note_content" in str(session.statements[7]) - assert "UPDATE search_index" in str(session.statements[8]) + assert "ORDER BY note_content.entity_id" in str(session.statements[2]) + assert "ORDER BY note_content.entity_id" in str(session.statements[3]) + assert "SELECT DISTINCT relation.from_id" in str(session.statements[4]) + assert "DELETE FROM search_index" in str(session.statements[5]) + assert "sqlite_master" in str(session.statements[6]) + assert "DELETE FROM entity" in str(session.statements[7]) + assert "UPDATE entity" in str(session.statements[8]) + assert "UPDATE note_content" in str(session.statements[9]) + assert "UPDATE search_index" in str(session.statements[10]) @pytest.mark.asyncio @@ -762,6 +767,8 @@ async def test_repository_project_index_maintenance_store_replaces_destination_w {"id": 20, "file_path": "archive/a.md", "checksum": "moved-checksum"}, ] ), + FakeProjectIndexResult(), # complete move-set NoteContent fence + FakeProjectIndexResult(), # replacement-delete NoteContent fence FakeProjectIndexResult(scalar_values=[99]), ] ) @@ -864,15 +871,16 @@ async def fake_scoped_session( ) ] assert content_updater.written == [(content_updater.seen_files[0], content_updater.updates[10])] - assert len(session.statements) == 6 - assert "checksum" in str(session.statements[2]) - assert "permalink" in str(session.statements[2]) - assert "markdown_content" in str(session.statements[3]) - assert "db_checksum" in str(session.statements[3]) - assert "file_checksum" in str(session.statements[3]) - assert "UPDATE search_index" in str(session.statements[4]) - assert "search_index.type" in str(session.statements[5]) - assert "permalink" in str(session.statements[5]) + assert len(session.statements) == 7 + assert "ORDER BY note_content.entity_id" in str(session.statements[2]) + assert "checksum" in str(session.statements[3]) + assert "permalink" in str(session.statements[3]) + assert "markdown_content" in str(session.statements[4]) + assert "db_checksum" in str(session.statements[4]) + assert "file_checksum" in str(session.statements[4]) + assert "UPDATE search_index" in str(session.statements[5]) + assert "search_index.type" in str(session.statements[6]) + assert "permalink" in str(session.statements[6]) @dataclass(frozen=True, slots=True) @@ -1174,6 +1182,7 @@ async def test_repository_project_index_maintenance_store_applies_delete_batch( {"id": 20, "file_path": "notes/b.md"}, ] ), + FakeProjectIndexResult(), # sorted NoteContent lock fence FakeProjectIndexResult(scalar_values=[99]), ] ) @@ -1208,12 +1217,13 @@ async def fake_scoped_session( relation_cleanup_entity_ids=frozenset({99}), missing_paths=("notes/missing.md",), ) - assert len(session.statements) == 5 + assert len(session.statements) == 6 assert "SELECT entity.id, entity.file_path" in str(session.statements[0]) - assert "SELECT DISTINCT relation.from_id" in str(session.statements[1]) - assert "DELETE FROM search_index" in str(session.statements[2]) - assert "sqlite_master" in str(session.statements[3]) - assert "DELETE FROM entity" in str(session.statements[4]) + assert "ORDER BY note_content.entity_id" in str(session.statements[1]) + assert "SELECT DISTINCT relation.from_id" in str(session.statements[2]) + assert "DELETE FROM search_index" in str(session.statements[3]) + assert "sqlite_master" in str(session.statements[4]) + assert "DELETE FROM entity" in str(session.statements[5]) @pytest.mark.asyncio @@ -1228,6 +1238,7 @@ async def test_repository_project_index_maintenance_store_deletes_vector_embeddi {"id": 10, "file_path": "notes/a.md"}, ] ), + FakeProjectIndexResult(), FakeProjectIndexResult(scalar_values=[]), FakeProjectIndexResult(), FakeProjectIndexResult( @@ -1302,6 +1313,7 @@ async def test_repository_project_index_maintenance_store_skips_vector_cleanup_w {"id": 10, "file_path": "notes/a.md"}, ] ), + FakeProjectIndexResult(), FakeProjectIndexResult(scalar_values=[]), FakeProjectIndexResult(), FakeProjectIndexResult(scalar_values=[]), diff --git a/tests/indexing/test_project_index_runtime.py b/tests/indexing/test_project_index_runtime.py index eaa3b4a0c..a8161821c 100644 --- a/tests/indexing/test_project_index_runtime.py +++ b/tests/indexing/test_project_index_runtime.py @@ -39,6 +39,7 @@ class StubUnresolvedRelation: from_id: int to_name: str relation_type: str = "related_to" + generation: int = 1 @dataclass(slots=True) @@ -329,12 +330,16 @@ async def test_project_index_runtime_resolves_forward_refs_and_refreshes_targets source_entity_id=10, target_entity_id=100, link_text="Target", + source_generation=1, + relation_type="related_to", ), ForwardReferenceUpdate( relation_id=3, source_entity_id=12, target_entity_id=200, link_text="Fails", + source_generation=1, + relation_type="related_to", ), ) assert set(refresher.calls) == {100, 200} diff --git a/tests/indexing/test_relation_persistence.py b/tests/indexing/test_relation_persistence.py new file mode 100644 index 000000000..3efda1abf --- /dev/null +++ b/tests/indexing/test_relation_persistence.py @@ -0,0 +1,455 @@ +"""Tests for generation-owned relation publication orchestration.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from collections.abc import Sequence +from datetime import UTC, datetime +from hashlib import sha256 +from typing import AsyncIterator, cast + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.indexing.change_detector import ChangeDetector +import basic_memory.indexing.relation_persistence as relation_persistence_module +from basic_memory.indexing.models import IndexedRelation +from basic_memory.indexing.relation_persistence import RelationGenerationPublisher +from basic_memory.models import Entity, Relation, RelationSearchRefresh +from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.note_content_repository import ( + AcceptedNoteContentWrite, + NoteContentRepository, +) +from basic_memory.repository.relation_repository import ( + AcceptedRelationWrite, + RelationRepository, + RelationGenerationWriteResult, +) + + +@dataclass(slots=True) +class RecordingRelationGenerationStore: + """Record the statement sequence produced by the publisher.""" + + begin_is_current: bool = True + generation_is_current: bool = True + calls: list[tuple[str, int, tuple[AcceptedRelationWrite, ...]]] = field(default_factory=list) + + async def begin_relation_generation_publication( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + assert session is not None + self.calls.append(("begin", generation, ())) + return RelationGenerationWriteResult(generation_is_current=self.begin_is_current) + + async def upsert_relation_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + relations: Sequence[AcceptedRelationWrite], + ) -> RelationGenerationWriteResult: + assert session is not None + self.calls.append(("upsert", generation, tuple(relations))) + return RelationGenerationWriteResult(generation_is_current=self.generation_is_current) + + async def cleanup_relation_generations( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + assert session is not None + self.calls.append(("cleanup", generation, ())) + return RelationGenerationWriteResult(generation_is_current=self.generation_is_current) + + +@pytest.mark.asyncio +async def test_relation_generation_publisher_commits_sorted_chunks_before_cleanup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Every bounded chunk and the final cleanup own a separate transaction.""" + sessions: list[AsyncSession] = [] + + @asynccontextmanager + async def fake_scoped_session( + session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + assert session_maker is not None + session = cast(AsyncSession, object()) + sessions.append(session) + yield session + + monkeypatch.setattr( + relation_persistence_module.db, + "scoped_session", + fake_scoped_session, + ) + store = RecordingRelationGenerationStore() + publisher = RelationGenerationPublisher( + relation_repository=store, + session_maker=cast(async_sessionmaker[AsyncSession], object()), + ) + relations = [ + IndexedRelation( + relation_type="links_to", + target_name=f"Target {index:03d}", + context=None, + target_id=42 if index == 0 else None, + ) + for index in reversed(range(251)) + ] + + generation_is_current = await publisher.publish( + entity_id=42, + generation=7, + relations=relations, + ) + + assert generation_is_current + assert [call[0] for call in store.calls] == ["begin", "upsert", "upsert", "cleanup"] + assert [len(call[2]) for call in store.calls] == [0, 250, 1, 0] + published_names = [ + relation.target_name + for operation, _, relation_chunk in store.calls + if operation == "upsert" + for relation in relation_chunk + ] + assert published_names == sorted(published_names) + published_targets = [ + relation.target_id + for operation, _, relation_chunk in store.calls + if operation == "upsert" + for relation in relation_chunk + ] + assert published_targets.count(42) == 1 + assert len(sessions) == 4 + assert len({id(session) for session in sessions}) == 4 + + +@pytest.mark.asyncio +async def test_relation_generation_publisher_stops_when_source_fence_is_stale( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A lost source claim cannot continue into later chunks or cleanup.""" + transaction_count = 0 + + @asynccontextmanager + async def fake_scoped_session( + session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + nonlocal transaction_count + assert session_maker is not None + transaction_count += 1 + yield cast(AsyncSession, object()) + + monkeypatch.setattr( + relation_persistence_module.db, + "scoped_session", + fake_scoped_session, + ) + store = RecordingRelationGenerationStore(generation_is_current=False) + publisher = RelationGenerationPublisher( + relation_repository=store, + session_maker=cast(async_sessionmaker[AsyncSession], object()), + ) + + generation_is_current = await publisher.publish( + entity_id=42, + generation=6, + relations=[IndexedRelation("links_to", "Target", None)], + ) + + assert not generation_is_current + assert [call[0] for call in store.calls] == ["begin", "upsert"] + assert transaction_count == 2 + + +@pytest.mark.asyncio +async def test_relation_generation_publisher_stops_when_publication_cannot_begin( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A generation that is already stale cannot create chunks or cleanup work.""" + transaction_count = 0 + + @asynccontextmanager + async def fake_scoped_session( + session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + nonlocal transaction_count + assert session_maker is not None + transaction_count += 1 + yield cast(AsyncSession, object()) + + monkeypatch.setattr( + relation_persistence_module.db, + "scoped_session", + fake_scoped_session, + ) + store = RecordingRelationGenerationStore(begin_is_current=False) + publisher = RelationGenerationPublisher( + relation_repository=store, + session_maker=cast(async_sessionmaker[AsyncSession], object()), + ) + + assert not await publisher.publish( + entity_id=42, + generation=6, + relations=[IndexedRelation("links_to", "Target", None)], + ) + assert [call[0] for call in store.calls] == ["begin"] + assert transaction_count == 1 + + +@pytest.mark.asyncio +async def test_relation_generation_publisher_deduplicates_pre_resolved_aliases( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Safe source aliases cannot collide in the resolved relation uniqueness domain.""" + + @asynccontextmanager + async def fake_scoped_session( + session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + assert session_maker is not None + yield cast(AsyncSession, object()) + + monkeypatch.setattr( + relation_persistence_module.db, + "scoped_session", + fake_scoped_session, + ) + store = RecordingRelationGenerationStore() + publisher = RelationGenerationPublisher( + relation_repository=store, + session_maker=cast(async_sessionmaker[AsyncSession], object()), + ) + + generation_is_current = await publisher.publish( + entity_id=42, + generation=7, + relations=[ + IndexedRelation("links_to", "source/path", "path alias", target_id=42), + IndexedRelation("links_to", "Source Title", "title alias", target_id=42), + IndexedRelation("documents", "Source Title", None, target_id=42), + ], + ) + + assert generation_is_current + assert store.calls == [ + ("begin", 7, ()), + ( + "upsert", + 7, + ( + AcceptedRelationWrite("documents", "Source Title", None, target_id=42), + AcceptedRelationWrite("links_to", "Source Title", "title alias", target_id=42), + ), + ), + ("cleanup", 7, ()), + ] + + +@pytest.mark.asyncio +async def test_relation_generation_publisher_rejects_non_self_pre_resolved_target() -> None: + """Ordinary targets remain resolver-owned even when a caller supplies an ID.""" + store = RecordingRelationGenerationStore() + publisher = RelationGenerationPublisher( + relation_repository=store, + session_maker=cast(async_sessionmaker[AsyncSession], object()), + ) + + with pytest.raises(ValueError, match="Only the source entity"): + await publisher.publish( + entity_id=42, + generation=7, + relations=[IndexedRelation("links_to", "Target", None, target_id=99)], + ) + + assert store.calls == [] + + +@pytest.mark.asyncio +async def test_failed_relation_publication_forces_change_detection_retry( + sample_entity: Entity, + entity_repository: EntityRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +) -> None: + """A committed retry marker re-drives unchanged bytes until publication completes.""" + content = "# Retry relation publication\n\n- links_to [[Target]]\n" + checksum = sha256(content.encode()).hexdigest() + observed_at = datetime.now(tz=UTC) + async with db.scoped_session(session_maker) as session: + entity = await entity_repository.get_by_id( + session, + sample_entity.id, + load_relations=False, + ) + assert entity is not None + entity.checksum = checksum + await NoteContentRepository(project_id=sample_entity.project_id).accept_write( + session, + AcceptedNoteContentWrite( + entity_id=sample_entity.id, + markdown_content=content, + db_version=1, + db_checksum=checksum, + last_source="test", + updated_at=observed_at, + ), + ) + + class _FailAfterPublicationBegins: + async def begin_relation_generation_publication( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + return await relation_repository.begin_relation_generation_publication( + session, + entity_id=entity_id, + generation=generation, + ) + + async def upsert_relation_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + relations: Sequence[AcceptedRelationWrite], + ) -> RelationGenerationWriteResult: + del session, entity_id, generation, relations + raise OSError("relation chunk write failed") + + async def cleanup_relation_generations( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + ) -> RelationGenerationWriteResult: + del session, entity_id, generation + raise AssertionError("failed publication cannot reach cleanup") + + failing_publisher = RelationGenerationPublisher( + relation_repository=_FailAfterPublicationBegins(), + session_maker=session_maker, + ) + with pytest.raises(OSError, match="relation chunk write failed"): + await failing_publisher.publish( + entity_id=sample_entity.id, + generation=1, + relations=[IndexedRelation("links_to", "Target", None)], + ) + + change_detector = ChangeDetector(entity_repository, session_maker) + assert await change_detector.load_indexed_file_checksums((sample_entity.file_path,)) == { + sample_entity.file_path: None + } + async with db.scoped_session(session_maker) as session: + publication_markers = list( + ( + await session.scalars( + select(RelationSearchRefresh).where( + RelationSearchRefresh.entity_id == sample_entity.id + ) + ) + ).all() + ) + assert [marker.publication_generation for marker in publication_markers] == [1] + + retry_publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + session_maker=session_maker, + ) + assert await retry_publisher.publish( + entity_id=sample_entity.id, + generation=1, + relations=[IndexedRelation("links_to", "Target", None)], + ) + assert await change_detector.load_indexed_file_checksums((sample_entity.file_path,)) == { + sample_entity.file_path: checksum + } + async with db.scoped_session(session_maker) as session: + refreshes = await relation_repository.list_pending_search_refreshes( + session, + entity_id=sample_entity.id, + ) + assert [refresh.entity_id for refresh in refreshes] == [sample_entity.id, sample_entity.id] + + +@pytest.mark.asyncio +async def test_generation_zero_relation_forces_generation_publication( + sample_entity: Entity, + entity_repository: EntityRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +) -> None: + """A rolling-deploy legacy write cannot make unchanged source bytes look complete.""" + content = "# Rolling relation write\n\n- links_to [[Target]]\n" + checksum = sha256(content.encode()).hexdigest() + async with db.scoped_session(session_maker) as session: + entity = await entity_repository.get_by_id( + session, + sample_entity.id, + load_relations=False, + ) + assert entity is not None + entity.checksum = checksum + await NoteContentRepository(project_id=sample_entity.project_id).accept_write( + session, + AcceptedNoteContentWrite( + entity_id=sample_entity.id, + markdown_content=content, + db_version=1, + db_checksum=checksum, + last_source="test", + updated_at=datetime.now(tz=UTC), + ), + ) + session.add( + Relation( + project_id=sample_entity.project_id, + from_id=sample_entity.id, + to_id=None, + to_name="Target", + relation_type="links_to", + generation=0, + ) + ) + + change_detector = ChangeDetector(entity_repository, session_maker) + assert await change_detector.load_indexed_file_checksums((sample_entity.file_path,)) == { + sample_entity.file_path: None + } + + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + session_maker=session_maker, + ) + assert await publisher.publish( + entity_id=sample_entity.id, + generation=1, + relations=[IndexedRelation("links_to", "Target", None)], + ) + + assert await change_detector.load_indexed_file_checksums((sample_entity.file_path,)) == { + sample_entity.file_path: checksum + } + async with db.scoped_session(session_maker) as session: + relations = await relation_repository.find_by_type(session, "links_to") + assert [(relation.to_name, relation.generation) for relation in relations] == [("Target", 1)] diff --git a/tests/indexing/test_relation_resolution.py b/tests/indexing/test_relation_resolution.py index 1617f9dad..53dd4de08 100644 --- a/tests/indexing/test_relation_resolution.py +++ b/tests/indexing/test_relation_resolution.py @@ -76,6 +76,7 @@ class FakeRelation: from_id: int to_name: str relation_type: str = "related_to" + generation: int = 1 @dataclass(frozen=True, slots=True) @@ -412,10 +413,10 @@ def test_relation_write_batch_plan_keeps_collision_domains_together() -> None: ResolvedRelationWrite( relation_id=relation_id, from_id=relation_id, + generation=1, original_target_name=f"Original {relation_id}", target_id=1_000 + relation_id, target_external_id=f"external-{1_000 + relation_id}", - target_name=f"Target {relation_id}", relation_type="related_to", ) for relation_id in range(1, RELATION_RESOLUTION_WRITE_BATCH_SIZE) @@ -424,19 +425,19 @@ def test_relation_write_batch_plan_keeps_collision_domains_together() -> None: ResolvedRelationWrite( relation_id=RELATION_RESOLUTION_WRITE_BATCH_SIZE, from_id=999, + generation=1, original_target_name="Alias A", target_id=2_000, target_external_id="external-2000", - target_name="Canonical B", relation_type="related_to", ), ResolvedRelationWrite( relation_id=RELATION_RESOLUTION_WRITE_BATCH_SIZE + 1, from_id=999, + generation=1, original_target_name="Alias B", target_id=2_001, target_external_id="external-2001", - target_name="Canonical A", relation_type="related_to", ), ] @@ -536,19 +537,19 @@ async def test_project_relation_resolution_uses_repository_runtime_and_counts_re ResolvedRelationWrite( relation_id=1, from_id=10, + generation=1, original_target_name="Target A", target_id=20, target_external_id="external-20", - target_name="Target A", relation_type="related_to", ), ResolvedRelationWrite( relation_id=2, from_id=11, + generation=1, original_target_name="Target B", target_id=21, target_external_id="external-21", - target_name="Target B", relation_type="related_to", ), ), diff --git a/tests/indexing/test_relation_resolution_scaling.py b/tests/indexing/test_relation_resolution_scaling.py index cea2db5e7..ec5ba7347 100644 --- a/tests/indexing/test_relation_resolution_scaling.py +++ b/tests/indexing/test_relation_resolution_scaling.py @@ -13,9 +13,13 @@ RelationResolutionNoteContent, RepositoryRelationResolutionRuntime, ) -from basic_memory.models import Entity, Relation +from basic_memory.models import Entity, NoteContent from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.repository.relation_repository import RelationRepository +from basic_memory.repository.relation_repository import ( + AcceptedRelationWrite, + RELATION_GENERATION_WRITE_STATEMENT_SIZE, + RelationRepository, +) from basic_memory.services.bulk_link_resolver import BulkLinkResolver @@ -93,15 +97,27 @@ async def test_postgres_ten_thousand_targets_use_a_bounded_query_budget( project_id=project_id, ), ) - for start in range(0, 10_000, 1_000): - await relation_repository.add_all_ignore_duplicates( + session.add( + NoteContent( + entity_id=source.id, + project_id=project_id, + external_id=source.external_id, + file_path=source.file_path, + markdown_content="# Relation Resolution Source\n", + db_version=1, + db_checksum="source-generation-1", + file_write_status="synced", + ) + ) + await session.flush() + for start in range(0, 10_000, RELATION_GENERATION_WRITE_STATEMENT_SIZE): + result = await relation_repository.upsert_relation_generation( session, - [ - Relation( - project_id=project_id, - from_id=source.id, - to_id=None, - to_name=( + entity_id=source.id, + generation=1, + relations=[ + AcceptedRelationWrite( + target_name=( target.title if target_index == 0 else f"Missing Target {target_index:05d}" @@ -109,9 +125,13 @@ async def test_postgres_ten_thousand_targets_use_a_bounded_query_budget( relation_type="related_to", context=None, ) - for target_index in range(start, start + 1_000) + for target_index in range( + start, + start + RELATION_GENERATION_WRITE_STATEMENT_SIZE, + ) ], ) + assert result.generation_is_current entity_indexer = RecordingEntityIndexer() runtime = RepositoryRelationResolutionRuntime( diff --git a/tests/indexing/test_relation_search_refresh_retry.py b/tests/indexing/test_relation_search_refresh_retry.py index 846a12b39..0326ee2b9 100644 --- a/tests/indexing/test_relation_search_refresh_retry.py +++ b/tests/indexing/test_relation_search_refresh_retry.py @@ -2,6 +2,7 @@ from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from hashlib import sha256 import pytest from sqlalchemy.ext.asyncio import AsyncSession @@ -10,7 +11,10 @@ from basic_memory.indexing.relation_resolution import RepositoryRelationResolutionRuntime from basic_memory.models import Entity, Relation, RelationSearchRefresh from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.repository.note_content_repository import NoteContentRepository +from basic_memory.repository.note_content_repository import ( + AcceptedNoteContentWrite, + NoteContentRepository, +) from basic_memory.repository.relation_repository import RelationRepository from basic_memory.schemas.search import SearchItemType @@ -75,7 +79,7 @@ async def test_clearing_observed_refresh_preserves_newer_work( @pytest.mark.asyncio -async def test_relation_refresh_retries_after_storage_read_failure_and_runtime_restart( +async def test_relation_refresh_retries_after_search_write_failure_and_runtime_restart( entity_repository: EntityRepository, relation_repository: RelationRepository, search_service, @@ -85,6 +89,8 @@ async def test_relation_refresh_retries_after_storage_read_failure_and_runtime_r ) -> None: """Committed relation work survives a failed refresh and a new runtime instance.""" now = datetime.now(timezone.utc) + source_content = "# Retry Source\n\nThe last valid search projection." + source_checksum = sha256(source_content.encode("utf-8")).hexdigest() source = Entity( project_id=test_project.id, title="Retry Source", @@ -108,6 +114,17 @@ async def test_relation_refresh_retries_after_storage_read_failure_and_runtime_r async with db.scoped_session(session_maker) as session: session.add_all([source, target]) await session.flush() + await NoteContentRepository(project_id=test_project.id).accept_write( + session, + AcceptedNoteContentWrite( + entity_id=source.id, + markdown_content=source_content, + db_version=1, + db_checksum=source_checksum, + last_source="test", + updated_at=now, + ), + ) session.add( Relation( project_id=test_project.id, @@ -115,6 +132,7 @@ async def test_relation_refresh_retries_after_storage_read_failure_and_runtime_r to_id=None, to_name=target.title, relation_type="relates_to", + generation=1, ) ) await session.flush() @@ -124,20 +142,28 @@ async def test_relation_refresh_retries_after_storage_read_failure_and_runtime_r async with db.scoped_session(session_maker) as session: indexed_source = await entity_repository.find_by_id(session, source_id) assert indexed_source is not None - source_content = "# Retry Source\n\nThe last valid search projection." await search_service.index_entity_data(indexed_source, content=source_content) - read_attempts = 0 + refresh_attempts = 0 + original_index_entities = search_service.index_entities - async def fail_once_read(entity: Entity) -> str: - nonlocal read_attempts - assert entity.id == source_id - read_attempts += 1 - if read_attempts == 1: - raise OSError("transient object-storage read failure") - return source_content + async def fail_once_index_entities( + entities: Sequence[Entity], + *, + content_by_entity_id: Mapping[int, str], + ) -> None: + nonlocal refresh_attempts + assert [entity.id for entity in entities] == [source_id] + assert content_by_entity_id == {source_id: source_content} + refresh_attempts += 1 + if refresh_attempts == 1: + raise OSError("transient search refresh failure") + await original_index_entities( + entities, + content_by_entity_id=content_by_entity_id, + ) - monkeypatch.setattr(search_service.file_service, "read_entity_content", fail_once_read) + monkeypatch.setattr(search_service, "index_entities", fail_once_index_entities) first_resolver = StaticLinkResolver({target.title: target}) first_runtime = RepositoryRelationResolutionRuntime( session_maker=session_maker, @@ -148,7 +174,7 @@ async def fail_once_read(entity: Entity) -> str: entity_indexer=search_service, ) - with pytest.raises(OSError, match="transient object-storage read failure"): + with pytest.raises(OSError, match="transient search refresh failure"): await first_runtime.resolve_relations() async with db.scoped_session(session_maker) as session: @@ -188,7 +214,7 @@ async def fail_once_read(entity: Entity) -> str: assert affected == {source_id} assert retry_resolver.calls == 0 - assert read_attempts == 2 + assert refresh_attempts == 2 async with db.scoped_session(session_maker) as session: remaining_refreshes = await relation_repository.list_pending_search_refreshes(session) assert remaining_refreshes == [] diff --git a/tests/repository/test_relation_repository.py b/tests/repository/test_relation_repository.py index 1eff1c0e6..f7f273c65 100644 --- a/tests/repository/test_relation_repository.py +++ b/tests/repository/test_relation_repository.py @@ -1,19 +1,25 @@ """Tests for the RelationRepository.""" from datetime import datetime, timezone +from unittest.mock import AsyncMock import pytest import pytest_asyncio from sqlalchemy import delete, select +from sqlalchemy.dialects import postgresql, sqlite from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession from basic_memory import db -from basic_memory.models import Entity, Project, Relation +from basic_memory.models import Entity, NoteContent, Project, Relation, RelationSearchRefresh from basic_memory.repository.relation_repository import ( AcceptedRelationWrite, + RELATION_GENERATION_WRITE_STATEMENT_SIZE, RelationRepository, ResolvedRelationWrite, ResolvedRelationWriteResult, + current_relation_generation_statement, + lock_note_content_before_entity_mutation, ) @@ -97,6 +103,36 @@ async def related_entity(entity_repository, session_maker): return await entity_repository.create(session, entity_data) +async def set_note_content_generation( + session_maker, + *, + source_entity: Entity, + test_project: Project, + generation: int, +) -> None: + """Create or advance the authoritative generation for one relation source.""" + async with db.scoped_session(session_maker) as session: + note_content = await session.get(NoteContent, source_entity.id) + if note_content is None: + session.add( + NoteContent( + entity_id=source_entity.id, + project_id=test_project.id, + external_id=source_entity.external_id, + file_path=source_entity.file_path, + markdown_content=f"# Generation {generation}\n", + db_version=generation, + db_checksum=f"checksum-{generation}", + file_write_status="synced", + ) + ) + return + + note_content.markdown_content = f"# Generation {generation}\n" + note_content.db_version = generation + note_content.db_checksum = f"checksum-{generation}" + + @pytest_asyncio.fixture(scope="function") async def sample_relation( relation_repository: RelationRepository, @@ -245,15 +281,23 @@ async def test_find_unresolved_relations( relation_repository: RelationRepository, sample_entity: Entity, related_entity: Entity, + test_project: Project, session_maker, ): """Test creating a new relation""" + await set_note_content_generation( + session_maker, + source_entity=sample_entity, + test_project=test_project, + generation=1, + ) relation_data = { "from_id": sample_entity.id, "to_id": None, "to_name": related_entity.title, "relation_type": "test_relation", "context": "test-context", + "generation": 1, } async with db.scoped_session(session_maker) as session: relation = await relation_repository.create(session, relation_data) @@ -410,300 +454,530 @@ async def test_delete_nonexistent_relation(relation_repository, session_maker): # ------------------------------------------------------------------------- -# Tests for add_all_ignore_duplicates +# Tests for generation-versioned relation persistence # ------------------------------------------------------------------------- -@pytest.mark.asyncio -async def test_add_all_ignore_duplicates_basic( - relation_repository: RelationRepository, - sample_entity: Entity, - related_entity: Entity, - session_maker, -): - """Test bulk inserting relations with ON CONFLICT DO NOTHING.""" - relations = [ - Relation( - from_id=sample_entity.id, - to_id=related_entity.id, - to_name=related_entity.title, - relation_type="links_to", - ), - Relation( - from_id=sample_entity.id, - to_id=related_entity.id, - to_name=related_entity.title, - relation_type="references", - ), - ] +def test_current_relation_generation_fence_is_portable() -> None: + """Postgres locks the source row while SQLite safely omits unsupported syntax.""" + statement = current_relation_generation_statement( + project_id=1, + entity_id=2, + generation=3, + ) - async with db.scoped_session(session_maker) as session: - inserted = await relation_repository.add_all_ignore_duplicates(session, relations) + postgres_sql = str(statement.compile(dialect=postgresql.dialect())) + sqlite_sql = str(statement.compile(dialect=sqlite.dialect())) - # Both should be inserted - assert inserted == 2 + assert "FOR UPDATE" in postgres_sql + assert "FOR UPDATE" not in sqlite_sql - # Verify they exist - found = await relation_repository.find_by_entities( - session, sample_entity.id, related_entity.id - ) - assert len(found) == 2 - relation_types = {r.relation_type for r in found} - assert relation_types == {"links_to", "references"} + +@pytest.mark.asyncio +async def test_bulk_note_content_fence_sorts_ids_and_is_portable() -> None: + """Bulk mutation fences share PostgreSQL locking and SQLite omission semantics.""" + session = AsyncMock(spec=AsyncSession) + + await lock_note_content_before_entity_mutation( + session, + project_id=1, + entity_ids=[3, 1, 2, 2], + ) + + statement = session.execute.await_args.args[0] + statement_params = statement.compile().params + assert [1, 2, 3] in statement_params.values() + assert "ORDER BY note_content.entity_id" in str(statement.compile(dialect=postgresql.dialect())) + assert "FOR UPDATE" in str(statement.compile(dialect=postgresql.dialect())) + assert "FOR UPDATE" not in str(statement.compile(dialect=sqlite.dialect())) + + session.reset_mock() + await lock_note_content_before_entity_mutation( + session, + project_id=1, + entity_ids=(), + ) + session.execute.assert_not_awaited() @pytest.mark.asyncio -async def test_add_all_ignore_duplicates_skips_duplicates( +async def test_upsert_relation_generation_rejects_empty_and_unbounded_chunks( relation_repository: RelationRepository, - sample_entity: Entity, - related_entity: Entity, session_maker, -): - """Test that duplicate relations are silently ignored.""" - # Same relation appearing multiple times (common when same [[link]] appears twice in doc) - relations = [ - Relation( - from_id=sample_entity.id, - to_id=None, # Unresolved - to_name="Some Target", - relation_type="links_to", - ), - Relation( - from_id=sample_entity.id, - to_id=None, - to_name="Some Target", # Duplicate! - relation_type="links_to", - ), - Relation( - from_id=sample_entity.id, - to_id=None, - to_name="Some Target", # Triple duplicate! - relation_type="links_to", - ), +) -> None: + """Publication statements require one bounded set of unique identities.""" + oversized_relations = [ + AcceptedRelationWrite( + relation_type="documents", + target_name=f"Target {index}", + context=None, + ) + for index in range(RELATION_GENERATION_WRITE_STATEMENT_SIZE + 1) ] - async with db.scoped_session(session_maker) as session: - inserted = await relation_repository.add_all_ignore_duplicates(session, relations) + with pytest.raises(ValueError, match="requires at least one relation"): + await relation_repository.upsert_relation_generation( + session, + entity_id=1, + generation=1, + relations=[], + ) + with pytest.raises(ValueError, match="exceeds the bounded statement size"): + await relation_repository.upsert_relation_generation( + session, + entity_id=1, + generation=1, + relations=oversized_relations, + ) - # Only 1 should be inserted (duplicates ignored) - assert inserted == 1 - # Verify only one exists - all_relations = await relation_repository.find_all(session) - matching = [r for r in all_relations if r.to_name == "Some Target"] - assert len(matching) == 1 +@pytest.mark.asyncio +async def test_upsert_relation_generation_resets_resolver_owned_target( + relation_repository: RelationRepository, + source_entity: Entity, + target_entity: Entity, + test_project: Project, + session_maker, +) -> None: + """A newer accepted generation owns context while resolution remains derived.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) + write = AcceptedRelationWrite( + relation_type="documents", + target_name="Target Alias", + context="generation one", + ) + async with db.scoped_session(session_maker) as session: + result = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity.id, + generation=1, + relations=[write], + ) + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + relation = (await relation_repository.find_by_type(session, "documents"))[0] + relation.to_id = target_entity.id + + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) + async with db.scoped_session(session_maker) as session: + result = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity.id, + generation=2, + relations=[ + AcceptedRelationWrite( + relation_type="documents", + target_name="Target Alias", + context="generation two", + ) + ], + ) -@pytest.mark.asyncio -async def test_add_all_ignore_duplicates_empty_list( - relation_repository: RelationRepository, session_maker -): - """Test with empty list returns 0.""" + assert result.generation_is_current async with db.scoped_session(session_maker) as session: - inserted = await relation_repository.add_all_ignore_duplicates(session, []) - assert inserted == 0 + relation = (await relation_repository.find_by_type(session, "documents"))[0] + + assert relation.generation == 2 + assert relation.context == "generation two" + assert relation.to_id is None + assert relation.to_name == "Target Alias" @pytest.mark.asyncio -async def test_add_all_ignore_duplicates_mixed( +async def test_upsert_relation_generation_persists_pre_resolved_self_target( relation_repository: RelationRepository, - sample_entity: Entity, - related_entity: Entity, + source_entity: Entity, + test_project: Project, session_maker, -): - """Test with mix of new and duplicate relations.""" - # First, insert one relation - first_relation = Relation( - from_id=sample_entity.id, - to_id=None, - to_name="Existing Target", - relation_type="links_to", +) -> None: + """Publication preserves the only target resolved safely from accepted bytes.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, ) - async with db.scoped_session(session_maker) as session: - await relation_repository.add_all_ignore_duplicates(session, [first_relation]) - # Now try to insert a mix of new and duplicate - relations = [ - Relation( - from_id=sample_entity.id, - to_id=None, - to_name="Existing Target", # Duplicate of first_relation - relation_type="links_to", - ), - Relation( - from_id=sample_entity.id, - to_id=None, - to_name="New Target 1", # New - relation_type="links_to", - ), - Relation( - from_id=sample_entity.id, - to_id=None, - to_name="New Target 2", # New - relation_type="references", - ), - ] - - inserted = await relation_repository.add_all_ignore_duplicates(session, relations) + async with db.scoped_session(session_maker) as session: + result = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity.id, + generation=1, + relations=[ + AcceptedRelationWrite( + relation_type="documents", + target_name=source_entity.file_path, + context=None, + target_id=source_entity.id, + ) + ], + ) - # Only 2 new ones should be inserted - assert inserted == 2 + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + relation = (await relation_repository.find_by_type(session, "documents"))[0] + unresolved = await relation_repository.find_unresolved_relations(session) - # Verify total count - all_relations = await relation_repository.find_all(session) - from_sample = [r for r in all_relations if r.from_id == sample_entity.id] - assert len(from_sample) == 3 # 1 existing + 2 new + assert relation.to_id == source_entity.id + assert relation.to_name == source_entity.file_path + assert unresolved == [] @pytest.mark.asyncio -async def test_add_all_ignore_duplicates_with_context( +async def test_upsert_relation_generation_replaces_pre_resolved_self_alias( relation_repository: RelationRepository, - sample_entity: Entity, - related_entity: Entity, + source_entity: Entity, + test_project: Project, session_maker, -): - """Test that context field is properly inserted.""" - relations = [ - Relation( - from_id=sample_entity.id, - to_id=related_entity.id, - to_name=related_entity.title, - relation_type="links_to", - context="some context here", - ), - ] - +) -> None: + """A newer authored alias replaces the same self-edge without crossing unique domains.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) async with db.scoped_session(session_maker) as session: - inserted = await relation_repository.add_all_ignore_duplicates(session, relations) - assert inserted == 1 + first = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity.id, + generation=1, + relations=[ + AcceptedRelationWrite( + relation_type="documents", + target_name=source_entity.file_path, + context=None, + target_id=source_entity.id, + ) + ], + ) + assert first.generation_is_current - # Verify context was saved - found = await relation_repository.find_by_entities( - session, sample_entity.id, related_entity.id + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) + async with db.scoped_session(session_maker) as session: + second = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity.id, + generation=2, + relations=[ + AcceptedRelationWrite( + relation_type="documents", + target_name=source_entity.title, + context="new alias", + target_id=source_entity.id, + ) + ], ) - assert len(found) == 1 - assert found[0].context == "some context here" + assert second.generation_is_current + + async with db.scoped_session(session_maker) as session: + relations = await relation_repository.find_by_type(session, "documents") + + assert len(relations) == 1 + assert relations[0].generation == 2 + assert relations[0].to_id == source_entity.id + assert relations[0].to_name == source_entity.title + assert relations[0].context == "new alias" @pytest.mark.asyncio -async def test_replace_accepted_outgoing_relations_inserts_unresolved( +async def test_stale_relation_generation_cannot_reinsert_absent_key_or_cleanup_newer_rows( relation_repository: RelationRepository, source_entity: Entity, + test_project: Project, session_maker, -): - """Accepted-write graph persistence inserts relations unresolved (to_id None).""" - writes = [ - AcceptedRelationWrite( - relation_type="works_at", - target_name="XSYS Target", - context="employment", - ), - AcceptedRelationWrite(relation_type="knows", target_name="Ada", context=None), - ] +) -> None: + """The source fence makes every statement from an already-stale writer inert.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) async with db.scoped_session(session_maker) as session: - await relation_repository.replace_accepted_outgoing_relations( - session, source_entity.id, writes + current = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity.id, + generation=2, + relations=[ + AcceptedRelationWrite( + relation_type="current", + target_name="Current Target", + context=None, + ) + ], ) + assert current.generation_is_current async with db.scoped_session(session_maker) as session: - relations = await relation_repository.find_by_type(session, "works_at") - knows = await relation_repository.find_by_type(session, "knows") + stale_upsert = await relation_repository.upsert_relation_generation( + session, + entity_id=source_entity.id, + generation=1, + relations=[ + AcceptedRelationWrite( + relation_type="removed", + target_name="Removed Target", + context=None, + ) + ], + ) + async with db.scoped_session(session_maker) as session: + stale_cleanup = await relation_repository.cleanup_relation_generations( + session, + entity_id=source_entity.id, + generation=1, + ) - assert len(relations) == 1 - works_at = relations[0] - assert works_at.from_id == source_entity.id - # Targets are written unresolved; the forward-reference job links to_id later. - assert works_at.to_id is None - assert works_at.to_name == "XSYS Target" - assert works_at.context == "employment" - assert len(knows) == 1 - assert knows[0].to_name == "Ada" + assert not stale_upsert.generation_is_current + assert not stale_cleanup.generation_is_current + async with db.scoped_session(session_maker) as session: + relations = await relation_repository.find_all(session) + + assert [(relation.relation_type, relation.generation) for relation in relations] == [ + ("current", 2) + ] @pytest.mark.asyncio -async def test_replace_accepted_outgoing_relations_persists_resolved_target( +async def test_newer_relation_generation_replaces_then_cleans_older_set( relation_repository: RelationRepository, source_entity: Entity, + test_project: Project, session_maker, -): - """Accepted self-links retain the safe target ID resolved by the runner.""" - write = AcceptedRelationWrite( - relation_type="documents", - target_name=source_entity.title, - context=None, - target_id=source_entity.id, +) -> None: + """A later generation updates retained identities before removing superseded rows.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, ) async with db.scoped_session(session_maker) as session: - await relation_repository.replace_accepted_outgoing_relations( + await relation_repository.upsert_relation_generation( session, - source_entity.id, - [write], + entity_id=source_entity.id, + generation=1, + relations=[ + AcceptedRelationWrite("old", "Old Target", None), + AcceptedRelationWrite("shared", "Shared Target", "old context"), + ], + ) + async with db.scoped_session(session_maker) as session: + await relation_repository.cleanup_relation_generations( + session, + entity_id=source_entity.id, + generation=1, ) + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) async with db.scoped_session(session_maker) as session: - relations = await relation_repository.find_by_entities( + current = await relation_repository.upsert_relation_generation( session, - source_entity.id, - source_entity.id, + entity_id=source_entity.id, + generation=2, + relations=[ + AcceptedRelationWrite("new", "New Target", None), + AcceptedRelationWrite("shared", "Shared Target", "new context"), + ], + ) + async with db.scoped_session(session_maker) as session: + cleanup = await relation_repository.cleanup_relation_generations( + session, + entity_id=source_entity.id, + generation=2, ) - assert len(relations) == 1 - assert relations[0].to_id == source_entity.id - assert relations[0].to_name == source_entity.title + assert current.generation_is_current + assert cleanup.generation_is_current + async with db.scoped_session(session_maker) as session: + relations = sorted(await relation_repository.find_all(session), key=lambda row: row.to_name) + + assert [ + (relation.relation_type, relation.to_name, relation.context, relation.generation) + for relation in relations + ] == [ + ("new", "New Target", None, 2), + ("shared", "Shared Target", "new context", 2), + ] @pytest.mark.asyncio -async def test_replace_accepted_outgoing_relations_replaces_existing_set( +async def test_empty_relation_generation_cleanup_removes_the_older_set( relation_repository: RelationRepository, source_entity: Entity, + test_project: Project, session_maker, -): - """A second accepted write replaces the prior outgoing relation set atomically.""" +) -> None: + """An accepted empty relation set still publishes through guarded cleanup.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) async with db.scoped_session(session_maker) as session: - await relation_repository.replace_accepted_outgoing_relations( + await relation_repository.add( session, - source_entity.id, - [AcceptedRelationWrite(relation_type="old_rel", target_name="Old", context=None)], + Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_name="Removed Target", + relation_type="removed", + generation=1, + ), ) async with db.scoped_session(session_maker) as session: - await relation_repository.replace_accepted_outgoing_relations( + cleanup = await relation_repository.cleanup_relation_generations( session, - source_entity.id, - [AcceptedRelationWrite(relation_type="new_rel", target_name="New", context=None)], + entity_id=source_entity.id, + generation=2, ) + assert cleanup.generation_is_current async with db.scoped_session(session_maker) as session: - old = await relation_repository.find_by_type(session, "old_rel") - new = await relation_repository.find_by_type(session, "new_rel") + assert await relation_repository.find_all(session) == [] - assert old == [] - assert [rel.to_name for rel in new] == ["New"] + +@pytest.mark.asyncio +async def test_apply_resolved_targets_accepts_empty_plan( + relation_repository: RelationRepository, + session_maker, +): + """An empty resolver batch has no transaction side effects.""" + async with db.scoped_session(session_maker) as session: + result = await relation_repository.apply_resolved_targets(session, []) + + assert result == ResolvedRelationWriteResult( + affected_entity_ids=frozenset(), + duplicate_relation_ids=(), + ) @pytest.mark.asyncio -async def test_replace_accepted_outgoing_relations_clears_when_empty( +async def test_legacy_relation_without_note_content_remains_resolvable( relation_repository: RelationRepository, source_entity: Entity, + target_entity: Entity, + test_project: Project, session_maker, -): - """An empty accepted relation set clears any prior outgoing rows for the entity.""" +) -> None: + """Migration-era generation-zero relations remain live until source bootstrap.""" + legacy_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=None, + to_name=target_entity.title, + relation_type="documents", + generation=0, + ) + async with db.scoped_session(session_maker) as session: + session.add(legacy_relation) + await session.flush() + relation_id = legacy_relation.id + + async with db.scoped_session(session_maker) as session: + unresolved = await relation_repository.find_unresolved_relations(session) + assert [relation.id for relation in unresolved] == [relation_id] + async with db.scoped_session(session_maker) as session: - await relation_repository.replace_accepted_outgoing_relations( + result = await relation_repository.apply_resolved_targets( session, - source_entity.id, - [AcceptedRelationWrite(relation_type="stale", target_name="Gone", context=None)], + [ + ResolvedRelationWrite( + relation_id=relation_id, + from_id=source_entity.id, + generation=0, + original_target_name=target_entity.title, + target_id=target_entity.id, + target_external_id=target_entity.external_id, + relation_type="documents", + ) + ], ) + assert result == ResolvedRelationWriteResult( + affected_entity_ids=frozenset({source_entity.id}), + duplicate_relation_ids=(), + ) + async with db.scoped_session(session_maker) as session: + relation = await relation_repository.find_by_id(session, relation_id) + + assert relation is not None + assert relation.to_id == target_entity.id + assert relation.generation == 0 + + +@pytest.mark.asyncio +async def test_legacy_relation_becomes_stale_after_note_content_bootstrap( + relation_repository: RelationRepository, + source_entity: Entity, + target_entity: Entity, + test_project: Project, + session_maker, +) -> None: + """Bootstrapping the source closes the generation-zero compatibility path.""" + legacy_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=None, + to_name=target_entity.title, + relation_type="documents", + generation=0, + ) async with db.scoped_session(session_maker) as session: - await relation_repository.replace_accepted_outgoing_relations(session, source_entity.id, []) + session.add(legacy_relation) + await session.flush() + relation_id = legacy_relation.id + + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) + write = ResolvedRelationWrite( + relation_id=relation_id, + from_id=source_entity.id, + generation=0, + original_target_name=target_entity.title, + target_id=target_entity.id, + target_external_id=target_entity.external_id, + relation_type="documents", + ) async with db.scoped_session(session_maker) as session: - remaining = await relation_repository.find_unresolved_relations_for_entity( - session, source_entity.id - ) + unresolved = await relation_repository.find_unresolved_relations(session) + result = await relation_repository.apply_resolved_targets(session, [write]) - assert remaining == [] + assert unresolved == [] + assert result == ResolvedRelationWriteResult( + affected_entity_ids=frozenset(), + duplicate_relation_ids=(), + stale_relation_ids=(relation_id,), + ) @pytest.mark.asyncio @@ -715,13 +989,20 @@ async def test_apply_resolved_targets_batches_updates_and_duplicate_cleanup( test_project: Project, session_maker, ): - """Canonical targets update together while redundant resolved edges are removed.""" + """Targets update together while redundant resolved edges are removed.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) accepted_target = Relation( project_id=test_project.id, from_id=source_entity.id, to_id=None, to_name="Target Alias", relation_type="documents", + generation=1, ) accepted_related = Relation( project_id=test_project.id, @@ -729,6 +1010,7 @@ async def test_apply_resolved_targets_batches_updates_and_duplicate_cleanup( to_id=None, to_name="Related Alias", relation_type="references", + generation=1, ) existing_edge = Relation( project_id=test_project.id, @@ -736,6 +1018,7 @@ async def test_apply_resolved_targets_batches_updates_and_duplicate_cleanup( to_id=target_entity.id, to_name=target_entity.title, relation_type="links_to", + generation=1, ) redundant_unresolved = Relation( project_id=test_project.id, @@ -743,6 +1026,7 @@ async def test_apply_resolved_targets_batches_updates_and_duplicate_cleanup( to_id=None, to_name="Duplicate Target Alias", relation_type="links_to", + generation=1, ) async with db.scoped_session(session_maker) as session: session.add_all([accepted_target, accepted_related, existing_edge, redundant_unresolved]) @@ -758,28 +1042,28 @@ async def test_apply_resolved_targets_batches_updates_and_duplicate_cleanup( ResolvedRelationWrite( relation_id=accepted_related_id, from_id=source_entity.id, + generation=1, original_target_name="Related Alias", target_id=related_entity.id, target_external_id=related_entity.external_id, - target_name=related_entity.title, relation_type="references", ), ResolvedRelationWrite( relation_id=redundant_unresolved_id, from_id=source_entity.id, + generation=1, original_target_name="Duplicate Target Alias", target_id=target_entity.id, target_external_id=target_entity.external_id, - target_name=target_entity.title, relation_type="links_to", ), ResolvedRelationWrite( relation_id=accepted_target_id, from_id=source_entity.id, + generation=1, original_target_name="Target Alias", target_id=target_entity.id, target_external_id=target_entity.external_id, - target_name=target_entity.title, relation_type="documents", ), ], @@ -797,10 +1081,10 @@ async def test_apply_resolved_targets_batches_updates_and_duplicate_cleanup( redundant = await relation_repository.find_by_id(session, redundant_unresolved_id) assert [(relation.to_id, relation.to_name) for relation in documents] == [ - (target_entity.id, target_entity.title) + (target_entity.id, "Target Alias") ] assert [(relation.to_id, relation.to_name) for relation in references] == [ - (related_entity.id, related_entity.title) + (related_entity.id, "Related Alias") ] assert [(relation.to_id, relation.to_name) for relation in links] == [ (target_entity.id, target_entity.title) @@ -809,7 +1093,7 @@ async def test_apply_resolved_targets_batches_updates_and_duplicate_cleanup( @pytest.mark.asyncio -async def test_apply_resolved_targets_preserves_canonical_name_swaps( +async def test_apply_resolved_targets_preserves_customer_aliases( relation_repository: RelationRepository, source_entity: Entity, target_entity: Entity, @@ -817,13 +1101,20 @@ async def test_apply_resolved_targets_preserves_canonical_name_swaps( test_project: Project, session_maker, ): - """Relations exchanging occupied names remain valid when planned together.""" + """Resolver-owned target backfill leaves customer-authored aliases unchanged.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) first_relation = Relation( project_id=test_project.id, from_id=source_entity.id, to_id=None, to_name=target_entity.title, relation_type="renames_to", + generation=1, ) second_relation = Relation( project_id=test_project.id, @@ -831,6 +1122,7 @@ async def test_apply_resolved_targets_preserves_canonical_name_swaps( to_id=None, to_name=related_entity.title, relation_type="renames_to", + generation=1, ) async with db.scoped_session(session_maker) as session: session.add_all([first_relation, second_relation]) @@ -845,19 +1137,19 @@ async def test_apply_resolved_targets_preserves_canonical_name_swaps( ResolvedRelationWrite( relation_id=first_relation_id, from_id=source_entity.id, + generation=1, original_target_name=target_entity.title, target_id=related_entity.id, target_external_id=related_entity.external_id, - target_name=related_entity.title, relation_type="renames_to", ), ResolvedRelationWrite( relation_id=second_relation_id, from_id=source_entity.id, + generation=1, original_target_name=related_entity.title, target_id=target_entity.id, target_external_id=target_entity.external_id, - target_name=target_entity.title, relation_type="renames_to", ), ], @@ -868,8 +1160,8 @@ async def test_apply_resolved_targets_preserves_canonical_name_swaps( relations = await relation_repository.find_by_type(session, "renames_to") assert {(relation.id, relation.to_id, relation.to_name) for relation in relations} == { - (first_relation_id, related_entity.id, related_entity.title), - (second_relation_id, target_entity.id, target_entity.title), + (first_relation_id, related_entity.id, target_entity.title), + (second_relation_id, target_entity.id, related_entity.title), } @@ -880,7 +1172,13 @@ async def test_apply_resolved_targets_bounds_sql_for_oversized_collision_domain( test_project: Project, session_maker, ): - """One large name-swap domain stays atomic without unbounded SQL expressions.""" + """One large collision domain stays bounded without rewriting customer aliases.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) domain_size = 1_100 now = datetime.now(timezone.utc) target_entities = [ @@ -906,6 +1204,7 @@ async def test_apply_resolved_targets_bounds_sql_for_oversized_collision_domain( to_id=None, to_name=target_entities[(index + 1) % domain_size].title, relation_type="large_domain", + generation=1, ) for index in range(domain_size) ] @@ -915,15 +1214,18 @@ async def test_apply_resolved_targets_bounds_sql_for_oversized_collision_domain( ResolvedRelationWrite( relation_id=relation.id, from_id=source_entity.id, + generation=1, original_target_name=relation.to_name, target_id=target.id, target_external_id=target.external_id, - target_name=target.title, relation_type=relation.relation_type, ) for relation, target in zip(unresolved_relations, target_entities, strict=True) ] - expected_targets = {(target.id, target.title) for target in target_entities} + expected_targets = { + (target.id, relation.to_name) + for relation, target in zip(unresolved_relations, target_entities, strict=True) + } async with db.scoped_session(session_maker) as session: result = await relation_repository.apply_resolved_targets(session, writes) @@ -949,6 +1251,259 @@ async def test_apply_resolved_targets_bounds_sql_for_oversized_collision_domain( assert resolved_targets == expected_targets +@pytest.mark.asyncio +async def test_apply_resolved_targets_is_inert_after_source_generation_advances( + relation_repository: RelationRepository, + source_entity: Entity, + target_entity: Entity, + test_project: Project, + session_maker, +): + """A resolver plan cannot mutate relation intent from an older accepted note.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) + unresolved_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=None, + to_name="Target Alias", + relation_type="documents", + generation=1, + ) + async with db.scoped_session(session_maker) as session: + session.add(unresolved_relation) + await session.flush() + relation_id = unresolved_relation.id + + stale_write = ResolvedRelationWrite( + relation_id=relation_id, + from_id=source_entity.id, + generation=1, + original_target_name=unresolved_relation.to_name, + target_id=target_entity.id, + target_external_id=target_entity.external_id, + relation_type=unresolved_relation.relation_type, + ) + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) + + async with db.scoped_session(session_maker) as session: + result = await relation_repository.apply_resolved_targets(session, [stale_write]) + + assert result == ResolvedRelationWriteResult( + affected_entity_ids=frozenset(), + duplicate_relation_ids=(), + stale_relation_ids=(relation_id,), + ) + async with db.scoped_session(session_maker) as session: + relation = await relation_repository.find_by_id(session, relation_id) + pending_refreshes = await relation_repository.list_pending_search_refreshes(session) + + assert relation is not None + assert relation.to_id is None + assert relation.to_name == "Target Alias" + assert pending_refreshes == [] + + +@pytest.mark.asyncio +async def test_apply_resolved_targets_rejects_relation_newer_than_source_generation( + relation_repository: RelationRepository, + source_entity: Entity, + target_entity: Entity, + test_project: Project, + session_maker, +): + """A relation ahead of authoritative note content fails as corrupted state.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) + future_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=None, + to_name="Future Alias", + relation_type="documents", + generation=2, + ) + async with db.scoped_session(session_maker) as session: + session.add(future_relation) + await session.flush() + relation_id = future_relation.id + + async with db.scoped_session(session_maker) as session: + with pytest.raises( + RuntimeError, + match="Relation generation cannot be newer than its source note_content", + ): + await relation_repository.apply_resolved_targets( + session, + [ + ResolvedRelationWrite( + relation_id=relation_id, + from_id=source_entity.id, + generation=1, + original_target_name="Future Alias", + target_id=target_entity.id, + target_external_id=target_entity.external_id, + relation_type="documents", + ) + ], + ) + + +@pytest.mark.asyncio +async def test_apply_resolved_targets_replaces_older_resolved_occupant( + relation_repository: RelationRepository, + source_entity: Entity, + target_entity: Entity, + test_project: Project, + session_maker, +): + """Current intent wins the resolved uniqueness key from an older generation.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) + older_resolved_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=target_entity.id, + to_name="Older Alias", + relation_type="documents", + generation=1, + ) + current_unresolved_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=None, + to_name="Current Alias", + relation_type="documents", + generation=2, + ) + async with db.scoped_session(session_maker) as session: + session.add_all([older_resolved_relation, current_unresolved_relation]) + await session.flush() + older_relation_id = older_resolved_relation.id + current_relation_id = current_unresolved_relation.id + + async with db.scoped_session(session_maker) as session: + result = await relation_repository.apply_resolved_targets( + session, + [ + ResolvedRelationWrite( + relation_id=current_relation_id, + from_id=source_entity.id, + generation=2, + original_target_name="Current Alias", + target_id=target_entity.id, + target_external_id=target_entity.external_id, + relation_type="documents", + ) + ], + ) + + assert result == ResolvedRelationWriteResult( + affected_entity_ids=frozenset({source_entity.id}), + duplicate_relation_ids=(), + ) + async with db.scoped_session(session_maker) as session: + older_relation = await relation_repository.find_by_id(session, older_relation_id) + current_relation = await relation_repository.find_by_id(session, current_relation_id) + + assert older_relation is None + assert current_relation is not None + assert current_relation.to_id == target_entity.id + assert current_relation.to_name == "Current Alias" + assert current_relation.generation == 2 + + +@pytest.mark.asyncio +async def test_apply_resolved_targets_chooses_lowest_id_for_current_alias_collision( + relation_repository: RelationRepository, + source_entity: Entity, + target_entity: Entity, + test_project: Project, + session_maker, +): + """Same-generation aliases resolving to one target choose a stable winner.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=3, + ) + first_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=None, + to_name="Alias A", + relation_type="documents", + generation=3, + ) + second_relation = Relation( + project_id=test_project.id, + from_id=source_entity.id, + to_id=None, + to_name="Alias B", + relation_type="documents", + generation=3, + ) + async with db.scoped_session(session_maker) as session: + session.add_all([first_relation, second_relation]) + await session.flush() + first_relation_id = first_relation.id + second_relation_id = second_relation.id + + writes = [ + ResolvedRelationWrite( + relation_id=second_relation_id, + from_id=source_entity.id, + generation=3, + original_target_name="Alias B", + target_id=target_entity.id, + target_external_id=target_entity.external_id, + relation_type="documents", + ), + ResolvedRelationWrite( + relation_id=first_relation_id, + from_id=source_entity.id, + generation=3, + original_target_name="Alias A", + target_id=target_entity.id, + target_external_id=target_entity.external_id, + relation_type="documents", + ), + ] + async with db.scoped_session(session_maker) as session: + result = await relation_repository.apply_resolved_targets(session, writes) + + assert result == ResolvedRelationWriteResult( + affected_entity_ids=frozenset({source_entity.id}), + duplicate_relation_ids=(second_relation_id,), + ) + async with db.scoped_session(session_maker) as session: + winner = await relation_repository.find_by_id(session, first_relation_id) + duplicate = await relation_repository.find_by_id(session, second_relation_id) + + assert winner is not None + assert winner.to_id == target_entity.id + assert winner.to_name == "Alias A" + assert duplicate is None + + @pytest.mark.asyncio async def test_apply_resolved_targets_skips_reused_target_identity( relation_repository: RelationRepository, @@ -958,12 +1513,19 @@ async def test_apply_resolved_targets_skips_reused_target_identity( session_maker, ): """A stale target ID cannot connect an edge to a replacement entity.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) unresolved_relation = Relation( project_id=test_project.id, from_id=source_entity.id, to_id=None, to_name="Original Target Alias", relation_type="documents", + generation=1, ) async with db.scoped_session(session_maker) as session: session.add(unresolved_relation) @@ -973,10 +1535,10 @@ async def test_apply_resolved_targets_skips_reused_target_identity( stale_write = ResolvedRelationWrite( relation_id=relation_id, from_id=source_entity.id, + generation=1, original_target_name=unresolved_relation.to_name, target_id=target_entity.id, target_external_id=target_entity.external_id, - target_name=target_entity.title, relation_type=unresolved_relation.relation_type, ) @@ -1027,12 +1589,19 @@ async def test_apply_resolved_targets_skips_reused_relation_identity( session_maker, ): """A stale command cannot resolve a replacement row that reused its database ID.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=1, + ) original_relation = Relation( project_id=test_project.id, from_id=source_entity.id, to_id=None, to_name="Original Alias", relation_type="documents", + generation=1, ) async with db.scoped_session(session_maker) as session: session.add(original_relation) @@ -1042,10 +1611,10 @@ async def test_apply_resolved_targets_skips_reused_relation_identity( stale_write = ResolvedRelationWrite( relation_id=reused_relation_id, from_id=source_entity.id, + generation=1, original_target_name=original_relation.to_name, target_id=target_entity.id, target_external_id=target_entity.external_id, - target_name=target_entity.title, relation_type=original_relation.relation_type, ) @@ -1062,6 +1631,7 @@ async def test_apply_resolved_targets_skips_reused_relation_identity( to_id=None, to_name="Replacement Alias", relation_type="supersedes", + generation=1, ) ) @@ -1082,3 +1652,39 @@ async def test_apply_resolved_targets_skips_reused_relation_identity( assert replacement.to_name == "Replacement Alias" assert replacement.relation_type == "supersedes" assert pending_refreshes == [] + + +@pytest.mark.asyncio +async def test_stale_generation_cannot_begin_relation_publication( + relation_repository: RelationRepository, + source_entity: Entity, + test_project: Project, + session_maker, +): + """A stale generation cannot create retry work that would mask a newer snapshot.""" + await set_note_content_generation( + session_maker, + source_entity=source_entity, + test_project=test_project, + generation=2, + ) + + async with db.scoped_session(session_maker) as session: + result = await relation_repository.begin_relation_generation_publication( + session, + entity_id=source_entity.id, + generation=1, + ) + + assert not result.generation_is_current + async with db.scoped_session(session_maker) as session: + markers = list( + ( + await session.scalars( + select(RelationSearchRefresh).where( + RelationSearchRefresh.entity_id == source_entity.id + ) + ) + ).all() + ) + assert markers == [] diff --git a/tests/services/test_entity_service_write_result.py b/tests/services/test_entity_service_write_result.py index 7e4eed47f..b0852b828 100644 --- a/tests/services/test_entity_service_write_result.py +++ b/tests/services/test_entity_service_write_result.py @@ -34,6 +34,37 @@ async def test_create_entity_with_content_returns_full_and_search_content( assert result.search_content == "Create body content" +@pytest.mark.asyncio +async def test_create_entity_publishes_relations_from_persisted_snapshot( + entity_service, + file_service, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A concurrent file replacement cannot stamp prepared relations with newer bytes.""" + persisted_content = "# Persisted Snapshot\n\n- links_to [[Persisted Target]]\n" + original_read = file_service.read_file_content + + async def replace_before_read(file_path) -> str: + (file_service.base_path / file_path).write_text(persisted_content, encoding="utf-8") + return await original_read(file_path) + + monkeypatch.setattr(file_service, "read_file_content", replace_before_read) + + result = await entity_service.create_entity_with_content( + EntitySchema( + title="Persisted Snapshot", + directory="notes", + note_type="note", + content="# Prepared Snapshot\n\n- links_to [[Prepared Target]]\n", + ) + ) + + assert result.content == persisted_content + assert [relation.to_name for relation in result.entity.outgoing_relations] == [ + "Persisted Target" + ] + + @pytest.mark.asyncio @pytest.mark.parametrize("permalink_line", ["permalink:", "permalink: null", 'permalink: ""']) async def test_create_entity_ignores_empty_frontmatter_permalink( diff --git a/tests/services/test_file_service.py b/tests/services/test_file_service.py index e00f8013b..ccd3292c8 100644 --- a/tests/services/test_file_service.py +++ b/tests/services/test_file_service.py @@ -174,7 +174,7 @@ async def test_write_unicode_content(tmp_path: Path, file_service: FileService): async def test_update_frontmatter_checksum_matches_windows_crlf_persisted_bytes( tmp_path: Path, file_service: FileService, monkeypatch ): - """Windows-style CRLF writes should hash the stored file, not the pre-write string.""" + """Windows-style CRLF writes return the exact payload and hash that reached disk.""" test_path = tmp_path / "note.md" test_path.write_text("# Note\nBody\n", encoding="utf-8") @@ -196,6 +196,8 @@ async def fake_write_file_atomic(path: Path, content: str) -> None: ) assert result.checksum == await file_service.compute_checksum(test_path) + assert result.content.encode("utf-8") == test_path.read_bytes() + assert "\r\n" in result.content with pytest.raises(FrozenInstanceError): setattr(result, "checksum", "changed") diff --git a/tests/services/test_upsert_entity_optimization.py b/tests/services/test_upsert_entity_optimization.py index 6b1ff206f..5e390850b 100644 --- a/tests/services/test_upsert_entity_optimization.py +++ b/tests/services/test_upsert_entity_optimization.py @@ -1,133 +1,26 @@ -"""Tests proving upsert_entity_from_markdown optimizations. - -Verifies that: -1. Redundant get_by_file_path call is eliminated (entity passed directly) -2. Final reload uses find_by_ids (PK lookup) instead of get_by_file_path (string lookup) -3. Telemetry sub-spans are emitted for each DB phase -4. Correctness is preserved for create, update, and edit flows -""" +"""Tests for the generation-safe EntityService write compatibility surface.""" from __future__ import annotations -from datetime import datetime, timezone -from pathlib import Path from types import SimpleNamespace -from typing import Any, TYPE_CHECKING +from typing import Any from unittest.mock import AsyncMock import pytest -from loguru import logger +from sqlalchemy import select -from basic_memory.markdown.schemas import ( - EntityFrontmatter, - EntityMarkdown, - Observation as MarkdownObservation, - Relation as MarkdownRelation, -) +from basic_memory import db +from basic_memory.models import NoteContent, Relation from basic_memory.schemas import Entity as EntitySchema from basic_memory.services.entity_service import EntityService -if TYPE_CHECKING: - from loguru import Record - -# --- Helpers --- - - -def _make_markdown( - title: str = "Test Entity", - observations: list[MarkdownObservation] | None = None, - relations: list[MarkdownRelation] | None = None, -) -> EntityMarkdown: - frontmatter = EntityFrontmatter(metadata={"title": title, "type": "note"}) - return EntityMarkdown( - frontmatter=frontmatter, - observations=observations or [], - relations=relations or [], - created=datetime.now(timezone.utc), - modified=datetime.now(timezone.utc), - ) - - -# --- Optimization 1: No redundant get_by_file_path in update_entity_relations --- - - -@pytest.mark.asyncio -async def test_upsert_update_does_not_refetch_entity(entity_service: EntityService, monkeypatch): - """update_entity_relations should NOT call get_by_file_path — entity is passed directly.""" - # Create an entity first - entity = await entity_service.create_entity( - EntitySchema( - title="Refetch Test", - directory="notes", - note_type="note", - content="# Refetch Test\n\n## Observations\n- [fact] some fact", - ) - ) - - # Spy on get_by_file_path calls - original_get_by_file_path = entity_service.repository.get_by_file_path - call_count = 0 - - async def spy_get_by_file_path(*args, **kwargs): - nonlocal call_count - call_count += 1 - return await original_get_by_file_path(*args, **kwargs) - - monkeypatch.setattr(entity_service.repository, "get_by_file_path", spy_get_by_file_path) - - # Run upsert with is_new=False — this calls update_entity_and_observations + update_entity_relations - markdown = _make_markdown( - title="Refetch Test", - observations=[MarkdownObservation(content="updated fact", category="fact")], - ) - await entity_service.upsert_entity_from_markdown(Path(entity.file_path), markdown, is_new=False) - - # update_entity_and_observations calls get_by_file_path once (to load the entity) - # update_entity_relations should NOT call it at all (entity passed directly) - assert call_count == 1, ( - f"Expected 1 get_by_file_path call (in update_entity_and_observations only), " - f"got {call_count}. update_entity_relations should not re-fetch." - ) - - -# --- Optimization 2: Final reload uses find_by_ids (PK) not get_by_file_path --- - - -@pytest.mark.asyncio -async def test_update_entity_relations_uses_pk_reload(entity_service: EntityService, monkeypatch): - """update_entity_relations should use find_by_ids for the final reload, not get_by_file_path.""" - entity = await entity_service.create_entity( - EntitySchema( - title="PK Reload Test", - directory="notes", - note_type="note", - content="# PK Reload Test", - ) - ) - - # Spy on find_by_ids calls - original_find_by_ids = entity_service.repository.find_by_ids - find_by_ids_calls = [] - - async def spy_find_by_ids(session, ids): - find_by_ids_calls.append(ids) - return await original_find_by_ids(session, ids) - - monkeypatch.setattr(entity_service.repository, "find_by_ids", spy_find_by_ids) - - markdown = _make_markdown(title="PK Reload Test") - await entity_service.upsert_entity_from_markdown(Path(entity.file_path), markdown, is_new=False) - - # update_entity_relations should call find_by_ids once with the entity's PK - assert len(find_by_ids_calls) == 1 - assert find_by_ids_calls[0] == [entity.id] - @pytest.mark.asyncio async def test_create_or_update_entity_uses_lightweight_exact_resolution( - entity_service: EntityService, monkeypatch -): - """create_or_update_entity should use strict lookups without eager relation loading.""" + entity_service: EntityService, + monkeypatch, +) -> None: + """Compatibility writes resolve exact file/permalink identities without graph loading.""" schema = EntitySchema( title="Create Or Update", directory="notes", @@ -157,306 +50,62 @@ async def fake_resolve_link(link_text: str, **kwargs): @pytest.mark.asyncio -async def test_upsert_with_relations_uses_lightweight_exact_resolution( - entity_service: EntityService, monkeypatch -): - """Relation target resolution should skip eager loading during upsert.""" - target = await entity_service.create_entity( - EntitySchema( - title="Lightweight Target", - directory="notes", - note_type="note", - content="# Lightweight Target", - ) - ) - source = await entity_service.create_entity( - EntitySchema( - title="Lightweight Source", - directory="notes", - note_type="note", - content="# Lightweight Source", - ) - ) - resolve_calls: list[tuple[str, dict[str, Any]]] = [] - - async def fake_resolve_link(link_text: str, **kwargs): - resolve_calls.append((link_text, kwargs)) - return target - - monkeypatch.setattr(entity_service.link_resolver, "resolve_link", fake_resolve_link) - - markdown = _make_markdown( - title="Lightweight Source", - relations=[MarkdownRelation(type="links_to", target="Lightweight Target")], - ) - await entity_service.upsert_entity_from_markdown(Path(source.file_path), markdown, is_new=False) - - assert len(resolve_calls) == 1 - link_text, kwargs = resolve_calls[0] - assert link_text == "Lightweight Target" - assert kwargs["strict"] is True - assert kwargs["load_relations"] is False - assert "session" in kwargs - - -@pytest.mark.asyncio -async def test_upsert_can_defer_relation_target_resolution( - entity_service: EntityService, monkeypatch -): - """Cloud one-file indexing can store unresolved relation rows for later repair.""" - await entity_service.create_entity( - EntitySchema( - title="Deferred Target", - directory="notes", - note_type="note", - content="# Deferred Target", - ) - ) - source = await entity_service.create_entity( - EntitySchema( - title="Deferred Source", - directory="notes", - note_type="note", - content="# Deferred Source", - ) - ) - resolve_link = AsyncMock(side_effect=AssertionError("relation lookup should be deferred")) - monkeypatch.setattr(entity_service.link_resolver, "resolve_link", resolve_link) - - markdown = _make_markdown( - title="Deferred Source", - relations=[MarkdownRelation(type="links_to", target="Deferred Target")], - ) - updated = await entity_service.upsert_entity_from_markdown( - Path(source.file_path), - markdown, - is_new=False, - resolve_relations=False, - ) - - resolve_link.assert_not_awaited() - outgoing = updated.outgoing_relations - assert len(outgoing) == 1 - assert outgoing[0].to_id is None - assert outgoing[0].to_name == "Deferred Target" - - -@pytest.mark.asyncio -async def test_upsert_deferred_relation_resolution_keeps_self_links_resolved( - entity_service: EntityService, monkeypatch -): - """Deferred relation mode should still resolve self-links without a target lookup.""" - source = await entity_service.create_entity( +async def test_entity_service_writes_publish_relations_by_note_generation( + entity_service: EntityService, + session_maker, +) -> None: + """The retained service surface cannot create generation-zero relation rows.""" + entity = await entity_service.create_entity( EntitySchema( - title="Deferred Self", + title="Generation Source", directory="notes", note_type="note", - content="# Deferred Self", - ) - ) - resolve_link = AsyncMock(side_effect=AssertionError("relation lookup should be deferred")) - monkeypatch.setattr(entity_service.link_resolver, "resolve_link", resolve_link) - - markdown = _make_markdown( - title="Deferred Self", - relations=[MarkdownRelation(type="links_to", target="Deferred Self")], - ) - updated = await entity_service.upsert_entity_from_markdown( - Path(source.file_path), - markdown, - is_new=False, - resolve_relations=False, - ) - - resolve_link.assert_not_awaited() - outgoing = updated.outgoing_relations - assert len(outgoing) == 1 - assert outgoing[0].to_id == source.id - assert outgoing[0].to_name == "Deferred Self" - - -@pytest.mark.asyncio -async def test_upsert_deferred_relation_resolution_does_not_guess_duplicate_titles( - entity_service: EntityService, monkeypatch -): - """Deferred relation mode should not treat duplicate titles as guaranteed self-links.""" - source = await entity_service.create_entity( - EntitySchema( - title="Duplicate Title", - directory="notes/source", - note_type="note", - content="# Duplicate Title", - ) - ) - await entity_service.create_entity( - EntitySchema( - title="Duplicate Title", - directory="notes/other", - note_type="note", - content="# Duplicate Title", + content=("# Generation Source\n\n## Relations\n- documents [[First Target]]\n"), ) ) - resolve_link = AsyncMock(side_effect=AssertionError("relation lookup should be deferred")) - monkeypatch.setattr(entity_service.link_resolver, "resolve_link", resolve_link) - - markdown = _make_markdown( - title="Duplicate Title", - relations=[MarkdownRelation(type="links_to", target="Duplicate Title")], - ) - updated = await entity_service.upsert_entity_from_markdown( - Path(source.file_path), - markdown, - is_new=False, - resolve_relations=False, - ) - - resolve_link.assert_not_awaited() - outgoing = updated.outgoing_relations - assert len(outgoing) == 1 - assert outgoing[0].to_id is None - assert outgoing[0].to_name == "Duplicate Title" - -@pytest.mark.asyncio -async def test_resolve_failure_is_logged_and_degrades_to_forward_reference( - entity_service: EntityService, monkeypatch -): - """A raising relation lookup must be logged (not silently swallowed) and still - degrade to a forward reference while the surrounding write succeeds.""" - source = await entity_service.create_entity( - EntitySchema( - title="Failing Resolve Source", - directory="notes", - note_type="note", - content="# Failing Resolve Source", + async with db.scoped_session(session_maker) as session: + note_content = await session.get(NoteContent, entity.id) + first_rows = list( + (await session.execute(select(Relation).where(Relation.from_id == entity.id))) + .scalars() + .all() ) - ) - - async def failing_resolve_link(link_text: str, **kwargs): - raise RuntimeError("lookup exploded") - monkeypatch.setattr(entity_service.link_resolver, "resolve_link", failing_resolve_link) - - records: list[Record] = [] - sink_id = logger.add(lambda message: records.append(message.record), level="WARNING") - try: - markdown = _make_markdown( - title="Failing Resolve Source", - relations=[MarkdownRelation(type="links_to", target="Missing Target")], - ) - updated = await entity_service.upsert_entity_from_markdown( - Path(source.file_path), - markdown, - is_new=False, - ) - finally: - logger.remove(sink_id) - - # The write survived the failed lookup and stored a forward reference. - outgoing = updated.outgoing_relations - assert len(outgoing) == 1 - assert outgoing[0].to_id is None - assert outgoing[0].to_name == "Missing Target" - - warnings = [ - record - for record in records - if "Relation target resolution failed for 'Missing Target'" in record["message"] + assert note_content is not None + assert note_content.db_version == 1 + assert [(row.to_name, row.to_id, row.generation) for row in first_rows] == [ + ("First Target", None, 1) ] - assert len(warnings) == 1 - assert warnings[0]["extra"]["error"] == "lookup exploded" - assert warnings[0]["extra"]["entity_id"] == source.id - - -# --- Correctness: full round-trip --- - -@pytest.mark.asyncio -async def test_upsert_update_preserves_observations(entity_service: EntityService): - """After upsert (update path), observations should be correctly replaced.""" - entity = await entity_service.create_entity( + await entity_service.update_entity( + entity, EntitySchema( - title="Obs Test", + title="Generation Source", directory="notes", note_type="note", - content="# Obs Test\n\n## Observations\n- [fact] original fact", - ) + content=("# Generation Source\n\n## Relations\n- documents [[Second Target]]\n"), + ), ) - assert len(entity.observations) == 1 - markdown = _make_markdown( - title="Obs Test", - observations=[ - MarkdownObservation(content="new fact 1", category="fact"), - MarkdownObservation(content="new fact 2", category="idea"), - ], - ) - updated = await entity_service.upsert_entity_from_markdown( - Path(entity.file_path), markdown, is_new=False - ) - - assert updated.id == entity.id - assert len(updated.observations) == 2 - obs_contents = {o.content for o in updated.observations} - assert obs_contents == {"new fact 1", "new fact 2"} - - -@pytest.mark.asyncio -async def test_upsert_update_preserves_relations(entity_service: EntityService): - """After upsert (update path), relations should be correctly replaced.""" - target = await entity_service.create_entity( - EntitySchema( - title="Relation Target", - directory="notes", - note_type="note", - content="# Relation Target", + async with db.scoped_session(session_maker) as session: + note_content = await session.get(NoteContent, entity.id) + second_rows = list( + (await session.execute(select(Relation).where(Relation.from_id == entity.id))) + .scalars() + .all() ) - ) - source = await entity_service.create_entity( - EntitySchema( - title="Relation Source", - directory="notes", - note_type="note", - content="# Relation Source\n\n## Relations\n- links_to [[Relation Target]]", - ) - ) - assert len(source.relations) == 1 - markdown = _make_markdown( - title="Relation Source", - relations=[MarkdownRelation(type="references", target="Relation Target")], - ) - updated = await entity_service.upsert_entity_from_markdown( - Path(source.file_path), markdown, is_new=False - ) - - assert updated.id == source.id - # Old relation replaced with new one - outgoing = [r for r in updated.relations if r.from_id == source.id] - assert len(outgoing) == 1 - assert outgoing[0].relation_type == "references" - assert outgoing[0].to_id == target.id - - -@pytest.mark.asyncio -async def test_upsert_create_path_works(entity_service: EntityService): - """The is_new=True path should still work correctly.""" - markdown = _make_markdown( - title="Create Path Test", - observations=[MarkdownObservation(content="a fact", category="fact")], - ) - result = await entity_service.upsert_entity_from_markdown( - Path("notes/create-path-test.md"), markdown, is_new=True - ) - - assert result.title == "Create Path Test" - assert len(result.observations) == 1 - assert result.observations[0].content == "a fact" + assert note_content is not None + assert note_content.db_version == 2 + assert [(row.to_name, row.to_id, row.generation) for row in second_rows] == [ + ("Second Target", None, 2) + ] @pytest.mark.asyncio -async def test_edit_entity_end_to_end(entity_service: EntityService): - """Full edit_entity flow uses optimized upsert and returns correct entity.""" +async def test_edit_entity_end_to_end(entity_service: EntityService) -> None: + """Compatibility edits still rebuild observations and finish the checksum.""" entity = await entity_service.create_entity( EntitySchema( title="Edit E2E", @@ -475,15 +124,15 @@ async def test_edit_entity_end_to_end(entity_service: EntityService): assert updated.id == entity.id assert len(updated.observations) == 1 assert updated.observations[0].content == "appended fact" - # Checksum should be set (not None) after edit completes assert updated.checksum is not None @pytest.mark.asyncio async def test_edit_entity_uses_lightweight_identifier_resolution( - entity_service: EntityService, monkeypatch -): - """edit_entity should resolve the target note without eager relation loading.""" + entity_service: EntityService, + monkeypatch, +) -> None: + """Compatibility edits resolve the note without eager relation loading.""" entity = await entity_service.create_entity( EntitySchema( title="Edit Lightweight", diff --git a/tests/test_note_content_migration.py b/tests/test_note_content_migration.py index b8f31f02a..602d57952 100644 --- a/tests/test_note_content_migration.py +++ b/tests/test_note_content_migration.py @@ -73,3 +73,112 @@ def test_alembic_upgrade_creates_note_content_table(tmp_path, monkeypatch): assert "ix_note_content_external_id" in indexes finally: connection.close() + + +def test_relation_generation_migration_backfills_source_db_version(tmp_path, monkeypatch): + """Existing relations inherit their source note generation during upgrade.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) + + database_path = tmp_path / "relation-generation-migration.db" + config = sqlite_alembic_config(database_path) + command.upgrade(config, "q0l1m2n3o4p5") + + connection = sqlite3.connect(database_path) + try: + timestamp = "2026-08-09 00:00:00" + connection.execute( + """ + INSERT INTO project ( + id, name, permalink, path, is_active, is_default, + created_at, updated_at, external_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (1, "test", "test", "/test", True, True, timestamp, timestamp, "project-1"), + ) + connection.executemany( + """ + INSERT INTO entity ( + id, title, note_type, content_type, file_path, + created_at, updated_at, project_id, external_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + ( + entity_id, + f"Source {entity_id}", + "note", + "text/markdown", + f"source-{entity_id}.md", + timestamp, + timestamp, + 1, + f"entity-{entity_id}", + ) + for entity_id in (1, 2) + ], + ) + connection.execute( + """ + INSERT INTO note_content ( + entity_id, project_id, external_id, file_path, markdown_content, + db_version, db_checksum, file_write_status, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 1, + 1, + "content-1", + "source-1.md", + "# Source 1", + 17, + "checksum", + "synced", + timestamp, + ), + ) + connection.executemany( + """ + INSERT INTO relation ( + id, from_id, to_name, relation_type, project_id + ) VALUES (?, ?, ?, ?, ?) + """, + [ + (1, 1, "Target A", "links_to", 1), + (2, 2, "Target B", "links_to", 1), + ], + ) + connection.commit() + finally: + connection.close() + + command.upgrade(config, "head") + + connection = sqlite3.connect(database_path) + try: + generation_column = next( + row + for row in connection.execute("PRAGMA table_info(relation)") + if row[1] == "generation" + ) + generations = connection.execute( + "SELECT id, generation FROM relation ORDER BY id" + ).fetchall() + indexes = {row[1] for row in connection.execute("PRAGMA index_list(relation)")} + refresh_columns = { + row[1]: row for row in connection.execute("PRAGMA table_info(relation_search_refresh)") + } + refresh_indexes = { + row[1] for row in connection.execute("PRAGMA index_list(relation_search_refresh)") + } + finally: + connection.close() + + assert generation_column[2].upper() == "BIGINT" + assert generation_column[3] == 1 + assert generation_column[4] == "0" + assert generations == [(1, 17), (2, 0)] + assert "ix_relation_project_from_generation" in indexes + assert refresh_columns["publication_generation"][2].upper() == "BIGINT" + assert refresh_columns["publication_generation"][3] == 0 + assert "ix_relation_search_refresh_project_publication_generation" in refresh_indexes diff --git a/tests/test_semantic_vector_index_migration.py b/tests/test_semantic_vector_index_migration.py index 0923d73cc..ba54f50dc 100644 --- a/tests/test_semantic_vector_index_migration.py +++ b/tests/test_semantic_vector_index_migration.py @@ -192,7 +192,7 @@ def test_upgrade_repairs_sqlite_state_from_duplicate_vector_revision( "created_at", } assert "ix_note_file_vacate_project_id" in indexes - assert version == ("q0l1m2n3o4p5",) + assert version == ("r1m2n3o4p5q6",) def test_downgrade_removes_manifest_state(monkeypatch) -> None: