diff --git a/src/basic_memory/api/app.py b/src/basic_memory/api/app.py index eca022b1b..32a338f7e 100644 --- a/src/basic_memory/api/app.py +++ b/src/basic_memory/api/app.py @@ -29,7 +29,7 @@ import logfire from basic_memory.cloud.note_content_materialization import drain_pending_materializations from basic_memory.config import init_api_logging -from basic_memory.deps.services import drain_background_tasks +from basic_memory.index.local_schedulers import drain_background_tasks from basic_memory.services.exceptions import EntityAlreadyExistsError from basic_memory.services.initialization import initialize_app from basic_memory.workspace_context import ( diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 65cd384bf..ec42eeeab 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -997,18 +997,19 @@ async def delete_directory( logger.info(f"API v2 request: delete_directory directory='{data.directory}'") try: - status_code, payload = await directory_delete_service.delete_directory( + result = await directory_delete_service.delete_directory( project_external_id=project_external_id, directory=data.directory, ) + payload = result.to_response_payload() logger.info( f"API v2 response: delete_directory " - f"total={payload.get('total_files')}, " - f"success={payload.get('successful_deletes')}, " - f"failed={payload.get('failed_deletes')}, " - f"file_delete_status={payload.get('file_delete_status')}" + f"total={payload['total_files']}, " + f"success={payload['successful_deletes']}, " + f"failed={payload['failed_deletes']}, " + f"file_delete_status={payload['file_delete_status']}" ) - return runtime_json_response(status_code=status_code, payload=payload) + return runtime_json_response(status_code=result.http_status_code, payload=payload) except DirectoryDeleteServiceError as error: logger.error(f"Error deleting directory: {error.detail}") diff --git a/src/basic_memory/api/v2/routers/project_router.py b/src/basic_memory/api/v2/routers/project_router.py index 3c8f1dc2c..3fd00fc9d 100644 --- a/src/basic_memory/api/v2/routers/project_router.py +++ b/src/basic_memory/api/v2/routers/project_router.py @@ -24,10 +24,10 @@ ProjectIndexCommandDep, ProjectIndexObserverDep, ProjectExternalIdPathDep, - ProjectIndexRouteRequest, SessionDep, SessionMakerDep, ) +from basic_memory.index.local_project import ProjectIndexRouteRequest from basic_memory.schemas import ProjectIndexStatusResponse from basic_memory.models import Project from basic_memory.repository.project_repository import ProjectRepository @@ -38,7 +38,11 @@ ProjectInfoResponse, ProjectStatusResponse, ) -from basic_memory.schemas.v2 import ProjectResolveRequest, ProjectResolveResponse +from basic_memory.schemas.v2 import ( + ProjectIndexResponse, + ProjectResolveRequest, + ProjectResolveResponse, +) from basic_memory.utils import normalize_project_path, generate_permalink router = APIRouter(prefix="/projects", tags=["project_management-v2"]) @@ -231,14 +235,14 @@ async def synchronize_projects( raise HTTPException(status_code=400, detail=str(e)) -@router.post("/{project_id}/index") +@router.post("/{project_id}/index", response_model=ProjectIndexResponse) async def index_project( project_index_command: ProjectIndexCommandDep, project_config: ProjectConfigV2ExternalDep, project_internal_id: ProjectExternalIdPathDep, force_full: bool = Query(False, description="Request a full project index run"), run_in_background: bool = Query(True, description="Run in background"), -): +) -> ProjectIndexResponse: """Run project-wide indexing through the event-index coordinator.""" return await project_index_command.index_project( ProjectIndexRouteRequest( diff --git a/src/basic_memory/cli/commands/command_utils.py b/src/basic_memory/cli/commands/command_utils.py index 81e93ba0d..7a07968a4 100644 --- a/src/basic_memory/cli/commands/command_utils.py +++ b/src/basic_memory/cli/commands/command_utils.py @@ -33,7 +33,7 @@ def run_with_cleanup(coro: Coroutine[Any, Any, T]) -> T: # at CLI import time — only when a command actually runs (#886). from basic_memory import db from basic_memory.cloud.note_content_materialization import drain_pending_materializations - from basic_memory.deps.services import drain_background_tasks + from basic_memory.index.local_schedulers import drain_background_tasks async def _with_cleanup() -> T: try: diff --git a/src/basic_memory/cloud/directory_deletes.py b/src/basic_memory/cloud/directory_deletes.py index 7efdce41e..e1b6b1bf7 100644 --- a/src/basic_memory/cloud/directory_deletes.py +++ b/src/basic_memory/cloud/directory_deletes.py @@ -14,6 +14,7 @@ from basic_memory import db from basic_memory.indexing.directory_delete_runner import ( DirectoryDeleteAcceptanceRequest, + DirectoryDeleteAcceptedResult, DirectoryDeleteRejected, DirectoryDeleteRejection, DirectoryDeleteRuntime, @@ -65,8 +66,12 @@ async def delete_directory( *, project_external_id: str, directory: str, - ) -> tuple[int, dict[str, object]]: - """Delete directory entities immediately and queue file cleanup in the background.""" + ) -> DirectoryDeleteAcceptedResult: + """Delete directory entities immediately and queue file cleanup in the background. + + The typed result carries the route status (``http_status_code``) and the + existing response contract (``to_response_payload``). + """ request = DirectoryDeleteAcceptanceRequest( project_external_id=project_external_id, directory=directory, @@ -102,8 +107,7 @@ async def delete_directory( sorted(accepted.relation_cleanup_entity_ids) ) - status_code = 500 if result.file_delete_status == "failed" else 200 - return status_code, result.to_response_payload() + return result @staticmethod def normalize_directory_path(directory: str) -> str: diff --git a/src/basic_memory/cloud/project_deletes.py b/src/basic_memory/cloud/project_deletes.py index 2f1104fe1..b9a79a20b 100644 --- a/src/basic_memory/cloud/project_deletes.py +++ b/src/basic_memory/cloud/project_deletes.py @@ -8,12 +8,10 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from basic_memory.indexing.project_delete_acceptance import ( - ProjectDeleteAcceptedProject, - ProjectDeleteAcceptedResult, -) +from basic_memory.indexing.project_delete_acceptance import ProjectDeleteAcceptedResult from basic_memory.models import Project from basic_memory.runtime.jobs import RuntimeJobId, RuntimeProjectDeleteJobRequest +from basic_memory.schemas.project_info import ProjectItem class ProjectDeleteAcceptanceError(Exception): @@ -106,7 +104,13 @@ async def delete_project( project_path=project.path, delete_notes=request.delete_notes, ) - old_project = ProjectDeleteAcceptedProject.from_source(project) + old_project = ProjectItem( + id=project.id, + external_id=project.external_id, + name=project.name, + path=project.path, + is_default=project.is_default or False, + ) project.is_active = False await session.commit() diff --git a/src/basic_memory/db.py b/src/basic_memory/db.py index 7cfd542c5..b73da2f93 100644 --- a/src/basic_memory/db.py +++ b/src/basic_memory/db.py @@ -139,31 +139,49 @@ def get_scoped_session_factory( @asynccontextmanager async def scoped_session( session_maker: async_sessionmaker[AsyncSession], + session: AsyncSession | None = None, ) -> AsyncGenerator[AsyncSession, None]: """ Get a scoped session with proper lifecycle management. + This is the one shared session-scope seam for services and indexing code. + It covers both real usage variants: + + - ``session`` provided: the caller-owned session is yielded unchanged and + the caller keeps commit/rollback ownership (composed multi-step writes). + - ``session`` omitted: a fresh task-scoped session is opened that commits + on success, rolls back on error, and always closes. + Args: session_maker: Session maker to create scoped sessions from + session: Optional caller-owned session to reuse instead of opening one """ + # Trigger: the caller already owns a transaction and passes its session in. + # Why: nested scopes must not commit or roll back mid-way through the + # caller's composed write; transaction ownership stays with the opener. + # Outcome: yield the session untouched and let the outermost scope finish it. + if session is not None: + yield session + return + factory = get_scoped_session_factory(session_maker) - session = factory() + owned_session = factory() try: # Only enable foreign keys for SQLite (Postgres has them enabled by default) # Detect database type from session's bind (engine) dialect - engine = session.get_bind() + engine = owned_session.get_bind() dialect_name = engine.dialect.name if dialect_name == "sqlite": - await session.execute(text("PRAGMA foreign_keys=ON")) + await owned_session.execute(text("PRAGMA foreign_keys=ON")) - yield session - await session.commit() + yield owned_session + await owned_session.commit() except Exception: - await session.rollback() + await owned_session.rollback() raise finally: - await session.close() + await owned_session.close() await factory.remove() diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index e2cf6283c..c849c7011 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -101,7 +101,6 @@ ProjectIndexSchedulerDep, get_project_index_command, ProjectIndexCommandDep, - ProjectIndexRouteRequest, get_search_reindex_scheduler, SearchReindexSchedulerDep, get_search_service, @@ -258,7 +257,6 @@ "ProjectIndexSchedulerDep", "get_project_index_command", "ProjectIndexCommandDep", - "ProjectIndexRouteRequest", "get_search_reindex_scheduler", "SearchReindexSchedulerDep", "get_search_service", diff --git a/src/basic_memory/deps/services.py b/src/basic_memory/deps/services.py index 324a98e92..29996a624 100644 --- a/src/basic_memory/deps/services.py +++ b/src/basic_memory/deps/services.py @@ -1,24 +1,22 @@ """Service dependency injection for basic-memory. -This module provides service-layer dependencies: +This module is the FastAPI composition root for the service layer: every +provider constructs services from repositories, config, and the local runtime +implementations that live in ``basic_memory.index`` — it defines no runtime +behavior of its own: - EntityParser, MarkdownProcessor - FileService, EntityService - SearchService, LinkResolver, ContextService - ProjectService, DirectoryService +- local note-content, indexing, and background-scheduler runtimes """ -import asyncio -from collections.abc import Sequence -from dataclasses import dataclass from pathlib import Path -from typing import Annotated, Any, Coroutine, Protocol +from typing import Annotated from fastapi import Depends from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from basic_memory import db -from basic_memory.config import BasicMemoryConfig from basic_memory.deps.config import AppConfigDep from basic_memory.deps.db import SessionMakerDep from basic_memory.deps.projects import ( @@ -41,11 +39,7 @@ SearchRepositoryV2Dep, SearchRepositoryV2ExternalDep, ) -from basic_memory.indexing.relation_resolution import ( - RelationResolutionRuntime, - RepositoryRelationResolutionRuntime, - resolve_project_relations, -) +from basic_memory.indexing.relation_resolution import RepositoryRelationResolutionRuntime from basic_memory.cloud import ( DirectoryDeleteService, LocalNoteContentMaterializationProvider, @@ -53,39 +47,45 @@ NoteContentQueryService, ) from basic_memory.index.local_dependencies import build_local_markdown_file_indexer -from basic_memory.index.local_project import LocalProjectIndexObservation, LocalProjectIndexRunner -from basic_memory.indexing.project_index_coordinator import ProjectIndexCoordinatorResult +from basic_memory.index.local_notes import ( + LocalAcceptedNotePreparerFactory, + LocalAcceptedNoteRepositories, + LocalCurrentNoteContentFreshener, + LocalDirectoryDeleteRelationCleanupRefresher, + LocalDirectoryFileDeleteEnqueuer, +) +from basic_memory.index.local_project import ( + LocalProjectIndexCommand, + LocalProjectIndexRunner, + ProjectIndexCommand, + ProjectIndexObserver, + ProjectIndexRunner, + ProjectIndexScheduler, +) + +from basic_memory.index.local_schedulers import ( + EntityVectorSyncScheduler, + LocalEntityVectorSyncScheduler, + LocalProjectIndexScheduler, + LocalRelationResolutionScheduler, + LocalSearchReindexScheduler, + RelationResolutionScheduler, + SearchReindexScheduler, +) from basic_memory.indexing.accepted_note_mutation_runner import ( AcceptedNoteMutationDependencies, AcceptedNoteMutationMovePolicy, - AcceptedNoteMutationPreparer, - build_default_accepted_note_repositories, ) from basic_memory.indexing.batch_indexer import BatchIndexer from basic_memory.indexing.directory_delete_runner import ( DirectoryDeleteRuntime, - DirectoryFileDeleteEnqueueError, RepositoryDirectoryDeleteAcceptanceStore, ) from basic_memory.indexing.index_file_runner import IndexFileExecutor from basic_memory.indexing.models import StorageIndexFileWriter -from basic_memory.indexing.note_file_delete_runner import run_note_file_delete -from basic_memory.file_utils import FileError from basic_memory.markdown import EntityParser from basic_memory.markdown.markdown_processor import MarkdownProcessor -from basic_memory.models import Project -from basic_memory.repository import ObservationRepository, RelationRepository -from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.repository.search_repository import create_search_repository -from basic_memory.runtime.cleanup import RuntimeFileDeleteResult, RuntimeNoteFileDeleteJobRequest -from basic_memory.runtime.storage import ( - RuntimeFileChecksum, - RuntimeFilePath, - runtime_content_type_is_markdown, -) -from basic_memory.schemas import ProjectIndexRunResponse from basic_memory.services import EntityService, ProjectService -from basic_memory.services.exceptions import FileOperationError from basic_memory.services.context_service import ContextService from basic_memory.services.directory_service import DirectoryService from basic_memory.services.file_service import FileService @@ -255,192 +255,9 @@ async def get_note_content_query_service( ] -@dataclass(frozen=True, slots=True) -class LocalAcceptedNotePreparerFactory: - """Construct prepare-only note semantics for local accepted-note mutations.""" - - session_maker: async_sessionmaker[AsyncSession] - app_config: BasicMemoryConfig - - def create_note_preparer(self, project: Project) -> AcceptedNoteMutationPreparer: - entity_parser = EntityParser(Path(project.path)) - markdown_processor = MarkdownProcessor(entity_parser, app_config=self.app_config) - file_service = FileService( - Path(project.path), - markdown_processor, - app_config=self.app_config, - ) - entity_repository = EntityRepository(project_id=project.id) - search_repository = create_search_repository( - self.session_maker, - project_id=project.id, - app_config=self.app_config, - ) - search_service = SearchService( - search_repository, - entity_repository, - file_service, - self.session_maker, - ) - link_resolver = LinkResolver( - entity_repository=entity_repository, - search_service=search_service, - session_maker=self.session_maker, - ) - return EntityService( - entity_repository=entity_repository, - observation_repository=ObservationRepository(project_id=project.id), - relation_repository=RelationRepository(project_id=project.id), - entity_parser=entity_parser, - file_service=file_service, - link_resolver=link_resolver, - session_maker=self.session_maker, - search_service=search_service, - app_config=self.app_config, - ) - - -class LocalCurrentNoteEntity(Protocol): - """Entity fields needed to refresh current markdown before route mutation.""" - - file_path: str - content_type: str - - -class LocalCurrentNoteEntityRepository(Protocol): - """Entity lookup needed by the local note-content freshener.""" - - async def get_by_external_id( - self, - session: AsyncSession, - external_id: str, - *, - load_relations: bool = True, - ) -> LocalCurrentNoteEntity | None: ... - - -class LocalCurrentNoteFileService(Protocol): - """Current file-state access needed before mutating accepted note content.""" - - async def exists( - self, - path: RuntimeFilePath, - ) -> bool: ... - - -class LocalCurrentNoteFileIndexer(Protocol): - """Single-file indexing capability used by the local note-content freshener.""" - - async def index_file( - self, - file_path: RuntimeFilePath, - *, - source: str, - ) -> object: ... - - -@dataclass(frozen=True, slots=True) -class LocalCurrentNoteContentFreshener: - """Converge directly-edited local markdown before accepted-note mutations.""" - - entity_repository: LocalCurrentNoteEntityRepository - file_service: LocalCurrentNoteFileService - file_indexer: LocalCurrentNoteFileIndexer - session_maker: async_sessionmaker[AsyncSession] - - async def freshen_note_content( - self, - *, - project_external_id: str, - entity_external_id: str, - ) -> None: - del project_external_id - - async with self.session_maker() as session: - entity = await self.entity_repository.get_by_external_id( - session, - entity_external_id, - load_relations=False, - ) - if entity is None or not runtime_content_type_is_markdown(entity): - return - file_path = entity.file_path - - if not await self.file_service.exists(file_path): - return - - await self.file_indexer.index_file( - file_path, - source="note-content-mutation-freshen", - ) - - # --- Directory Delete Runtime --- -@dataclass(frozen=True, slots=True) -class LocalNoteFileDeleteStorage: - """Adapt local FileService to guarded materialized-note cleanup.""" - - file_service: FileService - - async def exists(self, path: RuntimeFilePath) -> bool: - return await self.file_service.exists(path) - - async def compute_checksum(self, path: RuntimeFilePath) -> RuntimeFileChecksum: - return await self.file_service.compute_checksum(path) - - async def delete_file(self, path: RuntimeFilePath) -> None: - await self.file_service.delete_file(path) - - -@dataclass(frozen=True, slots=True) -class LocalDirectoryFileDeleteEnqueuer: - """Run accepted directory-delete file cleanup inline for the local runtime.""" - - file_service: FileService - - async def enqueue_directory_file_delete( - self, - request: RuntimeNoteFileDeleteJobRequest, - ) -> RuntimeFileDeleteResult | None: - # Local cleanup runs inline, so return the guarded delete result; a skipped - # delete (file changed before cleanup) must not be reported as a success. - try: - return await run_note_file_delete( - request, - storage=LocalNoteFileDeleteStorage(file_service=self.file_service), - ) - except (FileError, FileOperationError, OSError) as exc: - raise DirectoryFileDeleteEnqueueError(str(exc)) from exc - - -@dataclass(frozen=True, slots=True) -class LocalDirectoryDeleteRelationCleanupRefresher: - """Reindex surviving relation sources after an accepted directory delete. - - Their search_index relation rows went stale when the deleted targets' - rows cascaded away; reindexing each surviving source drops the danglers. - """ - - session_maker: async_sessionmaker[AsyncSession] - entity_repository: EntityRepository - search_service: SearchService - - async def refresh_relation_sources(self, entity_ids: Sequence[int]) -> None: - unique_entity_ids = sorted(set(entity_ids)) - if not unique_entity_ids: - return - - async with db.scoped_session(self.session_maker) as session: - entities = await self.entity_repository.find_by_ids(session, unique_entity_ids) - - # A source deleted between acceptance and this refresh has no search rows - # left to repair, so missing ids are skipped rather than treated as fatal. - for entity in entities: - await self.search_service.index_entity(entity) - - async def get_directory_delete_service( session_maker: SessionMakerDep, file_service: FileServiceV2ExternalDep, @@ -715,7 +532,7 @@ async def get_note_content_mutation_service( app_config: AppConfigDep, ) -> NoteContentMutationService: """Create the local accepted-note mutation facade for API routes.""" - accepted_note_repositories = build_default_accepted_note_repositories() + accepted_note_repositories = LocalAcceptedNoteRepositories() return NoteContentMutationService( session_maker=session_maker, mutation_dependencies=AcceptedNoteMutationDependencies( @@ -752,51 +569,6 @@ async def get_note_content_mutation_service( # --- Project Indexing --- -class ProjectIndexRunner(Protocol): - """Run project-wide indexing in the current process.""" - - async def index_project( - self, - project_id: int, - *, - force_full: bool = False, - ) -> ProjectIndexCoordinatorResult: ... - - -class ProjectIndexObserver(Protocol): - """Observe project files visible to the active runtime.""" - - async def observe_project(self, project_id: int) -> LocalProjectIndexObservation: ... - - -class ProjectIndexScheduler(Protocol): - """Schedule background project indexing.""" - - def schedule_project_index(self, *, project_id: int, force_full: bool = False) -> None: ... - - -@dataclass(frozen=True, slots=True) -class ProjectIndexRouteRequest: - """Route-level project-index command input.""" - - project_id: int - project_name: str - force_full: bool - run_in_background: bool - - -type ProjectIndexRouteResult = ProjectIndexRunResponse | dict[str, str] - - -class ProjectIndexCommand(Protocol): - """Handle a project-index route request.""" - - async def index_project( - self, - request: ProjectIndexRouteRequest, - ) -> ProjectIndexRouteResult: ... - - async def get_project_index_runner( project_repository: ProjectRepositoryDep, session_maker: SessionMakerDep, @@ -829,162 +601,6 @@ async def get_project_index_observer( # --- Background Work Schedulers --- -class EntityVectorSyncScheduler(Protocol): - """Schedule out-of-band semantic vector refreshes for note mutations.""" - - def schedule_entity_vector_sync(self, *, entity_id: int, project_id: int) -> None: ... - - -class SearchReindexScheduler(Protocol): - """Schedule a search-index rebuild for the active project.""" - - def schedule_search_reindex(self, *, project_id: int) -> None: ... - - -class EntityVectorSyncSearchService(Protocol): - async def sync_entity_vectors(self, entity_id: int) -> object: ... - - -class SearchReindexService(Protocol): - async def reindex_all(self) -> object: ... - - -def _log_task_failure(completed: asyncio.Task) -> None: - if completed.cancelled(): - return - try: - completed.result() - except asyncio.CancelledError: - return - except Exception as exc: # pragma: no cover - logger.exception("Background task failed", error=str(exc)) - - -# The event loop holds only weak references to tasks; without a strong reference -# a suspended background task can be garbage-collected mid-flight and silently -# never finish (asyncio.create_task docs: "Save a reference to the result"). -_background_tasks: set[asyncio.Task[object]] = set() - - -def _schedule_background_coroutine( - coroutine: Coroutine[Any, Any, object], - *, - test_mode: bool, -) -> None: - # Background tasks outlive pytest fixture cleanup and can race engine disposal. - # Focused tests call the scheduler classes directly with test_mode=False. - if test_mode: - coroutine.close() - return - - task = asyncio.create_task(coroutine) - _background_tasks.add(task) - task.add_done_callback(_background_tasks.discard) - task.add_done_callback(_log_task_failure) - - -async def drain_background_tasks() -> None: - """Await scheduled background work until none remains. - - One-shot CLI clients close the event loop right after the command coroutine - returns, which would cancel in-flight vector sync and relation resolution - scheduled by the write path — leaving semantic search stale until an - unrelated reindex. A task can schedule a follow-up task (the - relation-resolution dirty re-run), so drain in waves until no running task - remains. Failures are already logged by the done callback; the drain itself - never raises. - """ - while True: - # Filter on task state, not set membership: completed tasks are pruned - # by a call_soon done-callback that may not have run yet, and awaiting - # only already-done tasks never suspends — checking membership alone - # would busy-spin without ever letting that callback fire. - running = [task for task in _background_tasks if not task.done()] - if not running: - break - await asyncio.wait(running) - - -@dataclass(frozen=True, slots=True) -class LocalEntityVectorSyncScheduler: - search_service: EntityVectorSyncSearchService - test_mode: bool - - def schedule_entity_vector_sync(self, *, entity_id: int, project_id: int) -> None: - _ = project_id - _schedule_background_coroutine( - self.search_service.sync_entity_vectors(entity_id), - test_mode=self.test_mode, - ) - - -# Process-lifetime single-flight state: project ids with an index run already -# scheduled or in flight. Every POST .../index and the startup scan previously -# spawned an independent full coordinator run over the same rows; overlapping -# runs are also the trigger for move/delete races, so at most one run per -# project may be in flight. -_pending_project_index: set[int] = set() -# Projects whose index request arrived while a run was already in flight, with -# the strongest force_full seen. The in-flight run scanned a snapshot that may -# predate the new request, so exactly one trailing rerun starts when it -# finishes — mirroring the relation-resolution dirty bit above. -_dirty_project_index: dict[int, bool] = {} - - -@dataclass(frozen=True, slots=True) -class LocalProjectIndexScheduler: - """Run background project indexing with per-project single-flight coalescing.""" - - project_index_runner: ProjectIndexRunner - test_mode: bool - - def schedule_project_index(self, *, project_id: int, force_full: bool = False) -> None: - # Early-return in test mode BEFORE touching the pending set: the - # background coroutine (which clears the set) never runs under test mode, - # so adding here would leak the project id forever. - if self.test_mode: - return - # Coalesce: a run is already pending/in flight for this project. Mark it - # dirty (keeping the strongest force_full) so one follow-up run covers - # this request once the current run finishes, instead of racing it. - if project_id in _pending_project_index: - _dirty_project_index[project_id] = ( - _dirty_project_index.get(project_id, False) or force_full - ) - return - _pending_project_index.add(project_id) - _schedule_background_coroutine( - self._run_project_index(project_id, force_full=force_full), - test_mode=self.test_mode, - ) - - async def _run_project_index(self, project_id: int, *, force_full: bool) -> None: - try: - await self.project_index_runner.index_project(project_id, force_full=force_full) - finally: - rerun_force_full = _dirty_project_index.pop(project_id, None) - _pending_project_index.discard(project_id) - # Re-arm inside finally, outside the in-flight window (pending now - # cleared), so a request that raced the run gets its own pass even - # when this run raised — a failed run is exactly when the coalesced - # request most needs its retry. Bounded to one extra run per burst. - if rerun_force_full is not None: - self.schedule_project_index(project_id=project_id, force_full=rerun_force_full) - - -@dataclass(frozen=True, slots=True) -class LocalSearchReindexScheduler: - search_service: SearchReindexService - test_mode: bool - - def schedule_search_reindex(self, *, project_id: int) -> None: - _ = project_id - _schedule_background_coroutine( - self.search_service.reindex_all(), - test_mode=self.test_mode, - ) - - async def get_entity_vector_sync_scheduler( search_service: SearchServiceV2ExternalDep, app_config: AppConfigDep, @@ -1015,79 +631,6 @@ async def get_search_reindex_scheduler( ) -class RelationResolutionScheduler(Protocol): - """Schedule background forward-reference resolution after note mutations.""" - - def schedule_relation_resolution(self, *, project_id: int) -> None: ... - - -# Process-lifetime coalescing state: project ids with a relation-resolution -# pass already pending or in flight. A burst of writes collapses to a single -# offline pass instead of one whole-project relation scan per write — running a -# scan per write made the write path heavier and piled up under concurrency -# (see benchmarks/docs/write-load-benchmark.md). -_pending_relation_resolution: set[int] = set() -# Project ids whose forward references arrived while a pass was already scanning. -# The scan resolves whatever is unresolved when it reads the table, so a write that -# commits during the scan (after that read) would otherwise be missed until an -# unrelated later trigger. This dirty bit forces exactly one follow-up pass. -_dirty_relation_resolution: set[int] = set() - - -@dataclass(frozen=True, slots=True) -class LocalRelationResolutionScheduler: - """Back-resolve dangling forward references off the request path, coalesced. - - The MCP/API write path inline-indexes the materialized note but never - back-resolves inbound `[[wikilinks]]` whose target the new note now - satisfies (#1015). Resolution is a whole-project scan, so running it per - write is both wasteful and a real write-load cost. Instead each write only - enqueues: the first write of a burst schedules one debounced background pass - and every other write coalesces onto it (at most one pending pass per - project). The accept path stays light; reconciliation runs offline. No-op in - test mode, consistent with the other local schedulers. - """ - - relation_runtime: RelationResolutionRuntime - test_mode: bool - debounce_seconds: float = 0.5 - - def schedule_relation_resolution(self, *, project_id: int) -> None: - # Early-return in test mode BEFORE touching the pending set: the - # background coroutine (which clears the set) never runs under test mode, - # so adding here would leak the project id forever. - if self.test_mode: - return - # Coalesce: a pass is already pending/running for this project. Mark it - # dirty so a scan that has already read the table re-runs once more and - # picks up this write's rows, instead of dropping it (#1002 review). - if project_id in _pending_relation_resolution: - _dirty_relation_resolution.add(project_id) - return - _pending_relation_resolution.add(project_id) - _schedule_background_coroutine( - self._resolve_after_debounce(project_id), - test_mode=self.test_mode, - ) - - async def _resolve_after_debounce(self, project_id: int) -> None: - try: - # Debounce: let the burst settle so one pass covers all of it. - await asyncio.sleep(self.debounce_seconds) - # Writes up to here are covered by the scan we are about to run, so only - # writes that land DURING the scan should force a re-run. - _dirty_relation_resolution.discard(project_id) - await resolve_project_relations(self.relation_runtime) - finally: - rerun = project_id in _dirty_relation_resolution - _dirty_relation_resolution.discard(project_id) - _pending_relation_resolution.discard(project_id) - # Re-arm outside the in-flight window (pending now cleared) so a write that - # raced the scan gets its own pass. Bounded to one extra pass per burst. - if rerun: - self.schedule_relation_resolution(project_id=project_id) - - async def get_relation_resolution_scheduler( session_maker: SessionMakerDep, entity_repository: EntityRepositoryV2ExternalDep, @@ -1166,41 +709,6 @@ async def get_note_content_materialization_provider( ] -@dataclass(frozen=True, slots=True) -class LocalProjectIndexCommand: - project_index_runner: ProjectIndexRunner - project_index_scheduler: ProjectIndexScheduler - - async def index_project( - self, - request: ProjectIndexRouteRequest, - ) -> ProjectIndexRouteResult: - if request.run_in_background: - self.project_index_scheduler.schedule_project_index( - project_id=request.project_id, - force_full=request.force_full, - ) - logger.info( - f"Filesystem indexing initiated for project: {request.project_name} " - f"(force_full={request.force_full})" - ) - - return { - "status": "index_started", - "message": (f"Filesystem indexing initiated for project '{request.project_name}'"), - } - - result = await self.project_index_runner.index_project( - request.project_id, - force_full=request.force_full, - ) - logger.info( - f"Filesystem indexing completed for project: {request.project_name} " - f"(force_full={request.force_full})" - ) - return ProjectIndexRunResponse.from_result(result) - - async def get_project_index_command( project_index_runner: ProjectIndexRunnerDep, project_index_scheduler: ProjectIndexSchedulerDep, diff --git a/src/basic_memory/index/local_dependencies.py b/src/basic_memory/index/local_dependencies.py index daa416bee..69dac98bf 100644 --- a/src/basic_memory/index/local_dependencies.py +++ b/src/basic_memory/index/local_dependencies.py @@ -27,7 +27,7 @@ IndexMarkdownNoteContentReconciler, ) from basic_memory.indexing.index_batch_runtime import ( - DefaultIndexBatchRuntime, + IndexBatchRuntime, build_default_index_batch_runtime, ) from basic_memory.indexing.index_file_runner import ( @@ -223,7 +223,7 @@ async def dependencies_for_project(self, project: Project) -> LocalIndexProjectD class LocalIndexFileBatchIndexer(IndexFileBatchIndexer[IndexInputFile]): """Adapt the default loaded-file batch runtime to the file-batch job contract.""" - batch_runtime: DefaultIndexBatchRuntime[IndexInputFile] + batch_runtime: IndexBatchRuntime[Entity, IndexInputFile] async def index_files( self, diff --git a/src/basic_memory/index/local_notes.py b/src/basic_memory/index/local_notes.py new file mode 100644 index 000000000..ca3661d8f --- /dev/null +++ b/src/basic_memory/index/local_notes.py @@ -0,0 +1,254 @@ +"""Local runtime implementations for accepted-note mutations and note-file cleanup. + +The accepted-note mutation runner, note-content mutation service, and +directory-delete runtime are storage-neutral; this module supplies their +filesystem-backed local implementations. The FastAPI composition root in +``basic_memory.deps.services`` wires these into route dependencies; cloud +composes its own tenant-scoped equivalents behind the same protocols. +""" + +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.config import BasicMemoryConfig +from basic_memory.file_utils import FileError +from basic_memory.indexing.accepted_note_mutation_runner import AcceptedNoteMutationPreparer +from basic_memory.indexing.directory_delete_runner import DirectoryFileDeleteEnqueueError +from basic_memory.indexing.note_file_delete_runner import run_note_file_delete +from basic_memory.markdown import EntityParser +from basic_memory.markdown.markdown_processor import MarkdownProcessor +from basic_memory.models import Project +from basic_memory.repository import NoteContentRepository, ObservationRepository, RelationRepository +from basic_memory.repository.accepted_note_search_repository import AcceptedNoteSearchRepository +from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.search_repository import create_search_repository +from basic_memory.runtime.cleanup import RuntimeFileDeleteResult, RuntimeNoteFileDeleteJobRequest +from basic_memory.runtime.storage import ( + ProjectId, + RuntimeFileChecksum, + RuntimeFilePath, + runtime_content_type_is_markdown, +) +from basic_memory.services import EntityService +from basic_memory.services.exceptions import FileOperationError +from basic_memory.services.file_service import FileService +from basic_memory.services.link_resolver import LinkResolver +from basic_memory.services.search_service import SearchService + +# --- Accepted-Note Mutations --- + + +@dataclass(frozen=True, slots=True) +class LocalAcceptedNotePreparerFactory: + """Construct prepare-only note semantics for local accepted-note mutations.""" + + session_maker: async_sessionmaker[AsyncSession] + app_config: BasicMemoryConfig + + def create_note_preparer(self, project: Project) -> AcceptedNoteMutationPreparer: + entity_parser = EntityParser(Path(project.path)) + markdown_processor = MarkdownProcessor(entity_parser, app_config=self.app_config) + file_service = FileService( + Path(project.path), + markdown_processor, + app_config=self.app_config, + ) + entity_repository = EntityRepository(project_id=project.id) + search_repository = create_search_repository( + self.session_maker, + project_id=project.id, + app_config=self.app_config, + ) + search_service = SearchService( + search_repository, + entity_repository, + file_service, + self.session_maker, + ) + link_resolver = LinkResolver( + entity_repository=entity_repository, + search_service=search_service, + session_maker=self.session_maker, + ) + return EntityService( + entity_repository=entity_repository, + observation_repository=ObservationRepository(project_id=project.id), + relation_repository=RelationRepository(project_id=project.id), + entity_parser=entity_parser, + file_service=file_service, + link_resolver=link_resolver, + session_maker=self.session_maker, + search_service=search_service, + app_config=self.app_config, + ) + + +@dataclass(frozen=True, slots=True) +class LocalAcceptedNoteRepositories: + """Project-scoped core repositories for accepted-note mutations. + + One concrete bundle satisfies both the lookup and write repository + capabilities the accepted-note mutation runner consumes; cloud composes its + own tenant-scoped equivalent behind the same protocols. + """ + + def entity_repository(self, project_id: ProjectId) -> EntityRepository: + return EntityRepository(project_id=project_id) + + def pending_entity_repository(self, project_id: ProjectId) -> EntityRepository: + return EntityRepository(project_id=project_id) + + def note_content_repository(self, project_id: ProjectId) -> NoteContentRepository: + return NoteContentRepository(project_id=project_id) + + def search_repository(self, project_id: ProjectId) -> AcceptedNoteSearchRepository: + return AcceptedNoteSearchRepository(project_id=project_id) + + +# --- Current-Note Content Freshening --- + + +class LocalCurrentNoteEntity(Protocol): + """Entity fields needed to refresh current markdown before route mutation.""" + + file_path: str + content_type: str + + +class LocalCurrentNoteEntityRepository(Protocol): + """Entity lookup needed by the local note-content freshener.""" + + async def get_by_external_id( + self, + session: AsyncSession, + external_id: str, + *, + load_relations: bool = True, + ) -> LocalCurrentNoteEntity | None: ... + + +class LocalCurrentNoteFileService(Protocol): + """Current file-state access needed before mutating accepted note content.""" + + async def exists( + self, + path: RuntimeFilePath, + ) -> bool: ... + + +class LocalCurrentNoteFileIndexer(Protocol): + """Single-file indexing capability used by the local note-content freshener.""" + + async def index_file( + self, + file_path: RuntimeFilePath, + *, + source: str, + ) -> object: ... + + +@dataclass(frozen=True, slots=True) +class LocalCurrentNoteContentFreshener: + """Converge directly-edited local markdown before accepted-note mutations.""" + + entity_repository: LocalCurrentNoteEntityRepository + file_service: LocalCurrentNoteFileService + file_indexer: LocalCurrentNoteFileIndexer + session_maker: async_sessionmaker[AsyncSession] + + async def freshen_note_content( + self, + *, + project_external_id: str, + entity_external_id: str, + ) -> None: + del project_external_id + + async with self.session_maker() as session: + entity = await self.entity_repository.get_by_external_id( + session, + entity_external_id, + load_relations=False, + ) + if entity is None or not runtime_content_type_is_markdown(entity): + return + file_path = entity.file_path + + if not await self.file_service.exists(file_path): + return + + await self.file_indexer.index_file( + file_path, + source="note-content-mutation-freshen", + ) + + +# --- Directory-Delete File Cleanup --- + + +@dataclass(frozen=True, slots=True) +class LocalNoteFileDeleteStorage: + """Adapt local FileService to guarded materialized-note cleanup.""" + + file_service: FileService + + async def exists(self, path: RuntimeFilePath) -> bool: + return await self.file_service.exists(path) + + async def compute_checksum(self, path: RuntimeFilePath) -> RuntimeFileChecksum: + return await self.file_service.compute_checksum(path) + + async def delete_file(self, path: RuntimeFilePath) -> None: + await self.file_service.delete_file(path) + + +@dataclass(frozen=True, slots=True) +class LocalDirectoryFileDeleteEnqueuer: + """Run accepted directory-delete file cleanup inline for the local runtime.""" + + file_service: FileService + + async def enqueue_directory_file_delete( + self, + request: RuntimeNoteFileDeleteJobRequest, + ) -> RuntimeFileDeleteResult | None: + # Local cleanup runs inline, so return the guarded delete result; a skipped + # delete (file changed before cleanup) must not be reported as a success. + try: + return await run_note_file_delete( + request, + storage=LocalNoteFileDeleteStorage(file_service=self.file_service), + ) + except (FileError, FileOperationError, OSError) as exc: + raise DirectoryFileDeleteEnqueueError(str(exc)) from exc + + +@dataclass(frozen=True, slots=True) +class LocalDirectoryDeleteRelationCleanupRefresher: + """Reindex surviving relation sources after an accepted directory delete. + + Their search_index relation rows went stale when the deleted targets' + rows cascaded away; reindexing each surviving source drops the danglers. + """ + + session_maker: async_sessionmaker[AsyncSession] + entity_repository: EntityRepository + search_service: SearchService + + async def refresh_relation_sources(self, entity_ids: Sequence[int]) -> None: + unique_entity_ids = sorted(set(entity_ids)) + if not unique_entity_ids: + return + + async with db.scoped_session(self.session_maker) as session: + entities = await self.entity_repository.find_by_ids(session, unique_entity_ids) + + # A source deleted between acceptance and this refresh has no search rows + # left to repair, so missing ids are skipped rather than treated as fatal. + for entity in entities: + await self.search_service.index_entity(entity) diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index 749be4de0..748c5ae83 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -79,6 +79,11 @@ ) from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.storage import RuntimeFilePath +from basic_memory.schemas.project_index import ProjectIndexRunResponse +from basic_memory.schemas.v2.project_index import ( + ProjectIndexResponse, + ProjectIndexStartedResponse, +) from basic_memory.services import FileService from basic_memory.services.exceptions import FileOperationError @@ -704,6 +709,85 @@ async def index_project( ) +# --- Project-Index Route Commands --- + + +class ProjectIndexRunner(Protocol): + """Run project-wide indexing in the current process.""" + + async def index_project( + self, + project_id: int, + *, + force_full: bool = False, + ) -> ProjectIndexCoordinatorResult: ... + + +class ProjectIndexObserver(Protocol): + """Observe project files visible to the active runtime.""" + + async def observe_project(self, project_id: int) -> LocalProjectIndexObservation: ... + + +class ProjectIndexScheduler(Protocol): + """Schedule background project indexing.""" + + def schedule_project_index(self, *, project_id: int, force_full: bool = False) -> None: ... + + +@dataclass(frozen=True, slots=True) +class ProjectIndexRouteRequest: + """Route-level project-index command input.""" + + project_id: int + project_name: str + force_full: bool + run_in_background: bool + + +class ProjectIndexCommand(Protocol): + """Handle a project-index route request.""" + + async def index_project( + self, + request: ProjectIndexRouteRequest, + ) -> ProjectIndexResponse: ... + + +@dataclass(frozen=True, slots=True) +class LocalProjectIndexCommand: + project_index_runner: ProjectIndexRunner + project_index_scheduler: ProjectIndexScheduler + + async def index_project( + self, + request: ProjectIndexRouteRequest, + ) -> ProjectIndexResponse: + if request.run_in_background: + self.project_index_scheduler.schedule_project_index( + project_id=request.project_id, + force_full=request.force_full, + ) + logger.info( + f"Filesystem indexing initiated for project: {request.project_name} " + f"(force_full={request.force_full})" + ) + + return ProjectIndexStartedResponse( + message=f"Filesystem indexing initiated for project '{request.project_name}'", + ) + + result = await self.project_index_runner.index_project( + request.project_id, + force_full=request.force_full, + ) + logger.info( + f"Filesystem indexing completed for project: {request.project_name} " + f"(force_full={request.force_full})" + ) + return ProjectIndexRunResponse.from_result(result) + + async def run_local_project_index( request: RuntimeProjectIndexJobRequest, *, diff --git a/src/basic_memory/index/local_schedulers.py b/src/basic_memory/index/local_schedulers.py new file mode 100644 index 000000000..1bc03f716 --- /dev/null +++ b/src/basic_memory/index/local_schedulers.py @@ -0,0 +1,258 @@ +"""Background work scheduling for the local runtime. + +Note mutations schedule derived work — semantic vector sync, search reindex, +project indexing, and forward-reference resolution — off the request path. +This module owns the in-process task machinery and the local scheduler +implementations; the FastAPI composition root in ``basic_memory.deps.services`` +wires them into route dependencies. Cloud composes queue-backed equivalents +behind the same protocols. +""" + +import asyncio +from dataclasses import dataclass +from typing import Any, Coroutine, Protocol + +from loguru import logger + +from basic_memory.index.local_project import ProjectIndexRunner +from basic_memory.indexing.relation_resolution import ( + RelationResolutionRuntime, + resolve_project_relations, +) + +# --- Scheduler Capabilities --- + + +class EntityVectorSyncScheduler(Protocol): + """Schedule out-of-band semantic vector refreshes for note mutations.""" + + def schedule_entity_vector_sync(self, *, entity_id: int, project_id: int) -> None: ... + + +class SearchReindexScheduler(Protocol): + """Schedule a search-index rebuild for the active project.""" + + def schedule_search_reindex(self, *, project_id: int) -> None: ... + + +class RelationResolutionScheduler(Protocol): + """Schedule background forward-reference resolution after note mutations.""" + + def schedule_relation_resolution(self, *, project_id: int) -> None: ... + + +class EntityVectorSyncSearchService(Protocol): + async def sync_entity_vectors(self, entity_id: int) -> object: ... + + +class SearchReindexService(Protocol): + async def reindex_all(self) -> object: ... + + +# --- Background Task Machinery --- + + +def _log_task_failure(completed: asyncio.Task) -> None: + if completed.cancelled(): + return + try: + completed.result() + except asyncio.CancelledError: + return + except Exception as exc: # pragma: no cover + logger.exception("Background task failed", error=str(exc)) + + +# The event loop holds only weak references to tasks; without a strong reference +# a suspended background task can be garbage-collected mid-flight and silently +# never finish (asyncio.create_task docs: "Save a reference to the result"). +_background_tasks: set[asyncio.Task[object]] = set() + + +def _schedule_background_coroutine( + coroutine: Coroutine[Any, Any, object], + *, + test_mode: bool, +) -> None: + # Background tasks outlive pytest fixture cleanup and can race engine disposal. + # Focused tests call the scheduler classes directly with test_mode=False. + if test_mode: + coroutine.close() + return + + task = asyncio.create_task(coroutine) + _background_tasks.add(task) + task.add_done_callback(_background_tasks.discard) + task.add_done_callback(_log_task_failure) + + +async def drain_background_tasks() -> None: + """Await scheduled background work until none remains. + + One-shot CLI clients close the event loop right after the command coroutine + returns, which would cancel in-flight vector sync and relation resolution + scheduled by the write path — leaving semantic search stale until an + unrelated reindex. A task can schedule a follow-up task (the + relation-resolution dirty re-run), so drain in waves until no running task + remains. Failures are already logged by the done callback; the drain itself + never raises. + """ + while True: + # Filter on task state, not set membership: completed tasks are pruned + # by a call_soon done-callback that may not have run yet, and awaiting + # only already-done tasks never suspends — checking membership alone + # would busy-spin without ever letting that callback fire. + running = [task for task in _background_tasks if not task.done()] + if not running: + break + await asyncio.wait(running) + + +# --- Local Schedulers --- + + +@dataclass(frozen=True, slots=True) +class LocalEntityVectorSyncScheduler: + search_service: EntityVectorSyncSearchService + test_mode: bool + + def schedule_entity_vector_sync(self, *, entity_id: int, project_id: int) -> None: + _ = project_id + _schedule_background_coroutine( + self.search_service.sync_entity_vectors(entity_id), + test_mode=self.test_mode, + ) + + +# Process-lifetime single-flight state: project ids with an index run already +# scheduled or in flight. Every POST .../index and the startup scan previously +# spawned an independent full coordinator run over the same rows; overlapping +# runs are also the trigger for move/delete races, so at most one run per +# project may be in flight. +_pending_project_index: set[int] = set() +# Projects whose index request arrived while a run was already in flight, with +# the strongest force_full seen. The in-flight run scanned a snapshot that may +# predate the new request, so exactly one trailing rerun starts when it +# finishes — mirroring the relation-resolution dirty bit above. +_dirty_project_index: dict[int, bool] = {} + + +@dataclass(frozen=True, slots=True) +class LocalProjectIndexScheduler: + """Run background project indexing with per-project single-flight coalescing.""" + + project_index_runner: ProjectIndexRunner + test_mode: bool + + def schedule_project_index(self, *, project_id: int, force_full: bool = False) -> None: + # Early-return in test mode BEFORE touching the pending set: the + # background coroutine (which clears the set) never runs under test mode, + # so adding here would leak the project id forever. + if self.test_mode: + return + # Coalesce: a run is already pending/in flight for this project. Mark it + # dirty (keeping the strongest force_full) so one follow-up run covers + # this request once the current run finishes, instead of racing it. + if project_id in _pending_project_index: + _dirty_project_index[project_id] = ( + _dirty_project_index.get(project_id, False) or force_full + ) + return + _pending_project_index.add(project_id) + _schedule_background_coroutine( + self._run_project_index(project_id, force_full=force_full), + test_mode=self.test_mode, + ) + + async def _run_project_index(self, project_id: int, *, force_full: bool) -> None: + try: + await self.project_index_runner.index_project(project_id, force_full=force_full) + finally: + rerun_force_full = _dirty_project_index.pop(project_id, None) + _pending_project_index.discard(project_id) + # Re-arm inside finally, outside the in-flight window (pending now + # cleared), so a request that raced the run gets its own pass even + # when this run raised — a failed run is exactly when the coalesced + # request most needs its retry. Bounded to one extra run per burst. + if rerun_force_full is not None: + self.schedule_project_index(project_id=project_id, force_full=rerun_force_full) + + +@dataclass(frozen=True, slots=True) +class LocalSearchReindexScheduler: + search_service: SearchReindexService + test_mode: bool + + def schedule_search_reindex(self, *, project_id: int) -> None: + _ = project_id + _schedule_background_coroutine( + self.search_service.reindex_all(), + test_mode=self.test_mode, + ) + + +# Process-lifetime coalescing state: project ids with a relation-resolution +# pass already pending or in flight. A burst of writes collapses to a single +# offline pass instead of one whole-project relation scan per write — running a +# scan per write made the write path heavier and piled up under concurrency +# (see benchmarks/docs/write-load-benchmark.md). +_pending_relation_resolution: set[int] = set() +# Project ids whose forward references arrived while a pass was already scanning. +# The scan resolves whatever is unresolved when it reads the table, so a write that +# commits during the scan (after that read) would otherwise be missed until an +# unrelated later trigger. This dirty bit forces exactly one follow-up pass. +_dirty_relation_resolution: set[int] = set() + + +@dataclass(frozen=True, slots=True) +class LocalRelationResolutionScheduler: + """Back-resolve dangling forward references off the request path, coalesced. + + The MCP/API write path inline-indexes the materialized note but never + back-resolves inbound `[[wikilinks]]` whose target the new note now + satisfies (#1015). Resolution is a whole-project scan, so running it per + write is both wasteful and a real write-load cost. Instead each write only + enqueues: the first write of a burst schedules one debounced background pass + and every other write coalesces onto it (at most one pending pass per + project). The accept path stays light; reconciliation runs offline. No-op in + test mode, consistent with the other local schedulers. + """ + + relation_runtime: RelationResolutionRuntime + test_mode: bool + debounce_seconds: float = 0.5 + + def schedule_relation_resolution(self, *, project_id: int) -> None: + # Early-return in test mode BEFORE touching the pending set: the + # background coroutine (which clears the set) never runs under test mode, + # so adding here would leak the project id forever. + if self.test_mode: + return + # Coalesce: a pass is already pending/running for this project. Mark it + # dirty so a scan that has already read the table re-runs once more and + # picks up this write's rows, instead of dropping it (#1002 review). + if project_id in _pending_relation_resolution: + _dirty_relation_resolution.add(project_id) + return + _pending_relation_resolution.add(project_id) + _schedule_background_coroutine( + self._resolve_after_debounce(project_id), + test_mode=self.test_mode, + ) + + async def _resolve_after_debounce(self, project_id: int) -> None: + try: + # Debounce: let the burst settle so one pass covers all of it. + await asyncio.sleep(self.debounce_seconds) + # Writes up to here are covered by the scan we are about to run, so only + # writes that land DURING the scan should force a re-run. + _dirty_relation_resolution.discard(project_id) + await resolve_project_relations(self.relation_runtime) + finally: + rerun = project_id in _dirty_relation_resolution + _dirty_relation_resolution.discard(project_id) + _pending_relation_resolution.discard(project_id) + # Re-arm outside the in-flight window (pending now cleared) so a write that + # raced the scan gets its own pass. Bounded to one extra pass per burst. + if rerun: + self.schedule_relation_resolution(project_id=project_id) diff --git a/src/basic_memory/index/local_watch.py b/src/basic_memory/index/local_watch.py index 03686c5c6..45b162e3e 100644 --- a/src/basic_memory/index/local_watch.py +++ b/src/basic_memory/index/local_watch.py @@ -23,7 +23,7 @@ ) from basic_memory.runtime.storage import ( ProjectPath, - RuntimeStorageEventProcessingResult, + RuntimeJobCounts, StorageBucketName, StorageEventPayload, group_storage_events_by_bucket, @@ -33,7 +33,12 @@ class LocalWatchProjectSource(Protocol): - """Minimal project shape needed to build local watcher storage events.""" + """Minimal project shape needed to build local watcher storage events. + + Object-typed on purpose: downstream runtimes compose leaner project + projections (Path-typed paths, no identity attributes) against this seam, + so the helpers coerce and fall back instead of trusting the declared shape. + """ @property def path(self) -> object: ... @@ -237,25 +242,25 @@ def record_last_error(self) -> bool: def plan_local_watch_event_index_status_update( *, project_prefix: ProjectPath, - result: RuntimeStorageEventProcessingResult, + result: RuntimeJobCounts, ) -> LocalWatchEventIndexStatusUpdate: """Plan the local watch-status update for an event-index result.""" - if result.counts.failed: + if result.failed: return LocalWatchEventIndexStatusUpdate( path=project_prefix, status="error", - indexed_files_increment=result.counts.processed, - error_count_increment=result.counts.failed, + indexed_files_increment=result.processed, + error_count_increment=result.failed, error=( - f"event-index processed={result.counts.processed} " - f"failed={result.counts.failed} skipped={result.counts.skipped}" + f"event-index processed={result.processed} " + f"failed={result.failed} skipped={result.skipped}" ), ) return LocalWatchEventIndexStatusUpdate( path=project_prefix, status="success", - indexed_files_increment=result.counts.processed, + indexed_files_increment=result.processed, error_count_increment=0, ) @@ -291,11 +296,11 @@ async def run_local_watch_event_indexing( request: LocalWatchEventIndexRequest, *, runtime: StorageEventIndexRuntime, -) -> RuntimeStorageEventProcessingResult: +) -> RuntimeJobCounts: """Normalize local file changes and process them through storage-event indexing.""" event_source = LocalWatchStorageEventSource(request) events = event_source.events() - result = RuntimeStorageEventProcessingResult.empty() + result = RuntimeJobCounts() if isinstance(runtime, LocalWatchStorageEventIndexRuntime): move_processor = runtime.move_processor if move_processor is not None: diff --git a/src/basic_memory/index/storage_events.py b/src/basic_memory/index/storage_events.py index 8565a2296..a23e0bc6c 100644 --- a/src/basic_memory/index/storage_events.py +++ b/src/basic_memory/index/storage_events.py @@ -9,7 +9,7 @@ from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.storage import ( ProjectPath, - RuntimeStorageEventProcessingResult, + RuntimeJobCounts, StorageBucketName, StorageEventPayload, StorageEventSource, @@ -79,7 +79,7 @@ async def process_bucket_context_events( bucket_name: StorageBucketName, context: BucketContextT, events: tuple[StorageEventPayload, ...], - ) -> RuntimeStorageEventProcessingResult: + ) -> RuntimeJobCounts: """Process one bucket's storage events and return aggregate counts.""" async def bucket_failed( @@ -110,9 +110,9 @@ class StorageEventBucketIndexRuntime(Generic[BucketContextT]): async def run_storage_event_bucket_indexing( source: StorageEventSource, runtime: StorageEventBucketIndexRuntime[BucketContextT], -) -> RuntimeStorageEventProcessingResult: +) -> RuntimeJobCounts: """Resolve bucket contexts and aggregate provider-neutral bucket results.""" - result = RuntimeStorageEventProcessingResult.empty() + result = RuntimeJobCounts() for bucket_name, events in source.events_by_bucket().items(): if not events: @@ -144,10 +144,10 @@ async def run_storage_event_bucket_indexing( async def run_storage_event_indexing( events: Iterable[StorageEventPayload], runtime: StorageEventIndexRuntime, -) -> RuntimeStorageEventProcessingResult: +) -> RuntimeJobCounts: """Route normalized storage events by project and execute project-scoped operations.""" routing_plan = plan_runtime_storage_events_by_project(events) - result = RuntimeStorageEventProcessingResult.empty().add_counts(routing_plan.skipped_counts) + result = routing_plan.skipped_counts for project_batch in routing_plan.project_batches: project = await runtime.project_resolver.resolve_project(project_batch.project_path) diff --git a/src/basic_memory/index/watch_service.py b/src/basic_memory/index/watch_service.py index 2a0d5720a..c476e3ec8 100644 --- a/src/basic_memory/index/watch_service.py +++ b/src/basic_memory/index/watch_service.py @@ -5,8 +5,7 @@ import asyncio import os import time -from collections.abc import AsyncIterator, Sequence -from contextlib import asynccontextmanager +from collections.abc import Sequence from datetime import datetime from pathlib import Path from typing import Protocol @@ -122,12 +121,6 @@ def __init__( self.constrained_project = constrained_project self.console = Console(quiet=quiet) - @asynccontextmanager - async def _session_scope(self) -> AsyncIterator[AsyncSession]: - """Open a service-owned transaction.""" - async with db.scoped_session(self.session_maker) as session: - yield session - async def _schedule_restart(self, stop_event: asyncio.Event) -> None: """Schedule a watch cycle restart so project config changes are observed.""" await asyncio.sleep(self.app_config.watch_project_reload_interval) @@ -178,7 +171,7 @@ async def _watch_projects_cycle( async def _select_projects_to_watch(self) -> list[Project]: """Return locally syncable projects that this watcher instance owns.""" - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: projects = await self.project_repository.get_active_projects(session) if self.constrained_project: @@ -348,9 +341,9 @@ async def handle_changes(self, project: Project, changes: set[FileChange]) -> No duration_ms = int((time.time() - start_time) * 1000) logger.info( "Event-index file change processing completed, " - f"processed_files={result.counts.processed}, " - f"failed_files={result.counts.failed}, " - f"skipped_files={result.counts.skipped}, " + f"processed_files={result.processed}, " + f"failed_files={result.failed}, " + f"skipped_files={result.skipped}, " f"total_indexed_files={self.state.indexed_files}, " f"duration_ms={duration_ms}" ) diff --git a/src/basic_memory/indexing/accepted_note_mutation_runner.py b/src/basic_memory/indexing/accepted_note_mutation_runner.py index 569da16e8..7c5daff21 100644 --- a/src/basic_memory/indexing/accepted_note_mutation_runner.py +++ b/src/basic_memory/indexing/accepted_note_mutation_runner.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from datetime import UTC, datetime from enum import StrEnum -from typing import NoReturn, Protocol, cast +from typing import NoReturn, Protocol from uuid import UUID from sqlalchemy.exc import IntegrityError @@ -28,10 +28,8 @@ prepare_accepted_note_replace, ) from basic_memory.models import Entity, NoteContent, Project -from basic_memory.repository import NoteContentRepository, NoteContentVersionConflict +from basic_memory.repository import NoteContentVersionConflict from basic_memory.services.exceptions import EntityAlreadyExistsError -from basic_memory.repository.accepted_note_search_repository import AcceptedNoteSearchRepository -from basic_memory.repository.entity_repository import EntityRepository from basic_memory.runtime.note_content import ( RuntimeAcceptedNoteChange, RuntimeAcceptedNoteWriteConflictKind, @@ -267,36 +265,6 @@ def note_content_repository( ) -> AcceptedNoteMutationNoteContentRepository: ... -class AcceptedNoteRepositories( - AcceptedNoteMutationRepositories, - AcceptedNoteWriteRepositories, - Protocol, -): - """Repository capability set for DB-first accepted-note mutations.""" - - -@dataclass(frozen=True, slots=True) -class DefaultAcceptedNoteRepositories: - """Default core repositories for accepted-note mutation orchestration.""" - - def entity_repository(self, project_id: ProjectId) -> EntityRepository: - return EntityRepository(project_id=project_id) - - def pending_entity_repository(self, project_id: ProjectId) -> EntityRepository: - return EntityRepository(project_id=project_id) - - def note_content_repository(self, project_id: ProjectId) -> NoteContentRepository: - return NoteContentRepository(project_id=project_id) - - def search_repository(self, project_id: ProjectId) -> AcceptedNoteSearchRepository: - return AcceptedNoteSearchRepository(project_id=project_id) - - -def build_default_accepted_note_repositories() -> AcceptedNoteRepositories: - """Compose default repositories for accepted-note mutation orchestration.""" - return DefaultAcceptedNoteRepositories() - - @dataclass(frozen=True, slots=True) class AcceptedNoteMutationDependencies: """Dependencies required by accepted-note mutation orchestration.""" @@ -439,13 +407,11 @@ async def run_accepted_note_delete( load_relations=False, ) if entity is None: - return cast( - AcceptedNoteMutationChange, - await delete_accepted_note( - session, - project_id=project.id, - entity=None, - ), + return await delete_accepted_note( + session, + project_id=project.id, + entity=None, + repositories=dependencies.write_repositories, ) note_content = await load_accepted_note_content( @@ -455,14 +421,13 @@ async def run_accepted_note_delete( dependencies=dependencies, missing_kind=None, ) - accepted = await delete_accepted_note( + return await delete_accepted_note( session, project_id=project.id, entity=entity, note_content=note_content, repositories=dependencies.write_repositories, ) - return cast(AcceptedNoteMutationChange, accepted) async def _run_accepted_note_create( @@ -529,7 +494,7 @@ async def _run_accepted_note_create( updated_at=now, repositories=dependencies.write_repositories, ) - accepted = plan_accepted_note_write_change( + return plan_accepted_note_write_change( status_code=201, entity=entity, note_content=persisted.note_content, @@ -538,7 +503,6 @@ async def _run_accepted_note_create( actor_name=request.actor.name, fallback_source=request.source, ) - return cast(AcceptedNoteMutationChange, accepted) async def _run_accepted_note_update( @@ -670,7 +634,7 @@ async def _run_accepted_note_update( accepted_file_path=entity.file_path, repositories=dependencies.write_repositories, ) - accepted = plan_accepted_note_write_change( + return plan_accepted_note_write_change( status_code=201 if created else 200, entity=entity, note_content=persisted.note_content, @@ -680,7 +644,6 @@ async def _run_accepted_note_update( cleanup_after_write=persisted.previous_file_delete, fallback_source=request.source, ) - return cast(AcceptedNoteMutationChange, accepted) async def _run_accepted_note_edit( @@ -730,7 +693,7 @@ async def _run_accepted_note_edit( accepted_file_path=entity.file_path, repositories=dependencies.write_repositories, ) - accepted = plan_accepted_note_write_change( + return plan_accepted_note_write_change( status_code=200, entity=entity, note_content=persisted.note_content, @@ -739,7 +702,6 @@ async def _run_accepted_note_edit( actor_name=request.actor.name, fallback_source=request.source, ) - return cast(AcceptedNoteMutationChange, accepted) async def _run_accepted_note_move( @@ -821,7 +783,7 @@ async def _run_accepted_note_move( accepted_file_path=prepared_move.file_path, repositories=dependencies.write_repositories, ) - accepted = plan_accepted_note_write_change( + return plan_accepted_note_write_change( status_code=200, entity=entity, note_content=persisted.note_content, @@ -831,7 +793,6 @@ async def _run_accepted_note_move( cleanup_after_write=persisted.previous_file_delete, fallback_source=request.source, ) - return cast(AcceptedNoteMutationChange, accepted) async def load_accepted_note_mutation_project( diff --git a/src/basic_memory/indexing/accepted_note_write_runner.py b/src/basic_memory/indexing/accepted_note_write_runner.py index 2e4e5b9e2..6e9b6da39 100644 --- a/src/basic_memory/indexing/accepted_note_write_runner.py +++ b/src/basic_memory/indexing/accepted_note_write_runner.py @@ -17,12 +17,10 @@ build_accepted_note_search_row, ) from basic_memory.models import Entity, NoteContent -from basic_memory.repository import AcceptedNoteContentWrite, NoteContentRepository -from basic_memory.repository.accepted_note_search_repository import AcceptedNoteSearchRepository +from basic_memory.repository import AcceptedNoteContentWrite from basic_memory.repository.entity_repository import ( AcceptedPendingEntityWrite, EntityMetadata, - EntityRepository, ) from basic_memory.runtime.note_content import ( RuntimeAcceptedNoteChange, @@ -308,55 +306,6 @@ class AcceptedPersistedNoteWrite: previous_file_delete: RuntimePendingNoteFileDelete | None = None -def accepted_entity_repository_for_project( - project_id: ProjectId, -) -> AcceptedPendingEntityRepository: - """Create the core repository adapter for pending accepted entities.""" - return EntityRepository(project_id=project_id) - - -def accepted_note_content_repository_for_project( - project_id: ProjectId, -) -> AcceptedNoteContentRepository: - """Create the core repository adapter for accepted note_content rows.""" - return NoteContentRepository(project_id=project_id) - - -def accepted_note_search_repository_for_project( - project_id: ProjectId, -) -> AcceptedNoteSearchRowRepository: - """Create the core repository adapter for accepted-note search rows.""" - return AcceptedNoteSearchRepository(project_id=project_id) - - -@dataclass(frozen=True, slots=True) -class DefaultAcceptedNoteWriteRepositories: - """Default repository capability set for accepted-note write persistence.""" - - def pending_entity_repository( - self, - project_id: ProjectId, - ) -> AcceptedPendingEntityRepository: - return accepted_entity_repository_for_project(project_id) - - def note_content_repository( - self, - project_id: ProjectId, - ) -> AcceptedNoteContentRepository: - return accepted_note_content_repository_for_project(project_id) - - def search_repository( - self, - project_id: ProjectId, - ) -> AcceptedNoteSearchRowRepository: - return accepted_note_search_repository_for_project(project_id) - - -def build_default_accepted_note_write_repositories() -> AcceptedNoteWriteRepositories: - """Compose the default repository adapters for accepted-note write persistence.""" - return DefaultAcceptedNoteWriteRepositories() - - async def prepare_accepted_note_create( preparer: AcceptedNoteCreatePreparer, data: EntitySchema, @@ -543,11 +492,10 @@ async def create_accepted_pending_entity( now: datetime, user_profile_value: str | None, external_id: str | None = None, - repositories: AcceptedNoteWriteRepositories | None = None, + repositories: AcceptedNoteWriteRepositories, ) -> Entity: """Insert a prepared accepted entity row without materializing a file.""" - write_repositories = repositories or build_default_accepted_note_write_repositories() - repository = write_repositories.pending_entity_repository(project_id) + repository = repositories.pending_entity_repository(project_id) return await repository.create_pending_accepted_entity( session, accepted_pending_entity_write_from_prepared( @@ -588,11 +536,10 @@ async def accept_note_content_write( db_checksum: RuntimeNoteContentChecksum, last_source: RuntimeNoteChangeSource | None, updated_at: datetime, - repositories: AcceptedNoteWriteRepositories | None = None, + repositories: AcceptedNoteWriteRepositories, ) -> NoteContent: """Accept markdown into note_content before object storage catches up.""" - write_repositories = repositories or build_default_accepted_note_write_repositories() - repository = write_repositories.note_content_repository(entity.project_id) + repository = repositories.note_content_repository(entity.project_id) return await repository.accept_write( session, accepted_note_content_write_from_markdown( @@ -631,11 +578,10 @@ async def refresh_accepted_note_search_index( *, entity: AcceptedNoteSearchEntitySource, search_content: str, - repositories: AcceptedNoteWriteRepositories | None = None, + repositories: AcceptedNoteWriteRepositories, ) -> None: """Refresh the hot accepted-note search row inside the caller's transaction.""" - write_repositories = repositories or build_default_accepted_note_write_repositories() - repository = write_repositories.search_repository(entity.project_id) + repository = repositories.search_repository(entity.project_id) await repository.refresh_entity( session, accepted_note_search_row_from_entity(entity, search_content=search_content), @@ -647,11 +593,10 @@ async def delete_accepted_note_search_index( *, project_id: ProjectId, entity_id: RuntimeEntityId, - repositories: AcceptedNoteWriteRepositories | None = None, + repositories: AcceptedNoteWriteRepositories, ) -> None: """Remove all search rows for an accepted-note entity inside the caller's transaction.""" - write_repositories = repositories or build_default_accepted_note_write_repositories() - repository = write_repositories.search_repository(project_id) + repository = repositories.search_repository(project_id) await repository.delete_entity(session, entity_id) @@ -660,11 +605,10 @@ async def delete_accepted_note_vectors( *, project_id: ProjectId, entity_id: RuntimeEntityId, - repositories: AcceptedNoteWriteRepositories | None = None, + repositories: AcceptedNoteWriteRepositories, ) -> None: """Remove semantic vectors for an accepted-note entity inside the caller's transaction.""" - write_repositories = repositories or build_default_accepted_note_write_repositories() - repository = write_repositories.search_repository(project_id) + repository = repositories.search_repository(project_id) await repository.delete_entity_vectors(session, entity_id) @@ -680,10 +624,9 @@ async def persist_accepted_note_write( current_note_content: RuntimeAcceptedNoteContentWriteSource | None = None, existing_file_path: RuntimeFilePath | None = None, accepted_file_path: RuntimeFilePath | None = None, - repositories: AcceptedNoteWriteRepositories | None = None, + repositories: AcceptedNoteWriteRepositories, ) -> AcceptedPersistedNoteWrite: """Accept markdown into note_content and refresh search inside one transaction.""" - write_repositories = repositories or build_default_accepted_note_write_repositories() content_write = plan_accepted_note_content_write( project_id=entity.project_id, entity_id=entity.id, @@ -699,13 +642,13 @@ async def persist_accepted_note_write( db_checksum=db_checksum, last_source=last_source, updated_at=updated_at, - repositories=write_repositories, + repositories=repositories, ) await refresh_accepted_note_search_index( session, entity=entity, search_content=search_content, - repositories=write_repositories, + repositories=repositories, ) return AcceptedPersistedNoteWrite( note_content=note_content, @@ -728,7 +671,7 @@ async def delete_accepted_note( project_id: ProjectId, entity: AcceptedNoteDeleteEntitySource | None, note_content: RuntimeDeletedNoteFileChecksumSource | None = None, - repositories: AcceptedNoteWriteRepositories | None = None, + repositories: AcceptedNoteWriteRepositories, ) -> RuntimeAcceptedNoteChange[dict[str, object]]: """Plan an accepted note delete and remove the entity when it exists.""" accepted = plan_accepted_note_delete_change( diff --git a/src/basic_memory/indexing/batch_indexer.py b/src/basic_memory/indexing/batch_indexer.py index 523313ec8..99e5c384f 100644 --- a/src/basic_memory/indexing/batch_indexer.py +++ b/src/basic_memory/indexing/batch_indexer.py @@ -3,11 +3,10 @@ from __future__ import annotations import asyncio -from contextlib import asynccontextmanager from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import AsyncIterator, Awaitable, Callable, Mapping, TypeVar +from typing import Awaitable, Callable, Mapping, TypeVar from loguru import logger from sqlalchemy.exc import IntegrityError @@ -100,18 +99,6 @@ def __init__( self.file_writer = file_writer self.session_maker = session_maker - @asynccontextmanager - async def _session_scope( - self, session: AsyncSession | None = None - ) -> AsyncIterator[AsyncSession]: - """Use the caller's session or open a service-owned transaction.""" - if session is not None: - yield session - return - - async with db.scoped_session(self.session_maker) as owned_session: - yield owned_session - async def index_files( self, files: Mapping[str, IndexInputFile], @@ -184,7 +171,7 @@ async def index_files( max_concurrent=max_concurrent, ) - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: refreshed_entities = await self.entity_repository.find_by_ids( session, [prepared.entity_id for prepared in prepared_entities.values()] ) @@ -257,7 +244,7 @@ async def index_markdown_file( path=file.path, entity_id=persisted.entity.id, ): - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: refreshed = await self.entity_repository.find_by_ids(session, [persisted.entity.id]) if len(refreshed) != 1: # pragma: no cover raise ValueError(f"Failed to reload indexed entity for {file.path}") @@ -283,7 +270,7 @@ async def index_markdown_file( async def _get_file_path_to_permalink_map(self) -> dict[str, str | None]: """Load current file-path to permalink mappings in a service-owned session.""" - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: permalink_by_path: dict[str, str | None] = { path: permalink for path, permalink in ( @@ -444,7 +431,7 @@ async def _upsert_markdown_file(self, prepared: _PreparedMarkdownFile) -> _Prepa async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity: checksum = await self._resolve_checksum(file) - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: existing = await self.entity_repository.get_by_file_path( session, file.path, load_relations=False ) @@ -465,7 +452,7 @@ async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity: ) try: - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: created = await self.entity_repository.add(session, entity) entity_id = created.id except IntegrityError as exc: @@ -478,7 +465,7 @@ async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity: and "file_path" in message ) ): - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: existing = await self.entity_repository.get_by_file_path( session, file.path, @@ -494,7 +481,7 @@ async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity: else: entity_id = existing.id - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: updated = await self.entity_repository.update( session, entity_id, @@ -542,7 +529,7 @@ async def resolve_relation(relation: Relation) -> int: # link text, mismatching this with the sync_service forward-reference # path and producing confidently-wrong graph edges. See # sync_service.resolve_forward_references for the same change. - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: resolved_entity = await self.entity_service.link_resolver.resolve_link( relation.to_name, strict=True, session=session ) @@ -550,7 +537,7 @@ async def resolve_relation(relation: Relation) -> int: return 0 try: - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: await self.relation_repository.update( session, relation.id, @@ -560,7 +547,7 @@ async def resolve_relation(relation: Relation) -> int: }, ) except IntegrityError: - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: await self.relation_repository.delete(session, relation.id) return 1 except Exception as exc: # pragma: no cover - defensive logging @@ -586,7 +573,7 @@ async def resolve_relation(relation: Relation) -> int: async def _find_unresolved_relations_for_entity(self, entity_id: int): """Load unresolved relations for one entity in a service-owned session.""" - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: return await self.relation_repository.find_unresolved_relations_for_entity( session, entity_id ) @@ -626,7 +613,7 @@ async def _persist_markdown_file( resolve_relations: bool = True, reload_entity: bool = True, ) -> _PersistedMarkdownFile: - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: existing = await self.entity_repository.get_by_file_path( session, prepared.file.path, diff --git a/src/basic_memory/indexing/change_detector.py b/src/basic_memory/indexing/change_detector.py index 75857adff..ff9175194 100644 --- a/src/basic_memory/indexing/change_detector.py +++ b/src/basic_memory/indexing/change_detector.py @@ -18,6 +18,7 @@ FileMoveCandidate, StorageChecksumSource, plan_change_detection_snapshot, + plan_move_target_checksums, storage_checksums_from_sources, ) from basic_memory.indexing.file_index_checking import IndexedFileChecksumRepository @@ -148,18 +149,15 @@ async def detect_project_file_changes( with logfire.span("change_detector.detect_deletes"): all_db_paths = await store.load_all_indexed_paths() - candidate_snapshot = ChangeDetectionSnapshot( + move_target_checksums = plan_move_target_checksums( storage_checksum_by_path=storage_checksum_by_path, db_checksum_by_path=db_checksums, - all_db_paths=all_db_paths, ) with logfire.span( "change_detector.detect_moves", - candidate_count=len(candidate_snapshot.new_file_checksum_by_path), + candidate_count=len(move_target_checksums), ): - move_candidates = await store.load_move_candidates( - candidate_snapshot.new_file_checksum_by_path - ) + move_candidates = await store.load_move_candidates(move_target_checksums) snapshot = ChangeDetectionSnapshot( storage_checksum_by_path=storage_checksum_by_path, diff --git a/src/basic_memory/indexing/change_planning.py b/src/basic_memory/indexing/change_planning.py index 5e38e9d9b..5cab1aed7 100644 --- a/src/basic_memory/indexing/change_planning.py +++ b/src/basic_memory/indexing/change_planning.py @@ -69,19 +69,6 @@ def storage_paths(self) -> tuple[FileIndexPath, ...]: """Return observed storage paths in adapter-provided order.""" return tuple(self.storage_checksum_by_path) - @property - def new_file_checksum_by_path(self) -> dict[FileIndexPath, FileIndexChecksum]: - """Return storage objects that do not have an indexed row at the same path. - - Paths with an unknown (None) checksum are excluded: without content - evidence they cannot prove a move, so they stay plain new files. - """ - return { - path: checksum - for path, checksum in self.storage_checksum_by_path.items() - if path not in self.db_checksum_by_path and checksum is not None - } - def storage_checksums_from_sources( storage_files: Mapping[FileIndexPath, StorageChecksumSource], @@ -90,6 +77,26 @@ def storage_checksums_from_sources( return {path: file_info.checksum for path, file_info in storage_files.items()} +def plan_move_target_checksums( + *, + storage_checksum_by_path: Mapping[FileIndexPath, FileIndexChecksum | None], + db_checksum_by_path: Mapping[FileIndexPath, FileIndexChecksum | None], +) -> dict[FileIndexPath, FileIndexChecksum]: + """Return storage objects eligible to prove a move, keyed by destination path. + + Move destinations must be paths with no indexed row at all. A modified (or + null-checksum) path already has an entity; matching it to a deleted file's + checksum would redirect that entity onto the existing path and silently + drop the in-place edit. Unknown (None) storage checksums carry no content + evidence, so they cannot claim a move candidate either. + """ + return { + path: checksum + for path, checksum in storage_checksum_by_path.items() + if path not in db_checksum_by_path and checksum is not None + } + + def plan_change_detection_snapshot(snapshot: ChangeDetectionSnapshot) -> ChangeReport: """Classify changes from a typed runtime snapshot.""" return plan_file_changes( @@ -126,19 +133,11 @@ def plan_file_changes( continue unchanged_files.append(path) - # Move destinations must be paths with no indexed row at all. A modified (or - # null-checksum) path already has an entity; matching it to a deleted file's - # checksum would redirect that entity onto the existing path and silently - # drop the in-place edit. Unknown (None) storage checksums carry no content - # evidence, so they cannot claim a move candidate either. - move_target_checksum_by_path: dict[FileIndexPath, FileIndexChecksum] = {} - for path in new_files: - storage_checksum = storage_checksum_by_path[path] - if path in db_checksum_by_path or storage_checksum is None: - continue - move_target_checksum_by_path[path] = storage_checksum moved_files = plan_moved_files( - new_file_checksum_by_path=move_target_checksum_by_path, + new_file_checksum_by_path=plan_move_target_checksums( + storage_checksum_by_path=storage_checksum_by_path, + db_checksum_by_path=db_checksum_by_path, + ), storage_paths=storage_paths, move_candidates=move_candidates, ) diff --git a/src/basic_memory/indexing/directory_delete_runner.py b/src/basic_memory/indexing/directory_delete_runner.py index bcabe54c0..ac8d40aa2 100644 --- a/src/basic_memory/indexing/directory_delete_runner.py +++ b/src/basic_memory/indexing/directory_delete_runner.py @@ -5,7 +5,7 @@ from collections.abc import Sequence from dataclasses import dataclass from enum import StrEnum -from typing import Literal, Protocol +from typing import Literal, NotRequired, Protocol, TypedDict from sqlalchemy import bindparam, delete, select, text from sqlalchemy.ext.asyncio import AsyncSession @@ -265,6 +265,29 @@ class DirectoryDeleteFileFailure: reason: str +class DirectoryDeleteErrorPayload(TypedDict): + """One failed file delete in the existing route error contract.""" + + path: RuntimeFilePath + error: str + + +class DirectoryDeleteResponsePayload(TypedDict): + """Existing directory-delete route response contract. + + Extends the client-facing DirectoryDeleteResult schema with the runtime + ``file_delete_status``/``error`` fields that queued cleanups report. + """ + + total_files: int + successful_deletes: int + failed_deletes: int + deleted_files: list[RuntimeFilePath] + errors: list[DirectoryDeleteErrorPayload] + file_delete_status: DirectoryDeleteFileStatus + error: NotRequired[str] + + @dataclass(frozen=True, slots=True) class DirectoryDeleteAcceptedResult: """Existing directory-delete response shape before route serialization.""" @@ -315,7 +338,16 @@ def failed( error=error, ) - def to_response_payload(self) -> dict[str, object]: + @property + def http_status_code(self) -> int: + """Return the route status for this result. + + A failed cleanup enqueue leaves files on disk with their DB rows gone, + so the route reports a server error instead of a clean success. + """ + return 500 if self.file_delete_status == "failed" else 200 + + def to_response_payload(self) -> DirectoryDeleteResponsePayload: """Serialize to the current Basic Memory directory-delete response contract. A guarded cleanup that left files on disk (failed_files) is reported as @@ -323,7 +355,7 @@ def to_response_payload(self) -> dict[str, object]: callers see a clean success while stale files remain and later reappear. """ failed = len(self.failed_files) - payload: dict[str, object] = { + payload: DirectoryDeleteResponsePayload = { "total_files": len(self.deleted_files), "successful_deletes": len(self.deleted_files) - failed, "failed_deletes": failed, diff --git a/src/basic_memory/indexing/file_indexer.py b/src/basic_memory/indexing/file_indexer.py index 397d45236..c3889fe97 100644 --- a/src/basic_memory/indexing/file_indexer.py +++ b/src/basic_memory/indexing/file_indexer.py @@ -35,16 +35,16 @@ class IndexMarkdownEntity(Protocol): def id(self) -> int: ... @property - def external_id(self) -> object | None: ... + def external_id(self) -> str: ... @property - def title(self) -> object | None: ... + def title(self) -> str: ... @property - def permalink(self) -> object | None: ... + def permalink(self) -> str | None: ... @property - def checksum(self) -> object | None: ... + def checksum(self) -> str | None: ... class IndexMarkdownEntityRepository(Protocol): diff --git a/src/basic_memory/indexing/forward_reference_resolution.py b/src/basic_memory/indexing/forward_reference_resolution.py index 746eff59c..7ac378e5a 100644 --- a/src/basic_memory/indexing/forward_reference_resolution.py +++ b/src/basic_memory/indexing/forward_reference_resolution.py @@ -11,26 +11,13 @@ from basic_memory import db from basic_memory.indexing.link_resolution import LinkText, resolve_project_link_texts -from basic_memory.models import Entity, Relation - -type ForwardReferenceEntityId = int -type ForwardReferenceRelationId = int - - -class UnresolvedForwardReference(Protocol): - """Minimal unresolved relation shape needed for exact target planning.""" - - @property - def id(self) -> ForwardReferenceRelationId: - """Return the unresolved relation primary key.""" - - @property - def from_id(self) -> ForwardReferenceEntityId: - """Return the source entity id for the unresolved relation.""" - - @property - def to_name(self) -> LinkText | None: - """Return the unresolved target link text.""" +from basic_memory.indexing.relation_resolution import ( + EntityId, + RelationResolutionEntityIndexer, + RelationResolutionEntityRepository, + UnresolvedRelation, +) +from basic_memory.models import Relation class ForwardReferenceRelationSource(Protocol): @@ -38,7 +25,7 @@ class ForwardReferenceRelationSource(Protocol): async def list_unresolved_forward_references( self, - ) -> tuple[UnresolvedForwardReference, ...]: + ) -> tuple[UnresolvedRelation, ...]: """Return unresolved relation rows for one project.""" @@ -47,36 +34,18 @@ class ForwardReferenceEntityRefreshRuntime(Protocol): async def refresh_forward_reference_entity( self, - entity_id: ForwardReferenceEntityId, + entity_id: EntityId, ) -> bool: """Refresh one entity and return whether the entity still exists.""" -class ForwardReferenceEntityRepository(Protocol): - """Repository capability required to load forward-reference target entities.""" - - async def find_by_id( - self, - session: AsyncSession, - entity_id: ForwardReferenceEntityId, - ) -> Entity | None: - """Return one entity by id.""" - - -class ForwardReferenceEntityIndexer(Protocol): - """Search capability required to refresh one forward-reference target entity.""" - - async def index_entity(self, entity: Entity) -> object: - """Refresh one entity in the search index.""" - - @dataclass(frozen=True, slots=True) class ForwardReferenceUpdate: """One unresolved relation that can be filled with an exact target entity.""" - relation_id: ForwardReferenceRelationId - source_entity_id: ForwardReferenceEntityId - target_entity_id: ForwardReferenceEntityId + relation_id: int + source_entity_id: EntityId + target_entity_id: EntityId link_text: LinkText @@ -87,7 +56,7 @@ class ForwardReferenceResolutionPlan: unresolved_before: int link_texts: tuple[LinkText, ...] updates: tuple[ForwardReferenceUpdate, ...] - entity_ids_to_refresh: frozenset[ForwardReferenceEntityId] + entity_ids_to_refresh: frozenset[EntityId] @property def resolved_count(self) -> int: @@ -111,7 +80,7 @@ class ForwardReferenceResolutionRuntime(Protocol): async def resolve_forward_reference_link_texts( self, link_texts: Sequence[LinkText], - ) -> Mapping[LinkText, ForwardReferenceEntityId | None]: + ) -> Mapping[LinkText, EntityId | None]: """Resolve link texts to exact target entity ids.""" async def apply_forward_reference_updates( @@ -130,7 +99,7 @@ class RepositoryForwardReferenceRelationSource: async def list_unresolved_forward_references( self, - ) -> tuple[UnresolvedForwardReference, ...]: + ) -> tuple[UnresolvedRelation, ...]: async with db.scoped_session(self.session_maker) as session: result = await session.execute( select(Relation).where( @@ -151,7 +120,7 @@ class RepositoryForwardReferenceResolutionRuntime: async def resolve_forward_reference_link_texts( self, link_texts: Sequence[LinkText], - ) -> Mapping[LinkText, ForwardReferenceEntityId | None]: + ) -> Mapping[LinkText, EntityId | None]: return await resolve_project_link_texts( link_texts, session_maker=self.session_maker, @@ -184,12 +153,12 @@ class RepositoryForwardReferenceEntityRefreshRuntime: """Refresh forward-reference target entity search rows with explicit sessions.""" session_maker: async_sessionmaker[AsyncSession] - entity_repository: ForwardReferenceEntityRepository - entity_indexer: ForwardReferenceEntityIndexer + entity_repository: RelationResolutionEntityRepository + entity_indexer: RelationResolutionEntityIndexer async def refresh_forward_reference_entity( self, - entity_id: ForwardReferenceEntityId, + entity_id: EntityId, ) -> bool: async with db.scoped_session(self.session_maker) as session: entity = await self.entity_repository.find_by_id(session, entity_id) @@ -203,7 +172,7 @@ async def refresh_forward_reference_entity( class ForwardReferenceEntityRefreshFailure: """One target entity whose search refresh raised.""" - entity_id: ForwardReferenceEntityId + entity_id: EntityId error: Exception @@ -211,12 +180,12 @@ class ForwardReferenceEntityRefreshFailure: class ForwardReferenceEntityRefreshRun: """Search refresh results for target entities touched by forward refs.""" - successful_entity_ids: frozenset[ForwardReferenceEntityId] - missing_entity_ids: frozenset[ForwardReferenceEntityId] + successful_entity_ids: frozenset[EntityId] + missing_entity_ids: frozenset[EntityId] failures: tuple[ForwardReferenceEntityRefreshFailure, ...] @property - def failed_entity_ids(self) -> frozenset[ForwardReferenceEntityId]: + def failed_entity_ids(self) -> frozenset[EntityId]: """Return entity ids whose refresh raised.""" return frozenset(failure.entity_id for failure in self.failures) @@ -249,13 +218,13 @@ def remaining_count(self) -> int: return self.plan.remaining_count @property - def entity_ids_to_refresh(self) -> frozenset[ForwardReferenceEntityId]: + def entity_ids_to_refresh(self) -> frozenset[EntityId]: """Return exact target entity ids whose search rows should be refreshed.""" return self.plan.entity_ids_to_refresh def collect_forward_reference_link_texts( - unresolved_relations: Sequence[UnresolvedForwardReference], + unresolved_relations: Sequence[UnresolvedRelation], ) -> tuple[LinkText, ...]: """Collect unique unresolved link texts in first-seen order.""" link_texts: dict[LinkText, None] = {} @@ -266,12 +235,12 @@ def collect_forward_reference_link_texts( def plan_forward_reference_resolution( - unresolved_relations: Sequence[UnresolvedForwardReference], - resolved_targets: Mapping[LinkText, ForwardReferenceEntityId | None], + unresolved_relations: Sequence[UnresolvedRelation], + resolved_targets: Mapping[LinkText, EntityId | None], ) -> ForwardReferenceResolutionPlan: """Plan exact target updates for a batch of unresolved relation rows.""" updates: list[ForwardReferenceUpdate] = [] - entity_ids_to_refresh: set[ForwardReferenceEntityId] = set() + entity_ids_to_refresh: set[EntityId] = set() for relation in unresolved_relations: link_text = relation.to_name @@ -302,7 +271,7 @@ def plan_forward_reference_resolution( async def run_forward_reference_resolution( runtime: ForwardReferenceResolutionRuntime, - unresolved_relations: Sequence[UnresolvedForwardReference], + unresolved_relations: Sequence[UnresolvedRelation], ) -> ForwardReferenceResolutionRun: """Resolve link texts, apply exact relation updates, and return refresh targets.""" link_texts = collect_forward_reference_link_texts(unresolved_relations) @@ -323,11 +292,11 @@ async def run_forward_reference_resolution( async def run_forward_reference_entity_refresh( runtime: ForwardReferenceEntityRefreshRuntime, - entity_ids: Iterable[ForwardReferenceEntityId], + entity_ids: Iterable[EntityId], ) -> ForwardReferenceEntityRefreshRun: """Refresh forward-reference target search rows and report per-entity failures.""" - successful_entity_ids: set[ForwardReferenceEntityId] = set() - missing_entity_ids: set[ForwardReferenceEntityId] = set() + successful_entity_ids: set[EntityId] = set() + missing_entity_ids: set[EntityId] = set() failures: list[ForwardReferenceEntityRefreshFailure] = [] for entity_id in entity_ids: diff --git a/src/basic_memory/indexing/index_batch_runtime.py b/src/basic_memory/indexing/index_batch_runtime.py index b4c1fde0e..282a5d17d 100644 --- a/src/basic_memory/indexing/index_batch_runtime.py +++ b/src/basic_memory/indexing/index_batch_runtime.py @@ -24,11 +24,11 @@ StorageIndexFileWriter, ) from basic_memory.indexing.note_content_batch_reconciliation import ( - DefaultIndexedNoteContentTimestampProvider, IndexedNoteContentEntity, IndexedNoteContentEntityRepository, + IndexedNoteContentObservedAt, IndexedNoteContentReconciler, - IndexedNoteContentTimestampProvider, + indexed_note_content_observed_at, reconcile_indexed_note_content_batch, ) from basic_memory.indexing.note_content_reconciler import ( @@ -63,7 +63,7 @@ class IndexBatchRuntime[EntityT: IndexedNoteContentEntity, FileInfoT: LoadedInde entity_repository: IndexedNoteContentEntityRepository[EntityT] session_maker: async_sessionmaker[AsyncSession] note_content_reconciler: IndexedNoteContentReconciler[EntityT] - timestamp_provider: IndexedNoteContentTimestampProvider[FileInfoT] + timestamp_provider: IndexedNoteContentObservedAt[FileInfoT] = indexed_note_content_observed_at note_content_source: str = "index" # Optional canonical-file reader. When set, batch reconciliation re-reads each # file at reconcile time instead of trusting the scan snapshot, so a note @@ -111,30 +111,6 @@ def count_search_indexed_entities(indexed_entities: list[IndexedEntity]) -> int: return sum(1 for indexed in indexed_entities if indexed.markdown_content is not None) -@dataclass(frozen=True, slots=True) -class DefaultIndexBatchRuntime[FileInfoT: LoadedIndexFile]: - """Default batch-index runtime plus its concrete note-content reconciler.""" - - batch_runtime: IndexBatchRuntime[Entity, FileInfoT] - note_content_reconciler: NoteContentReconciler - - async def index_loaded_files( - self, - files: Mapping[str, FileInfoT], - *, - max_concurrent: int = 8, - parse_max_concurrent: int | None = None, - metadata_update_max_concurrent: int | None = None, - ) -> IndexingBatchResult: - """Index loaded files through the composed runtime.""" - return await self.batch_runtime.index_loaded_files( - files, - max_concurrent=max_concurrent, - parse_max_concurrent=parse_max_concurrent, - metadata_update_max_concurrent=metadata_update_max_concurrent, - ) - - def build_default_index_batch_runtime[FileInfoT: LoadedIndexFile]( *, project_id: ProjectId, @@ -147,7 +123,7 @@ def build_default_index_batch_runtime[FileInfoT: LoadedIndexFile]( content_type_provider: IndexContentTypeProvider, session_maker: async_sessionmaker[AsyncSession], file_reader: NoteContentReconcileFileReader | None = None, -) -> DefaultIndexBatchRuntime[FileInfoT]: +) -> IndexBatchRuntime[Entity, FileInfoT]: """Compose the default repository-backed batch index runtime. Hosted and local runtimes still own storage and session lifecycles. This @@ -173,15 +149,11 @@ def build_default_index_batch_runtime[FileInfoT: LoadedIndexFile]( file_writer=StorageIndexFileWriter(storage=frontmatter_storage), session_maker=session_maker, ) - return DefaultIndexBatchRuntime( - batch_runtime=IndexBatchRuntime( - batch_indexer=batch_indexer, - content_type_provider=content_type_provider, - entity_repository=entity_repository, - session_maker=session_maker, - note_content_reconciler=note_content_reconciler, - timestamp_provider=DefaultIndexedNoteContentTimestampProvider(), - file_reader=file_reader, - ), + return IndexBatchRuntime( + batch_indexer=batch_indexer, + content_type_provider=content_type_provider, + entity_repository=entity_repository, + session_maker=session_maker, note_content_reconciler=note_content_reconciler, + file_reader=file_reader, ) diff --git a/src/basic_memory/indexing/index_file_runner.py b/src/basic_memory/indexing/index_file_runner.py index c197c06a2..aab43b1db 100644 --- a/src/basic_memory/indexing/index_file_runner.py +++ b/src/basic_memory/indexing/index_file_runner.py @@ -39,25 +39,6 @@ class IndexFileObjectMetadata: metadata: RuntimeNoteObjectMetadataMap = field(default_factory=dict) -class IndexFileCurrentMetadata(Protocol): - """Storage metadata shape needed to build index-file object metadata.""" - - @property - def checksum(self) -> RuntimeFileChecksum: ... - - @property - def metadata(self) -> RuntimeNoteObjectMetadataMap: ... - - -class IndexFileCurrentMetadataSource(Protocol): - """Capability that loads current storage metadata for one file path.""" - - async def load_current_file_metadata( - self, - file_path: RuntimeFilePath, - ) -> IndexFileCurrentMetadata | None: ... - - class IndexFileRunnerChecker(Protocol): """Capability that decides whether an observed object needs indexing.""" @@ -73,25 +54,6 @@ async def load_current_file_metadata( ) -> IndexFileObjectMetadata | None: ... -@dataclass(frozen=True, slots=True) -class StorageIndexFileMetadataSource: - """Adapt a storage metadata loader to the index-file runner protocol.""" - - metadata_source: IndexFileCurrentMetadataSource - - async def load_current_file_metadata( - self, - file_path: RuntimeFilePath, - ) -> IndexFileObjectMetadata | None: - current_metadata = await self.metadata_source.load_current_file_metadata(file_path) - if current_metadata is None: - return None - return IndexFileObjectMetadata( - checksum=current_metadata.checksum, - metadata=current_metadata.metadata, - ) - - class IndexFileMaterializedNoteSource(Protocol): """Capability that loads accepted entity state for materialized-note checks.""" diff --git a/src/basic_memory/indexing/models.py b/src/basic_memory/indexing/models.py index 87249d623..3d97e841d 100644 --- a/src/basic_memory/indexing/models.py +++ b/src/basic_memory/indexing/models.py @@ -157,7 +157,12 @@ def from_fields( checksum: str, operation: FileIndexOperation, ) -> FileIndexResult: - """Validate entity fields loaded for a completed file-index result.""" + """Validate entity fields loaded for a completed file-index result. + + Identity fields arrive as raw ORM values; a blank or non-string value + is a broken index row, so the job fails here instead of publishing a + malformed result downstream. + """ return cls( file_path=file_path, entity_id=entity_id, @@ -435,11 +440,11 @@ def plan_index_file_note_live_update( def _required_index_file_note_live_update_text( - value: object, + value: str | None, *, field_name: str, ) -> str: - if not isinstance(value, str) or not value.strip(): + if value is None or not value.strip(): raise RuntimeError(f"Observed_object index_file result is missing {field_name}") return value.strip() @@ -465,7 +470,12 @@ def from_fields( checksum: object, file_path: str, ) -> CurrentMaterializedNoteEntity: - """Validate entity fields loaded for a current materialized note.""" + """Validate entity fields loaded for a current materialized note. + + The entity permalink is nullable in the ORM, but a current markdown note + must carry one for its live-update identity, so a missing permalink is a + broken index row rather than a plannable state. + """ return cls( entity_id=entity_id, external_id=_required_current_materialized_note_text( @@ -487,6 +497,17 @@ def from_fields( ) +def _required_current_materialized_note_text( + value: object, + *, + field_name: str, + file_path: str, +) -> str: + if not isinstance(value, str) or not value.strip(): + raise RuntimeError(f"Current entity for {file_path} is missing {field_name}") + return value.strip() + + @dataclass(frozen=True, slots=True) class CurrentMaterializedNotePlan: """Planned current-file result plus checksum diagnostics for adapter logging.""" @@ -520,17 +541,6 @@ class IndexedFileLiveUpdatePlan: operation: FileIndexOperation | None = None -def _required_current_materialized_note_text( - value: object, - *, - field_name: str, - file_path: str, -) -> str: - if not isinstance(value, str) or not value.strip(): - raise RuntimeError(f"Current entity for {file_path} is missing {field_name}") - return value.strip() - - def plan_current_materialized_note_result( *, reason: str, diff --git a/src/basic_memory/indexing/note_content_batch_reconciliation.py b/src/basic_memory/indexing/note_content_batch_reconciliation.py index efc56764f..250fc63e8 100644 --- a/src/basic_memory/indexing/note_content_batch_reconciliation.py +++ b/src/basic_memory/indexing/note_content_batch_reconciliation.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from typing import Protocol, TypeVar @@ -36,6 +36,12 @@ def last_modified(self) -> datetime | None: ... EntityT = TypeVar("EntityT", bound=IndexedNoteContentEntity) +# Injected timestamp seam: choose the observation time for one indexed markdown +# version. The default is indexed_note_content_observed_at below. +type IndexedNoteContentObservedAt[FileInfoT: IndexedNoteContentFileInfo] = Callable[ + [IndexedEntity, FileInfoT | None], datetime | None +] + class IndexingTask[ResultT](Protocol): """Retryable indexing follow-up task.""" @@ -44,28 +50,6 @@ async def run(self) -> ResultT: """Execute one fresh task attempt.""" -class IndexedNoteContentTimestampProvider[FileInfoT: IndexedNoteContentFileInfo](Protocol): - """Timestamp provider for indexed note_content reconciliation.""" - - def observed_at( - self, - indexed: IndexedEntity, - file_info: FileInfoT | None, - ) -> datetime | None: ... - - -@dataclass(frozen=True, slots=True) -class DefaultIndexedNoteContentTimestampProvider: - """Default timestamp provider for indexed markdown note_content.""" - - def observed_at( - self, - indexed: IndexedEntity, - file_info: IndexedNoteContentFileInfo | None, - ) -> datetime | None: - return indexed_note_content_observed_at(indexed, file_info) - - class IndexedNoteContentEntityRepository(Protocol[EntityT]): """Repository capability needed to reload indexed markdown entities.""" @@ -174,7 +158,7 @@ class IndexedNoteContentReconciliationTask[ entity_by_id: Mapping[int, EntityT] file_infos: Mapping[FileIndexPath, FileInfoT] note_content_reconciler: IndexedNoteContentReconciler[EntityT] - timestamp_provider: IndexedNoteContentTimestampProvider[FileInfoT] + timestamp_provider: IndexedNoteContentObservedAt[FileInfoT] source: str file_reader: NoteContentReconcileFileReader | None = None @@ -208,7 +192,7 @@ async def run(self) -> IndexedNoteContentReconciliationError | None: observed_at = fresh.last_modified else: markdown_content = self.indexed.markdown_content - observed_at = self.timestamp_provider.observed_at( + observed_at = self.timestamp_provider( self.indexed, self.file_infos.get(self.indexed.path), ) @@ -238,8 +222,8 @@ async def reconcile_indexed_note_content_batch[ entity_repository: IndexedNoteContentEntityRepository[EntityT], session_maker: async_sessionmaker[AsyncSession], note_content_reconciler: IndexedNoteContentReconciler[EntityT], - timestamp_provider: IndexedNoteContentTimestampProvider[FileInfoT], max_concurrent: int, + timestamp_provider: IndexedNoteContentObservedAt[FileInfoT] = indexed_note_content_observed_at, source: str = "index", file_reader: NoteContentReconcileFileReader | None = None, ) -> tuple[IndexedNoteContentReconciliationError, ...]: diff --git a/src/basic_memory/indexing/note_content_read_repair_runner.py b/src/basic_memory/indexing/note_content_read_repair_runner.py index 465b8358f..5aec82e1d 100644 --- a/src/basic_memory/indexing/note_content_read_repair_runner.py +++ b/src/basic_memory/indexing/note_content_read_repair_runner.py @@ -8,204 +8,32 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from basic_memory.indexing.note_content_reconciler import ( - NoteContentReconcileEntitySource, - NoteContentReconciler, -) +from basic_memory.indexing.note_content_reconciler import NoteContentReconciler from basic_memory.models import Entity, NoteContent, Project from basic_memory.repository import EntityRepository, NoteContentRepository, ProjectRepository from basic_memory.runtime.note_content import ( - RuntimeAcceptedNoteEntitySource, RuntimeAcceptedNoteResponse, RuntimeNoteContentReadAction, RuntimeNoteContentReadRepairStatus, RuntimeNoteContentResource, RuntimeNoteContentResponsePayload, RuntimeNoteContentState, - RuntimeNoteContentStateSource, plan_runtime_note_content_read, plan_runtime_note_content_read_repair, ) from basic_memory.runtime.storage import ( NoteExternalId, ProjectExternalId, - ProjectId, - ProjectPath, - RuntimeContentType, - RuntimeEntityId, - RuntimeFilePath, RuntimeNoteChangeSource, runtime_content_type_is_markdown, ) from basic_memory.schemas.v2.entity import EntityResponseV2 - -class NoteContentReadProjectSource(Protocol): - """Project identity needed for note-content read lookups.""" - - @property - def id(self) -> ProjectId: ... - - -class NoteContentReadRepairProjectSource(NoteContentReadProjectSource, Protocol): - """Project identity needed to repair note_content from a canonical file.""" - - @property - def path(self) -> ProjectPath: ... - - -class NoteContentReadEntitySource(NoteContentReconcileEntitySource, Protocol): - """Entity identity needed for note-content read lookups.""" - - @property - def content_type(self) -> RuntimeContentType: ... - - -class NoteContentReadResponseEntitySource( - NoteContentReadEntitySource, - RuntimeAcceptedNoteEntitySource, - Protocol, -): - """Entity shape needed for note-content response payloads.""" - - -class NoteContentReadRepairEntitySource(NoteContentReadEntitySource, Protocol): - """Entity identity needed to repair note_content from a canonical file.""" - - @property - def file_path(self) -> RuntimeFilePath: ... - - -class NoteContentReadProjectRepository[ProjectT: NoteContentReadProjectSource](Protocol): - """Repository capability for loading the project that owns a note read.""" - - async def get_by_external_id( - self, - session: AsyncSession, - external_id: ProjectExternalId, - ) -> ProjectT | None: ... - - -class NoteContentReadEntityRepository[EntityT: NoteContentReadEntitySource](Protocol): - """Repository capability for loading the entity that owns a note read.""" - - async def get_by_external_id( - self, - session: AsyncSession, - external_id: NoteExternalId, - ) -> EntityT | None: ... - - -class NoteContentReadNoteContentRepository[NoteContentT](Protocol): - """Repository capability for loading accepted note_content by entity.""" - - async def get_by_entity_id( - self, - session: AsyncSession, - entity_id: RuntimeEntityId, - ) -> NoteContentT | None: ... - - -class NoteContentReadRepairProjectRepository[ProjectT: NoteContentReadRepairProjectSource]( - NoteContentReadProjectRepository[ProjectT], Protocol -): - """Project repository capability for read repair targets.""" - - -class NoteContentReadRepairEntityRepository[EntityT: NoteContentReadRepairEntitySource]( - NoteContentReadEntityRepository[EntityT], - Protocol, -): - """Entity repository capability for read repair targets.""" - - -class NoteContentReadRepairNoteContentRepository[NoteContentT]( - NoteContentReadNoteContentRepository[NoteContentT], - Protocol, -): - """Note-content repository capability for read repair preflight.""" - - -class NoteContentReadRepairReconciler[EntityT: NoteContentReadRepairEntitySource](Protocol): - """Capability that applies one observed markdown file to note_content.""" - - async def reconcile( - self, - *, - entity: EntityT, - markdown_content: str, - observed_at: datetime | None, - source: RuntimeNoteChangeSource, - ) -> None: ... - - -class NoteContentReadRepairFileReader[ - ProjectT: NoteContentReadRepairProjectSource, - EntityT: NoteContentReadRepairEntitySource, -](Protocol): - """Capability that reads the canonical markdown file for read repair.""" - - async def read_note_content_repair_file( - self, - target: NoteContentReadRepairTarget[ProjectT, EntityT], - ) -> NoteContentReadRepairFile | None: ... - - -class NoteContentReadRepositories[ - ProjectT: NoteContentReadProjectSource, - EntityT: NoteContentReadEntitySource, - NoteContentT, -](Protocol): - """Repository capability set for hot note-content reads.""" - - def project_repository(self) -> NoteContentReadProjectRepository[ProjectT]: ... - - def entity_repository( - self, - project_id: ProjectId, - ) -> NoteContentReadEntityRepository[EntityT]: ... - - def note_content_repository( - self, - project_id: ProjectId, - ) -> NoteContentReadNoteContentRepository[NoteContentT]: ... - - -class NoteContentReadRepairRepositories[ - ProjectT: NoteContentReadRepairProjectSource, - EntityT: NoteContentReadRepairEntitySource, - NoteContentT, -](Protocol): - """Repository capability set for note-content read repair preflight.""" - - def project_repository(self) -> NoteContentReadRepairProjectRepository[ProjectT]: ... - - def entity_repository( - self, - project_id: ProjectId, - ) -> NoteContentReadRepairEntityRepository[EntityT]: ... - - def note_content_repository( - self, - project_id: ProjectId, - ) -> NoteContentReadRepairNoteContentRepository[NoteContentT]: ... - - -class NoteContentReadRepairReconcilerProvider[EntityT: NoteContentReadRepairEntitySource](Protocol): - """Capability that supplies the reconciler for one read-repair project.""" - - def reconciler( - self, - project_id: ProjectId, - session_maker: async_sessionmaker[AsyncSession], - ) -> NoteContentReadRepairReconciler[EntityT]: ... +# --- Read/repair value objects --- @dataclass(frozen=True, slots=True) -class NoteContentReadView[ - EntityT: NoteContentReadEntitySource, - NoteContentT, -]: +class NoteContentReadView[EntityT, NoteContentT]: """Joined entity plus accepted note_content used by hot note reads.""" entity: EntityT @@ -213,10 +41,7 @@ class NoteContentReadView[ @dataclass(frozen=True, slots=True) -class NoteContentReadRepairTarget[ - ProjectT: NoteContentReadRepairProjectSource, - EntityT: NoteContentReadRepairEntitySource, -]: +class NoteContentReadRepairTarget[ProjectT, EntityT]: """Storage object identity needed after DB preflight allows read repair.""" project: ProjectT @@ -232,10 +57,7 @@ class NoteContentReadRepairFile: @dataclass(frozen=True, slots=True) -class NoteContentReadRepairPreflight[ - ProjectT: NoteContentReadRepairProjectSource, - EntityT: NoteContentReadRepairEntitySource, -]: +class NoteContentReadRepairPreflight[ProjectT, EntityT]: """DB preflight result for a note-content read-repair attempt.""" status: RuntimeNoteContentReadRepairStatus @@ -273,92 +95,79 @@ def repaired(self) -> bool: } -def note_content_read_project_repository() -> NoteContentReadProjectRepository[Project]: - """Create the default project repository for note-content reads.""" - return ProjectRepository() - - -def note_content_read_entity_repository( - project_id: ProjectId, -) -> NoteContentReadEntityRepository[Entity]: - """Create the default entity repository for note-content reads.""" - return EntityRepository(project_id=project_id) - +class NoteContentReadRepairFileReader[ProjectT, EntityT](Protocol): + """Capability that reads the canonical markdown file for read repair. -def note_content_read_note_content_repository( - project_id: ProjectId, -) -> NoteContentReadNoteContentRepository[NoteContent]: - """Create the default note_content repository for note-content reads.""" - return NoteContentRepository(project_id=project_id) + This is the real storage seam: local runtimes read from the project + filesystem while hosted runtimes read from object storage. + """ + async def read_note_content_repair_file( + self, + target: NoteContentReadRepairTarget[ProjectT, EntityT], + ) -> NoteContentReadRepairFile | None: ... -@dataclass(frozen=True, slots=True) -class DefaultNoteContentReadRepositories: - """Default repository capability set for hot note-content reads.""" - - def project_repository(self) -> NoteContentReadProjectRepository[Project]: - return note_content_read_project_repository() - - def entity_repository(self, project_id: ProjectId) -> NoteContentReadEntityRepository[Entity]: - return note_content_read_entity_repository(project_id) - def note_content_repository( - self, - project_id: ProjectId, - ) -> NoteContentReadNoteContentRepository[NoteContent]: - return note_content_read_note_content_repository(project_id) +# --- Hot note-content reads --- -async def load_note_content_read_view[ - ProjectT: NoteContentReadProjectSource, - EntityT: NoteContentReadEntitySource, - NoteContentT, -]( +async def load_note_content_read_view_with_default_repositories( session: AsyncSession, *, project_external_id: ProjectExternalId, entity_external_id: NoteExternalId, - repositories: NoteContentReadRepositories[ProjectT, EntityT, NoteContentT], -) -> NoteContentReadView[EntityT, NoteContentT] | None: - """Load the DB view needed by hot note-content reads.""" - project_repository = repositories.project_repository() - project = await project_repository.get_by_external_id(session, project_external_id) +) -> NoteContentReadView[Entity, NoteContent] | None: + """Load the hot read view through the default Basic Memory repositories.""" + project = await ProjectRepository().get_by_external_id(session, project_external_id) if project is None: return None - entity_repository = repositories.entity_repository(project.id) - entity = await entity_repository.get_by_external_id(session, entity_external_id) + entity = await EntityRepository(project_id=project.id).get_by_external_id( + session, + entity_external_id, + ) if entity is None: return None note_content = None if runtime_content_type_is_markdown(entity): - note_content_repository = repositories.note_content_repository(project.id) - note_content = await note_content_repository.get_by_entity_id(session, entity.id) + note_content = await NoteContentRepository(project_id=project.id).get_by_entity_id( + session, + entity.id, + ) return NoteContentReadView(entity=entity, note_content=note_content) -async def load_note_content_read_view_with_default_repositories( - session: AsyncSession, - *, - project_external_id: ProjectExternalId, - entity_external_id: NoteExternalId, -) -> NoteContentReadView[Entity, NoteContent] | None: - """Load the hot read view through the default Basic Memory repositories.""" - return await load_note_content_read_view( - session, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - repositories=DefaultNoteContentReadRepositories(), +# EntityResponseV2 also carries the accepted note_content bookkeeping columns +# (versions, checksums, write status). Metadata-only note-content reads omit them +# so route payloads do not leak DB-internal write state; a test pins every name +# here to a real EntityResponseV2 field so the set cannot drift silently. +ENTITY_METADATA_PAYLOAD_EXCLUDE: frozenset[str] = frozenset( + { + "db_version", + "db_checksum", + "file_version", + "file_checksum", + "file_write_status", + "last_source", + "file_updated_at", + "last_materialization_error", + "sync_error", + } +) + + +def entity_metadata_response_payload(entity: Entity) -> RuntimeNoteContentResponsePayload: + """Serialize the metadata-only payload for a non-accepted note-content read.""" + return EntityResponseV2.model_validate(entity).model_dump( + mode="json", + exclude=set(ENTITY_METADATA_PAYLOAD_EXCLUDE), ) -def note_content_response_payload_from_read_view[ - EntityT: NoteContentReadResponseEntitySource, - NoteContentT: RuntimeNoteContentStateSource, -]( - view: NoteContentReadView[EntityT, NoteContentT] | None, +def note_content_response_payload_from_read_view( + view: NoteContentReadView[Entity, NoteContent] | None, ) -> RuntimeNoteContentResponsePayload | None: """Build the typed response payload for a loaded note-content read view.""" if view is None: @@ -366,20 +175,7 @@ def note_content_response_payload_from_read_view[ read_plan = plan_runtime_note_content_read(view.entity, view.note_content) if read_plan.action is RuntimeNoteContentReadAction.entity_metadata: - return EntityResponseV2.model_validate(read_plan.require_entity_metadata()).model_dump( - mode="json", - exclude={ - "db_version", - "db_checksum", - "file_version", - "file_checksum", - "file_write_status", - "last_source", - "file_updated_at", - "last_materialization_error", - "sync_error", - }, - ) + return entity_metadata_response_payload(read_plan.require_entity_metadata()) if read_plan.action is not RuntimeNoteContentReadAction.accepted_note: return None @@ -391,11 +187,8 @@ def note_content_response_payload_from_read_view[ ) -def note_content_resource_from_read_view[ - EntityT: NoteContentReadEntitySource, - NoteContentT: RuntimeNoteContentStateSource, -]( - view: NoteContentReadView[EntityT, NoteContentT] | None, +def note_content_resource_from_read_view( + view: NoteContentReadView[Entity, NoteContent] | None, ) -> RuntimeNoteContentResource | None: """Build the accepted markdown resource for a loaded note-content read view.""" if view is None: @@ -412,93 +205,30 @@ def note_content_resource_from_read_view[ ) -def note_content_read_repair_project_repository() -> NoteContentReadRepairProjectRepository[ - Project -]: - """Create the default project repository for note-content read repair.""" - return ProjectRepository() - - -def note_content_read_repair_entity_repository( - project_id: ProjectId, -) -> NoteContentReadRepairEntityRepository[Entity]: - """Create the default entity repository for note-content read repair.""" - return EntityRepository(project_id=project_id) - - -def note_content_read_repair_note_content_repository( - project_id: ProjectId, -) -> NoteContentReadRepairNoteContentRepository[NoteContent]: - """Create the default note_content repository for note-content read repair.""" - return NoteContentRepository(project_id=project_id) - - -def note_content_read_repair_reconciler( - project_id: ProjectId, - session_maker: async_sessionmaker[AsyncSession], -) -> NoteContentReadRepairReconciler[Entity]: - """Create the default note_content reconciler for read repair.""" - return NoteContentReconciler( - note_content_repository=NoteContentRepository(project_id=project_id), - session_maker=session_maker, - ) - - -@dataclass(frozen=True, slots=True) -class DefaultNoteContentReadRepairRepositories: - """Default repository capability set for note-content read repair preflight.""" - - def project_repository(self) -> NoteContentReadRepairProjectRepository[Project]: - return note_content_read_repair_project_repository() - - def entity_repository( - self, - project_id: ProjectId, - ) -> NoteContentReadRepairEntityRepository[Entity]: - return note_content_read_repair_entity_repository(project_id) +# --- Read repair for missing note_content rows --- - def note_content_repository( - self, - project_id: ProjectId, - ) -> NoteContentReadRepairNoteContentRepository[NoteContent]: - return note_content_read_repair_note_content_repository(project_id) - - -@dataclass(frozen=True, slots=True) -class DefaultNoteContentReadRepairReconcilerProvider: - """Default reconciler provider for note-content read repair.""" - - def reconciler( - self, - project_id: ProjectId, - session_maker: async_sessionmaker[AsyncSession], - ) -> NoteContentReadRepairReconciler[Entity]: - return note_content_read_repair_reconciler(project_id, session_maker) - -async def prepare_note_content_read_repair[ - ProjectT: NoteContentReadRepairProjectSource, - EntityT: NoteContentReadRepairEntitySource, - NoteContentT, -]( +async def prepare_note_content_read_repair_with_default_repositories( session: AsyncSession, *, project_external_id: ProjectExternalId, entity_external_id: NoteExternalId, - repositories: NoteContentReadRepairRepositories[ProjectT, EntityT, NoteContentT], -) -> NoteContentReadRepairPreflight[ProjectT, EntityT]: +) -> NoteContentReadRepairPreflight[Project, Entity]: """Load DB state and decide whether storage must be read for note_content repair.""" - project_repository = repositories.project_repository() - project = await project_repository.get_by_external_id(session, project_external_id) + project = await ProjectRepository().get_by_external_id(session, project_external_id) - entity: EntityT | None = None - note_content: NoteContentT | None = None + entity: Entity | None = None + note_content: NoteContent | None = None if project is not None: - entity_repository = repositories.entity_repository(project.id) - entity = await entity_repository.get_by_external_id(session, entity_external_id) + entity = await EntityRepository(project_id=project.id).get_by_external_id( + session, + entity_external_id, + ) if entity is not None and runtime_content_type_is_markdown(entity): - note_content_repository = repositories.note_content_repository(project.id) - note_content = await note_content_repository.get_by_entity_id(session, entity.id) + note_content = await NoteContentRepository(project_id=project.id).get_by_entity_id( + session, + entity.id, + ) repair_plan = plan_runtime_note_content_read_repair(project, entity, note_content) if not repair_plan.should_read_file: @@ -511,74 +241,14 @@ async def prepare_note_content_read_repair[ ) -async def prepare_note_content_read_repair_with_default_repositories( - session: AsyncSession, - *, - project_external_id: ProjectExternalId, - entity_external_id: NoteExternalId, -) -> NoteContentReadRepairPreflight[Project, Entity]: - """Prepare read repair through the default Basic Memory repositories.""" - return await prepare_note_content_read_repair( - session, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - repositories=DefaultNoteContentReadRepairRepositories(), - ) - - -async def apply_note_content_read_repair[ - ProjectT: NoteContentReadRepairProjectSource, - EntityT: NoteContentReadRepairEntitySource, -]( - target: NoteContentReadRepairTarget[ProjectT, EntityT], - *, - session_maker: async_sessionmaker[AsyncSession], - markdown_content: str, - observed_at: datetime | None, - source: RuntimeNoteChangeSource, - reconciler_provider: NoteContentReadRepairReconcilerProvider[EntityT], -) -> None: - """Apply observed storage markdown to note_content through the selected reconciler.""" - reconciler = reconciler_provider.reconciler(target.project.id, session_maker) - await reconciler.reconcile( - entity=target.entity, - markdown_content=markdown_content, - observed_at=observed_at, - source=source, - ) - - -async def apply_note_content_read_repair_with_default_reconciler( - target: NoteContentReadRepairTarget[Project, Entity], - *, - session_maker: async_sessionmaker[AsyncSession], - markdown_content: str, - observed_at: datetime | None, - source: RuntimeNoteChangeSource, -) -> None: - """Apply read repair through the default Basic Memory note_content reconciler.""" - await apply_note_content_read_repair( - target, - session_maker=session_maker, - markdown_content=markdown_content, - observed_at=observed_at, - source=source, - reconciler_provider=DefaultNoteContentReadRepairReconcilerProvider(), - ) - - -async def run_note_content_read_repair[ - ProjectT: NoteContentReadRepairProjectSource, - EntityT: NoteContentReadRepairEntitySource, -]( - preflight: NoteContentReadRepairPreflight[ProjectT, EntityT], +async def run_note_content_read_repair_with_default_reconciler( + preflight: NoteContentReadRepairPreflight[Project, Entity], *, session_maker: async_sessionmaker[AsyncSession], - file_reader: NoteContentReadRepairFileReader[ProjectT, EntityT] | None, + file_reader: NoteContentReadRepairFileReader[Project, Entity] | None, source: RuntimeNoteChangeSource, - reconciler_provider: NoteContentReadRepairReconcilerProvider[EntityT], ) -> NoteContentReadRepairRun: - """Run storage-neutral read repair after the database preflight decision.""" + """Run read repair through the default Basic Memory note_content reconciler.""" if not preflight.should_read_file: return NoteContentReadRepairRun(status=preflight.status) @@ -592,29 +262,14 @@ async def run_note_content_read_repair[ if repair_file.markdown_content is None: return NoteContentReadRepairRun(status=RuntimeNoteContentReadRepairStatus.empty_file) - await apply_note_content_read_repair( - target, + reconciler = NoteContentReconciler( + note_content_repository=NoteContentRepository(project_id=target.project.id), session_maker=session_maker, + ) + await reconciler.reconcile( + entity=target.entity, markdown_content=repair_file.markdown_content, observed_at=repair_file.observed_at, source=source, - reconciler_provider=reconciler_provider, ) return NoteContentReadRepairRun(status=RuntimeNoteContentReadRepairStatus.repaired) - - -async def run_note_content_read_repair_with_default_reconciler( - preflight: NoteContentReadRepairPreflight[Project, Entity], - *, - session_maker: async_sessionmaker[AsyncSession], - file_reader: NoteContentReadRepairFileReader[Project, Entity] | None, - source: RuntimeNoteChangeSource, -) -> NoteContentReadRepairRun: - """Run read repair through the default Basic Memory note_content reconciler.""" - return await run_note_content_read_repair( - preflight, - session_maker=session_maker, - file_reader=file_reader, - source=source, - reconciler_provider=DefaultNoteContentReadRepairReconcilerProvider(), - ) diff --git a/src/basic_memory/indexing/note_content_reconciler.py b/src/basic_memory/indexing/note_content_reconciler.py index 89f5d12d5..42d4228ff 100644 --- a/src/basic_memory/indexing/note_content_reconciler.py +++ b/src/basic_memory/indexing/note_content_reconciler.py @@ -2,8 +2,9 @@ from __future__ import annotations +from collections.abc import Callable from datetime import UTC, datetime -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Protocol, assert_never from loguru import logger @@ -120,23 +121,9 @@ def note_content_repository_for_project(project_id: ProjectId) -> NoteContentSto return NoteContentRepository(project_id=project_id) -class NoteContentRepositories(Protocol): - """Repository capability set needed by note-content materialization bookkeeping.""" - - def note_content_repository(self, project_id: ProjectId) -> NoteContentStateUpdateStore: ... - - -@dataclass(frozen=True, slots=True) -class DefaultNoteContentRepositories: - """Default repository capability set for note-content materialization.""" - - def note_content_repository(self, project_id: ProjectId) -> NoteContentStateUpdateStore: - return note_content_repository_for_project(project_id) - - -def build_default_note_content_repositories() -> NoteContentRepositories: - """Compose the default repository adapters for note-content materialization.""" - return DefaultNoteContentRepositories() +# Injected per-project store seam for materialization bookkeeping; the default +# is note_content_repository_for_project above. +type NoteContentStoreFactory = Callable[[ProjectId], NoteContentStateUpdateStore] @dataclass(frozen=True, slots=True) @@ -144,9 +131,7 @@ class RepositoryNoteMaterializationFailureMarker: """Repository-backed failure marker for accepted-note materialization enqueue failures.""" session_maker: async_sessionmaker[AsyncSession] - repositories: NoteContentRepositories = field( - default_factory=build_default_note_content_repositories - ) + note_content_store: NoteContentStoreFactory = note_content_repository_for_project async def mark_note_materialization_failed( self, @@ -155,36 +140,16 @@ async def mark_note_materialization_failed( entity_id: RuntimeEntityId, error_message: str, ) -> None: - """Record enqueue failure through the configured note_content repository.""" - await mark_note_materialization_enqueue_failed( - session_maker=self.session_maker, - project_id=project_id, - entity_id=entity_id, - error_message=error_message, - repositories=self.repositories, - ) - - -async def mark_note_materialization_enqueue_failed( - *, - session_maker: async_sessionmaker[AsyncSession], - project_id: ProjectId, - entity_id: RuntimeEntityId, - error_message: str, - repositories: NoteContentRepositories | None = None, - attempted_at: datetime | None = None, -) -> None: - """Mark accepted note content as failed when queue submission cannot start.""" - note_content_repositories = repositories or build_default_note_content_repositories() - async with session_maker() as session: - async with session.begin(): - await note_content_repositories.note_content_repository(project_id).update_state_fields( - session, - entity_id, - file_write_status="failed", - last_materialization_error=error_message, - last_materialization_attempt_at=attempted_at or datetime.now(tz=UTC), - ) + """Mark accepted note content as failed when queue submission cannot start.""" + async with self.session_maker() as session: + async with session.begin(): + await self.note_content_store(project_id).update_state_fields( + session, + entity_id, + file_write_status="failed", + last_materialization_error=error_message, + last_materialization_attempt_at=datetime.now(tz=UTC), + ) def note_content_state_from_model(note_content: NoteContent) -> NoteContentState: diff --git a/src/basic_memory/indexing/note_materialization_runner.py b/src/basic_memory/indexing/note_materialization_runner.py index 6b729824c..a757c474e 100644 --- a/src/basic_memory/indexing/note_materialization_runner.py +++ b/src/basic_memory/indexing/note_materialization_runner.py @@ -13,9 +13,9 @@ from basic_memory import db from basic_memory.indexing.note_content_reconciler import ( - NoteContentRepositories, + NoteContentStoreFactory, apply_note_content_update_plan, - build_default_note_content_repositories, + note_content_repository_for_project, note_content_state_from_model, ) from basic_memory.indexing.note_content_reconciliation import ( @@ -165,7 +165,7 @@ class NoteMaterializationContentSource(RuntimeNoteContentVersionSource, Protocol def markdown_content(self) -> str: ... @property - def file_checksum(self) -> object | None: ... + def file_checksum(self) -> RuntimeFileChecksum | None: ... class NoteMaterializationFileWriter(Protocol): @@ -280,10 +280,8 @@ def plan_note_materialization_preflight( plan_prepared_note_write( request=request, file_path=entity.file_path, - markdown_content=str(note_content.markdown_content), - previous_file_checksum=( - str(note_content.file_checksum) if note_content.file_checksum is not None else None - ), + markdown_content=note_content.markdown_content, + previous_file_checksum=note_content.file_checksum, attempted_at=attempted_at, ) ) @@ -418,9 +416,7 @@ class RepositoryNoteMaterializationPublisher: session_lock: NoteMaterializationSessionLock = field( default_factory=NoopNoteMaterializationSessionLock ) - repositories: NoteContentRepositories = field( - default_factory=build_default_note_content_repositories - ) + note_content_store: NoteContentStoreFactory = note_content_repository_for_project async def publish_written_file_state( self, @@ -455,14 +451,14 @@ async def publish_written_file_state( # concurrently, so guard every write on this db_version: if a newer # accepted write advanced the row between our read and our write, this # (now older) materialization must not revert the newer file_version. - expected_db_version = int(note_content.db_version) + expected_db_version = note_content.db_version if publish_plan.action is NoteMaterializationPublishAction.stale_file_path: return publish_plan.result if publish_plan.action is NoteMaterializationPublishAction.stale_db_version: await apply_note_content_update_plan( - self.repositories.note_content_repository(request.project_id), + self.note_content_store(request.project_id), session, request.entity_id, publish_plan.require_note_content_update(), @@ -486,7 +482,7 @@ async def publish_written_file_state( ) applied = await apply_note_content_update_plan( - self.repositories.note_content_repository(request.project_id), + self.note_content_store(request.project_id), session, request.entity_id, publish_plan.require_note_content_update(), @@ -521,9 +517,7 @@ class RepositoryNoteMaterializationStatusPublisher: session_lock: NoteMaterializationSessionLock = field( default_factory=NoopNoteMaterializationSessionLock ) - repositories: NoteContentRepositories = field( - default_factory=build_default_note_content_repositories - ) + note_content_store: NoteContentStoreFactory = note_content_repository_for_project async def publish_note_materialization_status( self, @@ -556,7 +550,7 @@ async def publish_note_materialization_status( return await apply_note_content_update_plan( - self.repositories.note_content_repository(request.project_id), + self.note_content_store(request.project_id), session, request.entity_id, plan, diff --git a/src/basic_memory/indexing/progress.py b/src/basic_memory/indexing/progress.py index d3fdbe9e0..651e22bcb 100644 --- a/src/basic_memory/indexing/progress.py +++ b/src/basic_memory/indexing/progress.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field from typing import Protocol, cast from pydantic import ( @@ -31,13 +30,34 @@ class VectorSyncBatchSummary(Protocol): class CheckpointModel(BaseModel): - """Shared base model for durable checkpoint JSON.""" + """Shared base model for durable checkpoint JSON. + + Subclasses that parse a slice of a larger metadata document rely on + ``extra="ignore"``. Runtime value models (IndexingResult, + VectorSyncProgress) override to ``extra="forbid"`` so a mistyped keyword + raises exactly as the dataclasses they replaced did; their legacy-document + tolerance lives in ``from_checkpoint_state``. + """ model_config = ConfigDict(extra="ignore") + @classmethod + def _known_checkpoint_fields(cls, state: object) -> object: + """Drop retired keys from a persisted checkpoint before validation.""" + if isinstance(state, Mapping): + state_payload = cast(Mapping[str, object], state) + return {k: v for k, v in state_payload.items() if k in cls.model_fields} + return state + + +class VectorSyncProgress(CheckpointModel): + """Durable progress snapshot for chunked vector sync runs. -class VectorSyncProgressState(CheckpointModel): - """JSON payload for durable vector sync progress.""" + This model is also the persisted checkpoint document: field names and the + dumped JSON shape must stay stable so older checkpoints keep restoring. + """ + + model_config = ConfigDict(extra="forbid") entity_ids: list[int] = Field(default_factory=list) next_index: int = 0 @@ -56,7 +76,7 @@ def dedupe_ids(cls, value: list[int]) -> list[int]: return list(dict.fromkeys(value)) @model_validator(mode="after") - def clamp_next_index(self) -> "VectorSyncProgressState": + def clamp_next_index(self) -> VectorSyncProgress: """Keep resume offsets inside the tracked entity list.""" self.next_index = min(self.next_index, len(self.entity_ids)) return self @@ -67,92 +87,29 @@ def entities_total(self) -> int: """Total entity ids tracked by this progress snapshot.""" return len(self.entity_ids) - -class IndexingResultState(CheckpointModel): - """JSON payload for durable aggregate indexing state.""" - - files_processed: int = 0 - files_unchanged: int = 0 - entities_created: int = 0 - entities_updated: int = 0 - entities_deleted: int = 0 - files_moved: int = 0 - forward_refs_resolved: int = 0 - relations_resolved: int = 0 - relations_unresolved: int = 0 - search_indexed: int = 0 - semantic_vector_entities_total: int = 0 - semantic_vectors_synced: int = 0 - semantic_vectors_failed: int = 0 - errors: list[tuple[str, str]] = Field(default_factory=list) - total_duration_seconds: float = 0.0 - change_detection_seconds: float = 0.0 - s3_download_seconds: float = 0.0 - file_processing_seconds: float = 0.0 - relation_resolution_seconds: float = 0.0 - search_indexing_seconds: float = 0.0 - semantic_vector_sync_seconds: float = 0.0 - semantic_vector_embed_seconds: float = 0.0 - semantic_vector_write_seconds: float = 0.0 - peak_rss_mib: float = 0.0 - batch_count: int = 0 - - @field_validator("errors", mode="before") - @classmethod - def normalize_errors(cls, value: object) -> object: - """Accept legacy tuple/list error payloads and normalize them.""" - if not isinstance(value, list): - return value - - normalized: list[tuple[str, str]] = [] - for item in value: - if isinstance(item, list | tuple) and len(item) == 2: - normalized.append((str(item[0]), str(item[1]))) - continue - if isinstance(item, Mapping): - item_payload = cast(Mapping[str, object], item) - path = item_payload.get("path") - error = item_payload.get("error") - if path is not None and error is not None: - normalized.append((str(path), str(error))) - return normalized - - -@dataclass(slots=True) -class VectorSyncProgress: - """Durable progress snapshot for chunked vector sync runs.""" - - entity_ids: list[int] = field(default_factory=list) - next_index: int = 0 - entities_synced: int = 0 - entities_failed: int = 0 - failed_entity_ids: list[int] = field(default_factory=list) - embedding_jobs_total: int = 0 - embed_seconds_total: float = 0.0 - write_seconds_total: float = 0.0 - elapsed_seconds: float = 0.0 - - @property - def entities_total(self) -> int: - """Total number of entity IDs tracked by this vector sync run.""" - return len(self.entity_ids) - - def without_entity_ids(self) -> "VectorSyncProgress": - """Return a progress snapshot without the static entity plan.""" - return VectorSyncProgress( - next_index=self.next_index, - entities_synced=self.entities_synced, - entities_failed=self.entities_failed, - failed_entity_ids=list(self.failed_entity_ids), - embedding_jobs_total=self.embedding_jobs_total, - embed_seconds_total=self.embed_seconds_total, - write_seconds_total=self.write_seconds_total, - elapsed_seconds=self.elapsed_seconds, + def without_entity_ids(self) -> VectorSyncProgress: + """Return a progress snapshot without the static entity plan. + + model_copy skips validation on purpose: dropping the plan must keep + the recorded counters (including next_index) exactly as they were. + """ + return self.model_copy( + update={ + "entity_ids": [], + "failed_entity_ids": list(self.failed_entity_ids), + } ) def to_checkpoint_state(self) -> dict[str, object]: - """Serialize vector progress into JSON-friendly workflow metadata.""" - return VectorSyncProgressState( + """Serialize vector progress into JSON-friendly workflow metadata. + + Constructed (not model_copy'd) on purpose: batch folds assign next_index + directly and without_entity_ids() drops the plan without revalidating, + so the write path must re-run the dedupe/clamp validators to keep the + persisted invariant next_index <= len(entity_ids) that external + checkpoint readers rely on. + """ + return VectorSyncProgress( entity_ids=list(self.entity_ids), next_index=self.next_index, entities_synced=self.entities_synced, @@ -165,28 +122,16 @@ def to_checkpoint_state(self) -> dict[str, object]: ).model_dump(mode="json") @classmethod - def from_checkpoint_state(cls, state: object) -> "VectorSyncProgress": + def from_checkpoint_state(cls, state: object) -> VectorSyncProgress: """Restore vector sync progress from workflow metadata.""" if state is None: return cls() try: - payload = VectorSyncProgressState.model_validate(state) + return cls.model_validate(cls._known_checkpoint_fields(state)) except ValidationError: return cls() - return cls( - entity_ids=payload.entity_ids, - next_index=payload.next_index, - entities_synced=payload.entities_synced, - entities_failed=payload.entities_failed, - failed_entity_ids=payload.failed_entity_ids, - embedding_jobs_total=payload.embedding_jobs_total, - embed_seconds_total=payload.embed_seconds_total, - write_seconds_total=payload.write_seconds_total, - elapsed_seconds=payload.elapsed_seconds, - ) - def initialize_vector_sync_progress( *, @@ -241,9 +186,15 @@ def apply_vector_sync_batch_result( return new_failed_entity_ids -@dataclass(slots=True) -class IndexingResult: - """Final result of an indexing operation.""" +class IndexingResult(CheckpointModel): + """Final result of an indexing operation. + + This model is also the persisted aggregate checkpoint for retry-safe + workflows: field names and the dumped JSON shape must stay stable so + older checkpoints keep restoring. + """ + + model_config = ConfigDict(extra="forbid") files_processed: int = 0 files_unchanged: int = 0 @@ -258,7 +209,7 @@ class IndexingResult: semantic_vector_entities_total: int = 0 semantic_vectors_synced: int = 0 semantic_vectors_failed: int = 0 - errors: list[tuple[str, str]] = field(default_factory=list) + errors: list[tuple[str, str]] = Field(default_factory=list) total_duration_seconds: float = 0.0 change_detection_seconds: float = 0.0 s3_download_seconds: float = 0.0 @@ -271,6 +222,34 @@ class IndexingResult: peak_rss_mib: float = 0.0 batch_count: int = 0 + @field_validator("errors", mode="before") + @classmethod + def normalize_errors(cls, value: object) -> object: + """Accept legacy tuple/list error payloads and normalize them.""" + if not isinstance(value, list): + return value + + normalized: list[tuple[str, str]] = [] + for item in value: + if isinstance(item, list | tuple) and len(item) == 2: + normalized.append((str(item[0]), str(item[1]))) + continue + if isinstance(item, Mapping): + item_payload = cast(Mapping[str, object], item) + path = item_payload.get("path") + error = item_payload.get("error") + if path is not None and error is not None: + normalized.append((str(path), str(error))) + continue + # Trigger: an error entry is neither a (path, error) pair nor the + # legacy mapping shape. + # Why: silently dropping it flips total_errors/success and hides + # the malformed producer; checkpoint restore already tolerates a + # raise here via its ValidationError fallback. + # Outcome: runtime construction fails fast on garbage entries. + raise ValueError(f"indexing error entries must be (path, error) pairs, got {item!r}") + return normalized + @property def total_errors(self) -> int: """Total number of errors.""" @@ -297,69 +276,50 @@ def avg_batch_duration(self) -> float: def to_checkpoint_state(self) -> dict[str, object]: """Serialize the durable aggregate result for retry-safe workflows.""" - return IndexingResultState( - files_processed=self.files_processed, - files_unchanged=self.files_unchanged, - entities_created=self.entities_created, - entities_updated=self.entities_updated, - entities_deleted=self.entities_deleted, - files_moved=self.files_moved, - forward_refs_resolved=self.forward_refs_resolved, - relations_resolved=self.relations_resolved, - relations_unresolved=self.relations_unresolved, - search_indexed=self.search_indexed, - semantic_vector_entities_total=self.semantic_vector_entities_total, - semantic_vectors_synced=self.semantic_vectors_synced, - semantic_vectors_failed=self.semantic_vectors_failed, - errors=list(self.errors), - total_duration_seconds=round(self.total_duration_seconds, 3), - change_detection_seconds=round(self.change_detection_seconds, 3), - s3_download_seconds=round(self.s3_download_seconds, 3), - file_processing_seconds=round(self.file_processing_seconds, 3), - relation_resolution_seconds=round(self.relation_resolution_seconds, 3), - search_indexing_seconds=round(self.search_indexing_seconds, 3), - semantic_vector_sync_seconds=round(self.semantic_vector_sync_seconds, 3), - semantic_vector_embed_seconds=round(self.semantic_vector_embed_seconds, 3), - semantic_vector_write_seconds=round(self.semantic_vector_write_seconds, 3), - peak_rss_mib=round(self.peak_rss_mib, 3), - batch_count=self.batch_count, - ).model_dump(mode="json") + rounded = self.model_copy( + update={ + "total_duration_seconds": round(self.total_duration_seconds, 3), + "change_detection_seconds": round(self.change_detection_seconds, 3), + "s3_download_seconds": round(self.s3_download_seconds, 3), + "file_processing_seconds": round(self.file_processing_seconds, 3), + "relation_resolution_seconds": round(self.relation_resolution_seconds, 3), + "search_indexing_seconds": round(self.search_indexing_seconds, 3), + "semantic_vector_sync_seconds": round(self.semantic_vector_sync_seconds, 3), + "semantic_vector_embed_seconds": round(self.semantic_vector_embed_seconds, 3), + "semantic_vector_write_seconds": round(self.semantic_vector_write_seconds, 3), + "peak_rss_mib": round(self.peak_rss_mib, 3), + } + ) + return rounded.model_dump(mode="json") @classmethod - def from_checkpoint_state(cls, state: object) -> "IndexingResult": + def from_checkpoint_state(cls, state: object) -> IndexingResult: """Restore cumulative indexing state from workflow metadata.""" if state is None: return cls() + known = cls._known_checkpoint_fields(state) + # Trigger: a persisted checkpoint carries error entries in a shape we + # no longer recognize. + # Why: restore is best-effort over historical documents — one garbage + # entry must not discard the whole checkpoint, while runtime + # construction (the validator below) stays fail-fast. + # Outcome: unrecognizable entries are dropped from the restored state. + if isinstance(known, dict): + known_payload = cast(dict[str, object], known) + raw_errors = known_payload.get("errors") + if isinstance(raw_errors, list): + known_payload["errors"] = [ + item + for item in cast(list[object], raw_errors) + if (isinstance(item, list | tuple) and len(item) == 2) + or ( + isinstance(item, Mapping) + and cast(Mapping[str, object], item).get("path") is not None + and cast(Mapping[str, object], item).get("error") is not None + ) + ] try: - payload = IndexingResultState.model_validate(state) + return cls.model_validate(known) except ValidationError: return cls() - - return cls( - files_processed=payload.files_processed, - files_unchanged=payload.files_unchanged, - entities_created=payload.entities_created, - entities_updated=payload.entities_updated, - entities_deleted=payload.entities_deleted, - files_moved=payload.files_moved, - forward_refs_resolved=payload.forward_refs_resolved, - relations_resolved=payload.relations_resolved, - relations_unresolved=payload.relations_unresolved, - search_indexed=payload.search_indexed, - semantic_vector_entities_total=payload.semantic_vector_entities_total, - semantic_vectors_synced=payload.semantic_vectors_synced, - semantic_vectors_failed=payload.semantic_vectors_failed, - errors=payload.errors, - total_duration_seconds=payload.total_duration_seconds, - change_detection_seconds=payload.change_detection_seconds, - s3_download_seconds=payload.s3_download_seconds, - file_processing_seconds=payload.file_processing_seconds, - relation_resolution_seconds=payload.relation_resolution_seconds, - search_indexing_seconds=payload.search_indexing_seconds, - semantic_vector_sync_seconds=payload.semantic_vector_sync_seconds, - semantic_vector_embed_seconds=payload.semantic_vector_embed_seconds, - semantic_vector_write_seconds=payload.semantic_vector_write_seconds, - peak_rss_mib=payload.peak_rss_mib, - batch_count=payload.batch_count, - ) diff --git a/src/basic_memory/indexing/project_delete_acceptance.py b/src/basic_memory/indexing/project_delete_acceptance.py index 3767ecb02..037762240 100644 --- a/src/basic_memory/indexing/project_delete_acceptance.py +++ b/src/basic_memory/indexing/project_delete_acceptance.py @@ -3,57 +3,16 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Literal, Protocol, Self +from typing import Literal, Self from basic_memory.runtime.jobs import RuntimeJobId, RuntimeProjectDeleteJobRequest +from basic_memory.schemas.project_info import ProjectItem type ProjectDeleteAcceptedStatus = Literal["success"] type ProjectDeleteAcceptedDeletionStatus = Literal["pending"] type ProjectDeleteAcceptedFileStatus = Literal["pending", "skipped"] -class ProjectDeleteAcceptedProjectSource(Protocol): - """Minimal project row shape needed for accepted-delete responses.""" - - id: int - external_id: str - name: str - path: str - is_default: bool | None - - -@dataclass(frozen=True, slots=True) -class ProjectDeleteAcceptedProject: - """Project snapshot returned as ``old_project`` after a soft delete.""" - - id: int - external_id: str - name: str - path: str - is_default: bool - - @classmethod - def from_source(cls, source: ProjectDeleteAcceptedProjectSource) -> Self: - """Snapshot the Basic Memory project fields exposed by delete responses.""" - return cls( - id=source.id, - external_id=source.external_id, - name=source.name, - path=source.path, - is_default=source.is_default or False, - ) - - def to_response_payload(self) -> dict[str, object]: - """Serialize to the existing Basic Memory project response shape.""" - return { - "id": self.id, - "external_id": self.external_id, - "name": self.name, - "path": self.path, - "is_default": self.is_default, - } - - @dataclass(frozen=True, slots=True) class ProjectDeleteAcceptedResult: """Accepted response for a project delete queued for background cleanup.""" @@ -61,7 +20,7 @@ class ProjectDeleteAcceptedResult: project_name: str job_id: RuntimeJobId file_delete_status: ProjectDeleteAcceptedFileStatus - old_project: ProjectDeleteAcceptedProject + old_project: ProjectItem status: ProjectDeleteAcceptedStatus = "success" deletion_status: ProjectDeleteAcceptedDeletionStatus = "pending" background: bool = True @@ -72,7 +31,7 @@ def queued( *, request: RuntimeProjectDeleteJobRequest, job_id: RuntimeJobId, - old_project: ProjectDeleteAcceptedProject, + old_project: ProjectItem, ) -> Self: """Build the accepted response for a queued project cleanup job.""" return cls( @@ -91,6 +50,11 @@ def to_response_payload(self) -> dict[str, object]: "file_delete_status": self.file_delete_status, "background": self.background, "job_id": str(self.job_id), - "old_project": self.old_project.to_response_payload(), + # ProjectItem also carries cloud-hosting metadata (display_name, + # is_private) that the accepted-delete response has never included; + # serialize only the persisted project fields so bytes stay stable. + "old_project": self.old_project.model_dump( + include={"id", "external_id", "name", "path", "is_default"} + ), "new_project": None, } diff --git a/src/basic_memory/indexing/project_delete_runner.py b/src/basic_memory/indexing/project_delete_runner.py index 4bce6084e..06f0a7ce0 100644 --- a/src/basic_memory/indexing/project_delete_runner.py +++ b/src/basic_memory/indexing/project_delete_runner.py @@ -90,20 +90,6 @@ class ProjectDeleteRepository(Protocol): async def delete(self, session: AsyncSession, entity_id: int) -> bool: ... -class ProjectDeleteRepositories(Protocol): - """Repository provider for project delete cleanup.""" - - def project_repository(self) -> ProjectDeleteRepository: ... - - -@dataclass(frozen=True, slots=True) -class DefaultProjectDeleteRepositories: - """Default repository provider for local project delete cleanup.""" - - def project_repository(self) -> ProjectDeleteRepository: - return ProjectRepository() - - async def load_project_file_snapshots( session: AsyncSession, *, @@ -205,9 +191,7 @@ class RepositoryProjectHardDeleter: """Repository-backed hard deleter for one inactive project.""" session_maker: async_sessionmaker[AsyncSession] - repositories: ProjectDeleteRepositories = field( - default_factory=DefaultProjectDeleteRepositories - ) + project_repository: ProjectDeleteRepository = field(default_factory=ProjectRepository) async def hard_delete_project( self, @@ -230,10 +214,7 @@ async def hard_delete_project( ) return ProjectHardDeleteOutcome.reactivated - deleted = await self.repositories.project_repository().delete( - session, - request.project_id, - ) + deleted = await self.project_repository.delete(session, request.project_id) return ProjectHardDeleteOutcome.deleted if deleted else ProjectHardDeleteOutcome.missing diff --git a/src/basic_memory/indexing/project_index_maintenance.py b/src/basic_memory/indexing/project_index_maintenance.py index 26757011e..11bde0d59 100644 --- a/src/basic_memory/indexing/project_index_maintenance.py +++ b/src/basic_memory/indexing/project_index_maintenance.py @@ -470,6 +470,166 @@ async def delete_project_index_entities( return relation_cleanup_entity_ids +@dataclass(frozen=True, slots=True) +class _MoveReplacementScreen: + """Move-batch rows that survive destination verification.""" + + target_rows: list[RowMapping] + replacement_rows: list[RowMapping] + dropped_move_paths: tuple[str, ...] + + +def _screen_replaced_move_targets( + *, + target_rows: list[RowMapping], + replacement_rows: list[RowMapping], + target_paths_by_old_path: dict[str, str], +) -> _MoveReplacementScreen: + """Drop planned moves whose destination row indexes different content. + + The move was planned by matching the destination file's checksum to the + source entity's indexed checksum, so that checksum is the only content a + replacement row may legitimately index. A mismatch means the destination + holds a concurrently created entity (e.g. an accepted-but-unmaterialized + note); deleting it would destroy that entity, so the move is dropped for + the next scan to reconcile. + """ + old_path_by_new_path = { + target_paths_by_old_path[str(row["file_path"])]: str(row["file_path"]) + for row in target_rows + } + expected_checksum_by_new_path = { + target_paths_by_old_path[str(row["file_path"])]: row["checksum"] for row in target_rows + } + verified_replacement_rows: list[RowMapping] = [] + dropped_new_paths: set[str] = set() + for replacement_row in replacement_rows: + replacement_path = str(replacement_row["file_path"]) + expected_checksum = expected_checksum_by_new_path.get(replacement_path) + if expected_checksum is not None and replacement_row["checksum"] == expected_checksum: + verified_replacement_rows.append(replacement_row) + continue + dropped_new_paths.add(replacement_path) + logger.warning( + "Dropping planned move: destination holds a concurrently created entity", + old_path=old_path_by_new_path.get(replacement_path), + new_path=replacement_path, + ) + + if not dropped_new_paths: + return _MoveReplacementScreen( + target_rows=target_rows, + replacement_rows=verified_replacement_rows, + dropped_move_paths=(), + ) + return _MoveReplacementScreen( + target_rows=[ + row + for row in target_rows + if target_paths_by_old_path[str(row["file_path"])] not in dropped_new_paths + ], + replacement_rows=verified_replacement_rows, + dropped_move_paths=tuple( + sorted(old_path_by_new_path[new_path] for new_path in dropped_new_paths) + ), + ) + + +@dataclass(frozen=True, slots=True) +class _MoveContentPlan: + """Planned frontmatter rewrites for one move batch, keyed by entity id.""" + + updates_by_entity_id: dict[int, ProjectIndexMovedFileContentUpdate] + moved_files_by_entity_id: dict[int, ProjectIndexMovedFile] + + @classmethod + def empty(cls) -> "_MoveContentPlan": + return cls(updates_by_entity_id={}, moved_files_by_entity_id={}) + + +@dataclass(frozen=True, slots=True) +class _MoveBatchUpdateValues: + """Parallel per-table CASE assignments for one move batch.""" + + entity_values: dict[str, object] + note_content_values: dict[str, object] + search_index_values: dict[str, object] + permalinks_by_entity_id: dict[int, str] + + +def _build_move_batch_update_values( + *, + target_paths_by_old_path: dict[str, str], + target_paths_by_entity_id: dict[int, str], + content_updates_by_entity_id: dict[int, ProjectIndexMovedFileContentUpdate], +) -> _MoveBatchUpdateValues: + """Assemble the parallel CASE assignments for entity/note_content/search_index. + + Every table repoints file_path in one statement; when content repair was + planned, the checksum/permalink/markdown columns join those same statements + so the batch transaction stamps rows that agree with the post-commit file + writes. + """ + entity_values: dict[str, object] = { + "file_path": case(target_paths_by_old_path, value=Entity.file_path) + } + note_content_values: dict[str, object] = { + "file_path": case(target_paths_by_entity_id, value=NoteContent.entity_id) + } + search_index_values: dict[str, object] = { + "file_path": case( + target_paths_by_entity_id, + value=PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id, + ) + } + permalinks_by_entity_id: dict[int, str] = {} + if content_updates_by_entity_id: + checksums_by_entity_id = { + entity_id: content_update.checksum + for entity_id, content_update in content_updates_by_entity_id.items() + } + markdown_by_entity_id = { + entity_id: content_update.markdown_content + for entity_id, content_update in content_updates_by_entity_id.items() + } + permalinks_by_entity_id = { + entity_id: content_update.permalink + for entity_id, content_update in content_updates_by_entity_id.items() + } + entity_values["checksum"] = case( + checksums_by_entity_id, + value=Entity.id, + else_=Entity.checksum, + ) + entity_values["permalink"] = case( + permalinks_by_entity_id, + value=Entity.id, + else_=Entity.permalink, + ) + note_content_values["db_checksum"] = case( + checksums_by_entity_id, + value=NoteContent.entity_id, + else_=NoteContent.db_checksum, + ) + note_content_values["file_checksum"] = case( + checksums_by_entity_id, + value=NoteContent.entity_id, + else_=NoteContent.file_checksum, + ) + note_content_values["markdown_content"] = case( + markdown_by_entity_id, + value=NoteContent.entity_id, + else_=NoteContent.markdown_content, + ) + + return _MoveBatchUpdateValues( + entity_values=entity_values, + note_content_values=note_content_values, + search_index_values=search_index_values, + permalinks_by_entity_id=permalinks_by_entity_id, + ) + + @dataclass(frozen=True, slots=True) class RepositoryProjectIndexMaintenanceStore: """Apply project-index move/delete maintenance with explicit sessions.""" @@ -499,101 +659,51 @@ async def apply_project_index_move_batch( target_paths_by_old_path = { move_target.old_path: move_target.new_path for move_target in move_batch.targets } - old_paths = tuple(target_paths_by_old_path) async with db.scoped_session(self.session_maker) as session: + # --- Load the indexed rows the batch may rewrite --- existing_paths_result = await session.execute( select(Entity.id, Entity.file_path, Entity.permalink, Entity.checksum).where( Entity.project_id == self.project_id, - Entity.file_path.in_(old_paths), + Entity.file_path.in_(tuple(target_paths_by_old_path)), ) ) target_rows = list(existing_paths_result.mappings().all()) - replaced_entity_ids: frozenset[int] = frozenset() - relation_cleanup_entity_ids: frozenset[int] = frozenset() - dropped_move_paths: tuple[str, ...] = () - content_updates_by_entity_id: dict[int, ProjectIndexMovedFileContentUpdate] = {} - replacement_rows: list[RowMapping] = [] - - if target_rows: - new_paths = tuple( - sorted({target_paths_by_old_path[str(row["file_path"])] for row in target_rows}) - ) - replacement_result = await session.execute( - select(Entity.id, Entity.file_path, Entity.checksum).where( - Entity.project_id == self.project_id, - Entity.file_path.in_(new_paths), - Entity.id.not_in(tuple(int(row["id"]) for row in target_rows)), - ) - ) - replacement_rows = list(replacement_result.mappings().all()) + replacement_rows = await self._load_move_replacement_rows( + session, + target_rows=target_rows, + target_paths_by_old_path=target_paths_by_old_path, + ) + # --- Screen destinations recreated concurrently --- + # See verify_replaced_move_targets above: only scan runtimes verify, + # and only when a row already occupies a destination path. + dropped_move_paths: tuple[str, ...] = () if self.verify_replaced_move_targets and replacement_rows: - old_path_by_new_path = { - target_paths_by_old_path[str(row["file_path"])]: str(row["file_path"]) - for row in target_rows - } - # The move was planned by matching the destination file's checksum - # to the source entity's indexed checksum, so that checksum is the - # only content a replacement row may legitimately index. - expected_checksum_by_new_path = { - target_paths_by_old_path[str(row["file_path"])]: row["checksum"] - for row in target_rows - } - verified_replacement_rows: list[RowMapping] = [] - dropped_new_paths: set[str] = set() - for replacement_row in replacement_rows: - replacement_path = str(replacement_row["file_path"]) - expected_checksum = expected_checksum_by_new_path.get(replacement_path) - if ( - expected_checksum is not None - and replacement_row["checksum"] == expected_checksum - ): - verified_replacement_rows.append(replacement_row) - continue - dropped_new_paths.add(replacement_path) - logger.warning( - "Dropping planned move: destination holds a concurrently created entity", - old_path=old_path_by_new_path.get(replacement_path), - new_path=replacement_path, - ) - replacement_rows = verified_replacement_rows - if dropped_new_paths: - dropped_move_paths = tuple( - sorted(old_path_by_new_path[new_path] for new_path in dropped_new_paths) - ) - target_rows = [ - row - for row in target_rows - if target_paths_by_old_path[str(row["file_path"])] not in dropped_new_paths - ] + replacement_screen = _screen_replaced_move_targets( + target_rows=target_rows, + replacement_rows=replacement_rows, + target_paths_by_old_path=target_paths_by_old_path, + ) + target_rows = replacement_screen.target_rows + replacement_rows = replacement_screen.replacement_rows + dropped_move_paths = replacement_screen.dropped_move_paths + # --- Plan provider-specific content repair inside the transaction --- updated_old_paths = frozenset(str(row["file_path"]) for row in target_rows) target_paths_by_entity_id = { int(row["id"]): target_paths_by_old_path[str(row["file_path"])] for row in target_rows } - planned_moved_files_by_entity_id: dict[int, ProjectIndexMovedFile] = {} - if self.move_content_updater is not None: - for row in target_rows: - entity_id = int(row["id"]) - old_path = str(row["file_path"]) - moved_file = ProjectIndexMovedFile( - entity_id=entity_id, - old_path=old_path, - new_path=target_paths_by_old_path[old_path], - old_permalink=( - str(row["permalink"]) if row["permalink"] is not None else None - ), - ) - content_update = await self.move_content_updater.plan_moved_file_content( - session, - moved_file, - ) - if content_update is not None: - content_updates_by_entity_id[entity_id] = content_update - planned_moved_files_by_entity_id[entity_id] = moved_file + content_plan = await self._plan_move_content_updates( + session, + target_rows=target_rows, + target_paths_by_old_path=target_paths_by_old_path, + ) + # --- Apply the batched replacement deletes and path/content updates --- + replaced_entity_ids: frozenset[int] = frozenset() + relation_cleanup_entity_ids: frozenset[int] = frozenset() if updated_old_paths: replaced_entity_ids = frozenset(int(row["id"]) for row in replacement_rows) relation_cleanup_entity_ids = await delete_project_index_entities( @@ -601,130 +711,21 @@ async def apply_project_index_move_batch( project_id=self.project_id, entity_ids=tuple(replaced_entity_ids), ) - - entity_update_values = { - "file_path": case( - target_paths_by_old_path, - value=Entity.file_path, - ) - } - note_content_update_values = { - "file_path": case( - target_paths_by_entity_id, - value=NoteContent.entity_id, - ) - } - search_index_update_values = { - "file_path": case( - target_paths_by_entity_id, - value=PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id, - ) - } - if content_updates_by_entity_id: - checksums_by_entity_id = { - entity_id: content_update.checksum - for entity_id, content_update in content_updates_by_entity_id.items() - } - markdown_by_entity_id = { - entity_id: content_update.markdown_content - for entity_id, content_update in content_updates_by_entity_id.items() - } - permalinks_by_entity_id = { - entity_id: content_update.permalink - for entity_id, content_update in content_updates_by_entity_id.items() - } - entity_update_values["checksum"] = case( - checksums_by_entity_id, - value=Entity.id, - else_=Entity.checksum, - ) - entity_update_values["permalink"] = case( - permalinks_by_entity_id, - value=Entity.id, - else_=Entity.permalink, - ) - note_content_update_values["db_checksum"] = case( - checksums_by_entity_id, - value=NoteContent.entity_id, - else_=NoteContent.db_checksum, - ) - note_content_update_values["file_checksum"] = case( - checksums_by_entity_id, - value=NoteContent.entity_id, - else_=NoteContent.file_checksum, - ) - note_content_update_values["markdown_content"] = case( - markdown_by_entity_id, - value=NoteContent.entity_id, - else_=NoteContent.markdown_content, - ) - - await session.execute( - update(Entity) - .where( - Entity.project_id == self.project_id, - Entity.file_path.in_(updated_old_paths), - ) - .values(**entity_update_values) - ) - await session.execute( - update(NoteContent) - .where( - NoteContent.project_id == self.project_id, - NoteContent.entity_id.in_(tuple(target_paths_by_entity_id)), - ) - .values(**note_content_update_values) - ) - await session.execute( - update(PROJECT_INDEX_SEARCH_INDEX_TABLE) - .where( - PROJECT_INDEX_SEARCH_INDEX_TABLE.c.project_id == self.project_id, - PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id.in_( - tuple(target_paths_by_entity_id) - ), - ) - .values(**search_index_update_values) + await self._execute_move_batch_updates( + session, + updated_old_paths=updated_old_paths, + target_paths_by_entity_id=target_paths_by_entity_id, + update_values=_build_move_batch_update_values( + target_paths_by_old_path=target_paths_by_old_path, + target_paths_by_entity_id=target_paths_by_entity_id, + content_updates_by_entity_id=content_plan.updates_by_entity_id, + ), ) - if content_updates_by_entity_id: - await session.execute( - update(PROJECT_INDEX_SEARCH_INDEX_TABLE) - .where( - PROJECT_INDEX_SEARCH_INDEX_TABLE.c.project_id == self.project_id, - PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id.in_( - tuple(content_updates_by_entity_id) - ), - PROJECT_INDEX_SEARCH_INDEX_TABLE.c.type == "entity", - ) - .values( - permalink=case( - permalinks_by_entity_id, - value=PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id, - ) - ) - ) - # Trigger: the batch committed with entity/note_content rows stamped from - # the planned markdown, and the files still hold their pre-move metadata. - # Why: writing files inside the transaction is not atomic with it — a - # rollback would revert the database while the on-disk frontmatter - # rewrites persisted, leaving files ahead of their indexed state. - # Outcome: writes happen only after a successful commit; a failed write - # leaves the file with a checksum that no longer matches its - # rows, which the next scan reconciles as a modified file. - if self.move_content_updater is not None: - for entity_id, content_update in content_updates_by_entity_id.items(): - try: - await self.move_content_updater.write_moved_file_content( - planned_moved_files_by_entity_id[entity_id], - content_update, - ) - except Exception as write_error: - logger.error( - "Failed to write moved file content after move batch commit", - path=planned_moved_files_by_entity_id[entity_id].new_path, - error=str(write_error), - ) + # --- Write planned file content after the commit --- + await self._write_moved_file_contents(content_plan) + # --- Report per-path outcomes --- missing_paths = tuple( move_target.old_path for move_target in move_batch.targets @@ -740,6 +741,154 @@ async def apply_project_index_move_batch( dropped_move_paths=dropped_move_paths, ) + async def _load_move_replacement_rows( + self, + session: AsyncSession, + *, + target_rows: list[RowMapping], + target_paths_by_old_path: dict[str, str], + ) -> list[RowMapping]: + """Load entities already occupying the batch's move destinations. + + Rows can appear there when the watcher legitimately moves onto an + existing indexed file, or when a racing event index created the moved + file at its new path first; survivors are deleted so the source entity + can take over the path. + """ + if not target_rows: + return [] + new_paths = tuple( + sorted({target_paths_by_old_path[str(row["file_path"])] for row in target_rows}) + ) + replacement_result = await session.execute( + select(Entity.id, Entity.file_path, Entity.checksum).where( + Entity.project_id == self.project_id, + Entity.file_path.in_(new_paths), + Entity.id.not_in(tuple(int(row["id"]) for row in target_rows)), + ) + ) + return list(replacement_result.mappings().all()) + + async def _plan_move_content_updates( + self, + session: AsyncSession, + *, + target_rows: list[RowMapping], + target_paths_by_old_path: dict[str, str], + ) -> _MoveContentPlan: + """Plan provider-specific frontmatter rewrites inside the batch transaction. + + Planning must not mutate storage: the batch can still roll back, and an + already-rewritten file would survive that rollback (see + ProjectIndexMoveContentUpdater). Runtimes without a content updater skip + content repair entirely. + """ + if self.move_content_updater is None: + return _MoveContentPlan.empty() + + updates_by_entity_id: dict[int, ProjectIndexMovedFileContentUpdate] = {} + moved_files_by_entity_id: dict[int, ProjectIndexMovedFile] = {} + for row in target_rows: + entity_id = int(row["id"]) + old_path = str(row["file_path"]) + moved_file = ProjectIndexMovedFile( + entity_id=entity_id, + old_path=old_path, + new_path=target_paths_by_old_path[old_path], + old_permalink=(str(row["permalink"]) if row["permalink"] is not None else None), + ) + content_update = await self.move_content_updater.plan_moved_file_content( + session, + moved_file, + ) + if content_update is not None: + updates_by_entity_id[entity_id] = content_update + moved_files_by_entity_id[entity_id] = moved_file + return _MoveContentPlan( + updates_by_entity_id=updates_by_entity_id, + moved_files_by_entity_id=moved_files_by_entity_id, + ) + + async def _execute_move_batch_updates( + self, + session: AsyncSession, + *, + updated_old_paths: frozenset[str], + target_paths_by_entity_id: dict[int, str], + update_values: _MoveBatchUpdateValues, + ) -> None: + """Run the batched UPDATE statements for one screened set of moves.""" + await session.execute( + update(Entity) + .where( + Entity.project_id == self.project_id, + Entity.file_path.in_(updated_old_paths), + ) + .values(**update_values.entity_values) + ) + await session.execute( + update(NoteContent) + .where( + NoteContent.project_id == self.project_id, + NoteContent.entity_id.in_(tuple(target_paths_by_entity_id)), + ) + .values(**update_values.note_content_values) + ) + await session.execute( + update(PROJECT_INDEX_SEARCH_INDEX_TABLE) + .where( + PROJECT_INDEX_SEARCH_INDEX_TABLE.c.project_id == self.project_id, + PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id.in_(tuple(target_paths_by_entity_id)), + ) + .values(**update_values.search_index_values) + ) + # Entity search rows carry a permalink column that only changes when + # content repair rewrote the note's permalink frontmatter. + if update_values.permalinks_by_entity_id: + await session.execute( + update(PROJECT_INDEX_SEARCH_INDEX_TABLE) + .where( + PROJECT_INDEX_SEARCH_INDEX_TABLE.c.project_id == self.project_id, + PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id.in_( + tuple(update_values.permalinks_by_entity_id) + ), + PROJECT_INDEX_SEARCH_INDEX_TABLE.c.type == "entity", + ) + .values( + permalink=case( + update_values.permalinks_by_entity_id, + value=PROJECT_INDEX_SEARCH_INDEX_TABLE.c.entity_id, + ) + ) + ) + + async def _write_moved_file_contents(self, content_plan: _MoveContentPlan) -> None: + """Write planned frontmatter rewrites once the batch has committed. + + Trigger: the batch committed with entity/note_content rows stamped from + the planned markdown, and the files still hold their pre-move metadata. + Why: writing files inside the transaction is not atomic with it — a + rollback would revert the database while the on-disk frontmatter + rewrites persisted, leaving files ahead of their indexed state. + Outcome: writes happen only after a successful commit; a failed write + leaves the file with a checksum that no longer matches its rows, which + the next scan reconciles as a modified file. + """ + if self.move_content_updater is None: + return + for entity_id, content_update in content_plan.updates_by_entity_id.items(): + try: + await self.move_content_updater.write_moved_file_content( + content_plan.moved_files_by_entity_id[entity_id], + content_update, + ) + except Exception as write_error: + logger.error( + "Failed to write moved file content after move batch commit", + path=content_plan.moved_files_by_entity_id[entity_id].new_path, + error=str(write_error), + ) + async def apply_project_index_delete_batch( self, delete_batch: ProjectIndexDeleteBatch, diff --git a/src/basic_memory/indexing/project_index_runtime.py b/src/basic_memory/indexing/project_index_runtime.py index 969021bd8..f3b19e81f 100644 --- a/src/basic_memory/indexing/project_index_runtime.py +++ b/src/basic_memory/indexing/project_index_runtime.py @@ -14,9 +14,6 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory.indexing.forward_reference_resolution import ( - ForwardReferenceEntityIndexer, - ForwardReferenceEntityRepository, - ForwardReferenceEntityRefreshFailure, ForwardReferenceEntityRefreshRun, ForwardReferenceEntityRefreshRuntime, ForwardReferenceRelationSource, @@ -28,13 +25,17 @@ run_forward_reference_entity_refresh, run_forward_reference_resolution, ) +from basic_memory.indexing.relation_resolution import ( + RelationResolutionEntityIndexer, + RelationResolutionEntityRepository, +) from basic_memory.indexing.progress import VectorSyncProgress from basic_memory.indexing.project_index_maintenance import ( ProjectIndexDeleteRun, + ProjectIndexMaintenanceRunner, ProjectIndexMoveRun, RepositoryProjectIndexMaintenanceStore, - run_project_index_delete_batches, - run_project_index_move_batches, + StoreProjectIndexMaintenanceRunner, ) from basic_memory.indexing.vector_sync_planning import ( CheckpointPhase, @@ -58,46 +59,6 @@ class ProjectIndexForwardReferenceRun: resolution: ForwardReferenceResolutionRun refresh: ForwardReferenceEntityRefreshRun - @property - def initial_count(self) -> int: - """Return how many unresolved relations were considered.""" - return self.resolution.unresolved_before - - @property - def unique_link_text_count(self) -> int: - """Return how many unique unresolved link texts were considered.""" - return len(self.resolution.link_texts) - - @property - def resolved_link_text_count(self) -> int: - """Return how many unique link texts resolved to a target.""" - return self.resolution.resolved_link_text_count - - @property - def resolved_count(self) -> int: - """Return how many relation rows were updated.""" - return self.resolution.resolved_count - - @property - def remaining_count(self) -> int: - """Return how many initially unresolved relation rows remain unresolved.""" - return self.resolution.remaining_count - - @property - def entity_ids_to_refresh(self) -> frozenset[EntityId]: - """Return exact target entities selected for search refresh.""" - return self.resolution.entity_ids_to_refresh - - @property - def successful_reindexed_entity_ids(self) -> frozenset[EntityId]: - """Return target entities whose search rows were refreshed.""" - return self.refresh.successful_entity_ids - - @property - def refresh_failures(self) -> tuple[ForwardReferenceEntityRefreshFailure, ...]: - """Return target entities whose search refresh raised.""" - return self.refresh.failures - @dataclass(frozen=True, slots=True) class ProjectIndexRuntime: @@ -106,8 +67,7 @@ class ProjectIndexRuntime: project_id: ProjectId vector_sync: VectorSyncExecutor vector_entity_source: VectorSyncEntitySource - move_store: RepositoryProjectIndexMaintenanceStore - delete_store: RepositoryProjectIndexMaintenanceStore + maintenance: ProjectIndexMaintenanceRunner forward_reference_relation_source: ForwardReferenceRelationSource forward_reference_resolution_runtime: ForwardReferenceResolutionRuntime forward_reference_entity_refresher: ForwardReferenceEntityRefreshRuntime @@ -166,10 +126,9 @@ async def run_move_batches( batch_size: int, ) -> ProjectIndexMoveRun: """Apply moved-file updates for the current project.""" - return await run_project_index_move_batches( - moved_files=dict(moved_files), + return await self.maintenance.run_move_batches( + moved_files=moved_files, batch_size=batch_size, - move_store=self.move_store, ) async def run_delete_batches( @@ -179,10 +138,9 @@ async def run_delete_batches( batch_size: int, ) -> ProjectIndexDeleteRun: """Delete file-backed entities for the current project.""" - return await run_project_index_delete_batches( - deleted_paths=list(deleted_paths), + return await self.maintenance.run_delete_batches( + deleted_paths=deleted_paths, batch_size=batch_size, - delete_store=self.delete_store, ) async def resolve_forward_references(self) -> ProjectIndexForwardReferenceRun: @@ -209,8 +167,8 @@ def build_default_project_index_runtime( project_id: ProjectId, session_maker: async_sessionmaker[AsyncSession], vector_sync: VectorSyncExecutor, - entity_repository: ForwardReferenceEntityRepository, - entity_indexer: ForwardReferenceEntityIndexer, + entity_repository: RelationResolutionEntityRepository, + entity_indexer: RelationResolutionEntityIndexer, ) -> ProjectIndexRuntime: """Compose the default repository-backed project-index runtime.""" vector_entity_source = RepositoryVectorSyncEntitySource( @@ -225,8 +183,10 @@ def build_default_project_index_runtime( project_id=project_id, vector_sync=vector_sync, vector_entity_source=vector_entity_source, - move_store=maintenance_store, - delete_store=maintenance_store, + maintenance=StoreProjectIndexMaintenanceRunner( + move_store=maintenance_store, + delete_store=maintenance_store, + ), forward_reference_relation_source=RepositoryForwardReferenceRelationSource( session_maker=session_maker, project_id=project_id, diff --git a/src/basic_memory/indexing/project_index_workflow.py b/src/basic_memory/indexing/project_index_workflow.py index 9f8ce2caa..f155a9f1a 100644 --- a/src/basic_memory/indexing/project_index_workflow.py +++ b/src/basic_memory/indexing/project_index_workflow.py @@ -11,6 +11,7 @@ apply_project_index_batch_job_results, project_index_file_outcome_from_job_result, ) +from basic_memory.indexing.progress import CheckpointModel from basic_memory.indexing.project_index_coordinator import ProjectIndexRequest from basic_memory.indexing.project_index_progress import ( ProjectIndexCounters, @@ -22,6 +23,7 @@ project_index_recorded_batches_from_metadata, should_emit_project_index_progress_event, ) +from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.workflows import WorkflowId @@ -67,192 +69,70 @@ class ProjectIndexWorkflowFailureUpdate: failed_event_data: dict[str, object] -type ProjectIndexWorkflowStartStatus = Literal["running", "complete"] - - @dataclass(frozen=True, slots=True) -class ProjectIndexWorkflowStartPlan: - """Portable decision for starting one project-index workflow.""" +class ProjectIndexWorkflowStartRunning: + """Non-terminal start: child batches remain to fan out.""" - status: ProjectIndexWorkflowStartStatus workflow_start: ProjectIndexWorkflowStart - completion_update: ProjectIndexWorkflowCompletionUpdate | None = None - - def __post_init__(self) -> None: - if self.status == "running": - if self.completion_update is not None: - raise ValueError("running start plans cannot include a completion update") - return - - if self.completion_update is None: - raise ValueError("complete start plans require a completion update") - - @classmethod - def running(cls, workflow_start: ProjectIndexWorkflowStart) -> Self: - """Return a non-terminal start plan.""" - return cls(status="running", workflow_start=workflow_start) - @classmethod - def complete( - cls, - *, - workflow_start: ProjectIndexWorkflowStart, - completion_update: ProjectIndexWorkflowCompletionUpdate, - ) -> Self: - """Return an immediately terminal start plan.""" - return cls( - status="complete", - workflow_start=workflow_start, - completion_update=completion_update, - ) - @property - def is_complete(self) -> bool: - """Return whether the workflow should complete immediately after starting.""" - return self.status == "complete" +@dataclass(frozen=True, slots=True) +class ProjectIndexWorkflowStartComplete: + """Terminal start: an empty project completes immediately after starting.""" - def require_completion_update(self) -> ProjectIndexWorkflowCompletionUpdate: - """Return the completion update or fail when this is a running plan.""" - if self.completion_update is None: - raise RuntimeError(f"{self.status} plan does not include a completion update") - return self.completion_update + workflow_start: ProjectIndexWorkflowStart + completion_update: ProjectIndexWorkflowCompletionUpdate -type ProjectIndexWorkflowRecordStatus = Literal["progress", "complete", "already_recorded"] +type ProjectIndexWorkflowStartPlan = ( + ProjectIndexWorkflowStartRunning | ProjectIndexWorkflowStartComplete +) @dataclass(frozen=True, slots=True) -class ProjectIndexWorkflowRecordPlan: - """Portable decision for applying one child result to aggregate workflow state.""" - - status: ProjectIndexWorkflowRecordStatus - progress_update: ProjectIndexWorkflowProgressUpdate | None = None - completion_update: ProjectIndexWorkflowCompletionUpdate | None = None +class ProjectIndexWorkflowRecordProgress: + """Running update: the child result advanced aggregate counters.""" - def __post_init__(self) -> None: - if self.status == "already_recorded": - if self.progress_update is not None or self.completion_update is not None: - raise ValueError("already_recorded plans cannot include updates") - return + progress_update: ProjectIndexWorkflowProgressUpdate - if self.progress_update is None: - raise ValueError(f"{self.status} plans require a progress update") - if self.status == "progress" and self.completion_update is not None: - raise ValueError("progress plans cannot include a completion update") - if self.status == "complete" and self.completion_update is None: - raise ValueError("complete plans require a completion update") +@dataclass(frozen=True, slots=True) +class ProjectIndexWorkflowRecordComplete: + """Terminal update: the child result finished the last outstanding file.""" - @classmethod - def progress( - cls, - progress_update: ProjectIndexWorkflowProgressUpdate, - ) -> Self: - """Return a running progress plan.""" - return cls(status="progress", progress_update=progress_update) + progress_update: ProjectIndexWorkflowProgressUpdate + completion_update: ProjectIndexWorkflowCompletionUpdate - @classmethod - def complete( - cls, - *, - progress_update: ProjectIndexWorkflowProgressUpdate, - completion_update: ProjectIndexWorkflowCompletionUpdate, - ) -> Self: - """Return a terminal success plan.""" - return cls( - status="complete", - progress_update=progress_update, - completion_update=completion_update, - ) - - @classmethod - def already_recorded(cls) -> Self: - """Return an idempotent no-op plan.""" - return cls(status="already_recorded") - @property - def is_complete(self) -> bool: - """Return whether this plan completes the workflow.""" - return self.status == "complete" +@dataclass(frozen=True, slots=True) +class ProjectIndexWorkflowAlreadyRecorded: + """Idempotent no-op: this batch already updated aggregate counters.""" - @property - def should_emit_progress_event(self) -> bool: - """Return whether the runtime should append a progress event.""" - return ( - self.status == "progress" - and self.progress_update is not None - and self.progress_update.should_emit_event - ) - def require_progress_update(self) -> ProjectIndexWorkflowProgressUpdate: - """Return the progress update or fail when this is an idempotent no-op.""" - if self.progress_update is None: - raise RuntimeError(f"{self.status} plan does not include a progress update") - return self.progress_update +type ProjectIndexWorkflowRecordPlan = ( + ProjectIndexWorkflowRecordProgress + | ProjectIndexWorkflowRecordComplete + | ProjectIndexWorkflowAlreadyRecorded +) - def require_completion_update(self) -> ProjectIndexWorkflowCompletionUpdate: - """Return the completion update or fail when the plan is not terminal.""" - if self.completion_update is None: - raise RuntimeError(f"{self.status} plan does not include a completion update") - return self.completion_update +@dataclass(frozen=True, slots=True) +class ProjectIndexStaleWorkflowKeepRunning: + """Non-terminal stale check: unfinished child jobs were observed.""" -type ProjectIndexStaleWorkflowStatus = Literal["keep_running", "fail"] + activity_update: ProjectIndexBatchJobActivityUpdate @dataclass(frozen=True, slots=True) -class ProjectIndexStaleWorkflowPlan: - """Portable decision for one stale project-index workflow check.""" - - status: ProjectIndexStaleWorkflowStatus - activity_update: ProjectIndexBatchJobActivityUpdate | None = None - failure_update: ProjectIndexWorkflowFailureUpdate | None = None - - def __post_init__(self) -> None: - if self.status == "keep_running": - if self.activity_update is None: - raise ValueError("keep_running plans require an activity update") - if self.failure_update is not None: - raise ValueError("keep_running plans cannot include a failure update") - return - - if self.failure_update is None: - raise ValueError("fail plans require a failure update") - if self.activity_update is not None: - raise ValueError("fail plans cannot include an activity update") +class ProjectIndexStaleWorkflowFail: + """Terminal stale check: no child activity remains, so the workflow fails.""" - @classmethod - def keep_running( - cls, - activity_update: ProjectIndexBatchJobActivityUpdate, - ) -> Self: - """Return a non-terminal activity update plan.""" - return cls(status="keep_running", activity_update=activity_update) - - @classmethod - def fail( - cls, - failure_update: ProjectIndexWorkflowFailureUpdate, - ) -> Self: - """Return a terminal stale-failure plan.""" - return cls(status="fail", failure_update=failure_update) + failure_update: ProjectIndexWorkflowFailureUpdate - @property - def should_fail(self) -> bool: - """Return whether this stale check should fail the workflow.""" - return self.status == "fail" - - def require_activity_update(self) -> ProjectIndexBatchJobActivityUpdate: - """Return the activity update or fail when this is a terminal plan.""" - if self.activity_update is None: - raise RuntimeError(f"{self.status} plan does not include an activity update") - return self.activity_update - def require_failure_update(self) -> ProjectIndexWorkflowFailureUpdate: - """Return the failure update or fail when this is a keep-running plan.""" - if self.failure_update is None: - raise RuntimeError(f"{self.status} plan does not include a failure update") - return self.failure_update +type ProjectIndexStaleWorkflowPlan = ( + ProjectIndexStaleWorkflowKeepRunning | ProjectIndexStaleWorkflowFail +) @dataclass(frozen=True, slots=True) @@ -299,6 +179,108 @@ class ProjectIndexBatchJobActivityUpdate: metadata: dict[str, object] +# --- Checkpoint metadata write models --- +# The workflow metadata document is validated with Pydantic on read (see +# project_index_progress). These models are the matching typed write side: +# field names and order define the persisted JSON shape, so builders dump +# them instead of mutating dict[str, object] by string key. + + +class ProjectIndexDiscoveryMetadata(CheckpointModel): + """Fan-out discovery facts recorded when a project-index workflow starts.""" + + total_files: int + batch_count: int + batch_size: int + discovered_at: str + + +class ProjectIndexWorkflowStartMetadata(CheckpointModel): + """Initial checkpoint metadata document for a project-index workflow.""" + + phase: Literal["indexing"] = "indexing" + progress: str + payload: dict[str, object] + discovery: ProjectIndexDiscoveryMetadata + counters: dict[str, int] + transport: dict[str, object] + + +class ProjectIndexWorkflowProgressMetadata(CheckpointModel): + """Checkpoint metadata fields rewritten by one running progress update.""" + + phase: Literal["indexing"] = "indexing" + progress: str + counters: dict[str, int] + # None means "leave any previously recorded batches untouched"; the field + # is dropped from the dump so per-file workflows never write the key. + recorded_batches: list[int] | None = None + + +class ProjectIndexWorkflowCompletionMetadata(CheckpointModel): + """Checkpoint metadata fields rewritten by terminal workflow success.""" + + phase: Literal["completed"] = "completed" + progress: str + counters: dict[str, int] + result: dict[str, int] + + +class ProjectIndexStaleDiagnostics(CheckpointModel): + """Diagnostics recorded when project-index batch fan-out stalls.""" + + reason: Literal["stale_project_index_batches"] = "stale_project_index_batches" + missing_batches: list[int] + recorded_batches: list[int] + # Despite the historical key name, this is the "legacy rows lack a + # batch_count" flag from ProjectIndexMissingBatches and persists as JSON + # true/false. + legacy_missing_batch_count: bool + last_heartbeat_at: str + stale_before: str + + +class ProjectIndexWorkflowFailureMetadata(CheckpointModel): + """Checkpoint metadata fields rewritten by terminal workflow failure.""" + + phase: Literal["failed"] = "failed" + progress: str + counters: dict[str, int] + diagnostics: ProjectIndexStaleDiagnostics + + +@dataclass(frozen=True, slots=True) +class ProjectIndexWorkflowAttemptEvent: + """Attempt event payload for one project-index workflow start. + + Queue transport identity is opaque to core: the owning runtime's transport + fields are spliced between the discovery counts and the project identity + to keep persisted event shapes stable. + """ + + progress: str + total_files: int + batch_count: int + batch_size: int + transport_event_data: Mapping[str, object] + project: ProjectRuntimeReference + + def to_event_data(self) -> dict[str, object]: + """Serialize to the persisted attempt event shape.""" + return { + "phase": "indexing", + "progress": self.progress, + "total_files": self.total_files, + "batch_count": self.batch_count, + "batch_size": self.batch_size, + **dict(self.transport_event_data), + "project_id": self.project.project_id, + "project_name": self.project.project_name, + "project_permalink": self.project.project_permalink, + "project_path": self.project.project_path, + } + + def build_project_index_batch_activity_update( *, metadata: Mapping[str, object], @@ -330,41 +312,35 @@ def build_project_index_workflow_start( Queue transport identity is opaque to core: the runtime that owns the queue passes its durable ``transport`` metadata dict and any transport fields it - wants merged into the attempt event (inserted between the discovery counts - and the project identity to keep persisted event shapes stable). + wants merged into the attempt event (see ProjectIndexWorkflowAttemptEvent). """ counters = initial_project_index_counters(total_files) progress = project_index_progress_text(counters) - payload = request.workflow_payload_metadata() - metadata: dict[str, object] = { - "phase": "indexing", - "progress": progress, - "payload": payload, - "discovery": { - "total_files": total_files, - "batch_count": batch_count, - "batch_size": batch_size, - "discovered_at": discovered_at, - }, - "counters": counters.to_metadata(), - "transport": dict(transport_metadata), - } + start_metadata = ProjectIndexWorkflowStartMetadata( + progress=progress, + payload=request.workflow_payload_metadata(), + discovery=ProjectIndexDiscoveryMetadata( + total_files=total_files, + batch_count=batch_count, + batch_size=batch_size, + discovered_at=discovered_at, + ), + counters=counters.to_metadata(), + transport=dict(transport_metadata), + ) + attempt_event = ProjectIndexWorkflowAttemptEvent( + progress=progress, + total_files=total_files, + batch_count=batch_count, + batch_size=batch_size, + transport_event_data=transport_event_data, + project=request.project, + ) return ProjectIndexWorkflowStart( counters=counters, progress=progress, - metadata=metadata, - attempt_event_data={ - "phase": "indexing", - "progress": progress, - "total_files": total_files, - "batch_count": batch_count, - "batch_size": batch_size, - **dict(transport_event_data), - "project_id": request.project.project_id, - "project_name": request.project.project_name, - "project_permalink": request.project.project_permalink, - "project_path": request.project.project_path, - }, + metadata=start_metadata.model_dump(), + attempt_event_data=attempt_event.to_event_data(), ) @@ -389,7 +365,7 @@ def plan_project_index_workflow_start( transport_event_data=transport_event_data, ) if total_files == 0: - return ProjectIndexWorkflowStartPlan.complete( + return ProjectIndexWorkflowStartComplete( workflow_start=workflow_start, completion_update=build_project_index_workflow_completion_update( metadata=workflow_start.metadata, @@ -397,7 +373,7 @@ def plan_project_index_workflow_start( progress=workflow_start.progress, ), ) - return ProjectIndexWorkflowStartPlan.running(workflow_start) + return ProjectIndexWorkflowStartRunning(workflow_start) def build_project_index_workflow_progress_update( @@ -409,12 +385,15 @@ def build_project_index_workflow_progress_update( """Build updated persisted metadata for a running project-index workflow.""" progress = project_index_progress_text(counters) counters_metadata = counters.to_metadata() + progress_metadata = ProjectIndexWorkflowProgressMetadata( + progress=progress, + counters=counters_metadata, + recorded_batches=( + list(recorded_batch_indexes) if recorded_batch_indexes is not None else None + ), + ) updated_metadata = dict(metadata) - updated_metadata["phase"] = "indexing" - updated_metadata["progress"] = progress - updated_metadata["counters"] = counters_metadata - if recorded_batch_indexes is not None: - updated_metadata["recorded_batches"] = list(recorded_batch_indexes) + updated_metadata.update(progress_metadata.model_dump(exclude_none=True)) return ProjectIndexWorkflowProgressUpdate( counters=counters, @@ -438,11 +417,13 @@ def build_project_index_workflow_completion_update( ) -> ProjectIndexWorkflowCompletionUpdate: """Build terminal success metadata for a project-index workflow.""" counters_metadata = counters.to_metadata() + completion_metadata = ProjectIndexWorkflowCompletionMetadata( + progress=progress, + counters=counters_metadata, + result=counters_metadata, + ) completed_metadata = dict(metadata) - completed_metadata["phase"] = "completed" - completed_metadata["progress"] = progress - completed_metadata["counters"] = counters_metadata - completed_metadata["result"] = counters_metadata + completed_metadata.update(completion_metadata.model_dump()) return ProjectIndexWorkflowCompletionUpdate( counters=counters, @@ -488,7 +469,7 @@ def plan_project_index_file_result_record( counters=counters, ) if counters.processed >= counters.total: - return ProjectIndexWorkflowRecordPlan.complete( + return ProjectIndexWorkflowRecordComplete( progress_update=progress_update, completion_update=build_project_index_workflow_completion_update( metadata=progress_update.metadata, @@ -496,7 +477,7 @@ def plan_project_index_file_result_record( progress=progress_update.progress, ), ) - return ProjectIndexWorkflowRecordPlan.progress(progress_update) + return ProjectIndexWorkflowRecordProgress(progress_update) def plan_project_index_batch_result_record( @@ -520,7 +501,7 @@ def plan_project_index_batch_result_record( results=results, ) if batch_update.already_recorded: - return ProjectIndexWorkflowRecordPlan.already_recorded() + return ProjectIndexWorkflowAlreadyRecorded() counters = batch_update.counters progress_update = build_project_index_workflow_progress_update( @@ -529,7 +510,7 @@ def plan_project_index_batch_result_record( recorded_batch_indexes=batch_update.recorded_batch_indexes, ) if batch_update.is_complete: - return ProjectIndexWorkflowRecordPlan.complete( + return ProjectIndexWorkflowRecordComplete( progress_update=progress_update, completion_update=build_project_index_workflow_completion_update( metadata=progress_update.metadata, @@ -537,7 +518,7 @@ def plan_project_index_batch_result_record( progress=progress_update.progress, ), ) - return ProjectIndexWorkflowRecordPlan.progress(progress_update) + return ProjectIndexWorkflowRecordProgress(progress_update) def plan_project_index_stale_workflow( @@ -551,7 +532,7 @@ def plan_project_index_stale_workflow( ) -> ProjectIndexStaleWorkflowPlan: """Plan how a runtime should update one stale project-index workflow.""" if active_batch_jobs.has_unfinished_jobs: - return ProjectIndexStaleWorkflowPlan.keep_running( + return ProjectIndexStaleWorkflowKeepRunning( build_project_index_batch_activity_update( metadata=metadata, activity=active_batch_jobs, @@ -564,7 +545,7 @@ def plan_project_index_stale_workflow( workflow_id=workflow_id, ) missing_batch_plan = project_index_missing_batches_from_metadata(metadata) - return ProjectIndexStaleWorkflowPlan.fail( + return ProjectIndexStaleWorkflowFail( build_project_index_workflow_stale_failure_update( metadata=metadata, counters=counters, @@ -583,32 +564,30 @@ def build_project_index_workflow_stale_failure_update( counters: ProjectIndexCounters, missing_batch_indexes: Sequence[int], recorded_batch_indexes: Sequence[int], - legacy_missing_batch_count: int, + legacy_missing_batch_count: bool, last_heartbeat_at: str, stale_before: str, ) -> ProjectIndexWorkflowFailureUpdate: """Build terminal failure metadata for stale project-index batch fan-out.""" missing_batches = list(missing_batch_indexes) - recorded_batches = list(recorded_batch_indexes) if legacy_missing_batch_count: error_message = "Project index stalled with legacy batch metadata" else: error_message = f"Project index stalled with {len(missing_batches)} unreported batch(es)" progress = f"Project index stalled after {counters.processed}/{counters.total} files" - diagnostics: dict[str, object] = { - "reason": "stale_project_index_batches", - "missing_batches": missing_batches, - "recorded_batches": recorded_batches, - "legacy_missing_batch_count": legacy_missing_batch_count, - "last_heartbeat_at": last_heartbeat_at, - "stale_before": stale_before, - } - counters_metadata = counters.to_metadata() + failure_metadata = ProjectIndexWorkflowFailureMetadata( + progress=progress, + counters=counters.to_metadata(), + diagnostics=ProjectIndexStaleDiagnostics( + missing_batches=missing_batches, + recorded_batches=list(recorded_batch_indexes), + legacy_missing_batch_count=legacy_missing_batch_count, + last_heartbeat_at=last_heartbeat_at, + stale_before=stale_before, + ), + ) failed_metadata = dict(metadata) - failed_metadata["phase"] = "failed" - failed_metadata["progress"] = progress - failed_metadata["counters"] = counters_metadata - failed_metadata["diagnostics"] = diagnostics + failed_metadata.update(failure_metadata.model_dump()) return ProjectIndexWorkflowFailureUpdate( counters=counters, @@ -620,6 +599,6 @@ def build_project_index_workflow_stale_failure_update( "progress": progress, "payload": failed_metadata.get("payload") or {}, "error": error_message, - "diagnostics": diagnostics, + "diagnostics": failed_metadata["diagnostics"], }, ) diff --git a/src/basic_memory/indexing/relation_resolution.py b/src/basic_memory/indexing/relation_resolution.py index b24528e06..daeadc6e7 100644 --- a/src/basic_memory/indexing/relation_resolution.py +++ b/src/basic_memory/indexing/relation_resolution.py @@ -14,31 +14,23 @@ from basic_memory import db from basic_memory.indexing.models import IndexFileJobStatus -from basic_memory.models import Entity +from basic_memory.models import Entity, Relation type EntityId = int type AffectedEntityIds = set[EntityId] RESOLVE_RELATIONS_DEBOUNCE_SECONDS = 10 -class RelationResolutionPass(Protocol): - """Capability that performs one relation-resolution pass.""" +class RelationResolutionRuntime(Protocol): + """Capability that owns relation resolution for one project.""" async def resolve_relations(self) -> AffectedEntityIds: """Resolve currently visible relations and return affected source entity IDs.""" - -class UnresolvedRelationCounter(Protocol): - """Capability that counts currently unresolved relations.""" - async def count_unresolved_relations(self) -> int: """Return the current unresolved relation count.""" -class RelationResolutionRuntime(RelationResolutionPass, UnresolvedRelationCounter, Protocol): - """Capability that owns relation resolution for one project.""" - - class UnresolvedRelation(Protocol): """Unresolved relation fields required by the resolver.""" @@ -71,16 +63,21 @@ async def find_unresolved_relations_for_entity( ) -> Sequence[UnresolvedRelation]: """Return unresolved relations for one source entity.""" + # Positional-only parameters: the concrete implementation is the generic + # model repository, whose parameters are named for entities. `/` lets this + # contract name the relation id honestly without renaming the shared + # repository method. async def update( self, session: AsyncSession, - entity_id: int, - entity_data: dict[str, object], - ) -> object | None: - """Apply resolved target fields to one relation.""" + relation_id: int, + resolved_target_fields: dict[str, int | str], + /, + ) -> Relation | None: + """Apply resolved target fields (to_id, to_name) to one relation row.""" - async def delete(self, session: AsyncSession, entity_id: int) -> bool: - """Delete one redundant unresolved relation.""" + async def delete(self, session: AsyncSession, relation_id: int, /) -> bool: + """Delete one redundant unresolved relation row.""" class RelationResolutionEntityRepository(Protocol): @@ -236,7 +233,13 @@ def routing_headers(self, headers: Mapping[str, str] | None = None) -> dict[str, @dataclass(frozen=True, slots=True) class ProjectIndexRelationResolutionContext: - """Project-index completion facts needed to queue relation resolution.""" + """Project-index completion facts needed to queue relation resolution. + + The wide identity types are deliberate: downstream runtimes rebuild this + context from legacy workflow metadata, where project_id may arrive as a + string and either field may be missing. Planning coerces or skips instead + of pushing malformed identity into a queue request. + """ project_id: int | str | None project_path: str | None @@ -269,7 +272,13 @@ async def resolve_project_index_completion_relations( *, max_passes: int = 3, ) -> ResolveRelationsResult | None: - """Run the final relation-resolution pass for a completed project index.""" + """Run the final relation-resolution pass for a completed project index. + + The context names the project so queue-based runtimes can plan an enqueue + from the same completion facts; the inline path resolves directly against + the already project-scoped runtime. A context without complete project + identity plans no request, so the resolution pass is skipped. + """ request = plan_project_index_completion_relation_resolution(context) if request is None: return None @@ -308,57 +317,40 @@ def resolved(self) -> int: return max(0, self.unresolved_before - self.remaining) -async def resolve_relations_until_stable( +async def resolve_project_relations( + runtime: RelationResolutionRuntime, *, - resolver: RelationResolutionPass, - unresolved_counter: UnresolvedRelationCounter, max_passes: int = 3, ) -> ResolveRelationsResult: - """Resolve all relations visible to the supplied capabilities. + """Resolve all resolvable forward references for one project runtime. - The loop deliberately runs one confirming pass after a productive pass. This - lets queue workers catch writes that committed while the first pass was still - running, while the pass cap keeps a noisy resolver from looping forever. + One pass resolves every relation that is unresolved at the moment it reads + the table, and the loop deliberately runs one confirming pass after a + productive pass. Queued runtimes can coalesce concurrent writes onto an + in-flight resolve job, so the confirming pass catches writes that committed + while the first pass was still running, while the pass cap keeps a noisy + resolver from looping forever. Relations left after a stable pass are + genuine forward references and remain unresolved until their target note + exists. """ - unresolved_before = await unresolved_counter.count_unresolved_relations() + unresolved_before = await runtime.count_unresolved_relations() affected_entities: AffectedEntityIds = set() passes = 0 while passes < max_passes: - affected = await resolver.resolve_relations() + affected = await runtime.resolve_relations() passes += 1 affected_entities |= affected if not affected: break - remaining = await unresolved_counter.count_unresolved_relations() - return ResolveRelationsResult( + result = ResolveRelationsResult( unresolved_before=unresolved_before, - remaining=remaining, + remaining=await runtime.count_unresolved_relations(), passes=passes, affected_entities=len(affected_entities), ) - - -async def resolve_project_relations( - runtime: RelationResolutionRuntime, - *, - max_passes: int = 3, -) -> ResolveRelationsResult: - """Resolve all resolvable forward references for one project runtime. - - One pass resolves every relation that is unresolved at the moment it reads - the table. Queued runtimes can coalesce concurrent writes onto an in-flight - resolve job, so run until one pass changes nothing or the pass cap is - reached. Relations left after a stable pass are genuine forward references - and remain unresolved until their target note exists. - """ - result = await resolve_relations_until_stable( - resolver=runtime, - unresolved_counter=runtime, - max_passes=max_passes, - ) logger.info( "Resolved project relations", unresolved_before=result.unresolved_before, diff --git a/src/basic_memory/mcp/server.py b/src/basic_memory/mcp/server.py index d5121504a..fda3b6b6f 100644 --- a/src/basic_memory/mcp/server.py +++ b/src/basic_memory/mcp/server.py @@ -14,7 +14,7 @@ from basic_memory.cli.auth import CLIAuth from basic_memory.cloud.note_content_materialization import drain_pending_materializations from basic_memory.db import scoped_session -from basic_memory.deps.services import drain_background_tasks +from basic_memory.index.local_schedulers import drain_background_tasks from basic_memory.mcp.client_info import MCPClientInfoMiddleware from basic_memory.mcp.container import McpContainer, set_container from basic_memory.services.initialization import initialize_app diff --git a/src/basic_memory/runtime/job_payloads.py b/src/basic_memory/runtime/job_payloads.py index f0bd54de4..d8562acab 100644 --- a/src/basic_memory/runtime/job_payloads.py +++ b/src/basic_memory/runtime/job_payloads.py @@ -1,9 +1,7 @@ """Pydantic boundary models for portable runtime worker payloads.""" from collections.abc import Mapping -from dataclasses import dataclass -from datetime import timedelta -from typing import Protocol, Self +from typing import Self from uuid import UUID from pydantic import BaseModel, field_validator @@ -11,10 +9,7 @@ from basic_memory.runtime.cleanup import RuntimeNoteFileDeleteJobRequest from basic_memory.runtime.jobs import ( JobEntrypoint, - JobRuntime, - RuntimeJobId, RuntimeJobRequest, - RuntimeJobRequestSource, runtime_job_request_from_source, ) from basic_memory.runtime.note_content import RuntimeNoteMaterializationJobRequest @@ -29,69 +24,6 @@ MATERIALIZE_NOTE_FILE_ENTRYPOINT: JobEntrypoint = "materialize_note_file" -class RuntimeSerializedJobPayload(Protocol): - """Validated payload that can cross a runtime worker boundary.""" - - def model_dump_json(self) -> str: ... - - -class RuntimeJobPayloadSource(Protocol): - """Validated payload that owns concrete runtime job request construction.""" - - def runtime_job_request( - self, - *, - headers: Mapping[str, str] | None = None, - ) -> RuntimeJobRequest: ... - - -async def enqueue_runtime_job_payload( - runtime: JobRuntime, - payload: RuntimeJobPayloadSource, - *, - headers: Mapping[str, str] | None = None, -) -> RuntimeJobId: - """Queue one validated payload through the selected runtime adapter.""" - return await runtime.enqueue(payload.runtime_job_request(headers=headers)) - - -class RuntimeJobPayloadSerializer[RequestT: RuntimeJobRequestSource](Protocol): - """Capability that validates and serializes a runtime job request payload.""" - - def serialize(self, request: RequestT) -> RuntimeSerializedJobPayload: - """Return a validated payload ready for queue serialization.""" - - -@dataclass(frozen=True, slots=True) -class RuntimePayloadJobEnqueuer[RequestT: RuntimeJobRequestSource]: - """Queue a typed runtime request after validating its serialized payload.""" - - runtime: JobRuntime - entrypoint: JobEntrypoint - payload_serializer: RuntimeJobPayloadSerializer[RequestT] - - async def enqueue( - self, - request: RequestT, - *, - headers: Mapping[str, str] | None = None, - priority: int = 0, - execute_after: timedelta | None = None, - ) -> RuntimeJobId: - """Validate, serialize, and enqueue one runtime request.""" - payload = self.payload_serializer.serialize(request) - return await self.runtime.enqueue( - runtime_job_request_from_source( - request, - entrypoint=self.entrypoint, - payload=payload.model_dump_json().encode("utf-8"), - headers=headers, - priority=priority, - execute_after=execute_after, - ) - ) - - class RuntimeNoteFileDeleteJobPayload(BaseModel): """Serialized worker payload for materialized note-file cleanup.""" diff --git a/src/basic_memory/runtime/note_content.py b/src/basic_memory/runtime/note_content.py index a97eada8a..3efdd472e 100644 --- a/src/basic_memory/runtime/note_content.py +++ b/src/basic_memory/runtime/note_content.py @@ -136,7 +136,13 @@ def project_id(self) -> ProjectId: ... class RuntimeDeletedNoteEntitySource(RuntimeContentTypeSource, Protocol): - """Minimal deleted-note entity shape needed before row cleanup.""" + """Minimal deleted-note entity shape needed before row cleanup. + + The wide identity types are deliberate: downstream runtimes feed loosely + typed entity projections through this seam, so the delete live-update + identity is validated where the reference is built rather than trusted + from the declared shape. + """ @property def external_id(self) -> object | None: ... @@ -159,7 +165,7 @@ class RuntimeDeletedNoteEntityChecksumSource(Protocol): """Minimal deleted-note entity shape needed to guard file cleanup.""" @property - def checksum(self) -> object | None: ... + def checksum(self) -> RuntimeFileChecksum | None: ... class RuntimeDeletedNoteFileDeleteEntitySource( @@ -177,7 +183,7 @@ class RuntimeDeletedNoteFileChecksumSource(Protocol): """Minimal note_content shape needed to guard file cleanup.""" @property - def file_checksum(self) -> object | None: ... + def file_checksum(self) -> RuntimeFileChecksum | None: ... class RuntimeNoteContentStateSource(Protocol): @@ -212,7 +218,12 @@ def last_materialization_error(self) -> str | None: ... class RuntimePendingNoteMaterializationSource(Protocol): - """Minimal note_content row shape needed to queue materialization work.""" + """Minimal note_content row shape needed to queue materialization work. + + Input-typed on purpose: downstream runtimes replay these values from + persisted job payloads where a driver may deliver a string, so planning + coerces instead of trusting the declared shape. + """ @property def db_version(self) -> RuntimeNoteContentVersionInput: ... @@ -236,11 +247,16 @@ class RuntimeMaterializedNoteSource(Protocol): """Minimal note_content row shape needed to clean up a materialized file.""" @property - def file_checksum(self) -> object | None: ... + def file_checksum(self) -> RuntimeFileChecksum | None: ... class RuntimeNoteContentDbVersionSource(Protocol): - """Minimal note_content row shape needed to advance accepted DB versions.""" + """Minimal note_content row shape needed to advance accepted DB versions. + + db_version stays input-typed on purpose: downstream runtimes replay these + values from persisted job payloads where a driver may deliver a string, so + the version helpers coerce instead of trusting the declared shape. + """ @property def db_version(self) -> RuntimeNoteContentVersionInput: ... @@ -428,10 +444,8 @@ def select_deleted_note_file_checksum( ) -> RuntimeFileChecksum | None: """Choose the best accepted file checksum to guard deleted-note cleanup.""" if note_content is not None and note_content.file_checksum is not None: - return str(note_content.file_checksum) - if entity.checksum is not None: - return str(entity.checksum) - return None + return note_content.file_checksum + return entity.checksum def required_runtime_deleted_note_text( @@ -542,11 +556,7 @@ def plan_previous_materialized_note_file_delete( current_note_content: RuntimeMaterializedNoteSource | None, ) -> RuntimePendingNoteFileDelete | None: """Return old-file cleanup work when a moved note has materialized file state.""" - file_checksum = ( - str(current_note_content.file_checksum) - if current_note_content is not None and current_note_content.file_checksum is not None - else None - ) + file_checksum = current_note_content.file_checksum if current_note_content is not None else None return plan_previous_note_file_delete( project_id=project_id, entity_id=entity_id, diff --git a/src/basic_memory/runtime/storage.py b/src/basic_memory/runtime/storage.py index 900ae0202..7ec529c9e 100644 --- a/src/basic_memory/runtime/storage.py +++ b/src/basic_memory/runtime/storage.py @@ -282,51 +282,6 @@ def plan_runtime_storage_event_operations( return tuple(plan_runtime_storage_event_operation(event) for event in events) -@dataclass(frozen=True, slots=True) -class RuntimeStorageEventProcessingResult: - """Internal storage-event processing result for adapter handoffs.""" - - counts: RuntimeJobCounts - - @classmethod - def empty(cls) -> Self: - return cls(counts=RuntimeJobCounts()) - - @classmethod - def from_counts( - cls, - *, - processed: int = 0, - failed: int = 0, - skipped: int = 0, - ) -> Self: - return cls( - counts=RuntimeJobCounts( - processed=processed, - failed=failed, - skipped=skipped, - ) - ) - - def add(self, other: RuntimeStorageEventProcessingResult) -> Self: - return type(self)(counts=self.counts.add(other.counts)) - - def add_counts(self, counts: RuntimeJobCounts) -> Self: - return type(self)(counts=self.counts.add(counts)) - - def with_processed(self, count: int = 1) -> Self: - return type(self)(counts=self.counts.with_processed(count)) - - def with_failed(self, count: int = 1) -> Self: - return type(self)(counts=self.counts.with_failed(count)) - - def with_skipped(self, count: int = 1) -> Self: - return type(self)(counts=self.counts.with_skipped(count)) - - def as_dict(self) -> dict[str, int]: - return self.counts.as_dict() - - @dataclass(frozen=True, slots=True) class RuntimeStorageFileIndexRequest: """Typed request for indexing one observed runtime storage object.""" diff --git a/src/basic_memory/runtime/storage_events.py b/src/basic_memory/runtime/storage_events.py index c698d434a..f85b8e4a0 100644 --- a/src/basic_memory/runtime/storage_events.py +++ b/src/basic_memory/runtime/storage_events.py @@ -5,9 +5,9 @@ from typing import Protocol from basic_memory.runtime.storage import ( + RuntimeJobCounts, RuntimeStorageEventOperation, RuntimeStorageEventOperationKind, - RuntimeStorageEventProcessingResult, StorageBucketName, StorageEtag, StorageEventName, @@ -87,9 +87,9 @@ def events_by_bucket(self) -> dict[StorageBucketName, tuple[StorageEventPayload, async def run_runtime_storage_event_operations( events: Iterable[StorageEventPayload], processor: RuntimeStorageEventOperationProcessor, -) -> RuntimeStorageEventProcessingResult: +) -> RuntimeJobCounts: """Execute normalized storage event operations and count per-event outcomes.""" - result = RuntimeStorageEventProcessingResult.empty() + result = RuntimeJobCounts() for operation in plan_runtime_storage_event_operations(events): try: diff --git a/src/basic_memory/schemas/v2/__init__.py b/src/basic_memory/schemas/v2/__init__.py index 386c630a2..bd737f44b 100644 --- a/src/basic_memory/schemas/v2/__init__.py +++ b/src/basic_memory/schemas/v2/__init__.py @@ -17,6 +17,10 @@ GraphResponse, OrphanEntitiesResponse, ) +from basic_memory.schemas.v2.project_index import ( + ProjectIndexResponse, + ProjectIndexStartedResponse, +) from basic_memory.schemas.v2.resource import ( CreateResourceRequest, UpdateResourceRequest, @@ -37,6 +41,8 @@ "GraphNode", "GraphResponse", "OrphanEntitiesResponse", + "ProjectIndexResponse", + "ProjectIndexStartedResponse", "CreateResourceRequest", "UpdateResourceRequest", "ResourceResponse", diff --git a/src/basic_memory/schemas/v2/project_index.py b/src/basic_memory/schemas/v2/project_index.py new file mode 100644 index 000000000..a656281aa --- /dev/null +++ b/src/basic_memory/schemas/v2/project_index.py @@ -0,0 +1,20 @@ +"""V2 project-index route response schemas.""" + +from typing import Literal + +from pydantic import BaseModel, Field + +from basic_memory.schemas.project_index import ProjectIndexRunResponse + + +class ProjectIndexStartedResponse(BaseModel): + """Acknowledgement that a project-index run was scheduled in the background.""" + + status: Literal["index_started"] = "index_started" + message: str = Field(description="Human-readable scheduling confirmation") + + +# One project-index route, two outcomes: run_in_background schedules a run and +# acknowledges it; a foreground request runs the coordinator inline and reports +# its counts. +type ProjectIndexResponse = ProjectIndexRunResponse | ProjectIndexStartedResponse diff --git a/src/basic_memory/services/context_service.py b/src/basic_memory/services/context_service.py index 9ab2937fb..701729975 100644 --- a/src/basic_memory/services/context_service.py +++ b/src/basic_memory/services/context_service.py @@ -2,10 +2,9 @@ from __future__ import annotations -from contextlib import asynccontextmanager from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Any, AsyncIterator, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, List, Optional, Tuple, TYPE_CHECKING from loguru import logger @@ -102,13 +101,11 @@ def __init__( self.link_resolver = link_resolver self.session_maker = session_maker - @asynccontextmanager - async def _session_scope(self) -> AsyncIterator[AsyncSession]: - """Open a service-owned transaction for core repository reads.""" + def _require_session_maker(self) -> async_sessionmaker[AsyncSession]: + """Fail fast when a session-opening path runs without a session maker.""" if self.session_maker is None: # pragma: no cover raise ValueError("session_maker is required for ContextService") - async with db.scoped_session(self.session_maker) as session: - yield session + return self.session_maker async def build_context( self, @@ -189,7 +186,7 @@ async def build_context( ) if not primary and self.link_resolver: - async with self._session_scope() as session: + async with db.scoped_session(self._require_session_maker()) as session: entity = await self.link_resolver.resolve_link( path, use_search=True, @@ -251,7 +248,7 @@ async def build_context( phase="load_observations", result_count=len(entity_ids), ): - async with self._session_scope() as session: + async with db.scoped_session(self._require_session_maker()) as session: observations_by_entity = await self.observation_repository.find_by_entities( session, entity_ids ) diff --git a/src/basic_memory/services/directory_service.py b/src/basic_memory/services/directory_service.py index af63c9220..e128a2eab 100644 --- a/src/basic_memory/services/directory_service.py +++ b/src/basic_memory/services/directory_service.py @@ -3,9 +3,8 @@ import fnmatch import logging import os -from contextlib import asynccontextmanager from datetime import datetime -from typing import AsyncIterator, Dict, List, Optional, Sequence +from typing import Dict, List, Optional, Sequence from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -45,17 +44,11 @@ def __init__( self.entity_repository = entity_repository self.session_maker = session_maker - @asynccontextmanager - async def _session_scope(self) -> AsyncIterator[AsyncSession]: - """Open a service-owned transaction.""" - async with db.scoped_session(self.session_maker) as session: - yield session - async def get_directory_tree(self) -> DirectoryNode: """Build a hierarchical directory tree from indexed files.""" # Get all files from DB (flat list) - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entity_rows = await self.entity_repository.find_all(session) # Create a root directory node @@ -130,7 +123,7 @@ async def get_directory_structure(self) -> DirectoryNode: DirectoryNode tree containing only folders (type="directory") """ # Get unique directories without loading entities - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: directories = await self.entity_repository.get_distinct_directories(session) # Create a root directory node @@ -196,7 +189,7 @@ async def list_directory( # Optimize: Query only entities in the target directory # instead of loading the entire tree dir_prefix = dir_name.lstrip("/") - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entity_rows = await self.entity_repository.find_by_directory_prefix(session, dir_prefix) # Build a partial tree from only the relevant entities diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 83e66cdc8..4158fd815 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1,12 +1,11 @@ """Service for managing entities in the database.""" from collections.abc import Callable -from contextlib import asynccontextmanager from copy import deepcopy from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, AsyncIterator, List, Optional, Sequence, Tuple, Union +from typing import Any, List, Optional, Sequence, Tuple, Union import frontmatter import yaml @@ -218,18 +217,6 @@ def __init__( # Default returns None for local/CLI usage. Cloud overrides this to read from UserContext. self.get_user_id: Callable[[], Optional[str]] = lambda: None - @asynccontextmanager - async def _session_scope( - self, session: AsyncSession | None = None - ) -> AsyncIterator[AsyncSession]: - """Use the caller's session or open a service-owned transaction.""" - if session is not None: - yield session - return - - async with db.scoped_session(self.session_maker) as owned_session: - yield owned_session - async def detect_file_path_conflicts( self, file_path: str, @@ -258,7 +245,7 @@ async def detect_file_path_conflicts( # Load only file paths. Conflict detection is on the hot write path and # does not need observations or relations. - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: existing_paths = await self.repository.get_all_file_paths(active_session) # Use the enhanced conflict detection utility @@ -287,7 +274,7 @@ async def resolve_permalink( file_path_str = Path(file_path).as_posix() # Check for potential file path conflicts before resolving permalink - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: conflicts = await self.detect_file_path_conflicts( file_path_str, skip_check=skip_conflict_check, session=active_session ) @@ -881,7 +868,7 @@ async def create_entity(self, schema: EntitySchema) -> EntityModel: async def create_entity_with_content(self, schema: EntitySchema) -> EntityWriteResult: """Create a new entity and return both the entity row and written markdown.""" logger.debug(f"Creating entity: {schema.title}") - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: # --- Prepare Accepted State --- # Derive the canonical markdown/entity fields before touching the filesystem. prepared = await self.prepare_create_entity_content(schema, session=session) @@ -925,7 +912,7 @@ async def update_entity_with_content( f"Updating entity with permalink: {entity.permalink} content-type: {schema.content_type}" ) - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: # --- Read Current File State --- # Full replacements merge with existing frontmatter, so local mode still needs the current # file contents as input to the prepare step. @@ -1026,7 +1013,7 @@ async def delete_entity(self, permalink_or_id: str | int) -> bool: # Trigger: repository.delete returns False when entity is already gone (NoResultFound) # Why: concurrent delete_directory requests can race to delete the same entity # Outcome: treat as success since the entity is deleted either way - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: deleted = await self.repository.delete(session, entity.id) if not deleted: logger.info("Entity already removed from DB", entity_id=permalink_or_id) @@ -1039,7 +1026,7 @@ async def delete_entity(self, permalink_or_id: str | int) -> bool: async def get_by_permalink(self, permalink: str) -> EntityModel: """Get entity by type and name combination.""" logger.debug(f"Getting entity by permalink: {permalink}") - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: db_entity = await self.repository.get_by_permalink(session, permalink) if not db_entity: raise EntityNotFoundError(f"Entity not found: {permalink}") @@ -1048,18 +1035,18 @@ async def get_by_permalink(self, permalink: str) -> EntityModel: async def get_entities_by_id(self, ids: List[int]) -> Sequence[EntityModel]: """Get specific entities and their relationships.""" logger.debug(f"Getting entities: {ids}") - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: return await self.repository.find_by_ids(session, ids) async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]: """Get specific nodes and their relationships.""" logger.debug(f"Getting entities permalinks: {permalinks}") - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: return await self.repository.find_by_permalinks(session, permalinks) async def delete_entity_by_file_path(self, file_path: Union[str, Path]) -> None: """Delete entity by file path.""" - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: await self.repository.delete_by_file_path(session, str(file_path)) async def create_entity_from_markdown( @@ -1089,7 +1076,7 @@ async def create_entity_from_markdown( model.created_by = user_id model.last_updated_by = user_id - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: # Use UPSERT to handle conflicts cleanly try: return await self.repository.upsert_entity(active_session, model) @@ -1112,7 +1099,7 @@ async def update_entity_and_observations( """ logger.debug(f"Updating entity and observations: {file_path}") - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: if existing_entity is not None: db_entity = await self.repository.get_by_id( active_session, @@ -1196,7 +1183,7 @@ async def upsert_entity_from_markdown( session: AsyncSession | None = None, ) -> EntityModel: """Create/update entity and relations from parsed markdown.""" - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: if is_new: created = await self.create_entity_from_markdown( file_path, markdown, session=active_session @@ -1234,7 +1221,7 @@ async def update_entity_relations( entity_id = entity.id logger.debug(f"Updating relations for entity: {entity.file_path}") - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: # Clear existing relations first await self.relation_repository.delete_outgoing_relations_from_entity( active_session, entity_id @@ -1344,7 +1331,7 @@ async def _resolve_deferred_self_relation( # Title-only links are ambiguous because Basic Memory allows duplicate titles. # Collapse them to self only when the title lookup proves this source is the sole candidate; # otherwise leave the relation unresolved so we do not create a wrong permanent edge. - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: title_matches = await self.repository.get_by_title( active_session, clean_target, load_relations=False ) @@ -1412,7 +1399,7 @@ async def edit_entity_with_content( file_path = Path(entity.file_path) current_content, _ = await self.file_service.read_file(file_path) - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: # --- Prepare Against Explicit Base Content --- # The edit operation is the semantic step; file/DB writes below are just persistence of that # accepted result. @@ -1777,7 +1764,7 @@ async def move_entity( updates["checksum"] = new_checksum # 9. Update database - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: updated_entity = await self.repository.update(session, entity.id, updates) if not updated_entity: raise ValueError(f"Failed to update entity in database: {entity.id}") @@ -1831,7 +1818,7 @@ async def move_directory( destination_directory = destination_directory.strip("/") # Find all entities in the source directory - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entities = await self.repository.find_by_directory_prefix(session, source_directory) if not entities: @@ -1916,7 +1903,7 @@ async def delete_directory( directory = directory.strip("/") # Find all entities in the directory - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entities = await self.repository.find_by_directory_prefix(session, directory) if not entities: diff --git a/src/basic_memory/services/link_resolver.py b/src/basic_memory/services/link_resolver.py index b17ef6c8f..a3e375ff4 100644 --- a/src/basic_memory/services/link_resolver.py +++ b/src/basic_memory/services/link_resolver.py @@ -1,8 +1,7 @@ """Service and helpers for resolving markdown links and permalink-like identifiers.""" import uuid as uuid_mod -from contextlib import asynccontextmanager -from typing import Any, AsyncIterator, Optional, Tuple, Dict +from typing import Any, Optional, Tuple, Dict from loguru import logger from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -99,18 +98,6 @@ def __init__( self._entity_repository_cache: Dict[int, EntityRepository] = {} self._search_service_cache: Dict[int, SearchService] = {} - @asynccontextmanager - async def _session_scope( - self, session: AsyncSession | None = None - ) -> AsyncIterator[AsyncSession]: - """Use the caller's session or open a service-owned transaction.""" - if session is not None: - yield session - return - - async with db.scoped_session(self.session_maker) as owned_session: - yield owned_session - async def resolve_link( self, link_text: str, @@ -137,7 +124,7 @@ async def resolve_link( explicit_project_reference = "::" in clean_text clean_text = normalize_project_reference(clean_text) - async with self._session_scope(session) as active_session: + async with db.scoped_session(self.session_maker, session) as active_session: # --- External ID Resolution --- # Try external_id first if identifier looks like a UUID. # Canonicalize to lowercase-hyphen form so uppercase or unhyphenated diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index ae123af41..d4f5be525 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -3,10 +3,9 @@ import asyncio import ast import re -from contextlib import asynccontextmanager from dataclasses import dataclass from datetime import datetime -from typing import Any, AsyncIterator, List, Optional, Set, Dict +from typing import Any, List, Optional, Set, Dict from dateparser import parse from fastapi import BackgroundTasks @@ -126,18 +125,6 @@ def __init__( self.file_service = file_service self.session_maker = session_maker - @asynccontextmanager - async def _session_scope( - self, session: AsyncSession | None = None - ) -> AsyncIterator[AsyncSession]: - """Use the caller's session or open a service-owned transaction.""" - if session is not None: - yield session - return - - async with db.scoped_session(self.session_maker) as owned_session: - yield owned_session - async def init_search_index(self): """Create FTS5 virtual table if it doesn't exist.""" await self.repository.init_search_index() @@ -165,7 +152,7 @@ async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) # Reindex all entities logger.debug("Indexing entities") - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entities = await self.entity_repository.find_all(session) for entity in entities: await self.index_entity(entity, background_tasks) @@ -561,7 +548,7 @@ async def index_entity_data( async def sync_entity_vectors(self, entity_id: int) -> None: """Refresh vector chunks for one entity in repositories that support semantic indexing.""" - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entity = await self.entity_repository.find_by_id(session, entity_id) if entity is None: await self._clear_entity_vectors(entity_id) @@ -586,7 +573,7 @@ async def sync_entity_vectors_batch( entities_failed=0, ) - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entities_by_id = { entity.id: entity for entity in await self.entity_repository.find_by_ids(session, entity_ids) @@ -680,7 +667,7 @@ async def reindex_vectors(self, progress_callback=None, force_full: bool = False Returns: dict with stats: total_entities, embedded, skipped, errors """ - async with self._session_scope() as session: + async with db.scoped_session(self.session_maker) as session: entities = await self.entity_repository.find_all(session) entity_ids = [entity.id for entity in entities] diff --git a/tests/api/v2/test_knowledge_router.py b/tests/api/v2/test_knowledge_router.py index bb37f0158..73a09fa37 100644 --- a/tests/api/v2/test_knowledge_router.py +++ b/tests/api/v2/test_knowledge_router.py @@ -1349,6 +1349,8 @@ async def test_delete_directory_v2_success(client: AsyncClient, v2_project_url): assert result.successful_deletes == 3 assert result.failed_deletes == 0 assert len(result.deleted_files) == 3 + # Runtime cleanup fields ride alongside the client schema in the raw payload. + assert response.json()["file_delete_status"] == "pending" # Verify entity is no longer accessible get_response = await client.get( @@ -1366,10 +1368,16 @@ async def test_delete_directory_v2_empty_directory(client: AsyncClient, v2_proje response = await client.post(f"{v2_project_url}/knowledge/delete-directory", json=delete_data) assert response.status_code == 200 - result = DirectoryDeleteResult.model_validate(response.json()) - assert result.total_files == 0 - assert result.successful_deletes == 0 - assert result.failed_deletes == 0 + # Exact snapshot of the route JSON: the typed result must keep serializing + # the existing directory-delete response contract byte-for-byte. + assert response.json() == { + "total_files": 0, + "successful_deletes": 0, + "failed_deletes": 0, + "deleted_files": [], + "errors": [], + "file_delete_status": "complete", + } @pytest.mark.asyncio diff --git a/tests/api/v2/test_project_index_command.py b/tests/api/v2/test_project_index_command.py index 7b8a7a58f..f39c43cb4 100644 --- a/tests/api/v2/test_project_index_command.py +++ b/tests/api/v2/test_project_index_command.py @@ -6,16 +6,17 @@ from basic_memory.api.v2.routers.project_router import index_project from basic_memory.config import ProjectConfig -from basic_memory.deps.services import ProjectIndexRouteRequest +from basic_memory.index.local_project import ProjectIndexRouteRequest +from basic_memory.schemas.v2 import ProjectIndexResponse, ProjectIndexStartedResponse class RecordingProjectIndexCommand: def __init__(self) -> None: self.request: ProjectIndexRouteRequest | None = None - async def index_project(self, request: ProjectIndexRouteRequest) -> dict[str, str]: + async def index_project(self, request: ProjectIndexRouteRequest) -> ProjectIndexResponse: self.request = request - return {"status": "delegated"} + return ProjectIndexStartedResponse(message="delegated") @pytest.mark.asyncio @@ -31,7 +32,7 @@ async def test_project_index_route_delegates_to_command_dependency() -> None: run_in_background=False, ) - assert response == {"status": "delegated"} + assert response == ProjectIndexStartedResponse(message="delegated") assert command.request is not None assert command.request.project_id == 5 assert command.request.project_name == "moby-dick" diff --git a/tests/api/v2/test_project_router.py b/tests/api/v2/test_project_router.py index a771248a3..8779ae6f0 100644 --- a/tests/api/v2/test_project_router.py +++ b/tests/api/v2/test_project_router.py @@ -380,6 +380,44 @@ async def test_project_index_uses_event_indexer_not_sync_service( assert "new" not in data +@pytest.mark.asyncio +async def test_project_index_foreground_response_payload_snapshot( + client: AsyncClient, + test_project: Project, + v2_projects_url, +): + """The typed response union must keep the foreground payload byte-identical.""" + note_path = Path(test_project.path) / "incoming" / "index-snapshot.md" + note_path.parent.mkdir(parents=True, exist_ok=True) + note_path.write_text("# Index Snapshot\n\nOne indexable file.\n", encoding="utf-8") + + response = await client.post( + f"{v2_projects_url}/{test_project.external_id}/index", + params={"run_in_background": False}, + ) + + assert response.status_code == 200 + assert response.content == ( + b'{"total_files":1,"enqueued_files":1,"enqueued_batches":1,"deleted_files":0}' + ) + + +@pytest.mark.asyncio +async def test_project_index_background_response_payload_snapshot( + client: AsyncClient, + test_project: Project, + v2_projects_url, +): + """The typed response union must keep the background payload byte-identical.""" + response = await client.post(f"{v2_projects_url}/{test_project.external_id}/index") + + assert response.status_code == 200 + expected_message = f"Filesystem indexing initiated for project '{test_project.name}'" + assert response.content == ( + f'{{"status":"index_started","message":"{expected_message}"}}'.encode() + ) + + @pytest.mark.asyncio async def test_project_status_uses_event_index_report_not_sync_service( client: AsyncClient, diff --git a/tests/cli/test_command_utils.py b/tests/cli/test_command_utils.py index 517ad306c..c8b6d1e40 100644 --- a/tests/cli/test_command_utils.py +++ b/tests/cli/test_command_utils.py @@ -2,7 +2,7 @@ import basic_memory.cloud.note_content_materialization as note_content_materialization import basic_memory.db as db -import basic_memory.deps.services as deps_services +import basic_memory.index.local_schedulers as local_schedulers from basic_memory.cli.commands.command_utils import run_with_cleanup @@ -27,7 +27,7 @@ async def fake_shutdown() -> None: "drain_pending_materializations", fake_drain_materializations, ) - monkeypatch.setattr(deps_services, "drain_background_tasks", fake_drain_background) + monkeypatch.setattr(local_schedulers, "drain_background_tasks", fake_drain_background) monkeypatch.setattr(db, "shutdown_db", fake_shutdown) async def work() -> int: diff --git a/tests/cloud/test_cloud_services.py b/tests/cloud/test_cloud_services.py index 8b4e9f19b..0766d7e54 100644 --- a/tests/cloud/test_cloud_services.py +++ b/tests/cloud/test_cloud_services.py @@ -769,12 +769,13 @@ async def test_directory_delete_service_uses_injected_runtime_and_session_maker( ), ) - status_code, payload = await service.delete_directory( + result = await service.delete_directory( project_external_id="project-123", directory="/notes/", ) - assert status_code == 200 + assert result.http_status_code == 200 + payload = result.to_response_payload() assert payload["file_delete_status"] == "pending" assert payload["deleted_files"] == ["notes/example.md"] assert enqueuer.requests == [ @@ -826,12 +827,12 @@ async def delete_directory_entities( ), ) - status_code, _ = await service.delete_directory( + result = await service.delete_directory( project_external_id="project-123", directory="/notes/", ) - assert status_code == 200 + assert result.http_status_code == 200 # Ids arrive sorted so reindex order is deterministic. assert refresher.refreshed == [[42, 99]] diff --git a/tests/cloud/test_project_deletes.py b/tests/cloud/test_project_deletes.py index b77504fa1..a768bbfbe 100644 --- a/tests/cloud/test_project_deletes.py +++ b/tests/cloud/test_project_deletes.py @@ -12,6 +12,7 @@ from basic_memory.models import Base as BasicMemoryBase from basic_memory.models import Project from basic_memory.runtime.jobs import RuntimeJobId, RuntimeProjectDeleteJobRequest +from basic_memory.schemas.project_info import ProjectItem from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.pool import StaticPool @@ -35,7 +36,8 @@ async def tenant_session_maker() -> AsyncGenerator[async_sessionmaker[AsyncSessi async def create_project( session_maker: async_sessionmaker[AsyncSession], *, - is_default: bool = False, + # None mirrors the nullable column default; the accepted response maps it to False. + is_default: bool | None = None, ) -> Project: async with session_maker() as session: project = Project( @@ -106,7 +108,13 @@ async def test_project_delete_acceptance_soft_deletes_and_queues_runtime_request ] assert result.to_response_payload()["job_id"] == "123" assert result.to_response_payload()["file_delete_status"] == "pending" - assert result.old_project.external_id == "project-main" + assert result.old_project == ProjectItem( + id=project.id, + external_id="project-main", + name="Main", + path="basic-memory", + is_default=False, + ) @pytest.mark.asyncio diff --git a/tests/indexing/test_default_accepted_note_repositories.py b/tests/index/test_local_accepted_note_repositories.py similarity index 53% rename from tests/indexing/test_default_accepted_note_repositories.py rename to tests/index/test_local_accepted_note_repositories.py index 72c35cada..44037e511 100644 --- a/tests/indexing/test_default_accepted_note_repositories.py +++ b/tests/index/test_local_accepted_note_repositories.py @@ -1,19 +1,22 @@ -"""Tests for the default accepted-note repository provider.""" +"""Tests for the local accepted-note repository bundle.""" -from basic_memory.indexing.accepted_note_mutation_runner import ( - DefaultAcceptedNoteRepositories, - build_default_accepted_note_repositories, -) +from basic_memory.index.local_notes import LocalAcceptedNoteRepositories +from basic_memory.indexing.accepted_note_mutation_runner import AcceptedNoteMutationRepositories +from basic_memory.indexing.accepted_note_write_runner import AcceptedNoteWriteRepositories from basic_memory.repository import NoteContentRepository from basic_memory.repository.accepted_note_search_repository import AcceptedNoteSearchRepository from basic_memory.repository.entity_repository import EntityRepository -def test_build_default_accepted_note_repositories_wires_core_repositories() -> None: - """The default provider should satisfy lookup and write repository needs.""" - repositories = build_default_accepted_note_repositories() +def test_local_accepted_note_repositories_wires_project_scoped_repositories() -> None: + """One concrete bundle satisfies lookup and write repository needs per project.""" + repositories = LocalAcceptedNoteRepositories() + + # The same bundle instance serves both capability seams of the mutation runner. + lookup_repositories: AcceptedNoteMutationRepositories = repositories + write_repositories: AcceptedNoteWriteRepositories = repositories + assert lookup_repositories is write_repositories - assert isinstance(repositories, DefaultAcceptedNoteRepositories) assert isinstance(repositories.entity_repository(7), EntityRepository) assert repositories.entity_repository(7).project_id == 7 assert isinstance(repositories.pending_entity_repository(8), EntityRepository) diff --git a/tests/index/test_local_project_index.py b/tests/index/test_local_project_index.py index 828ba8fb2..0a90688a3 100644 --- a/tests/index/test_local_project_index.py +++ b/tests/index/test_local_project_index.py @@ -58,7 +58,7 @@ ResolvedRelationTarget, UnresolvedRelation, ) -from basic_memory.models import Entity, Project +from basic_memory.models import Entity, Project, Relation from basic_memory.repository import EntityRepository from basic_memory.repository.note_content_repository import NoteContentRepository from basic_memory.runtime.jobs import ( @@ -2457,12 +2457,13 @@ async def find_unresolved_relations_for_entity( async def update( self, session: AsyncSession, - entity_id: int, - entity_data: dict[str, object], - ) -> object | None: + relation_id: int, + resolved_target_fields: dict[str, int | str], + /, + ) -> Relation | None: return None - async def delete(self, session: AsyncSession, entity_id: int) -> bool: + async def delete(self, session: AsyncSession, relation_id: int, /) -> bool: return False diff --git a/tests/deps/test_task_failure_callback.py b/tests/index/test_local_scheduler_task_failure.py similarity index 77% rename from tests/deps/test_task_failure_callback.py rename to tests/index/test_local_scheduler_task_failure.py index 601702efe..29ea7c204 100644 --- a/tests/deps/test_task_failure_callback.py +++ b/tests/index/test_local_scheduler_task_failure.py @@ -5,7 +5,7 @@ import pytest -from basic_memory.deps.services import _log_task_failure +from basic_memory.index.local_schedulers import _log_task_failure @pytest.mark.asyncio @@ -18,7 +18,7 @@ async def slow(): with pytest.raises(asyncio.CancelledError): await task - with patch("basic_memory.deps.services.logger.exception") as mock_exc: + with patch("basic_memory.index.local_schedulers.logger.exception") as mock_exc: _log_task_failure(task) mock_exc.assert_not_called() @@ -32,7 +32,7 @@ async def boom(): with pytest.raises(ValueError): await task - with patch("basic_memory.deps.services.logger.exception") as mock_exc: + with patch("basic_memory.index.local_schedulers.logger.exception") as mock_exc: _log_task_failure(task) mock_exc.assert_called_once() assert "sync failed" in str(mock_exc.call_args) diff --git a/tests/services/test_task_scheduler_semantic.py b/tests/index/test_local_schedulers.py similarity index 94% rename from tests/services/test_task_scheduler_semantic.py rename to tests/index/test_local_schedulers.py index 92f2f5464..75bcd154f 100644 --- a/tests/services/test_task_scheduler_semantic.py +++ b/tests/index/test_local_schedulers.py @@ -6,7 +6,7 @@ import pytest from basic_memory.indexing.project_index_coordinator import ProjectIndexCoordinatorResult -from basic_memory.deps.services import ( +from basic_memory.index.local_schedulers import ( LocalEntityVectorSyncScheduler, LocalProjectIndexScheduler, LocalRelationResolutionScheduler, @@ -57,7 +57,7 @@ async def test_entity_vector_scheduler_maps_to_search_service(): def _clear_project_index_scheduler_state() -> None: - from basic_memory.deps.services import _dirty_project_index, _pending_project_index + from basic_memory.index.local_schedulers import _dirty_project_index, _pending_project_index _pending_project_index.clear() _dirty_project_index.clear() @@ -105,7 +105,7 @@ async def test_project_index_scheduler_coalesces_requests_during_in_flight_run() """While a run is in flight, new requests must not start a second concurrent run over the same rows; they coalesce to exactly one trailing rerun that keeps the strongest force_full seen.""" - from basic_memory.deps.services import _dirty_project_index, _pending_project_index + from basic_memory.index.local_schedulers import _dirty_project_index, _pending_project_index _clear_project_index_scheduler_state() runner = GatedProjectIndexRunner() @@ -154,7 +154,7 @@ async def index_project( async def test_project_index_scheduler_reruns_coalesced_request_after_failed_run(): """A request coalesced behind a run that raises must still get its rerun — a failed run is exactly when the coalesced request most needs its retry.""" - from basic_memory.deps.services import _dirty_project_index, _pending_project_index + from basic_memory.index.local_schedulers import _dirty_project_index, _pending_project_index _clear_project_index_scheduler_state() runner = FailingThenGatedProjectIndexRunner() @@ -196,7 +196,7 @@ async def test_project_index_scheduler_single_flight_is_per_project(): @pytest.mark.asyncio async def test_project_index_scheduler_is_noop_in_test_mode(): """Test mode must suppress the run without leaking a pending marker.""" - from basic_memory.deps.services import _pending_project_index + from basic_memory.index.local_schedulers import _pending_project_index _clear_project_index_scheduler_state() runner = StubProjectIndexRunner() @@ -239,7 +239,7 @@ async def resolve_relations(self, entity_id: int | None = None) -> set[int]: @pytest.mark.asyncio async def test_relation_resolution_scheduler_runs_project_resolution(): """A single write schedules one debounced project resolution pass.""" - from basic_memory.deps.services import _pending_relation_resolution + from basic_memory.index.local_schedulers import _pending_relation_resolution _pending_relation_resolution.clear() runtime = StubRelationResolutionRuntime() @@ -260,7 +260,7 @@ async def test_relation_resolution_scheduler_runs_project_resolution(): @pytest.mark.asyncio async def test_relation_resolution_scheduler_coalesces_a_burst(): """A burst of writes collapses to a single project resolution pass.""" - from basic_memory.deps.services import _pending_relation_resolution + from basic_memory.index.local_schedulers import _pending_relation_resolution _pending_relation_resolution.clear() runtime = StubRelationResolutionRuntime() @@ -282,7 +282,7 @@ async def test_relation_resolution_scheduler_coalesces_a_burst(): async def test_relation_resolution_scheduler_reruns_for_write_during_pass(): """A write that commits while a pass is scanning must trigger a follow-up pass, not be dropped by coalescing (the scan already read the unresolved rows).""" - from basic_memory.deps.services import ( + from basic_memory.index.local_schedulers import ( _dirty_relation_resolution, _pending_relation_resolution, ) @@ -343,7 +343,7 @@ async def test_drain_background_tasks_awaits_scheduled_work(): async def test_drain_background_tasks_covers_follow_up_tasks(): """A drained task can schedule a follow-up (the relation-resolution dirty re-run); the drain must wait for that wave too, not just the first snapshot.""" - from basic_memory.deps.services import ( + from basic_memory.index.local_schedulers import ( _dirty_relation_resolution, _pending_relation_resolution, ) @@ -384,7 +384,7 @@ async def resolve_relations(self, entity_id: int | None = None) -> set[int]: @pytest.mark.asyncio async def test_relation_resolution_scheduler_is_noop_in_test_mode(): """Test mode should suppress the background resolution pass entirely.""" - from basic_memory.deps.services import _pending_relation_resolution + from basic_memory.index.local_schedulers import _pending_relation_resolution _pending_relation_resolution.clear() runtime = StubRelationResolutionRuntime() diff --git a/tests/index/test_local_watch_orchestration.py b/tests/index/test_local_watch_orchestration.py index ad4a05e8f..745c95ef4 100644 --- a/tests/index/test_local_watch_orchestration.py +++ b/tests/index/test_local_watch_orchestration.py @@ -33,8 +33,8 @@ ) from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.storage import ( + RuntimeJobCounts, RuntimeStorageEventOperation, - RuntimeStorageEventProcessingResult, ) @@ -199,6 +199,8 @@ def test_local_watch_request_builds_storage_prefix_from_project(tmp_path: Path) note_path = project_root / "notes" / "a.md" note_path.parent.mkdir() note_path.write_text("# A\n", encoding="utf-8") + # No permalink/name attributes at all: a structural caller without the + # optional identity shape falls back to the root directory name. project = SimpleNamespace(path=str(project_root)) request = LocalWatchEventIndexRequest.from_project_changes( @@ -213,6 +215,23 @@ def test_local_watch_request_builds_storage_prefix_from_project(tmp_path: Path) assert request.project_prefix == "configured-project-root" assert [event.object_key for event in source.events()] == ["configured-project-root/notes/a.md"] + # None-valued identity attributes also fall back to the directory name, + # and a Path-typed project path is coerced rather than crashing. + none_identity_request = LocalWatchEventIndexRequest.from_project_changes( + project=SimpleNamespace(path=project_root, permalink=None, name=None), + changes=((Change.added, str(note_path)),), + ) + + assert none_identity_request.project_root == project_root.resolve() + assert none_identity_request.project_prefix == "configured-project-root" + + # A project without a usable path has no watch root at all. + with pytest.raises(ValueError, match="requires path"): + LocalWatchEventIndexRequest.from_project_changes( + project=SimpleNamespace(path=""), + changes=(), + ) + def test_local_watch_request_uses_project_permalink_for_duplicate_leaf_roots( tmp_path: Path, @@ -453,7 +472,7 @@ def test_local_watch_path_visibility_uses_project_relative_hidden_parts(tmp_path def test_local_watch_status_update_plans_success() -> None: update = plan_local_watch_event_index_status_update( project_prefix="configured-project-root", - result=RuntimeStorageEventProcessingResult.empty().with_processed(2).with_skipped(1), + result=RuntimeJobCounts(processed=2, skipped=1), ) assert update.path == "configured-project-root" @@ -468,10 +487,7 @@ def test_local_watch_status_update_plans_success() -> None: def test_local_watch_status_update_plans_failure_details() -> None: update = plan_local_watch_event_index_status_update( project_prefix="configured-project-root", - result=RuntimeStorageEventProcessingResult.empty() - .with_processed(1) - .with_failed(2) - .with_skipped(3), + result=RuntimeJobCounts(processed=1, failed=2, skipped=3), ) assert update.path == "configured-project-root" diff --git a/tests/index/test_storage_event_orchestration.py b/tests/index/test_storage_event_orchestration.py index 9c49c59f4..cbf9f7a5a 100644 --- a/tests/index/test_storage_event_orchestration.py +++ b/tests/index/test_storage_event_orchestration.py @@ -17,8 +17,8 @@ ) from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.storage import ( + RuntimeJobCounts, RuntimeStorageEventOperation, - RuntimeStorageEventProcessingResult, StorageBucketName, StorageEventPayload, StorageObjectIdentity, @@ -149,13 +149,13 @@ async def process_bucket_context_events( bucket_name: StorageBucketName, context: BucketRuntimeContext, events: tuple[StorageEventPayload, ...], - ) -> RuntimeStorageEventProcessingResult: + ) -> RuntimeJobCounts: self.calls.append( (bucket_name, context.runtime_name, tuple(event.object_key for event in events)) ) if bucket_name == self.fail_bucket: raise RuntimeError("bucket context failed") - return RuntimeStorageEventProcessingResult.from_counts(processed=len(events)) + return RuntimeJobCounts(processed=len(events)) async def bucket_failed( self, diff --git a/tests/indexing/test_accepted_note_write_runner.py b/tests/indexing/test_accepted_note_write_runner.py index c2591c6a9..a406c6fc0 100644 --- a/tests/indexing/test_accepted_note_write_runner.py +++ b/tests/indexing/test_accepted_note_write_runner.py @@ -13,14 +13,12 @@ from basic_memory.indexing.accepted_note_search import AcceptedNoteSearchRow from basic_memory.indexing.accepted_note_write_runner import ( - DefaultAcceptedNoteWriteRepositories, AcceptedNoteWriteRepositories, accept_note_content_write, accepted_note_content_write_from_markdown, accepted_note_search_row_from_entity, accepted_pending_entity_write_from_prepared, apply_accepted_prepared_entity_fields, - build_default_accepted_note_write_repositories, create_accepted_pending_entity, delete_accepted_note, delete_accepted_note_entity, @@ -33,10 +31,8 @@ delete_accepted_note_search_index, ) from basic_memory.models import Entity, NoteContent -from basic_memory.repository import AcceptedNoteContentWrite, NoteContentRepository -from basic_memory.repository.accepted_note_search_repository import AcceptedNoteSearchRepository +from basic_memory.repository import AcceptedNoteContentWrite from basic_memory.repository.entity_repository import AcceptedPendingEntityWrite -from basic_memory.repository.entity_repository import EntityRepository from basic_memory.schemas.base import Entity as EntitySchema @@ -369,15 +365,6 @@ def _note_content() -> NoteContent: ) -def test_build_default_accepted_note_write_repositories_wires_core_repositories() -> None: - repositories = build_default_accepted_note_write_repositories() - - assert isinstance(repositories, DefaultAcceptedNoteWriteRepositories) - assert isinstance(repositories.pending_entity_repository(7), EntityRepository) - assert isinstance(repositories.note_content_repository(7), NoteContentRepository) - assert isinstance(repositories.search_repository(7), AcceptedNoteSearchRepository) - - @pytest.mark.asyncio async def test_prepare_accepted_note_create_hashes_prepared_markdown() -> None: session = cast(AsyncSession, object()) @@ -810,10 +797,12 @@ async def test_delete_accepted_note_entity_deletes_via_session() -> None: async def test_delete_accepted_note_plans_missing_response_without_deleting() -> None: session = _DeleteSession() + # The fail-fast provider proves a missing entity touches no repository. accepted = await delete_accepted_note( cast(AsyncSession, session), project_id=7, entity=None, + repositories=_repository_provider(), ) assert session.deleted == [] diff --git a/tests/indexing/test_change_planning.py b/tests/indexing/test_change_planning.py index 46d117bc6..7bd453cfb 100644 --- a/tests/indexing/test_change_planning.py +++ b/tests/indexing/test_change_planning.py @@ -15,6 +15,7 @@ FileMoveCandidate, plan_change_detection_snapshot, plan_file_changes, + plan_move_target_checksums, storage_checksums_from_sources, ) from basic_memory.indexing.change_detector import ( @@ -49,7 +50,10 @@ def test_plan_change_detection_snapshot_maps_typed_runtime_state() -> None: move_candidates=(FileMoveCandidate(path="old/moved.md", checksum="moved-checksum"),), ) - assert snapshot.new_file_checksum_by_path == { + assert plan_move_target_checksums( + storage_checksum_by_path=snapshot.storage_checksum_by_path, + db_checksum_by_path=snapshot.db_checksum_by_path, + ) == { "new/moved.md": "moved-checksum", "new.md": "new-file-checksum", } @@ -83,15 +87,14 @@ def test_plan_file_changes_keeps_unobservable_files_out_of_deletes() -> None: ) -def test_new_file_checksum_by_path_excludes_unknown_checksums() -> None: +def test_plan_move_target_checksums_excludes_unknown_checksums() -> None: """Unknown checksums carry no content evidence, so they cannot claim a move.""" - snapshot = ChangeDetectionSnapshot( + move_target_checksums = plan_move_target_checksums( storage_checksum_by_path={"new.md": "new-checksum", "unreadable.md": None}, db_checksum_by_path={}, - all_db_paths=(), ) - assert snapshot.new_file_checksum_by_path == {"new.md": "new-checksum"} + assert move_target_checksums == {"new.md": "new-checksum"} def test_plan_file_changes_detects_new_modified_unchanged_and_deleted_files() -> None: diff --git a/tests/indexing/test_directory_delete_runner.py b/tests/indexing/test_directory_delete_runner.py index b6f4efde7..06ea8b254 100644 --- a/tests/indexing/test_directory_delete_runner.py +++ b/tests/indexing/test_directory_delete_runner.py @@ -149,6 +149,7 @@ def directory_snapshot( def test_directory_delete_result_serializes_empty_complete_shape() -> None: result = DirectoryDeleteAcceptedResult.complete() + assert result.http_status_code == 200 assert result.to_response_payload() == { "total_files": 0, "successful_deletes": 0, @@ -228,6 +229,8 @@ def test_directory_delete_result_serializes_failed_enqueue_error() -> None: error="queue unavailable", ) + # Files remain on disk with their DB rows gone: the route must report 500. + assert result.http_status_code == 500 assert result.to_response_payload() == { "total_files": 1, "successful_deletes": 1, diff --git a/tests/indexing/test_forward_reference_resolution.py b/tests/indexing/test_forward_reference_resolution.py index 03b48f395..60f6e4833 100644 --- a/tests/indexing/test_forward_reference_resolution.py +++ b/tests/indexing/test_forward_reference_resolution.py @@ -25,11 +25,13 @@ from basic_memory.models import Entity -@dataclass(frozen=True, slots=True) +# Not frozen: UnresolvedRelation declares plain (writable) attribute members. +@dataclass(slots=True) class StubUnresolvedRelation: id: int from_id: int - to_name: str | None + to_name: str + relation_type: str = "related_to" class RecordingForwardReferenceRuntime: @@ -126,7 +128,7 @@ def test_collect_forward_reference_link_texts_dedupes_in_first_seen_order() -> N StubUnresolvedRelation(id=1, from_id=10, to_name="Target"), StubUnresolvedRelation(id=2, from_id=11, to_name="Other"), StubUnresolvedRelation(id=3, from_id=12, to_name="Target"), - StubUnresolvedRelation(id=4, from_id=13, to_name=None), + StubUnresolvedRelation(id=4, from_id=13, to_name=""), StubUnresolvedRelation(id=5, from_id=14, to_name=""), ] @@ -138,7 +140,7 @@ def test_plan_forward_reference_resolution_filters_only_exact_safe_updates() -> StubUnresolvedRelation(id=1, from_id=10, to_name="Target"), StubUnresolvedRelation(id=2, from_id=11, to_name="Missing"), StubUnresolvedRelation(id=3, from_id=12, to_name="Self"), - StubUnresolvedRelation(id=4, from_id=13, to_name=None), + StubUnresolvedRelation(id=4, from_id=13, to_name=""), StubUnresolvedRelation(id=5, from_id=14, to_name="Target"), ] @@ -459,7 +461,7 @@ async def test_run_forward_reference_resolution_skips_resolution_without_link_te result = await run_forward_reference_resolution( runtime, - (StubUnresolvedRelation(id=1, from_id=10, to_name=None),), + (StubUnresolvedRelation(id=1, from_id=10, to_name=""),), ) assert result.resolved_link_text_count == 0 diff --git a/tests/indexing/test_index_batch_runtime.py b/tests/indexing/test_index_batch_runtime.py index 61ccafbcd..fa85d5ab5 100644 --- a/tests/indexing/test_index_batch_runtime.py +++ b/tests/indexing/test_index_batch_runtime.py @@ -15,7 +15,6 @@ from basic_memory.config import BasicMemoryConfig from basic_memory.indexing.batch_indexer import BatchIndexer from basic_memory.indexing.index_batch_runtime import ( - DefaultIndexBatchRuntime, IndexBatchRuntime, build_default_index_batch_runtime, count_search_indexed_entities, @@ -121,16 +120,14 @@ async def reconcile( raise RuntimeError(f"note_content failed for {entity.id}") -@dataclass(slots=True) -class RecordingIndexedNoteContentTimestampProvider: - def observed_at( - self, - indexed: IndexedEntity, - file_info: FakeFileInfo | None, - ) -> datetime | None: - _ = indexed - assert file_info is not None - return file_info.last_modified +def recording_indexed_note_content_timestamps( + indexed: IndexedEntity, + file_info: FakeFileInfo | None, +) -> datetime | None: + """Test stand-in for the injected IndexedNoteContentObservedAt callable.""" + _ = indexed + assert file_info is not None + return file_info.last_modified @pytest.mark.asyncio @@ -194,7 +191,7 @@ async def fake_scoped_session( entity_repository=repository, session_maker=session_maker, note_content_reconciler=reconciler, - timestamp_provider=RecordingIndexedNoteContentTimestampProvider(), + timestamp_provider=recording_indexed_note_content_timestamps, ) files = { "ok.md": FakeFileInfo( @@ -291,17 +288,13 @@ def test_build_default_index_batch_runtime_composes_repository_backed_stack() -> session_maker=session_maker, ) - assert isinstance(runtime, DefaultIndexBatchRuntime) + assert isinstance(runtime, IndexBatchRuntime) assert isinstance(runtime.note_content_reconciler, NoteContentReconciler) + assert runtime.content_type_provider is content_type_provider + assert runtime.entity_repository is entity_repository + assert runtime.session_maker is session_maker - batch_runtime = runtime.batch_runtime - assert isinstance(batch_runtime, IndexBatchRuntime) - assert batch_runtime.content_type_provider is content_type_provider - assert batch_runtime.entity_repository is entity_repository - assert batch_runtime.session_maker is session_maker - assert batch_runtime.note_content_reconciler is runtime.note_content_reconciler - - batch_indexer = batch_runtime.batch_indexer + batch_indexer = runtime.batch_indexer assert isinstance(batch_indexer, BatchIndexer) assert batch_indexer.app_config is app_config assert batch_indexer.entity_service is entity_service @@ -355,7 +348,7 @@ async def index_entity_data(self, entity: Entity, content: str | None = None) -> # batch_indexer is typed as the IndexInputBatchExecutor protocol (no writer # attribute); reach the concrete BatchIndexer to exercise its composed writer. - await cast(BatchIndexer, runtime.batch_runtime.batch_indexer).search_service.index_entity_data( + await cast(BatchIndexer, runtime.batch_indexer).search_service.index_entity_data( cast(Entity, _NonMarkdownEntity()) ) diff --git a/tests/indexing/test_index_file_runner.py b/tests/indexing/test_index_file_runner.py index f99150176..689e7967a 100644 --- a/tests/indexing/test_index_file_runner.py +++ b/tests/indexing/test_index_file_runner.py @@ -2,7 +2,6 @@ from collections.abc import AsyncIterator, Sequence from contextlib import asynccontextmanager -from dataclasses import dataclass from typing import cast import pytest @@ -16,10 +15,8 @@ FileIndexTarget, ) from basic_memory.indexing.index_file_runner import ( - IndexFileCurrentMetadataSource, IndexFileObjectMetadata, RepositoryCurrentMaterializedNoteSource, - StorageIndexFileMetadataSource, run_index_file, ) from basic_memory.indexing.index_file_runtime import IndexFileRuntimeRequest @@ -68,27 +65,6 @@ async def load_current_file_metadata(self, file_path: str) -> IndexFileObjectMet return self.metadata -@dataclass(frozen=True, slots=True) -class FakeStorageMetadata: - checksum: str - metadata: dict[str, str] - - -@dataclass(slots=True) -class FakeStorageMetadataSource: - calls: list[str] - - async def load_current_file_metadata( - self, - file_path: str, - ) -> FakeStorageMetadata | None: - self.calls.append(file_path) - return FakeStorageMetadata( - checksum="etag-1", - metadata={NOTE_OBJECT_SOURCE_METADATA: "api"}, - ) - - class FakeMaterializedNoteSource: def __init__(self, entity: CurrentMaterializedNoteEntity | None) -> None: self.entity = entity @@ -198,21 +174,6 @@ def indexed_file() -> FileIndexResult: ) -@pytest.mark.asyncio -async def test_storage_index_file_metadata_source_maps_storage_metadata() -> None: - calls: list[str] = [] - metadata_source: IndexFileCurrentMetadataSource = FakeStorageMetadataSource(calls) - source = StorageIndexFileMetadataSource(metadata_source=metadata_source) - - result = await source.load_current_file_metadata("notes/a.md") - - assert result == IndexFileObjectMetadata( - checksum="etag-1", - metadata={NOTE_OBJECT_SOURCE_METADATA: "api"}, - ) - assert calls == ["notes/a.md"] - - @pytest.mark.asyncio async def test_repository_current_materialized_note_source_loads_entity( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/indexing/test_models.py b/tests/indexing/test_models.py index a1e47b5b0..b780bcf1c 100644 --- a/tests/indexing/test_models.py +++ b/tests/indexing/test_models.py @@ -85,9 +85,9 @@ def test_file_index_result_from_fields_validates_required_entity_text(): result = FileIndexResult.from_fields( file_path="notes/a.md", entity_id=42, - external_id="note-42", - title="A Note", - permalink="notes/a-note", + external_id=" note-42 ", + title=" A Note ", + permalink=" notes/a-note ", checksum="checksum-1", operation=FileIndexOperation.created, ) @@ -114,6 +114,42 @@ def test_file_index_result_from_fields_validates_required_entity_text(): ) +def test_file_index_result_from_fields_validates_optional_permalink_text(): + result = FileIndexResult.from_fields( + file_path="notes/a.md", + entity_id=42, + external_id="note-42", + title="A Note", + permalink=None, + checksum="checksum-1", + operation=FileIndexOperation.created, + ) + + assert result.permalink is None + + with pytest.raises(RuntimeError, match="Indexed entity for notes/a.md has invalid permalink"): + FileIndexResult.from_fields( + file_path="notes/a.md", + entity_id=42, + external_id="note-42", + title="A Note", + permalink=123, + checksum="checksum-1", + operation=FileIndexOperation.created, + ) + + with pytest.raises(RuntimeError, match="Indexed entity for notes/a.md has blank permalink"): + FileIndexResult.from_fields( + file_path="notes/a.md", + entity_id=42, + external_id="note-42", + title="A Note", + permalink=" ", + checksum="checksum-1", + operation=FileIndexOperation.created, + ) + + def test_index_file_job_result_carries_live_update_metadata(): result = IndexFileJobResult( status=IndexFileJobStatus.processed, @@ -648,6 +684,58 @@ def test_project_index_outcomes_from_file_job_results_update_batch_counters(): ) +def test_current_materialized_note_entity_from_fields_requires_indexed_permalink(): + with pytest.raises(RuntimeError, match="Current entity for notes/a.md is missing permalink"): + CurrentMaterializedNoteEntity.from_fields( + entity_id=42, + external_id="note-42", + title="A Note", + permalink=None, + checksum="checksum-1", + file_path="notes/a.md", + ) + + +def test_current_materialized_note_entity_from_fields_validates_identity_text(): + entity = CurrentMaterializedNoteEntity.from_fields( + entity_id=42, + external_id=" note-42 ", + title=" A Note ", + permalink=" notes/a-note ", + checksum="checksum-1", + file_path="notes/a.md", + ) + + assert entity == CurrentMaterializedNoteEntity( + entity_id=42, + external_id="note-42", + title="A Note", + permalink="notes/a-note", + checksum="checksum-1", + ) + + no_checksum = CurrentMaterializedNoteEntity.from_fields( + entity_id=42, + external_id="note-42", + title="A Note", + permalink="notes/a-note", + checksum=None, + file_path="notes/a.md", + ) + + assert no_checksum.checksum is None + + with pytest.raises(RuntimeError, match="Current entity for notes/a.md is missing title"): + CurrentMaterializedNoteEntity.from_fields( + entity_id=42, + external_id="note-42", + title=" ", + permalink="notes/a-note", + checksum="checksum-1", + file_path="notes/a.md", + ) + + def test_plan_current_materialized_note_result_preserves_trusted_live_update_metadata(): entity = CurrentMaterializedNoteEntity.from_fields( entity_id=42, diff --git a/tests/indexing/test_note_content_batch_reconciliation.py b/tests/indexing/test_note_content_batch_reconciliation.py index 3035b577f..60e03d25a 100644 --- a/tests/indexing/test_note_content_batch_reconciliation.py +++ b/tests/indexing/test_note_content_batch_reconciliation.py @@ -17,7 +17,6 @@ from basic_memory import db, file_utils from basic_memory.indexing.models import IndexedEntity from basic_memory.indexing.note_content_batch_reconciliation import ( - DefaultIndexedNoteContentTimestampProvider, indexed_note_content_observed_at, reconcile_indexed_note_content_batch, run_indexing_tasks_with_retries, @@ -73,10 +72,12 @@ async def run(self) -> str: @dataclass(slots=True) -class RecordingIndexedNoteContentTimestampProvider: +class RecordingIndexedNoteContentTimestamps: + """Recording stand-in for the injected IndexedNoteContentObservedAt callable.""" + calls: list[tuple[str, FakeFileInfo | None]] - def observed_at( + def __call__( self, indexed: IndexedEntity, file_info: FakeFileInfo | None, @@ -110,7 +111,7 @@ async def test_reconcile_indexed_note_content_batch_reports_per_file_errors( repository = FakeEntityRepository([FakeEntity(id=42), FakeEntity(id=43)]) reconcile = AsyncMock() observed_at = datetime(2026, 6, 19, 14, 0, tzinfo=UTC) - timestamp_provider = RecordingIndexedNoteContentTimestampProvider(calls=[]) + timestamp_provider = RecordingIndexedNoteContentTimestamps(calls=[]) @asynccontextmanager async def fake_scoped_session( @@ -308,7 +309,7 @@ async def fake_scoped_session( entity_repository=repository, session_maker=session_maker, note_content_reconciler=cast(Any, SimpleNamespace(reconcile=reconcile)), - timestamp_provider=RecordingIndexedNoteContentTimestampProvider(calls=[]), + timestamp_provider=RecordingIndexedNoteContentTimestamps(calls=[]), max_concurrent=1, source="index", file_reader=StubReconcileFileReader(StubReconcileFile(content=None, last_modified=None)), @@ -384,7 +385,6 @@ async def test_batch_reader_reconciles_fresh_content_not_scan_snapshot( entity_repository=EntityRepository(project_id=test_project.id), session_maker=session_maker, note_content_reconciler=reconciler, - timestamp_provider=DefaultIndexedNoteContentTimestampProvider(), max_concurrent=1, source="index", file_reader=reader, @@ -460,7 +460,6 @@ async def test_batch_without_reader_reverts_to_scan_snapshot( entity_repository=EntityRepository(project_id=test_project.id), session_maker=session_maker, note_content_reconciler=reconciler, - timestamp_provider=DefaultIndexedNoteContentTimestampProvider(), max_concurrent=1, source="index", ) diff --git a/tests/indexing/test_note_content_read_repair_runner.py b/tests/indexing/test_note_content_read_repair_runner.py index 63060967a..7b4966a01 100644 --- a/tests/indexing/test_note_content_read_repair_runner.py +++ b/tests/indexing/test_note_content_read_repair_runner.py @@ -1,319 +1,254 @@ +"""Tests for repository-backed note-content read and read-repair handoffs.""" + from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime -from typing import cast import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from basic_memory import db, file_utils from basic_memory.indexing.note_content_read_repair_runner import ( + ENTITY_METADATA_PAYLOAD_EXCLUDE, NoteContentReadRepairFile, NoteContentReadRepairPreflight, - NoteContentReadRepairReconcilerProvider, - NoteContentReadRepairRepositories, - NoteContentReadView, - NoteContentReadRepositories, NoteContentReadRepairTarget, - apply_note_content_read_repair, - load_note_content_read_view, + NoteContentReadView, + load_note_content_read_view_with_default_repositories, note_content_resource_from_read_view, note_content_response_payload_from_read_view, - prepare_note_content_read_repair, - run_note_content_read_repair, + prepare_note_content_read_repair_with_default_repositories, + run_note_content_read_repair_with_default_reconciler, ) +from basic_memory.models import Entity, NoteContent, Project +from basic_memory.repository import EntityRepository, NoteContentRepository from basic_memory.runtime.note_content import ( RuntimeAcceptedNoteResponse, RuntimeNoteContentReadRepairStatus, RuntimeNoteContentResource, ) - - -@dataclass(frozen=True, slots=True) -class _Project: - id: int - path: str - - -@dataclass(frozen=True, slots=True) -class _Entity: - id: int - content_type: str - file_path: str - external_id: str = "note-456" - title: str = "Read note" - note_type: str = "note" - entity_metadata: dict[str, object] | None = None - permalink: str | None = "main/notes/read" - content: str | None = None - observations: tuple[object, ...] = () - relations: tuple[object, ...] = () - created_at: datetime = datetime(2026, 4, 13, 12, 0, tzinfo=UTC) - updated_at: datetime = datetime(2026, 4, 13, 12, 5, tzinfo=UTC) - created_by: str | None = "creator" - last_updated_by: str | None = "editor" - - -@dataclass(frozen=True, slots=True) -class _NoteContent: - markdown_content: str - db_version: int = 4 - db_checksum: str = "db-checksum" - file_version: int | None = 3 - file_checksum: str | None = "file-checksum" - file_write_status: str = "synced" - last_source: str | None = "api" - file_updated_at: datetime | None = datetime(2026, 4, 13, 13, 0, tzinfo=UTC) - last_materialization_error: str | None = None - - -class _ProjectRepository: - def __init__(self, project: _Project | None) -> None: - self.project = project - self.external_ids: list[str] = [] - - async def get_by_external_id( - self, - session: AsyncSession, - external_id: str, - ) -> _Project | None: - assert session is not None - self.external_ids.append(external_id) - return self.project - - -class _EntityRepository: - def __init__(self, entity: _Entity | None) -> None: - self.entity = entity - self.external_ids: list[str] = [] - - async def get_by_external_id( - self, - session: AsyncSession, - external_id: str, - ) -> _Entity | None: - assert session is not None - self.external_ids.append(external_id) - return self.entity - - -class _NoteContentRepository: - def __init__(self, note_content: _NoteContent | None) -> None: - self.note_content = note_content - self.entity_ids: list[int] = [] - - async def get_by_entity_id( - self, - session: AsyncSession, - entity_id: int, - ) -> _NoteContent | None: - assert session is not None - self.entity_ids.append(entity_id) - return self.note_content - - -@dataclass(frozen=True, slots=True) -class _ReadRepositories: - project_repository_result: _ProjectRepository - entity_repository_result: _EntityRepository - note_content_repository_result: _NoteContentRepository - - def project_repository(self) -> _ProjectRepository: - return self.project_repository_result - - def entity_repository(self, project_id: int) -> _EntityRepository: - _ = project_id - return self.entity_repository_result - - def note_content_repository(self, project_id: int) -> _NoteContentRepository: - _ = project_id - return self.note_content_repository_result - - -class _FileReader: - def __init__(self, repair_file: NoteContentReadRepairFile | None) -> None: - self.repair_file = repair_file - self.targets: list[NoteContentReadRepairTarget[_Project, _Entity]] = [] +from basic_memory.schemas.v2.entity import EntityResponseV2 + + +async def create_note_content_row( + session_maker: async_sessionmaker[AsyncSession], + project: Project, + entity: Entity, + markdown_content: str = "# Read\n", +) -> None: + """Persist an accepted note_content row for one markdown entity.""" + checksum = await file_utils.compute_checksum(markdown_content) + now = datetime.now(tz=UTC) + repository = NoteContentRepository(project_id=project.id) + async with db.scoped_session(session_maker) as session: + await repository.create( + session, + { + "entity_id": entity.id, + "project_id": entity.project_id, + "external_id": entity.external_id, + "file_path": entity.file_path, + "markdown_content": markdown_content, + "db_version": 4, + "db_checksum": checksum, + "file_version": 3, + "file_checksum": checksum, + "file_write_status": "synced", + "last_source": "api", + "updated_at": now, + "file_updated_at": now, + "last_materialization_error": None, + "last_materialization_attempt_at": None, + }, + ) + + +async def create_image_entity( + session_maker: async_sessionmaker[AsyncSession], + project: Project, +) -> Entity: + """Persist a non-markdown entity that has no note_content row.""" + now = datetime.now(tz=UTC) + entity_repository = EntityRepository(project_id=project.id) + async with db.scoped_session(session_maker) as session: + return await entity_repository.create( + session, + { + "project_id": project.id, + "title": "diagram.png", + "note_type": "file", + "permalink": None, + "file_path": "images/diagram.png", + "content_type": "image/png", + "created_at": now, + "updated_at": now, + }, + ) + + +async def load_read_view( + session_maker: async_sessionmaker[AsyncSession], + project: Project, + entity: Entity, +) -> NoteContentReadView[Entity, NoteContent] | None: + async with db.scoped_session(session_maker) as session: + return await load_note_content_read_view_with_default_repositories( + session, + project_external_id=project.external_id, + entity_external_id=entity.external_id, + ) + + +@dataclass(slots=True) +class StubReadRepairFileReader: + """Return a fixed canonical file for the repair target under test.""" + + repair_file: NoteContentReadRepairFile | None + targets: list[NoteContentReadRepairTarget[Project, Entity]] = field(default_factory=list) async def read_note_content_repair_file( self, - target: NoteContentReadRepairTarget[_Project, _Entity], + target: NoteContentReadRepairTarget[Project, Entity], ) -> NoteContentReadRepairFile | None: self.targets.append(target) return self.repair_file -class _NeverReconciler: - async def reconcile( - self, - *, - entity: _Entity, - markdown_content: str, - observed_at: datetime | None, - source: str, - ) -> None: - raise AssertionError("unexpected read repair reconciliation") +# --- Hot note-content reads --- -@dataclass(frozen=True, slots=True) -class _FailingReconcilerProvider: - message: str +@pytest.mark.asyncio +async def test_load_note_content_read_view_returns_markdown_entity_with_content( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + await create_note_content_row(session_maker, test_project, sample_entity) - def reconciler( - self, - project_id: int, - session_maker: async_sessionmaker[AsyncSession], - ) -> _NeverReconciler: - _ = project_id - _ = session_maker - pytest.fail(self.message) + view = await load_read_view(session_maker, test_project, sample_entity) + assert view is not None + assert view.entity.id == sample_entity.id + assert view.note_content is not None + assert view.note_content.markdown_content == "# Read\n" -def test_note_content_read_providers_name_repository_and_repair_behavior() -> None: - """Read repair should use behavior providers, not Callable factory aliases.""" - project_repository = _ProjectRepository(_Project(id=7, path="/app/data/main")) - entity_repository = _EntityRepository( - _Entity(id=42, content_type="text/markdown", file_path="notes/read.md") - ) - note_content_repository = _NoteContentRepository(_NoteContent(markdown_content="# Read\n")) - test_session_maker = cast(async_sessionmaker[AsyncSession], object()) - - class FakeReconciler: - async def reconcile( - self, - *, - entity: _Entity, - markdown_content: str, - observed_at: datetime | None, - source: str, - ) -> None: - return None - - class ReadRepositories: - def project_repository(self) -> _ProjectRepository: - return project_repository - - def entity_repository(self, project_id: int) -> _EntityRepository: - assert project_id == 7 - return entity_repository - - def note_content_repository(self, project_id: int) -> _NoteContentRepository: - assert project_id == 7 - return note_content_repository - - class ReconcilerProvider: - def reconciler( - self, - project_id: int, - session_maker: async_sessionmaker[AsyncSession], - ) -> FakeReconciler: - assert project_id == 7 - assert session_maker is test_session_maker - return FakeReconciler() - - read_repositories: NoteContentReadRepositories[_Project, _Entity, _NoteContent] = ( - ReadRepositories() - ) - repair_repositories: NoteContentReadRepairRepositories[_Project, _Entity, _NoteContent] = ( - ReadRepositories() - ) - reconciler_provider: NoteContentReadRepairReconcilerProvider[_Entity] = ReconcilerProvider() +@pytest.mark.asyncio +async def test_load_note_content_read_view_returns_none_when_project_is_missing( + session_maker: async_sessionmaker[AsyncSession], + sample_entity: Entity, +) -> None: + async with db.scoped_session(session_maker) as session: + view = await load_note_content_read_view_with_default_repositories( + session, + project_external_id="missing-project", + entity_external_id=sample_entity.external_id, + ) - assert read_repositories.project_repository() is project_repository - assert read_repositories.entity_repository(7) is entity_repository - assert repair_repositories.note_content_repository(7) is note_content_repository - assert reconciler_provider.reconciler(7, test_session_maker) is not None + assert view is None @pytest.mark.asyncio -async def test_load_note_content_read_view_returns_markdown_entity_with_content() -> None: - session = cast(AsyncSession, object()) - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/read.md") - note_content = _NoteContent(markdown_content="# Read\n") - project_repository = _ProjectRepository(project) - entity_repository = _EntityRepository(entity) - note_content_repository = _NoteContentRepository(note_content) - - view = await load_note_content_read_view( - session, - project_external_id="project-123", - entity_external_id="note-456", - repositories=_ReadRepositories( - project_repository_result=project_repository, - entity_repository_result=entity_repository, - note_content_repository_result=note_content_repository, - ), - ) +async def test_load_note_content_read_view_returns_none_when_entity_is_missing( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, +) -> None: + async with db.scoped_session(session_maker) as session: + view = await load_note_content_read_view_with_default_repositories( + session, + project_external_id=test_project.external_id, + entity_external_id="missing-note", + ) - assert view == NoteContentReadView(entity=entity, note_content=note_content) - assert project_repository.external_ids == ["project-123"] - assert entity_repository.external_ids == ["note-456"] - assert note_content_repository.entity_ids == [42] + assert view is None -def test_note_content_response_payload_from_read_view_returns_accepted_note_response() -> None: - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/read.md") - note_content = _NoteContent(markdown_content="# Read\n") +@pytest.mark.asyncio +async def test_load_note_content_read_view_skips_note_lookup_for_non_markdown( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, +) -> None: + image_entity = await create_image_entity(session_maker, test_project) - payload = note_content_response_payload_from_read_view( - NoteContentReadView(entity=entity, note_content=note_content) - ) + view = await load_read_view(session_maker, test_project, image_entity) + + assert view is not None + assert view.entity.id == image_entity.id + assert view.note_content is None + + +@pytest.mark.asyncio +async def test_note_content_response_payload_returns_accepted_note_response( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + await create_note_content_row(session_maker, test_project, sample_entity) + view = await load_read_view(session_maker, test_project, sample_entity) + + payload = note_content_response_payload_from_read_view(view) assert isinstance(payload, RuntimeAcceptedNoteResponse) - assert payload.external_id == "note-456" + assert payload.external_id == sample_entity.external_id assert payload.markdown_content == "# Read\n" assert payload.db_version == 4 - assert payload.db_checksum == "db-checksum" assert payload.file_write_status == "synced" -def test_note_content_response_payload_from_read_view_returns_entity_payload_for_non_markdown() -> ( - None -): - entity = _Entity( - id=42, - content_type="image/png", - file_path="images/diagram.png", - title="diagram.png", - note_type="file", - permalink="main/images/diagram", - ) +@pytest.mark.asyncio +async def test_note_content_response_payload_returns_entity_payload_for_non_markdown( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, +) -> None: + image_entity = await create_image_entity(session_maker, test_project) + view = await load_read_view(session_maker, test_project, image_entity) - payload = note_content_response_payload_from_read_view( - NoteContentReadView(entity=entity, note_content=None) - ) + payload = note_content_response_payload_from_read_view(view) assert payload is not None assert not isinstance(payload, RuntimeAcceptedNoteResponse) payload_dict = dict(payload) - assert payload_dict["external_id"] == "note-456" + assert payload_dict["external_id"] == image_entity.external_id assert payload_dict["title"] == "diagram.png" assert payload_dict["note_type"] == "file" assert payload_dict["content_type"] == "image/png" assert payload_dict["file_path"] == "images/diagram.png" - assert "db_version" not in payload_dict + # The payload is the full EntityResponseV2 dump minus exactly the excluded + # note_content bookkeeping fields. + assert set(payload_dict) == set(EntityResponseV2.model_fields) - ENTITY_METADATA_PAYLOAD_EXCLUDE -def test_note_content_resource_from_read_view_returns_accepted_markdown_resource() -> None: - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/read.md") - note_content = _NoteContent(markdown_content="# Read\n") +def test_entity_metadata_payload_exclusions_name_real_response_fields() -> None: + """Every excluded name must be a real EntityResponseV2 field, or exclusion drifts.""" + assert ENTITY_METADATA_PAYLOAD_EXCLUDE <= EntityResponseV2.model_fields.keys() - resource = note_content_resource_from_read_view( - NoteContentReadView(entity=entity, note_content=note_content) - ) + +@pytest.mark.asyncio +async def test_note_content_resource_returns_accepted_markdown_resource( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + await create_note_content_row(session_maker, test_project, sample_entity) + view = await load_read_view(session_maker, test_project, sample_entity) + + resource = note_content_resource_from_read_view(view) assert isinstance(resource, RuntimeNoteContentResource) assert resource.content == "# Read\n" assert resource.content_type == "text/markdown" -def test_note_content_read_payload_helpers_return_none_for_missing_view_or_content() -> None: - markdown_without_content = NoteContentReadView( - entity=_Entity(id=42, content_type="text/markdown", file_path="notes/read.md"), - note_content=None, - ) +@pytest.mark.asyncio +async def test_note_content_read_payload_helpers_return_none_for_missing_view_or_content( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + markdown_without_content = await load_read_view(session_maker, test_project, sample_entity) + assert markdown_without_content is not None + assert markdown_without_content.note_content is None assert note_content_response_payload_from_read_view(None) is None assert note_content_response_payload_from_read_view(markdown_without_content) is None @@ -321,93 +256,54 @@ def test_note_content_read_payload_helpers_return_none_for_missing_view_or_conte assert note_content_resource_from_read_view(markdown_without_content) is None -@pytest.mark.asyncio -async def test_load_note_content_read_view_returns_none_when_project_is_missing() -> None: - session = cast(AsyncSession, object()) - project_repository = _ProjectRepository(None) - - view = await load_note_content_read_view( - session, - project_external_id="project-123", - entity_external_id="note-456", - repositories=_ReadRepositories( - project_repository_result=project_repository, - entity_repository_result=_EntityRepository(None), - note_content_repository_result=_NoteContentRepository(None), - ), - ) - - assert view is None - assert project_repository.external_ids == ["project-123"] - +# --- Read repair for missing note_content rows --- -@pytest.mark.asyncio -async def test_load_note_content_read_view_skips_note_lookup_for_non_markdown() -> None: - session = cast(AsyncSession, object()) - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="image/png", file_path="images/diagram.png") - - view = await load_note_content_read_view( - session, - project_external_id="project-123", - entity_external_id="note-456", - repositories=_ReadRepositories( - project_repository_result=_ProjectRepository(project), - entity_repository_result=_EntityRepository(entity), - note_content_repository_result=_NoteContentRepository(None), - ), - ) - assert view == NoteContentReadView(entity=entity, note_content=None) +async def prepare_read_repair( + session_maker: async_sessionmaker[AsyncSession], + *, + project_external_id: str, + entity_external_id: str, +) -> NoteContentReadRepairPreflight[Project, Entity]: + async with db.scoped_session(session_maker) as session: + return await prepare_note_content_read_repair_with_default_repositories( + session, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) @pytest.mark.asyncio -async def test_prepare_note_content_read_repair_returns_storage_target_for_missing_row() -> None: - session = cast(AsyncSession, object()) - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/repair.md") - project_repository = _ProjectRepository(project) - entity_repository = _EntityRepository(entity) - note_content_repository = _NoteContentRepository(None) - - preflight = await prepare_note_content_read_repair( - session, - project_external_id="project-123", - entity_external_id="note-456", - repositories=_ReadRepositories( - project_repository_result=project_repository, - entity_repository_result=entity_repository, - note_content_repository_result=note_content_repository, - ), +async def test_prepare_note_content_read_repair_returns_storage_target_for_missing_row( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + preflight = await prepare_read_repair( + session_maker, + project_external_id=test_project.external_id, + entity_external_id=sample_entity.external_id, ) assert preflight.status is RuntimeNoteContentReadRepairStatus.read_file assert preflight.should_read_file - assert preflight.require_target() == NoteContentReadRepairTarget( - project=project, - entity=entity, - ) - assert project_repository.external_ids == ["project-123"] - assert entity_repository.external_ids == ["note-456"] - assert note_content_repository.entity_ids == [42] + target = preflight.require_target() + assert target.project.id == test_project.id + assert target.entity.id == sample_entity.id @pytest.mark.asyncio -async def test_prepare_note_content_read_repair_reports_existing_row_as_repaired() -> None: - session = cast(AsyncSession, object()) - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/repair.md") - note_content_repository = _NoteContentRepository(_NoteContent(markdown_content="# Present\n")) - - preflight = await prepare_note_content_read_repair( - session, - project_external_id="project-123", - entity_external_id="note-456", - repositories=_ReadRepositories( - project_repository_result=_ProjectRepository(project), - entity_repository_result=_EntityRepository(entity), - note_content_repository_result=note_content_repository, - ), +async def test_prepare_note_content_read_repair_reports_existing_row_as_repaired( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + await create_note_content_row(session_maker, test_project, sample_entity) + + preflight = await prepare_read_repair( + session_maker, + project_external_id=test_project.external_id, + entity_external_id=sample_entity.external_id, ) assert preflight.status is RuntimeNoteContentReadRepairStatus.already_present @@ -418,20 +314,16 @@ async def test_prepare_note_content_read_repair_reports_existing_row_as_repaired @pytest.mark.asyncio -async def test_prepare_note_content_read_repair_skips_note_lookup_for_non_markdown() -> None: - session = cast(AsyncSession, object()) - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="image/png", file_path="images/diagram.png") - - preflight = await prepare_note_content_read_repair( - session, - project_external_id="project-123", - entity_external_id="note-456", - repositories=_ReadRepositories( - project_repository_result=_ProjectRepository(project), - entity_repository_result=_EntityRepository(entity), - note_content_repository_result=_NoteContentRepository(None), - ), +async def test_prepare_note_content_read_repair_skips_non_markdown_entities( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, +) -> None: + image_entity = await create_image_entity(session_maker, test_project) + + preflight = await prepare_read_repair( + session_maker, + project_external_id=test_project.external_id, + entity_external_id=image_entity.external_id, ) assert preflight.status is RuntimeNoteContentReadRepairStatus.entity_missing @@ -440,73 +332,38 @@ async def test_prepare_note_content_read_repair_skips_note_lookup_for_non_markdo @pytest.mark.asyncio -async def test_apply_note_content_read_repair_uses_project_reconciler() -> None: - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/repair.md") - target = NoteContentReadRepairTarget(project=project, entity=entity) - test_session_maker = cast(async_sessionmaker[AsyncSession], object()) - observed_at = datetime(2026, 4, 13, 15, 0, tzinfo=UTC) - calls: list[tuple[_Entity, str, datetime | None, str]] = [] - factory_calls: list[tuple[int, async_sessionmaker[AsyncSession]]] = [] - - class FakeReconciler: - async def reconcile( - self, - *, - entity: _Entity, - markdown_content: str, - observed_at: datetime | None, - source: str, - ) -> None: - calls.append((entity, markdown_content, observed_at, source)) - - class FakeReconcilerProvider: - def reconciler( - self, - project_id: int, - session_maker: async_sessionmaker[AsyncSession], - ) -> FakeReconciler: - factory_calls.append((project_id, session_maker)) - return FakeReconciler() - - await apply_note_content_read_repair( - target, - session_maker=test_session_maker, - markdown_content="# Repaired\n", - observed_at=observed_at, - source="read_repair", - reconciler_provider=FakeReconcilerProvider(), +async def test_prepare_note_content_read_repair_reports_missing_project( + session_maker: async_sessionmaker[AsyncSession], + sample_entity: Entity, +) -> None: + preflight = await prepare_read_repair( + session_maker, + project_external_id="missing-project", + entity_external_id=sample_entity.external_id, ) - assert factory_calls == [(7, test_session_maker)] - assert calls == [(entity, "# Repaired\n", observed_at, "read_repair")] + assert preflight.status is RuntimeNoteContentReadRepairStatus.project_missing + assert not preflight.should_read_file @pytest.mark.asyncio -async def test_run_note_content_read_repair_returns_preflight_status_without_file_read() -> None: - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/repair.md") - preflight = await prepare_note_content_read_repair( - cast(AsyncSession, object()), - project_external_id="project-123", - entity_external_id="note-456", - repositories=_ReadRepositories( - project_repository_result=_ProjectRepository(project), - entity_repository_result=_EntityRepository(entity), - note_content_repository_result=_NoteContentRepository( - _NoteContent(markdown_content="# Present\n") - ), - ), +async def test_run_note_content_read_repair_returns_preflight_status_without_file_read( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + await create_note_content_row(session_maker, test_project, sample_entity) + preflight = await prepare_read_repair( + session_maker, + project_external_id=test_project.external_id, + entity_external_id=sample_entity.external_id, ) - run = await run_note_content_read_repair( + run = await run_note_content_read_repair_with_default_reconciler( preflight, - session_maker=cast(async_sessionmaker[AsyncSession], object()), + session_maker=session_maker, file_reader=None, source="read_repair", - reconciler_provider=_FailingReconcilerProvider( - "already-present repair should not reconcile" - ), ) assert run.status is RuntimeNoteContentReadRepairStatus.already_present @@ -514,21 +371,42 @@ async def test_run_note_content_read_repair_returns_preflight_status_without_fil @pytest.mark.asyncio -async def test_run_note_content_read_repair_reports_missing_file() -> None: - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/repair.md") - target = NoteContentReadRepairTarget(project=project, entity=entity) - file_reader = _FileReader(None) - - run = await run_note_content_read_repair( - preflight=NoteContentReadRepairPreflight( +async def test_run_note_content_read_repair_requires_file_reader( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + preflight = NoteContentReadRepairPreflight( + status=RuntimeNoteContentReadRepairStatus.read_file, + target=NoteContentReadRepairTarget(project=test_project, entity=sample_entity), + ) + + with pytest.raises(RuntimeError, match="requires a file reader"): + await run_note_content_read_repair_with_default_reconciler( + preflight, + session_maker=session_maker, + file_reader=None, + source="read_repair", + ) + + +@pytest.mark.asyncio +async def test_run_note_content_read_repair_reports_missing_file( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + target = NoteContentReadRepairTarget(project=test_project, entity=sample_entity) + file_reader = StubReadRepairFileReader(None) + + run = await run_note_content_read_repair_with_default_reconciler( + NoteContentReadRepairPreflight( status=RuntimeNoteContentReadRepairStatus.read_file, target=target, ), - session_maker=cast(async_sessionmaker[AsyncSession], object()), + session_maker=session_maker, file_reader=file_reader, source="read_repair", - reconciler_provider=_FailingReconcilerProvider("missing files should not reconcile"), ) assert run.status is RuntimeNoteContentReadRepairStatus.file_missing @@ -537,20 +415,19 @@ async def test_run_note_content_read_repair_reports_missing_file() -> None: @pytest.mark.asyncio -async def test_run_note_content_read_repair_reports_empty_file() -> None: - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/repair.md") - target = NoteContentReadRepairTarget(project=project, entity=entity) - - run = await run_note_content_read_repair( - preflight=NoteContentReadRepairPreflight( +async def test_run_note_content_read_repair_reports_empty_file( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + run = await run_note_content_read_repair_with_default_reconciler( + NoteContentReadRepairPreflight( status=RuntimeNoteContentReadRepairStatus.read_file, - target=target, + target=NoteContentReadRepairTarget(project=test_project, entity=sample_entity), ), - session_maker=cast(async_sessionmaker[AsyncSession], object()), - file_reader=_FileReader(NoteContentReadRepairFile(None, observed_at=None)), + session_maker=session_maker, + file_reader=StubReadRepairFileReader(NoteContentReadRepairFile(None, observed_at=None)), source="read_repair", - reconciler_provider=_FailingReconcilerProvider("empty files should not reconcile"), ) assert run.status is RuntimeNoteContentReadRepairStatus.empty_file @@ -558,46 +435,36 @@ async def test_run_note_content_read_repair_reports_empty_file() -> None: @pytest.mark.asyncio -async def test_run_note_content_read_repair_applies_observed_markdown() -> None: - project = _Project(id=7, path="/app/data/main") - entity = _Entity(id=42, content_type="text/markdown", file_path="notes/repair.md") - target = NoteContentReadRepairTarget(project=project, entity=entity) - test_session_maker = cast(async_sessionmaker[AsyncSession], object()) +async def test_run_note_content_read_repair_applies_observed_markdown( + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + sample_entity: Entity, +) -> None: + """A successful repair reconciles the observed markdown into note_content.""" + markdown_content = "# Repaired\n" observed_at = datetime(2026, 4, 13, 15, 0, tzinfo=UTC) - calls: list[tuple[_Entity, str, datetime | None, str]] = [] - - class FakeReconciler: - async def reconcile( - self, - *, - entity: _Entity, - markdown_content: str, - observed_at: datetime | None, - source: str, - ) -> None: - calls.append((entity, markdown_content, observed_at, source)) - - class FakeReconcilerProvider: - def reconciler( - self, - project_id: int, - session_maker: async_sessionmaker[AsyncSession], - ) -> FakeReconciler: - assert project_id == 7 - assert session_maker is test_session_maker - return FakeReconciler() - - run = await run_note_content_read_repair( - preflight=NoteContentReadRepairPreflight( - status=RuntimeNoteContentReadRepairStatus.read_file, - target=target, + preflight = await prepare_read_repair( + session_maker, + project_external_id=test_project.external_id, + entity_external_id=sample_entity.external_id, + ) + assert preflight.should_read_file + + run = await run_note_content_read_repair_with_default_reconciler( + preflight, + session_maker=session_maker, + file_reader=StubReadRepairFileReader( + NoteContentReadRepairFile(markdown_content, observed_at=observed_at) ), - session_maker=test_session_maker, - file_reader=_FileReader(NoteContentReadRepairFile("# Repaired\n", observed_at=observed_at)), source="read_repair", - reconciler_provider=FakeReconcilerProvider(), ) assert run.status is RuntimeNoteContentReadRepairStatus.repaired assert run.repaired - assert calls == [(entity, "# Repaired\n", observed_at, "read_repair")] + repository = NoteContentRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + row = await repository.get_by_entity_id(session, sample_entity.id) + assert row is not None + assert row.markdown_content == markdown_content + assert row.db_checksum == await file_utils.compute_checksum(markdown_content) + assert row.last_source == "read_repair" diff --git a/tests/indexing/test_note_content_reconciler.py b/tests/indexing/test_note_content_reconciler.py index 2a7dd397b..707696a33 100644 --- a/tests/indexing/test_note_content_reconciler.py +++ b/tests/indexing/test_note_content_reconciler.py @@ -3,7 +3,6 @@ from __future__ import annotations from contextlib import asynccontextmanager -from dataclasses import dataclass from datetime import UTC, datetime from types import SimpleNamespace from typing import Any, cast @@ -18,13 +17,10 @@ NoteContentMaterializedCurrent, ) from basic_memory.indexing.note_content_reconciler import ( - DefaultNoteContentRepositories, NoteContentReconciler, - NoteContentRepositories, RepositoryNoteMaterializationFailureMarker, apply_note_content_update_plan, - build_default_note_content_repositories, - mark_note_materialization_enqueue_failed, + note_content_repository_for_project, reconcile_note_content_for_entity, ) from basic_memory.models import Entity @@ -60,27 +56,6 @@ def begin(self) -> FakeTransaction: return FakeTransaction() -@dataclass(frozen=True, slots=True) -class FakeNoteContentRepositories: - repository_type: type - - def note_content_repository(self, project_id: int) -> Any: - return self.repository_type(project_id) - - -def test_note_content_repositories_name_materialization_behavior() -> None: - """Materialization bookkeeping should use behavior methods, not Callable aliases.""" - - class FakeProvider: - def note_content_repository(self, project_id: int) -> NoteContentRepository: - assert project_id == 7 - return NoteContentRepository(project_id=project_id) - - repositories: NoteContentRepositories = FakeProvider() - - assert isinstance(repositories.note_content_repository(7), NoteContentRepository) - - @pytest.mark.asyncio async def test_reconciler_converges_after_concurrent_create_conflict() -> None: """A concurrent repair winner should not make the losing worker fail permanently.""" @@ -258,56 +233,11 @@ async def test_apply_note_content_update_plan_marks_materialization_status() -> ) -@pytest.mark.asyncio -async def test_mark_note_materialization_enqueue_failed_uses_repository_factory() -> None: - """Queue failure bookkeeping should stay in core repository adapter code.""" - attempted_at = datetime(2026, 4, 13, 14, 59, tzinfo=UTC) - session = FakeRepositorySession() - repository_calls: list[tuple[int, FakeRepositorySession, int, dict[str, object]]] = [] - - def session_maker() -> FakeRepositorySession: - return session - - class FakeNoteContentRepository: - def __init__(self, project_id: int) -> None: - self.project_id = project_id - - async def update_state_fields( - self, - session: Any, - entity_id: int, - **updates: object, - ) -> None: - repository_calls.append((self.project_id, session, entity_id, updates)) - - await mark_note_materialization_enqueue_failed( - session_maker=cast(Any, session_maker), - project_id=7, - entity_id=42, - error_message="pgq unavailable", - repositories=FakeNoteContentRepositories(FakeNoteContentRepository), - attempted_at=attempted_at, - ) - - assert repository_calls == [ - ( - 7, - session, - 42, - { - "file_write_status": "failed", - "last_materialization_error": "pgq unavailable", - "last_materialization_attempt_at": attempted_at, - }, - ) - ] - - @pytest.mark.asyncio async def test_repository_failure_marker_records_materialization_enqueue_failure() -> None: """The failure-marker protocol adapter should be usable by enqueue runners.""" session = FakeRepositorySession() - repository_calls: list[dict[str, object]] = [] + repository_calls: list[tuple[int, int, dict[str, object]]] = [] def session_maker() -> FakeRepositorySession: return session @@ -323,12 +253,11 @@ async def update_state_fields( **updates: object, ) -> None: assert session is not None - assert entity_id == 42 - repository_calls.append(updates) + repository_calls.append((self.project_id, entity_id, updates)) marker = RepositoryNoteMaterializationFailureMarker( session_maker=cast(Any, session_maker), - repositories=FakeNoteContentRepositories(FakeNoteContentRepository), + note_content_store=lambda project_id: cast(Any, FakeNoteContentRepository(project_id)), ) await marker.mark_note_materialization_failed( @@ -338,17 +267,20 @@ async def update_state_fields( ) assert len(repository_calls) == 1 - assert repository_calls[0]["file_write_status"] == "failed" - assert repository_calls[0]["last_materialization_error"] == "pgq unavailable" - assert isinstance(repository_calls[0]["last_materialization_attempt_at"], datetime) + project_id, entity_id, updates = repository_calls[0] + assert project_id == 7 + assert entity_id == 42 + assert updates["file_write_status"] == "failed" + assert updates["last_materialization_error"] == "pgq unavailable" + assert isinstance(updates["last_materialization_attempt_at"], datetime) -def test_default_note_content_repositories_use_core_repository() -> None: +def test_failure_marker_defaults_to_core_note_content_repository() -> None: """The default materialization contract should stay backed by core repositories.""" - repositories = build_default_note_content_repositories() + marker = RepositoryNoteMaterializationFailureMarker(session_maker=cast(Any, object())) - assert isinstance(repositories, DefaultNoteContentRepositories) - assert isinstance(repositories.note_content_repository(7), NoteContentRepository) + assert marker.note_content_store is note_content_repository_for_project + assert isinstance(note_content_repository_for_project(7), NoteContentRepository) @pytest.mark.asyncio diff --git a/tests/indexing/test_note_materialization_runner.py b/tests/indexing/test_note_materialization_runner.py index 3819cdbc6..34109ae6e 100644 --- a/tests/indexing/test_note_materialization_runner.py +++ b/tests/indexing/test_note_materialization_runner.py @@ -214,15 +214,6 @@ async def create( raise AssertionError("repository adapter tests should not create note_content") -@dataclass(frozen=True, slots=True) -class RecordingNoteContentRepositories: - repository: RecordingNoteContentRepository - - def note_content_repository(self, project_id: int) -> RecordingNoteContentRepository: - _ = project_id - return self.repository - - @dataclass(frozen=True, slots=True) class FakeFileMetadata: modified_at: datetime @@ -541,7 +532,7 @@ async def test_repository_note_materialization_publisher_updates_current_written result = await RepositoryNoteMaterializationPublisher( session_maker=cast(async_sessionmaker[AsyncSession], object()), session_lock=session_lock, - repositories=RecordingNoteContentRepositories(repository), + note_content_store=lambda project_id: repository, ).publish_written_file_state(request, prepared, written) assert result == RuntimeNoteMaterializationResult( @@ -572,6 +563,63 @@ async def test_repository_note_materialization_publisher_updates_current_written assert session.flush_count == 1 +@pytest.mark.asyncio +async def test_repository_note_materialization_publisher_records_stale_written_file() -> None: + """A newer accepted version at publish time records the written file as pending + without touching the entity row; the newer version's own materialization owns + the final state.""" + request = materialization_request() + prepared = prepared_write(request) + written = written_file() + entity = materialization_entity() + # The accepted note advanced past the requested db_version between the file + # write and this publish. + note_content = materialization_note_content(db_version=5, db_checksum="newer-db-checksum") + session = FakeRepositorySession(entity=entity, note_content=note_content) + session_lock = FakeSessionLock() + repository = RecordingNoteContentRepository() + scoped_session = RecordingScopedSession( + scoped_session=FakeScopedSession(session), + opened_session_makers=[], + ) + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr( + "basic_memory.indexing.note_materialization_runner.db.scoped_session", + scoped_session, + ) + result = await RepositoryNoteMaterializationPublisher( + session_maker=cast(async_sessionmaker[AsyncSession], object()), + session_lock=session_lock, + note_content_store=lambda project_id: repository, + ).publish_written_file_state(request, prepared, written) + + assert result == RuntimeNoteMaterializationResult( + entity_id=42, + status=RuntimeNoteMaterializationStatus.stale, + reason="file written but newer accepted note remains pending: 42", + file_path="notes/a.md", + file_checksum="new-file-sum", + ) + assert repository.calls == [ + ( + cast(AsyncSession, session), + 42, + { + "expected_db_version": 5, + "file_version": 4, + "file_checksum": "new-file-sum", + "file_write_status": "pending", + "file_updated_at": written.file_updated_at, + "last_materialization_error": None, + "last_materialization_attempt_at": prepared.attempted_at, + }, + ) + ] + # The entity metadata update belongs to the newer version's publish. + assert session.flush_count == 0 + + @pytest.mark.asyncio async def test_repository_note_materialization_status_publisher_records_conflict() -> None: request = materialization_request() @@ -595,7 +643,7 @@ async def test_repository_note_materialization_status_publisher_records_conflict await RepositoryNoteMaterializationStatusPublisher( session_maker=cast(async_sessionmaker[AsyncSession], object()), session_lock=session_lock, - repositories=RecordingNoteContentRepositories(repository), + note_content_store=lambda project_id: repository, ).publish_note_materialization_status( request, NoteMaterializationStatusPublication( diff --git a/tests/indexing/test_progress_models.py b/tests/indexing/test_progress_models.py index cb3fdb9df..898c27034 100644 --- a/tests/indexing/test_progress_models.py +++ b/tests/indexing/test_progress_models.py @@ -2,6 +2,9 @@ from dataclasses import dataclass, field +import pytest +from pydantic import ValidationError + from basic_memory.indexing.progress import ( IndexingResult, VectorSyncProgress, @@ -23,6 +26,41 @@ class BatchSummary: write_seconds_total: float = 0.0 +def test_vector_sync_progress_checkpoint_state_shape_is_stable() -> None: + """The dumped checkpoint document must stay byte-identical across releases.""" + progress = VectorSyncProgress( + entity_ids=[11, 22], + next_index=2, + entities_synced=2, + entities_failed=1, + failed_entity_ids=[22], + embedding_jobs_total=40, + embed_seconds_total=12.346, + write_seconds_total=1.234, + elapsed_seconds=15.679, + ) + + state = progress.to_checkpoint_state() + + expected = { + "entity_ids": [11, 22], + "next_index": 2, + "entities_synced": 2, + "entities_failed": 1, + "failed_entity_ids": [22], + "embedding_jobs_total": 40, + "embed_seconds_total": 12.346, + "write_seconds_total": 1.234, + "elapsed_seconds": 15.679, + "entities_total": 2, + } + assert state == expected + assert list(state) == list(expected) + # Old checkpoints carry the computed entities_total field; restoring must + # ignore it and rebuild the identical progress value. + assert VectorSyncProgress.from_checkpoint_state(expected) == progress + + def test_vector_sync_progress_checkpoint_round_trip() -> None: progress = VectorSyncProgress( entity_ids=[11, 22, 22], @@ -72,6 +110,26 @@ def test_vector_sync_progress_without_entity_ids_keeps_counters_only() -> None: assert compact.failed_entity_ids == [22] +def test_vector_sync_progress_checkpoint_write_reruns_dedupe_and_clamp() -> None: + """Post-construction mutation must not leak an unclamped offset into the checkpoint.""" + progress = VectorSyncProgress(entity_ids=[11, 22], next_index=1) + apply_vector_sync_batch_result( + progress, + BatchSummary(entities_synced=1, entities_failed=0), + next_index=5, + elapsed_seconds=1.0, + ) + + # The in-memory offset keeps the raw batch value; the persisted document clamps. + assert progress.next_index == 5 + assert progress.to_checkpoint_state()["next_index"] == 2 + + compact = progress.without_entity_ids() + + assert compact.next_index == 5 + assert compact.to_checkpoint_state()["next_index"] == 0 + + def test_vector_sync_progress_recovers_empty_progress_from_missing_or_invalid_state() -> None: missing = VectorSyncProgress.from_checkpoint_state(None) invalid = VectorSyncProgress.from_checkpoint_state({"entity_ids": "not a list"}) @@ -155,6 +213,56 @@ def test_apply_vector_sync_batch_result_updates_progress_and_reports_new_failure assert new_failed_entity_ids == [33] +def test_indexing_result_checkpoint_state_shape_is_stable() -> None: + """The dumped checkpoint document must stay byte-identical across releases.""" + result = IndexingResult( + files_processed=3, + files_unchanged=4, + entities_created=5, + entities_deleted=1, + relations_resolved=7, + semantic_vectors_synced=9, + errors=[("a.md", "bad frontmatter")], + total_duration_seconds=12.346, + semantic_vector_sync_seconds=4.568, + peak_rss_mib=512.988, + batch_count=2, + ) + + state = result.to_checkpoint_state() + + expected = { + "files_processed": 3, + "files_unchanged": 4, + "entities_created": 5, + "entities_updated": 0, + "entities_deleted": 1, + "files_moved": 0, + "forward_refs_resolved": 0, + "relations_resolved": 7, + "relations_unresolved": 0, + "search_indexed": 0, + "semantic_vector_entities_total": 0, + "semantic_vectors_synced": 9, + "semantic_vectors_failed": 0, + "errors": [["a.md", "bad frontmatter"]], + "total_duration_seconds": 12.346, + "change_detection_seconds": 0.0, + "s3_download_seconds": 0.0, + "file_processing_seconds": 0.0, + "relation_resolution_seconds": 0.0, + "search_indexing_seconds": 0.0, + "semantic_vector_sync_seconds": 4.568, + "semantic_vector_embed_seconds": 0.0, + "semantic_vector_write_seconds": 0.0, + "peak_rss_mib": 512.988, + "batch_count": 2, + } + assert state == expected + assert list(state) == list(expected) + assert IndexingResult.from_checkpoint_state(expected) == result + + def test_indexing_result_checkpoint_round_trip() -> None: result = IndexingResult( files_processed=3, @@ -222,3 +330,41 @@ def test_indexing_result_recovers_empty_result_from_missing_or_invalid_state() - assert missing == IndexingResult() assert invalid == IndexingResult() + + +def test_runtime_construction_rejects_unknown_fields() -> None: + """A mistyped keyword must raise, as the replaced dataclasses did.""" + with pytest.raises(ValidationError): + IndexingResult.model_validate({"files_processsed": 3}) + with pytest.raises(ValidationError): + VectorSyncProgress.model_validate({"next_indx": 1}) + + +def test_runtime_construction_rejects_malformed_error_entries() -> None: + """Silently dropping a malformed error entry would flip success to True.""" + with pytest.raises(ValidationError): + IndexingResult.model_validate({"errors": [{"path": "a.md"}]}) + + +def test_checkpoint_restore_tolerates_retired_fields_and_legacy_error_shapes() -> None: + """Old checkpoint documents keep restoring after fields are retired.""" + restored = IndexingResult.from_checkpoint_state( + { + "files_processed": 2, + "errors": [{"path": "a.md", "error": "boom"}], + "retired_field": "ignored", + } + ) + assert restored.files_processed == 2 + assert restored.errors == [("a.md", "boom")] + + progress = VectorSyncProgress.from_checkpoint_state( + {"entity_ids": [1, 2], "next_index": 1, "retired_field": True} + ) + assert progress.entity_ids == [1, 2] + assert progress.next_index == 1 + + +def test_checkpoint_restore_falls_back_on_garbage_state() -> None: + """A checkpoint that cannot validate restores to a fresh state.""" + assert IndexingResult.from_checkpoint_state({"errors": [{"path": "a.md"}]}) == IndexingResult() diff --git a/tests/indexing/test_project_delete_acceptance.py b/tests/indexing/test_project_delete_acceptance.py index 7cfad133c..0370a6593 100644 --- a/tests/indexing/test_project_delete_acceptance.py +++ b/tests/indexing/test_project_delete_acceptance.py @@ -1,22 +1,8 @@ """Tests for portable project-delete acceptance response values.""" -from dataclasses import dataclass - -from basic_memory.indexing.project_delete_acceptance import ( - ProjectDeleteAcceptedProject, - ProjectDeleteAcceptedResult, -) +from basic_memory.indexing.project_delete_acceptance import ProjectDeleteAcceptedResult from basic_memory.runtime.jobs import RuntimeProjectDeleteJobRequest - - -# Not frozen: ProjectDeleteAcceptedProjectSource declares plain (writable) attribute members. -@dataclass(slots=True) -class ProjectSource: - id: int - external_id: str - name: str - path: str - is_default: bool | None +from basic_memory.schemas.project_info import ProjectItem def project_delete_request(*, delete_notes: bool) -> RuntimeProjectDeleteJobRequest: @@ -29,48 +15,26 @@ def project_delete_request(*, delete_notes: bool) -> RuntimeProjectDeleteJobRequ ) -def test_project_delete_accepted_project_snapshots_basic_memory_shape() -> None: - project = ProjectDeleteAcceptedProject.from_source( - ProjectSource( - id=101, - external_id="project-main", - name="Main", - path="basic-memory", - is_default=None, - ) - ) - - assert project == ProjectDeleteAcceptedProject( +def old_project_item() -> ProjectItem: + return ProjectItem( id=101, external_id="project-main", name="Main", path="basic-memory", is_default=False, ) - assert project.to_response_payload() == { - "id": 101, - "external_id": "project-main", - "name": "Main", - "path": "basic-memory", - "is_default": False, - } def test_project_delete_accepted_result_serializes_existing_pending_response() -> None: - old_project = ProjectDeleteAcceptedProject( - id=101, - external_id="project-main", - name="Main", - path="basic-memory", - is_default=False, - ) - result = ProjectDeleteAcceptedResult.queued( request=project_delete_request(delete_notes=True), job_id=123, - old_project=old_project, + old_project=old_project_item(), ) + # Exact snapshot of the accepted-delete response contract: old_project must + # carry only the persisted project fields, never ProjectItem's cloud-hosting + # metadata (display_name, is_private). assert result.to_response_payload() == { "message": "Project 'Main' deletion queued", "status": "success", @@ -93,13 +57,7 @@ def test_project_delete_accepted_result_marks_file_delete_skipped() -> None: result = ProjectDeleteAcceptedResult.queued( request=project_delete_request(delete_notes=False), job_id="job-1", - old_project=ProjectDeleteAcceptedProject( - id=101, - external_id="project-main", - name="Main", - path="basic-memory", - is_default=False, - ), + old_project=old_project_item(), ) assert result.file_delete_status == "skipped" diff --git a/tests/indexing/test_project_delete_runner.py b/tests/indexing/test_project_delete_runner.py index 409bb9856..b13d40f44 100644 --- a/tests/indexing/test_project_delete_runner.py +++ b/tests/indexing/test_project_delete_runner.py @@ -1,7 +1,6 @@ """Tests for portable project-delete cleanup orchestration.""" from collections.abc import AsyncGenerator -from dataclasses import dataclass from datetime import UTC, datetime import pytest @@ -10,9 +9,7 @@ from sqlalchemy.pool import StaticPool from basic_memory.indexing.project_delete_runner import ( - DefaultProjectDeleteRepositories, ProjectDeletePreflightResult, - ProjectDeleteRepositories, ProjectHardDeleteOutcome, RepositoryProjectDeletePreflight, RepositoryProjectHardDeleter, @@ -99,14 +96,6 @@ async def delete(self, session: AsyncSession, entity_id: int) -> bool: return self.deleted -@dataclass(frozen=True, slots=True) -class FakeProjectDeleteRepositories: - repository: FakeProjectDeleteRepository - - def project_repository(self) -> FakeProjectDeleteRepository: - return self.repository - - def project_delete_request( *, project_id: int = 101, @@ -350,27 +339,29 @@ async def test_repository_project_hard_deleter_aborts_when_project_reactivated( @pytest.mark.asyncio -async def test_repository_project_hard_deleter_uses_repository_provider( +async def test_repository_project_hard_deleter_uses_injected_project_repository( project_delete_session_maker: async_sessionmaker[AsyncSession], ) -> None: project = await create_project_with_note(project_delete_session_maker, is_active=False) request = project_delete_request(project_id=project.id) repository = FakeProjectDeleteRepository(deleted=False) - repositories: ProjectDeleteRepositories = FakeProjectDeleteRepositories(repository) outcome = await RepositoryProjectHardDeleter( session_maker=project_delete_session_maker, - repositories=repositories, + project_repository=repository, ).hard_delete_project(request) assert outcome is ProjectHardDeleteOutcome.missing assert repository.entity_ids == [project.id] -def test_default_project_delete_repositories_builds_project_repository() -> None: - repositories: ProjectDeleteRepositories = DefaultProjectDeleteRepositories() +@pytest.mark.asyncio +async def test_repository_project_hard_deleter_defaults_to_core_project_repository( + project_delete_session_maker: async_sessionmaker[AsyncSession], +) -> None: + hard_deleter = RepositoryProjectHardDeleter(session_maker=project_delete_session_maker) - assert repositories.project_repository().__class__.__name__ == "ProjectRepository" + assert hard_deleter.project_repository.__class__.__name__ == "ProjectRepository" @pytest.mark.asyncio diff --git a/tests/indexing/test_project_index_maintenance.py b/tests/indexing/test_project_index_maintenance.py index b43c1ee8b..e83bd635a 100644 --- a/tests/indexing/test_project_index_maintenance.py +++ b/tests/indexing/test_project_index_maintenance.py @@ -561,6 +561,53 @@ async def fake_scoped_session( assert "UPDATE search_index" in str(session.statements[4]) +@pytest.mark.asyncio +async def test_repository_project_index_maintenance_store_move_batch_handles_empty_work( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A batch with no targets returns before touching the database, and a batch + whose paths match no indexed rows reports them missing without deleting or + updating anything.""" + session_maker = cast(async_sessionmaker[AsyncSession], object()) + session = FakeProjectIndexSession(results=[FakeProjectIndexResult()]) + + @asynccontextmanager + async def fake_scoped_session( + scoped_session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[FakeProjectIndexSession]: + yield session + + monkeypatch.setattr( + project_index_maintenance_module.db, + "scoped_session", + fake_scoped_session, + ) + + store = RepositoryProjectIndexMaintenanceStore( + session_maker=session_maker, + project_id=42, + ) + + empty_result = await store.apply_project_index_move_batch( + ProjectIndexMoveBatch(completed_batches=1, targets=()) + ) + assert empty_result == ProjectIndexMoveBatchResult(updated_files=0) + assert session.statements == [] + + result = await store.apply_project_index_move_batch( + ProjectIndexMoveBatch( + completed_batches=1, + targets=(ProjectIndexMoveTarget("notes/gone.md", "archive/gone.md"),), + ) + ) + assert result == ProjectIndexMoveBatchResult( + updated_files=0, + missing_paths=("notes/gone.md",), + ) + # Only the target-row select ran: no replacement lookup, deletes, or updates. + assert len(session.statements) == 1 + + @pytest.mark.asyncio async def test_repository_project_index_maintenance_store_deletes_replaced_move_targets( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/indexing/test_project_index_runtime.py b/tests/indexing/test_project_index_runtime.py index 15466be36..533d33a7d 100644 --- a/tests/indexing/test_project_index_runtime.py +++ b/tests/indexing/test_project_index_runtime.py @@ -23,6 +23,7 @@ ProjectIndexMoveBatch, ProjectIndexMoveBatchResult, RepositoryProjectIndexMaintenanceStore, + StoreProjectIndexMaintenanceRunner, ) from basic_memory.indexing.vector_sync_planning import RepositoryVectorSyncEntitySource from basic_memory.indexing.forward_reference_resolution import ( @@ -32,11 +33,13 @@ ) -@dataclass(frozen=True, slots=True) +# Not frozen: UnresolvedRelation declares plain (writable) attribute members. +@dataclass(slots=True) class StubUnresolvedRelation: id: int from_id: int - to_name: str | None + to_name: str + relation_type: str = "related_to" @dataclass(slots=True) @@ -168,7 +171,7 @@ async def find_by_id(self, session, entity_id: int): @dataclass(slots=True) class NoopEntityIndexer: - async def index_entity(self, entity) -> object: + async def index_entity(self, entity) -> None: msg = "index_entity should not be called by the construction test" raise AssertionError(msg) @@ -186,8 +189,10 @@ def make_runtime( project_id=7, vector_sync=NoopVectorSync(), vector_entity_source=vector_entity_source or RecordingVectorEntitySource(), - move_store=move_store or _make_maintenance_store(), - delete_store=delete_store or _make_maintenance_store(), + maintenance=StoreProjectIndexMaintenanceRunner( + move_store=move_store or _make_maintenance_store(), + delete_store=delete_store or _make_maintenance_store(), + ), forward_reference_relation_source=relation_source or RecordingForwardReferenceRelationSource(relations=()), forward_reference_resolution_runtime=resolution_runtime @@ -217,8 +222,9 @@ def test_build_default_project_index_runtime_composes_repository_backed_runtime( assert isinstance(runtime.vector_entity_source, RepositoryVectorSyncEntitySource) assert runtime.vector_entity_source.session_maker is session_maker assert runtime.vector_entity_source.project_id == 7 - assert isinstance(runtime.move_store, RepositoryProjectIndexMaintenanceStore) - assert runtime.move_store is runtime.delete_store + assert isinstance(runtime.maintenance, StoreProjectIndexMaintenanceRunner) + assert isinstance(runtime.maintenance.move_store, RepositoryProjectIndexMaintenanceStore) + assert runtime.maintenance.move_store is runtime.maintenance.delete_store assert isinstance( runtime.forward_reference_relation_source, RepositoryForwardReferenceRelationSource, @@ -333,11 +339,11 @@ async def test_project_index_runtime_resolves_forward_refs_and_refreshes_targets ), ) assert set(refresher.calls) == {100, 200} - assert run.initial_count == 3 - assert run.unique_link_text_count == 3 - assert run.resolved_link_text_count == 2 - assert run.resolved_count == 2 - assert run.remaining_count == 1 - assert run.entity_ids_to_refresh == frozenset({100, 200}) - assert run.successful_reindexed_entity_ids == frozenset({100}) - assert [failure.entity_id for failure in run.refresh_failures] == [200] + assert run.resolution.unresolved_before == 3 + assert run.resolution.link_texts == ("Target", "Broken", "Fails") + assert run.resolution.resolved_link_text_count == 2 + assert run.resolution.resolved_count == 2 + assert run.resolution.remaining_count == 1 + assert run.resolution.entity_ids_to_refresh == frozenset({100, 200}) + assert run.refresh.successful_entity_ids == frozenset({100}) + assert [failure.entity_id for failure in run.refresh.failures] == [200] diff --git a/tests/indexing/test_project_index_workflow.py b/tests/indexing/test_project_index_workflow.py index a8261a132..beb715746 100644 --- a/tests/indexing/test_project_index_workflow.py +++ b/tests/indexing/test_project_index_workflow.py @@ -1,23 +1,26 @@ """Tests for portable project-index workflow metadata planning.""" +import json from dataclasses import dataclass from uuid import UUID -import pytest - from basic_memory.indexing.models import IndexFileJobResult, IndexFileJobStatus from basic_memory.indexing.project_index_coordinator import ProjectIndexRequest from basic_memory.indexing.project_index_progress import ProjectIndexCounters from basic_memory.indexing.project_index_workflow import ( ProjectIndexBatchJobActivity, ProjectIndexBatchJobActivityUpdate, - ProjectIndexStaleWorkflowPlan, + ProjectIndexStaleWorkflowFail, + ProjectIndexStaleWorkflowKeepRunning, + ProjectIndexWorkflowAlreadyRecorded, ProjectIndexWorkflowCompletionUpdate, ProjectIndexWorkflowFailureUpdate, ProjectIndexWorkflowProgressUpdate, - ProjectIndexWorkflowRecordPlan, + ProjectIndexWorkflowRecordComplete, + ProjectIndexWorkflowRecordProgress, ProjectIndexWorkflowStart, - ProjectIndexWorkflowStartPlan, + ProjectIndexWorkflowStartComplete, + ProjectIndexWorkflowStartRunning, build_project_index_batch_activity_update, build_project_index_workflow_completion_update, build_project_index_workflow_progress_update, @@ -198,6 +201,27 @@ def test_project_index_workflow_start_builds_existing_metadata_and_attempt_event "project_path": "project", }, ) + # Persisted document key order is part of the stable checkpoint shape. + assert list(start.metadata) == [ + "phase", + "progress", + "payload", + "discovery", + "counters", + "transport", + ] + assert list(start.attempt_event_data) == [ + "phase", + "progress", + "total_files", + "batch_count", + "batch_size", + "queue_job_id", + "project_id", + "project_name", + "project_permalink", + "project_path", + ] def test_project_index_workflow_start_plan_keeps_nonempty_workflows_running() -> None: @@ -228,9 +252,7 @@ def test_project_index_workflow_start_plan_keeps_nonempty_workflows_running() -> transport_event_data={"queue_job_id": "123"}, ) - assert plan.status == "running" - assert plan.is_complete is False - assert plan.completion_update is None + assert isinstance(plan, ProjectIndexWorkflowStartRunning) assert plan.workflow_start.progress == "Indexed 0/4 files, 0 succeeded" assert plan.workflow_start.metadata["phase"] == "indexing" assert plan.workflow_start.metadata["transport"] == { @@ -238,8 +260,6 @@ def test_project_index_workflow_start_plan_keeps_nonempty_workflows_running() -> "entrypoint": "index_project", "queue_job_id": "123", } - with pytest.raises(RuntimeError, match="does not include a completion update"): - plan.require_completion_update() def test_project_index_workflow_start_plan_completes_empty_projects() -> None: @@ -270,7 +290,7 @@ def test_project_index_workflow_start_plan_completes_empty_projects() -> None: transport_event_data={"queue_job_id": None}, ) - assert plan == ProjectIndexWorkflowStartPlan.complete( + assert plan == ProjectIndexWorkflowStartComplete( workflow_start=ProjectIndexWorkflowStart( counters=ProjectIndexCounters( total=0, @@ -396,8 +416,8 @@ def test_project_index_workflow_start_plan_completes_empty_projects() -> None: }, ), ) - assert plan.is_complete is True - assert plan.require_completion_update().metadata["phase"] == "completed" + assert isinstance(plan, ProjectIndexWorkflowStartComplete) + assert plan.completion_update.metadata["phase"] == "completed" def test_project_index_workflow_progress_update_builds_metadata_and_event_data() -> None: @@ -582,11 +602,9 @@ def test_project_index_file_result_record_plan_builds_progress_update() -> None: ), ) - assert plan.status == "progress" - assert plan.is_complete is False - assert plan.should_emit_progress_event is True - assert plan.completion_update is None - progress_update = plan.require_progress_update() + assert isinstance(plan, ProjectIndexWorkflowRecordProgress) + progress_update = plan.progress_update + assert progress_update.should_emit_event is True assert progress_update.counters == ProjectIndexCounters( total=2, processed=1, @@ -596,6 +614,8 @@ def test_project_index_file_result_record_plan_builds_progress_update() -> None: ) assert progress_update.metadata["phase"] == "indexing" assert progress_update.metadata["progress"] == "Indexed 1/2 files, 1 succeeded" + # Per-file workflows never track batch structure; the key must stay absent. + assert "recorded_batches" not in progress_update.metadata assert progress_update.progress_event_data == { "phase": "indexing", "progress": "Indexed 1/2 files, 1 succeeded", @@ -626,10 +646,9 @@ def test_project_index_file_result_record_plan_builds_completion_update() -> Non ), ) - assert plan.status == "complete" - assert plan.is_complete is True - progress_update = plan.require_progress_update() - completion_update = plan.require_completion_update() + assert isinstance(plan, ProjectIndexWorkflowRecordComplete) + progress_update = plan.progress_update + completion_update = plan.completion_update assert progress_update.metadata["phase"] == "indexing" assert completion_update.counters == ProjectIndexCounters( total=1, @@ -685,10 +704,34 @@ def test_project_index_batch_result_record_plan_ignores_recorded_batches() -> No ], ) - assert plan == ProjectIndexWorkflowRecordPlan.already_recorded() - assert plan.should_emit_progress_event is False - with pytest.raises(RuntimeError, match="does not include a progress update"): - plan.require_progress_update() + assert plan == ProjectIndexWorkflowAlreadyRecorded() + + +def test_project_index_batch_result_record_plan_builds_progress_update() -> None: + workflow_id = UUID("22222222-2222-2222-2222-222222222222") + + plan = plan_project_index_batch_result_record( + metadata=project_index_record_metadata(total=3, recorded_batches=[]), + workflow_id=workflow_id, + batch_index=0, + batch_count=2, + results=[ + IndexFileJobResult( + status=IndexFileJobStatus.processed, + reason="file indexed: notes/a.md", + ) + ], + ) + + assert isinstance(plan, ProjectIndexWorkflowRecordProgress) + assert plan.progress_update.counters == ProjectIndexCounters( + total=3, + processed=1, + succeeded=1, + missing=0, + failed=0, + ) + assert plan.progress_update.metadata["recorded_batches"] == [0] def test_project_index_batch_result_record_plan_builds_completion_update() -> None: @@ -712,10 +755,9 @@ def test_project_index_batch_result_record_plan_builds_completion_update() -> No ], ) - assert plan.status == "complete" - assert plan.is_complete is True - progress_update = plan.require_progress_update() - completion_update = plan.require_completion_update() + assert isinstance(plan, ProjectIndexWorkflowRecordComplete) + progress_update = plan.progress_update + completion_update = plan.completion_update assert progress_update.metadata["recorded_batches"] == [0, 1] assert completion_update.metadata["phase"] == "completed" assert completion_update.metadata["recorded_batches"] == [0, 1] @@ -752,10 +794,8 @@ def test_project_index_stale_workflow_plan_keeps_active_batches_running() -> Non stale_before="2026-06-19T10:25:30+00:00", ) - assert plan.status == "keep_running" - assert plan.should_fail is False - assert plan.failure_update is None - assert plan.require_activity_update() == ProjectIndexBatchJobActivityUpdate( + assert isinstance(plan, ProjectIndexStaleWorkflowKeepRunning) + assert plan.activity_update == ProjectIndexBatchJobActivityUpdate( activity=active_batch_jobs, metadata={ "phase": "indexing", @@ -788,8 +828,6 @@ def test_project_index_stale_workflow_plan_keeps_active_batches_running() -> Non }, }, ) - with pytest.raises(RuntimeError, match="does not include a failure update"): - plan.require_failure_update() def test_project_index_stale_workflow_plan_builds_failure_update() -> None: @@ -814,11 +852,11 @@ def test_project_index_stale_workflow_plan_builds_failure_update() -> None: "reason": "stale_project_index_batches", "missing_batches": [1], "recorded_batches": [0], - "legacy_missing_batch_count": 0, + "legacy_missing_batch_count": False, "last_heartbeat_at": "2026-06-19T10:20:30+00:00", "stale_before": "2026-06-19T10:25:30+00:00", } - assert plan == ProjectIndexStaleWorkflowPlan.fail( + assert plan == ProjectIndexStaleWorkflowFail( ProjectIndexWorkflowFailureUpdate( counters=ProjectIndexCounters( total=100, @@ -866,9 +904,6 @@ def test_project_index_stale_workflow_plan_builds_failure_update() -> None: }, ) ) - assert plan.should_fail is True - with pytest.raises(RuntimeError, match="does not include an activity update"): - plan.require_activity_update() def test_project_index_workflow_stale_failure_update_builds_metadata_and_event_data() -> None: @@ -900,7 +935,7 @@ def test_project_index_workflow_stale_failure_update_builds_metadata_and_event_d counters=counters, missing_batch_indexes=(1,), recorded_batch_indexes=(0,), - legacy_missing_batch_count=0, + legacy_missing_batch_count=False, last_heartbeat_at="2026-06-19T10:20:30+00:00", stale_before="2026-06-19T10:25:30+00:00", ) @@ -909,7 +944,7 @@ def test_project_index_workflow_stale_failure_update_builds_metadata_and_event_d "reason": "stale_project_index_batches", "missing_batches": [1], "recorded_batches": [0], - "legacy_missing_batch_count": 0, + "legacy_missing_batch_count": False, "last_heartbeat_at": "2026-06-19T10:20:30+00:00", "stale_before": "2026-06-19T10:25:30+00:00", } @@ -947,3 +982,45 @@ def test_project_index_workflow_stale_failure_update_builds_metadata_and_event_d "diagnostics": diagnostics, }, ) + + +def test_project_index_workflow_stale_failure_update_flags_legacy_batch_metadata() -> None: + counters = ProjectIndexCounters( + total=100, + processed=50, + succeeded=49, + missing=1, + failed=0, + ) + + update = build_project_index_workflow_stale_failure_update( + metadata={ + "phase": "indexing", + "payload": {"project_id": 42}, + "counters": { + "total": 100, + "processed": 50, + "succeeded": 49, + "missing": 1, + "failed": 0, + }, + }, + counters=counters, + missing_batch_indexes=(), + recorded_batch_indexes=(), + legacy_missing_batch_count=True, + last_heartbeat_at="2026-06-19T10:20:30+00:00", + stale_before="2026-06-19T10:25:30+00:00", + ) + + assert update.error_message == "Project index stalled with legacy batch metadata" + assert update.metadata["diagnostics"] == { + "reason": "stale_project_index_batches", + "missing_batches": [], + "recorded_batches": [], + "legacy_missing_batch_count": True, + "last_heartbeat_at": "2026-06-19T10:20:30+00:00", + "stale_before": "2026-06-19T10:25:30+00:00", + } + # The legacy marker persists as a raw JSON boolean, not a count. + assert '"legacy_missing_batch_count": true' in json.dumps(update.metadata["diagnostics"]) diff --git a/tests/indexing/test_relation_resolution.py b/tests/indexing/test_relation_resolution.py index b959f0dd4..298a25564 100644 --- a/tests/indexing/test_relation_resolution.py +++ b/tests/indexing/test_relation_resolution.py @@ -19,50 +19,29 @@ plan_project_index_completion_relation_resolution, resolve_project_index_completion_relations, resolve_project_relations, - resolve_relations_until_stable, ) from basic_memory.indexing.models import IndexFileJobStatus -from basic_memory.models import Entity - - -class StubUnresolvedRelationCounter: - """Returns scripted unresolved relation counts, in call order.""" - - def __init__(self, counts: list[int]) -> None: - self._counts = counts - self.calls = 0 - - async def count_unresolved_relations(self) -> int: - index = min(self.calls, len(self._counts) - 1) - self.calls += 1 - return self._counts[index] - - -class StubRelationResolutionPass: - """Returns scripted affected entity sets, in pass order.""" - - def __init__(self, affected_per_pass: list[set[int]]) -> None: - self._affected_per_pass = affected_per_pass - self.calls = 0 - - async def resolve_relations(self) -> set[int]: - index = min(self.calls, len(self._affected_per_pass) - 1) - self.calls += 1 - return self._affected_per_pass[index] +from basic_memory.models import Entity, Relation class StubRelationResolutionRuntime: """Relation-resolution runtime with scripted counters and pass results.""" def __init__(self, counts: list[int], affected_per_pass: list[set[int]]) -> None: - self.counter = StubUnresolvedRelationCounter(counts) - self.resolver = StubRelationResolutionPass(affected_per_pass) + self._counts = counts + self._affected_per_pass = affected_per_pass + self.counter_calls = 0 + self.resolve_calls = 0 async def count_unresolved_relations(self) -> int: - return await self.counter.count_unresolved_relations() + index = min(self.counter_calls, len(self._counts) - 1) + self.counter_calls += 1 + return self._counts[index] async def resolve_relations(self) -> set[int]: - return await self.resolver.resolve_relations() + index = min(self.resolve_calls, len(self._affected_per_pass) - 1) + self.resolve_calls += 1 + return self._affected_per_pass[index] class FakeSession: @@ -107,7 +86,7 @@ def __init__( self._unresolved_per_call = unresolved_per_call self._fail_update_ids = fail_update_ids or set() self.calls = 0 - self.updates: list[tuple[int, dict[str, object]]] = [] + self.updates: list[tuple[int, dict[str, int | str]]] = [] self.deletes: list[int] = [] async def find_unresolved_relations( @@ -134,18 +113,19 @@ async def find_unresolved_relations_for_entity( async def update( self, session: AsyncSession, - entity_id: int, - entity_data: dict[str, object], - ) -> object | None: + relation_id: int, + resolved_target_fields: dict[str, int | str], + /, + ) -> Relation | None: assert isinstance(session, FakeSession) - if entity_id in self._fail_update_ids: + if relation_id in self._fail_update_ids: raise IntegrityError("update relation", {}, Exception("duplicate relation")) - self.updates.append((entity_id, entity_data)) - return object() + self.updates.append((relation_id, resolved_target_fields)) + return None - async def delete(self, session: AsyncSession, entity_id: int) -> bool: + async def delete(self, session: AsyncSession, relation_id: int, /) -> bool: assert isinstance(session, FakeSession) - self.deletes.append(entity_id) + self.deletes.append(relation_id) return True @@ -277,8 +257,8 @@ async def test_project_index_completion_relation_resolution_runs_shared_pass() - passes=2, affected_entities=1, ) - assert runtime.counter.calls == 2 - assert runtime.resolver.calls == 2 + assert runtime.counter_calls == 2 + assert runtime.resolve_calls == 2 skipped_runtime = StubRelationResolutionRuntime([2, 0], [{10}]) assert ( @@ -291,8 +271,8 @@ async def test_project_index_completion_relation_resolution_runs_shared_pass() - ) is None ) - assert skipped_runtime.counter.calls == 0 - assert skipped_runtime.resolver.calls == 0 + assert skipped_runtime.counter_calls == 0 + assert skipped_runtime.resolve_calls == 0 def test_index_file_relation_resolution_plan_requires_incremental_processed_file() -> None: @@ -320,13 +300,9 @@ def test_index_file_relation_resolution_plan_requires_incremental_processed_file @pytest.mark.asyncio async def test_resolves_until_a_stable_pass_changes_nothing() -> None: - counter = StubUnresolvedRelationCounter([3, 1]) - resolver = StubRelationResolutionPass([{10, 11}, set()]) + runtime = StubRelationResolutionRuntime([3, 1], [{10, 11}, set()]) - result = await resolve_relations_until_stable( - resolver=resolver, - unresolved_counter=counter, - ) + result = await resolve_project_relations(runtime) assert result == ResolveRelationsResult( unresolved_before=3, @@ -335,40 +311,31 @@ async def test_resolves_until_a_stable_pass_changes_nothing() -> None: affected_entities=2, ) assert result.resolved == 2 - assert resolver.calls == 2 - assert counter.calls == 2 + assert runtime.resolve_calls == 2 + assert runtime.counter_calls == 2 @pytest.mark.asyncio async def test_stops_immediately_when_no_relations_resolve() -> None: - counter = StubUnresolvedRelationCounter([1, 1]) - resolver = StubRelationResolutionPass([set()]) + runtime = StubRelationResolutionRuntime([1, 1], [set()]) - result = await resolve_relations_until_stable( - resolver=resolver, - unresolved_counter=counter, - ) + result = await resolve_project_relations(runtime) assert result.passes == 1 assert result.resolved == 0 assert result.remaining == 1 - assert resolver.calls == 1 + assert runtime.resolve_calls == 1 @pytest.mark.asyncio async def test_resolution_loop_is_bounded_by_max_passes() -> None: - counter = StubUnresolvedRelationCounter([2, 0]) - resolver = StubRelationResolutionPass([{1}]) + runtime = StubRelationResolutionRuntime([2, 0], [{1}]) - result = await resolve_relations_until_stable( - resolver=resolver, - unresolved_counter=counter, - max_passes=3, - ) + result = await resolve_project_relations(runtime, max_passes=3) assert result.passes == 3 assert result.remaining == 0 - assert resolver.calls == 3 + assert runtime.resolve_calls == 3 @pytest.mark.asyncio diff --git a/tests/runtime/test_deleted_note_file_checksum.py b/tests/runtime/test_deleted_note_file_checksum.py index 0de8ba127..901cff5ab 100644 --- a/tests/runtime/test_deleted_note_file_checksum.py +++ b/tests/runtime/test_deleted_note_file_checksum.py @@ -7,12 +7,12 @@ @dataclass(frozen=True, slots=True) class _NoteContentFileState: - file_checksum: object | None + file_checksum: str | None @dataclass(frozen=True, slots=True) class _EntityFileState: - checksum: object | None + checksum: str | None def test_select_deleted_note_file_checksum_prefers_materialized_note_content() -> None: diff --git a/tests/runtime/test_deleted_note_response.py b/tests/runtime/test_deleted_note_response.py index 3a941a17d..f255c3682 100644 --- a/tests/runtime/test_deleted_note_response.py +++ b/tests/runtime/test_deleted_note_response.py @@ -8,6 +8,7 @@ RuntimeDeletedNoteResponse, RuntimePendingNoteFileDelete, plan_accepted_note_delete_change, + runtime_deleted_note_permalink, ) from basic_memory.runtime.storage import RUNTIME_MARKDOWN_CONTENT_TYPE @@ -27,13 +28,13 @@ class _DeletedFileEntity: title: object | None permalink: object | None file_path: str - checksum: object | None + checksum: str | None content_type: str = RUNTIME_MARKDOWN_CONTENT_TYPE @dataclass(frozen=True, slots=True) class _DeletedNoteContent: - file_checksum: object | None + file_checksum: str | None def test_runtime_deleted_note_response_builds_pending_file_delete_payload() -> None: @@ -80,6 +81,13 @@ def test_runtime_deleted_note_response_uses_file_path_when_permalink_is_missing( } +def test_runtime_deleted_note_permalink_requires_a_usable_fallback_path() -> None: + # A markdown entity with neither a permalink nor a real file path has no + # stable identity to publish in the delete payload. + with pytest.raises(RuntimeError, match="missing permalink"): + runtime_deleted_note_permalink(None, file_path=" ") + + @pytest.mark.parametrize( ("entity", "message"), [ diff --git a/tests/runtime/test_note_materialization_request_matching.py b/tests/runtime/test_note_materialization_request_matching.py index 0d3e3dc42..7cdb9b795 100644 --- a/tests/runtime/test_note_materialization_request_matching.py +++ b/tests/runtime/test_note_materialization_request_matching.py @@ -31,7 +31,15 @@ def _request( ) +def test_note_content_matches_materialization_request_accepts_matching_row(): + note_content = _NoteContentVersion(db_version=4, db_checksum="12345") + + assert note_content_matches_materialization_request(note_content, _request()) + + def test_note_content_matches_materialization_request_with_coerced_runtime_values(): + # Replayed job payloads can deliver a string db_version / non-str checksum; + # matching coerces instead of silently treating the row as stale. note_content = _NoteContentVersion(db_version="4", db_checksum=12345) assert note_content_matches_materialization_request(note_content, _request()) diff --git a/tests/runtime/test_pending_note_materialization.py b/tests/runtime/test_pending_note_materialization.py index f74a2508d..6c7b25a99 100644 --- a/tests/runtime/test_pending_note_materialization.py +++ b/tests/runtime/test_pending_note_materialization.py @@ -55,6 +55,23 @@ def test_plan_pending_note_materialization_uses_fallback_source_when_missing() - ) +def test_plan_pending_note_materialization_coerces_replayed_payload_values() -> None: + materialization = plan_pending_note_materialization( + project_id=7, + entity_id=42, + note_content=_NoteContentState( + db_version="4", + db_checksum="db-checksum", + last_source=None, + ), + fallback_source="api", + ) + + assert materialization.db_version == 4 + assert materialization.db_checksum == "db-checksum" + assert materialization.source == "api" + + def test_plan_pending_note_materialization_prefers_note_source() -> None: materialization = plan_pending_note_materialization( project_id=7, diff --git a/tests/runtime/test_previous_materialized_note_file_delete.py b/tests/runtime/test_previous_materialized_note_file_delete.py index 6635e447c..19c54324f 100644 --- a/tests/runtime/test_previous_materialized_note_file_delete.py +++ b/tests/runtime/test_previous_materialized_note_file_delete.py @@ -7,7 +7,7 @@ def test_plan_previous_materialized_note_file_delete_uses_note_content_file_checksum(): - note_content = SimpleNamespace(file_checksum=12345) + note_content = SimpleNamespace(file_checksum="old-file-checksum") cleanup = plan_previous_materialized_note_file_delete( project_id=7, @@ -21,7 +21,7 @@ def test_plan_previous_materialized_note_file_delete_uses_note_content_file_chec project_id=7, entity_id=42, file_path="notes/old.md", - file_checksum="12345", + file_checksum="old-file-checksum", # The accepted destination rides along so the local delete adapter can # skip a case-only rename that aliases old and new on disk (P0 guard). live_file_path="notes/new.md", diff --git a/tests/runtime/test_runtime_job_payloads.py b/tests/runtime/test_runtime_job_payloads.py index e50cdfd76..875124e49 100644 --- a/tests/runtime/test_runtime_job_payloads.py +++ b/tests/runtime/test_runtime_job_payloads.py @@ -1,8 +1,5 @@ """Tests for portable runtime worker payload boundaries.""" -from collections.abc import Mapping -from dataclasses import dataclass, field -from datetime import timedelta from uuid import UUID import pytest @@ -11,55 +8,14 @@ from basic_memory.runtime.job_payloads import ( DELETE_NOTE_FILE_ENTRYPOINT, MATERIALIZE_NOTE_FILE_ENTRYPOINT, - RuntimeJobPayloadSerializer, - RuntimeJobPayloadSource, RuntimeNoteFileDeleteJobPayload, RuntimeNoteMaterializationJobPayload, - RuntimePayloadJobEnqueuer, - enqueue_runtime_job_payload, ) from basic_memory.runtime.jobs import RuntimeJobRequest from basic_memory.runtime.note_content import RuntimeNoteMaterializationJobRequest from basic_memory.runtime.note_object_metadata import NOTE_OBJECT_ACTOR_KIND_MCP_CLIENT -@dataclass(slots=True) -class FakeJobRuntime: - """Runtime double that records the concrete queue request it receives.""" - - job_id: str = "job-1" - requests: list[RuntimeJobRequest] = field(default_factory=list) - - async def enqueue(self, request: RuntimeJobRequest) -> str: - self.requests.append(request) - return self.job_id - - -class FakeRuntimeJobPayload: - """Payload double that owns concrete runtime request construction.""" - - def __init__(self, request: RuntimeJobRequest) -> None: - self.request = request - self.headers: Mapping[str, str] | None = None - - def runtime_job_request( - self, - *, - headers: Mapping[str, str] | None = None, - ) -> RuntimeJobRequest: - self.headers = headers - return self.request - - -@dataclass(frozen=True, slots=True) -class NoteFileDeletePayloadSerializer: - def serialize( - self, - request: RuntimeNoteFileDeleteJobRequest, - ) -> RuntimeNoteFileDeleteJobPayload: - return RuntimeNoteFileDeleteJobPayload.from_runtime_request(request) - - def test_runtime_note_file_delete_job_payload_round_trips_runtime_request() -> None: """The Pydantic worker payload preserves the queue-neutral delete request.""" runtime_request = RuntimeNoteFileDeleteJobRequest( @@ -99,75 +55,6 @@ def test_runtime_note_file_delete_job_payload_builds_runtime_queue_request() -> ) -@pytest.mark.asyncio -async def test_runtime_payload_job_enqueuer_validates_serializes_and_queues() -> None: - """The typed enqueuer builds the concrete job request without queue-specific code.""" - runtime_request = RuntimeNoteFileDeleteJobRequest( - project_id=101, - entity_id=42, - file_path="notes/a.md", - file_checksum="file-sum", - ) - payload = RuntimeNoteFileDeleteJobPayload.from_runtime_request(runtime_request) - execute_after = timedelta(seconds=5) - runtime = FakeJobRuntime(job_id="job-42") - payload_serializer: RuntimeJobPayloadSerializer[RuntimeNoteFileDeleteJobRequest] = ( - NoteFileDeletePayloadSerializer() - ) - enqueuer = RuntimePayloadJobEnqueuer( - runtime=runtime, - entrypoint="delete_note_file", - payload_serializer=payload_serializer, - ) - - job_id = await enqueuer.enqueue( - runtime_request, - headers={"source": "test"}, - priority=3, - execute_after=execute_after, - ) - - assert job_id == "job-42" - assert runtime.requests == [ - RuntimeJobRequest( - entrypoint="delete_note_file", - payload=payload.model_dump_json().encode("utf-8"), - priority=3, - execute_after=execute_after, - dedupe_key=runtime_request.dedupe_key(), - headers={ - "source": "test", - "project_id": str(runtime_request.project_id), - }, - ) - ] - - -@pytest.mark.asyncio -async def test_enqueue_runtime_job_payload_uses_payload_owned_request_builder() -> None: - """Queueable payloads keep special request semantics while adapters stay generic.""" - request = RuntimeJobRequest( - entrypoint="custom_entrypoint", - payload=b"{}", - dedupe_key="custom-dedupe", - headers={"origin": "custom"}, - execute_after=timedelta(seconds=10), - ) - payload_source: RuntimeJobPayloadSource = FakeRuntimeJobPayload(request) - runtime = FakeJobRuntime(job_id="job-99") - - job_id = await enqueue_runtime_job_payload( - runtime, - payload_source, - headers={"source": "test"}, - ) - - assert job_id == "job-99" - assert isinstance(payload_source, FakeRuntimeJobPayload) - assert payload_source.headers == {"source": "test"} - assert runtime.requests == [request] - - def test_runtime_note_materialization_job_payload_round_trips_runtime_request() -> None: """The Pydantic worker payload preserves the queue-neutral materialization request.""" runtime_request = RuntimeNoteMaterializationJobRequest( diff --git a/tests/test_runtime.py b/tests/test_runtime.py index c4be71d71..9b303b56f 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -92,7 +92,6 @@ RuntimeJobCounts, RuntimeStorageEventOperation, RuntimeStorageEventOperationKind, - RuntimeStorageEventProcessingResult, RuntimeStorageEventProjectBatch, RuntimeStorageEventRoutingPlan, RuntimeStorageEventSkipReason, @@ -755,28 +754,19 @@ def test_runtime_workflow_status_helpers_match_job_status_vocabulary(self): assert parse_runtime_workflow_id("not-a-workflow-id") is None def test_runtime_job_counts_are_immutable_accumulators(self): - result = RuntimeJobCounts().with_processed(2).with_failed().add(RuntimeJobCounts(skipped=3)) - - assert result.as_dict() == {"processed": 2, "failed": 1, "skipped": 3} - - def test_runtime_storage_event_processing_result_wraps_counts_for_internal_handoffs(self): result = ( - RuntimeStorageEventProcessingResult.empty() + RuntimeJobCounts() .with_processed(2) .with_failed() - .add(RuntimeStorageEventProcessingResult.from_counts(skipped=3)) + .with_skipped(2) + .add(RuntimeJobCounts(skipped=1)) ) - assert result.counts == RuntimeJobCounts(processed=2, failed=1, skipped=3) + assert result == RuntimeJobCounts(processed=2, failed=1, skipped=3) assert result.as_dict() == {"processed": 2, "failed": 1, "skipped": 3} - assert result.add_counts(RuntimeJobCounts(processed=4)).as_dict() == { - "processed": 6, - "failed": 1, - "skipped": 3, - } with pytest.raises(FrozenInstanceError): - setattr(result, "counts", RuntimeJobCounts()) + setattr(result, "processed", 0) def test_runtime_deleted_note_reference_validates_live_update_identity(self): reference = RuntimeDeletedNoteReference.from_entity( @@ -806,6 +796,20 @@ def test_runtime_deleted_note_reference_validates_live_update_identity(self): file_path="notes/deleted.md", ) + # A markdown entity indexed without a permalink still needs a stable + # live-update identity, so the file path stands in for it. + fallback_reference = RuntimeDeletedNoteReference.from_entity( + FakeDeletedNoteEntity( + id=1, + external_id="note-1", + title="Deleted note", + permalink=None, + ), + file_path="notes/deleted.md", + ) + + assert fallback_reference.permalink == "notes/deleted.md" + def test_runtime_external_file_delete_plan_distinguishes_adapter_work(self): entity = FakeDeletedNoteEntity( id=7,